diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d42b58784..2b21adc101 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,10 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re ### Fixed +### Added + +- [#5220](https://github.com/thanos-io/thanos/pull/5220) Query Frontend: Add `--query-frontend.forward-header` flag, forward headers to downstream querier. + ### Changed - [#5205](https://github.com/thanos-io/thanos/pull/5205) Rule: Add ruler labels as external labels in stateless ruler mode. diff --git a/cmd/thanos/query_frontend.go b/cmd/thanos/query_frontend.go index 007e44a2e5..b6a85d3a5d 100644 --- a/cmd/thanos/query_frontend.go +++ b/cmd/thanos/query_frontend.go @@ -134,6 +134,8 @@ func registerQueryFrontend(app *extkingpin.App) { "If multiple headers match the request, the first matching arg specified will take precedence. "+ "If no headers match 'anonymous' will be used.").PlaceHolder("").StringsVar(&cfg.orgIdHeaders) + cmd.Flag("query-frontend.forward-header", "List of headers forwarded by the query-frontend to downstream queriers, default is empty").PlaceHolder("").StringsVar(&cfg.ForwardHeaders) + cmd.Flag("log.request.decision", "Deprecation Warning - This flag would be soon deprecated, and replaced with `request.logging-config`. Request Logging for logging the start and end of requests. By default this flag is disabled. LogFinishCall : Logs the finish call of the requests. LogStartAndFinishCall : Logs the start and finish call of the requests. NoLogCall : Disable request logging.").Default("").EnumVar(&cfg.RequestLoggingDecision, "NoLogCall", "LogFinishCall", "LogStartAndFinishCall", "") reqLogConfig := extkingpin.RegisterRequestLoggingFlags(cmd) diff --git a/docs/components/query-frontend.md b/docs/components/query-frontend.md index 41959b558d..5f6d997cf5 100644 --- a/docs/components/query-frontend.md +++ b/docs/components/query-frontend.md @@ -148,6 +148,19 @@ Keys which denote a duration are strings that can end with `s` or `m` to indicat You can find the default values [here](https://github.com/thanos-io/thanos/blob/55cb8ca38b3539381dc6a781e637df15c694e50a/pkg/exthttp/transport.go#L12-L27). +## Forward Headers to Downstream Queriers + +`--query-frontend.forward-header` flag provides list of request headers forwarded by query frontend to downstream queriers. + +If downstream queriers need basic authentication to access, we can run query-frontend: + +```bash +thanos query-frontend \ + --http-address "0.0.0.0:9090" \ + --query-frontend.forward-header "Authorization" + --query-frontend.downstream-url=":" +``` + ## Flags ```$ mdox-exec="thanos query-frontend --help" @@ -233,6 +246,9 @@ Flags: --query-frontend.downstream-url="http://localhost:9090" URL of downstream Prometheus Query compatible API. + --query-frontend.forward-header= ... + List of headers forwarded by the query-frontend + to downstream queriers, default is empty --query-frontend.log-queries-longer-than=0 Log queries that are slower than the specified duration. Set to 0 to disable. Set to < 0 to diff --git a/pkg/queryfrontend/config.go b/pkg/queryfrontend/config.go index 7d49bc53f5..d0b24596b8 100644 --- a/pkg/queryfrontend/config.go +++ b/pkg/queryfrontend/config.go @@ -203,6 +203,7 @@ type Config struct { CacheCompression string RequestLoggingDecision string DownstreamURL string + ForwardHeaders []string } // QueryRangeConfig holds the config for query range tripperware. diff --git a/pkg/queryfrontend/labels_codec.go b/pkg/queryfrontend/labels_codec.go index 8596fc3ac4..342737556b 100644 --- a/pkg/queryfrontend/labels_codec.go +++ b/pkg/queryfrontend/labels_codec.go @@ -107,7 +107,7 @@ func (c labelsCodec) MergeResponse(responses ...queryrange.Response) (queryrange } } -func (c labelsCodec) DecodeRequest(_ context.Context, r *http.Request, _ []string) (queryrange.Request, error) { +func (c labelsCodec) DecodeRequest(_ context.Context, r *http.Request, forwardHeaders []string) (queryrange.Request, error) { if err := r.ParseForm(); err != nil { return nil, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) } @@ -118,9 +118,9 @@ func (c labelsCodec) DecodeRequest(_ context.Context, r *http.Request, _ []strin ) switch op := getOperation(r); op { case labelNamesOp, labelValuesOp: - req, err = c.parseLabelsRequest(r, op) + req, err = c.parseLabelsRequest(r, op, forwardHeaders) case seriesOp: - req, err = c.parseSeriesRequest(r) + req, err = c.parseSeriesRequest(r, forwardHeaders) } if err != nil { return nil, err @@ -167,6 +167,12 @@ func (c labelsCodec) EncodeRequest(ctx context.Context, r queryrange.Request) (* req.Header.Set("Content-Type", "application/x-www-form-urlencoded") } + for _, hv := range thanosReq.Headers { + for _, v := range hv.Values { + req.Header.Add(hv.Name, v) + } + } + case *ThanosSeriesRequest: var params = url.Values{ "start": []string{encodeTime(thanosReq.Start)}, @@ -187,6 +193,11 @@ func (c labelsCodec) EncodeRequest(ctx context.Context, r queryrange.Request) (* return nil, httpgrpc.Errorf(http.StatusBadRequest, "error creating request: %s", err.Error()) } req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + for _, hv := range thanosReq.Headers { + for _, v := range hv.Values { + req.Header.Add(hv.Name, v) + } + } default: return nil, httpgrpc.Errorf(http.StatusInternalServerError, "invalid request format") @@ -271,7 +282,7 @@ func (c labelsCodec) EncodeResponse(ctx context.Context, res queryrange.Response return &resp, nil } -func (c labelsCodec) parseLabelsRequest(r *http.Request, op string) (queryrange.Request, error) { +func (c labelsCodec) parseLabelsRequest(r *http.Request, op string, forwardHeaders []string) (queryrange.Request, error) { var ( result ThanosLabelsRequest err error @@ -312,10 +323,20 @@ func (c labelsCodec) parseLabelsRequest(r *http.Request, op string) (queryrange. } } + // Include the specified headers from http request in prometheusRequest. + for _, header := range forwardHeaders { + for h, hv := range r.Header { + if strings.EqualFold(h, header) { + result.Headers = append(result.Headers, &RequestHeader{Name: h, Values: hv}) + break + } + } + } + return &result, nil } -func (c labelsCodec) parseSeriesRequest(r *http.Request) (queryrange.Request, error) { +func (c labelsCodec) parseSeriesRequest(r *http.Request, forwardHeaders []string) (queryrange.Request, error) { var ( result ThanosSeriesRequest err error @@ -358,6 +379,16 @@ func (c labelsCodec) parseSeriesRequest(r *http.Request) (queryrange.Request, er } } + // Include the specified headers from http request in prometheusRequest. + for _, header := range forwardHeaders { + for h, hv := range r.Header { + if strings.EqualFold(h, header) { + result.Headers = append(result.Headers, &RequestHeader{Name: h, Values: hv}) + break + } + } + } + return &result, nil } diff --git a/pkg/queryfrontend/queryrange_codec.go b/pkg/queryfrontend/queryrange_codec.go index fb524a29ce..a0366e5d62 100644 --- a/pkg/queryfrontend/queryrange_codec.go +++ b/pkg/queryfrontend/queryrange_codec.go @@ -53,7 +53,7 @@ func NewThanosQueryRangeCodec(partialResponse bool) *queryRangeCodec { } } -func (c queryRangeCodec) DecodeRequest(_ context.Context, r *http.Request, _ []string) (queryrange.Request, error) { +func (c queryRangeCodec) DecodeRequest(_ context.Context, r *http.Request, forwardHeaders []string) (queryrange.Request, error) { var ( result ThanosQueryRangeRequest err error @@ -126,6 +126,14 @@ func (c queryRangeCodec) DecodeRequest(_ context.Context, r *http.Request, _ []s } } + for _, header := range forwardHeaders { + for h, hv := range r.Header { + if strings.EqualFold(h, header) { + result.Headers = append(result.Headers, &RequestHeader{Name: h, Values: hv}) + break + } + } + } return &result, nil } @@ -161,7 +169,11 @@ func (c queryRangeCodec) EncodeRequest(ctx context.Context, r queryrange.Request return nil, httpgrpc.Errorf(http.StatusBadRequest, "error creating request: %s", err.Error()) } req.Header.Set("Content-Type", "application/x-www-form-urlencoded") - + for _, hv := range thanosReq.Headers { + for _, v := range hv.Values { + req.Header.Add(hv.Name, v) + } + } return req.WithContext(ctx), nil } diff --git a/pkg/queryfrontend/request.go b/pkg/queryfrontend/request.go index bb11b680d6..0164eac91f 100644 --- a/pkg/queryfrontend/request.go +++ b/pkg/queryfrontend/request.go @@ -19,6 +19,11 @@ type ThanosRequest interface { GetStoreMatchers() [][]*labels.Matcher } +type RequestHeader struct { + Name string + Values []string +} + type ThanosQueryRangeRequest struct { Path string Start int64 @@ -33,6 +38,7 @@ type ThanosQueryRangeRequest struct { ReplicaLabels []string StoreMatchers [][]*labels.Matcher CachingOptions queryrange.CachingOptions + Headers []*RequestHeader } // GetStart returns the start timestamp of the request in milliseconds. @@ -107,6 +113,7 @@ type ThanosLabelsRequest struct { StoreMatchers [][]*labels.Matcher PartialResponse bool CachingOptions queryrange.CachingOptions + Headers []*RequestHeader } // GetStart returns the start timestamp of the request in milliseconds. @@ -178,6 +185,7 @@ type ThanosSeriesRequest struct { Matchers [][]*labels.Matcher StoreMatchers [][]*labels.Matcher CachingOptions queryrange.CachingOptions + Headers []*RequestHeader } // GetStart returns the start timestamp of the request in milliseconds. diff --git a/pkg/queryfrontend/roundtrip.go b/pkg/queryfrontend/roundtrip.go index 570d315339..405e93c293 100644 --- a/pkg/queryfrontend/roundtrip.go +++ b/pkg/queryfrontend/roundtrip.go @@ -51,13 +51,13 @@ func NewTripperware(config Config, reg prometheus.Registerer, logger log.Logger) labelsCodec := NewThanosLabelsCodec(config.LabelsConfig.PartialResponseStrategy, config.DefaultTimeRange) queryRangeTripperware, err := newQueryRangeTripperware(config.QueryRangeConfig, queryRangeLimits, queryRangeCodec, - prometheus.WrapRegistererWith(prometheus.Labels{"tripperware": "query_range"}, reg), logger) + prometheus.WrapRegistererWith(prometheus.Labels{"tripperware": "query_range"}, reg), logger, config.ForwardHeaders) if err != nil { return nil, err } labelsTripperware, err := newLabelsTripperware(config.LabelsConfig, labelsLimits, labelsCodec, - prometheus.WrapRegistererWith(prometheus.Labels{"tripperware": "labels"}, reg), logger) + prometheus.WrapRegistererWith(prometheus.Labels{"tripperware": "labels"}, reg), logger, config.ForwardHeaders) if err != nil { return nil, err } @@ -138,6 +138,7 @@ func newQueryRangeTripperware( codec *queryRangeCodec, reg prometheus.Registerer, logger log.Logger, + forwardHeaders []string, ) (queryrange.Tripperware, error) { queryRangeMiddleware := []queryrange.Middleware{queryrange.NewLimitsMiddleware(limits)} m := queryrange.NewInstrumentMiddlewareMetrics(reg) @@ -203,7 +204,7 @@ func newQueryRangeTripperware( } return func(next http.RoundTripper) http.RoundTripper { - rt := queryrange.NewRoundTripper(next, codec, nil, queryRangeMiddleware...) + rt := queryrange.NewRoundTripper(next, codec, forwardHeaders, queryRangeMiddleware...) return queryrange.RoundTripFunc(func(r *http.Request) (*http.Response, error) { return rt.RoundTrip(r) }) @@ -218,6 +219,7 @@ func newLabelsTripperware( codec *labelsCodec, reg prometheus.Registerer, logger log.Logger, + forwardHeaders []string, ) (queryrange.Tripperware, error) { labelsMiddleware := []queryrange.Middleware{} m := queryrange.NewInstrumentMiddlewareMetrics(reg) @@ -265,7 +267,7 @@ func newLabelsTripperware( ) } return func(next http.RoundTripper) http.RoundTripper { - rt := queryrange.NewRoundTripper(next, codec, nil, labelsMiddleware...) + rt := queryrange.NewRoundTripper(next, codec, forwardHeaders, labelsMiddleware...) return queryrange.RoundTripFunc(func(r *http.Request) (*http.Response, error) { return rt.RoundTrip(r) })