diff --git a/prometheus/promhttp/instrument_client.go b/prometheus/promhttp/instrument_client.go index 861b4d21c..097aff2df 100644 --- a/prometheus/promhttp/instrument_client.go +++ b/prometheus/promhttp/instrument_client.go @@ -38,11 +38,11 @@ func (rt RoundTripperFunc) RoundTrip(r *http.Request) (*http.Response, error) { // // See the example for ExampleInstrumentRoundTripperDuration for example usage. func InstrumentRoundTripperInFlight(gauge prometheus.Gauge, next http.RoundTripper) RoundTripperFunc { - return RoundTripperFunc(func(r *http.Request) (*http.Response, error) { + return func(r *http.Request) (*http.Response, error) { gauge.Inc() defer gauge.Dec() return next.RoundTrip(r) - }) + } } // InstrumentRoundTripperCounter is a middleware that wraps the provided @@ -59,22 +59,29 @@ func InstrumentRoundTripperInFlight(gauge prometheus.Gauge, next http.RoundTripp // If the wrapped RoundTripper panics or returns a non-nil error, the Counter // is not incremented. // +// Use with WithExemplarFromContext to instrument the exemplars on the counter of requests. +// // See the example for ExampleInstrumentRoundTripperDuration for example usage. func InstrumentRoundTripperCounter(counter *prometheus.CounterVec, next http.RoundTripper, opts ...Option) RoundTripperFunc { - rtOpts := &option{} + rtOpts := defaultOptions() for _, o := range opts { - o(rtOpts) + o.apply(rtOpts) } code, method := checkLabels(counter) - return RoundTripperFunc(func(r *http.Request) (*http.Response, error) { + return func(r *http.Request) (*http.Response, error) { resp, err := next.RoundTrip(r) if err == nil { + exemplarAdd( + counter.With(labels(code, method, r.Method, resp.StatusCode, rtOpts.extraMethods...)), + 1, + rtOpts.getExemplarFn(r.Context()), + ) counter.With(labels(code, method, r.Method, resp.StatusCode, rtOpts.extraMethods...)).Inc() } return resp, err - }) + } } // InstrumentRoundTripperDuration is a middleware that wraps the provided @@ -94,24 +101,30 @@ func InstrumentRoundTripperCounter(counter *prometheus.CounterVec, next http.Rou // If the wrapped RoundTripper panics or returns a non-nil error, no values are // reported. // +// Use with WithExemplarFromContext to instrument the exemplars on the duration histograms. +// // Note that this method is only guaranteed to never observe negative durations // if used with Go1.9+. func InstrumentRoundTripperDuration(obs prometheus.ObserverVec, next http.RoundTripper, opts ...Option) RoundTripperFunc { - rtOpts := &option{} + rtOpts := defaultOptions() for _, o := range opts { - o(rtOpts) + o.apply(rtOpts) } code, method := checkLabels(obs) - return RoundTripperFunc(func(r *http.Request) (*http.Response, error) { + return func(r *http.Request) (*http.Response, error) { start := time.Now() resp, err := next.RoundTrip(r) if err == nil { - obs.With(labels(code, method, r.Method, resp.StatusCode, rtOpts.extraMethods...)).Observe(time.Since(start).Seconds()) + exemplarObserve( + obs.With(labels(code, method, r.Method, resp.StatusCode, rtOpts.extraMethods...)), + time.Since(start).Seconds(), + rtOpts.getExemplarFn(r.Context()), + ) } return resp, err - }) + } } // InstrumentTrace is used to offer flexibility in instrumenting the available @@ -149,7 +162,7 @@ type InstrumentTrace struct { // // See the example for ExampleInstrumentRoundTripperDuration for example usage. func InstrumentRoundTripperTrace(it *InstrumentTrace, next http.RoundTripper) RoundTripperFunc { - return RoundTripperFunc(func(r *http.Request) (*http.Response, error) { + return func(r *http.Request) (*http.Response, error) { start := time.Now() trace := &httptrace.ClientTrace{ @@ -231,5 +244,5 @@ func InstrumentRoundTripperTrace(it *InstrumentTrace, next http.RoundTripper) Ro r = r.WithContext(httptrace.WithClientTrace(r.Context(), trace)) return next.RoundTrip(r) - }) + } } diff --git a/prometheus/promhttp/instrument_client_test.go b/prometheus/promhttp/instrument_client_test.go index aab8dbe8d..c5edeb280 100644 --- a/prometheus/promhttp/instrument_client_test.go +++ b/prometheus/promhttp/instrument_client_test.go @@ -18,14 +18,18 @@ import ( "log" "net/http" "net/http/httptest" + "reflect" + "sort" "strings" "testing" "time" "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "google.golang.org/protobuf/proto" ) -func makeInstrumentedClient() (*http.Client, *prometheus.Registry) { +func makeInstrumentedClient(opts ...Option) (*http.Client, *prometheus.Registry) { client := http.DefaultClient client.Timeout = 1 * time.Second @@ -91,13 +95,91 @@ func makeInstrumentedClient() (*http.Client, *prometheus.Registry) { client.Transport = InstrumentRoundTripperInFlight(inFlightGauge, InstrumentRoundTripperCounter(counter, InstrumentRoundTripperTrace(trace, - InstrumentRoundTripperDuration(histVec, http.DefaultTransport), + InstrumentRoundTripperDuration(histVec, http.DefaultTransport, opts...), ), - ), + opts...), ) return client, reg } +func labelsToLabelPair(l prometheus.Labels) []*dto.LabelPair { + ret := make([]*dto.LabelPair, 0, len(l)) + for k, v := range l { + ret = append(ret, &dto.LabelPair{Name: proto.String(k), Value: proto.String(v)}) + } + sort.Slice(ret, func(i, j int) bool { + return *ret[i].Name < *ret[j].Name + }) + return ret +} + +func assetMetricAndExemplars( + t *testing.T, + reg *prometheus.Registry, + expectedNumMetrics int, + expectedExemplar []*dto.LabelPair, +) { + t.Helper() + + mfs, err := reg.Gather() + if err != nil { + t.Fatal(err) + } + if want, got := expectedNumMetrics, len(mfs); want != got { + t.Fatalf("unexpected number of metric families gathered, want %d, got %d", want, got) + } + + for _, mf := range mfs { + if len(mf.Metric) == 0 { + t.Errorf("metric family %s must not be empty", mf.GetName()) + } + for _, m := range mf.GetMetric() { + if c := m.GetCounter(); c != nil { + if len(expectedExemplar) == 0 { + if c.Exemplar != nil { + t.Errorf("expected no exemplar on the counter %v%v, got %v", mf.GetName(), m.Label, c.Exemplar.String()) + } + continue + } + + if c.Exemplar == nil { + t.Errorf("expected exemplar %v on the counter %v%v, got none", expectedExemplar, mf.GetName(), m.Label) + continue + } + if got := c.Exemplar.Label; !reflect.DeepEqual(expectedExemplar, got) { + t.Errorf("expected exemplar %v on the counter %v%v, got %v", expectedExemplar, mf.GetName(), m.Label, got) + } + continue + } + if h := m.GetHistogram(); h != nil { + found := false + for _, b := range h.GetBucket() { + if len(expectedExemplar) == 0 { + if b.Exemplar != nil { + t.Errorf("expected no exemplar on histogram %v%v bkt %v, got %v", mf.GetName(), m.Label, b.GetUpperBound(), b.Exemplar.String()) + } + continue + } + + if b.Exemplar == nil { + continue + } + if got := b.Exemplar.Label; !reflect.DeepEqual(expectedExemplar, got) { + t.Errorf("expected exemplar %v on the histogram %v%v on bkt %v, got %v", expectedExemplar, mf.GetName(), m.Label, b.GetUpperBound(), got) + continue + } + found = true + break + } + + if len(expectedExemplar) > 0 && !found { + t.Errorf("expected exemplar %v on at least one bucket of the histogram %v%v, got none", expectedExemplar, mf.GetName(), m.Label) + } + } + } + } +} + func TestClientMiddlewareAPI(t *testing.T) { client, reg := makeInstrumentedClient() backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -111,21 +193,28 @@ func TestClientMiddlewareAPI(t *testing.T) { } defer resp.Body.Close() - mfs, err := reg.Gather() + assetMetricAndExemplars(t, reg, 3, nil) +} + +func TestClientMiddlewareAPI_WithExemplars(t *testing.T) { + exemplar := prometheus.Labels{"traceID": "example situation observed by this metric"} + + client, reg := makeInstrumentedClient(WithExemplarFromContext(func(_ context.Context) prometheus.Labels { return exemplar })) + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer backend.Close() + + resp, err := client.Get(backend.URL) if err != nil { t.Fatal(err) } - if want, got := 3, len(mfs); want != got { - t.Fatalf("unexpected number of metric families gathered, want %d, got %d", want, got) - } - for _, mf := range mfs { - if len(mf.Metric) == 0 { - t.Errorf("metric family %s must not be empty", mf.GetName()) - } - } + defer resp.Body.Close() + + assetMetricAndExemplars(t, reg, 3, labelsToLabelPair(exemplar)) } -func TestClientMiddlewareAPIWithRequestContext(t *testing.T) { +func TestClientMiddlewareAPI_WithRequestContext(t *testing.T) { client, reg := makeInstrumentedClient() backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) diff --git a/prometheus/promhttp/instrument_server.go b/prometheus/promhttp/instrument_server.go index ad7a6e598..b35d0939a 100644 --- a/prometheus/promhttp/instrument_server.go +++ b/prometheus/promhttp/instrument_server.go @@ -14,7 +14,6 @@ package promhttp import ( - "context" "errors" "net/http" "strconv" @@ -79,9 +78,9 @@ func InstrumentHandlerInFlight(g prometheus.Gauge, next http.Handler) http.Handl // Note that this method is only guaranteed to never observe negative durations // if used with Go1.9+. func InstrumentHandlerDuration(obs prometheus.ObserverVec, next http.Handler, opts ...Option) http.HandlerFunc { - mwOpts := &option{getExemplarFn: func(ctx context.Context) prometheus.Labels { return nil }} + hOpts := defaultOptions() for _, o := range opts { - o(mwOpts) + o.apply(hOpts) } code, method := checkLabels(obs) @@ -93,9 +92,9 @@ func InstrumentHandlerDuration(obs prometheus.ObserverVec, next http.Handler, op next.ServeHTTP(d, r) exemplarObserve( - obs.With(labels(code, method, r.Method, d.Status(), mwOpts.extraMethods...)), + obs.With(labels(code, method, r.Method, d.Status(), hOpts.extraMethods...)), time.Since(now).Seconds(), - mwOpts.getExemplarFn(r.Context()), + hOpts.getExemplarFn(r.Context()), ) } } @@ -105,9 +104,9 @@ func InstrumentHandlerDuration(obs prometheus.ObserverVec, next http.Handler, op next.ServeHTTP(w, r) exemplarObserve( - obs.With(labels(code, method, r.Method, 0, mwOpts.extraMethods...)), + obs.With(labels(code, method, r.Method, 0, hOpts.extraMethods...)), time.Since(now).Seconds(), - mwOpts.getExemplarFn(r.Context()), + hOpts.getExemplarFn(r.Context()), ) } } @@ -130,9 +129,9 @@ func InstrumentHandlerDuration(obs prometheus.ObserverVec, next http.Handler, op // // See the example for InstrumentHandlerDuration for example usage. func InstrumentHandlerCounter(counter *prometheus.CounterVec, next http.Handler, opts ...Option) http.HandlerFunc { - mwOpts := &option{getExemplarFn: func(ctx context.Context) prometheus.Labels { return nil }} + hOpts := defaultOptions() for _, o := range opts { - o(mwOpts) + o.apply(hOpts) } code, method := checkLabels(counter) @@ -143,9 +142,9 @@ func InstrumentHandlerCounter(counter *prometheus.CounterVec, next http.Handler, next.ServeHTTP(d, r) exemplarAdd( - counter.With(labels(code, method, r.Method, d.Status(), mwOpts.extraMethods...)), + counter.With(labels(code, method, r.Method, d.Status(), hOpts.extraMethods...)), 1, - mwOpts.getExemplarFn(r.Context()), + hOpts.getExemplarFn(r.Context()), ) } } @@ -153,9 +152,9 @@ func InstrumentHandlerCounter(counter *prometheus.CounterVec, next http.Handler, return func(w http.ResponseWriter, r *http.Request) { next.ServeHTTP(w, r) exemplarAdd( - counter.With(labels(code, method, r.Method, 0, mwOpts.extraMethods...)), + counter.With(labels(code, method, r.Method, 0, hOpts.extraMethods...)), 1, - mwOpts.getExemplarFn(r.Context()), + hOpts.getExemplarFn(r.Context()), ) } } @@ -183,9 +182,9 @@ func InstrumentHandlerCounter(counter *prometheus.CounterVec, next http.Handler, // // See the example for InstrumentHandlerDuration for example usage. func InstrumentHandlerTimeToWriteHeader(obs prometheus.ObserverVec, next http.Handler, opts ...Option) http.HandlerFunc { - mwOpts := &option{getExemplarFn: func(ctx context.Context) prometheus.Labels { return nil }} + hOpts := defaultOptions() for _, o := range opts { - o(mwOpts) + o.apply(hOpts) } code, method := checkLabels(obs) @@ -194,9 +193,9 @@ func InstrumentHandlerTimeToWriteHeader(obs prometheus.ObserverVec, next http.Ha now := time.Now() d := newDelegator(w, func(status int) { exemplarObserve( - obs.With(labels(code, method, r.Method, status, mwOpts.extraMethods...)), + obs.With(labels(code, method, r.Method, status, hOpts.extraMethods...)), time.Since(now).Seconds(), - mwOpts.getExemplarFn(r.Context()), + hOpts.getExemplarFn(r.Context()), ) }) next.ServeHTTP(d, r) @@ -223,26 +222,33 @@ func InstrumentHandlerTimeToWriteHeader(obs prometheus.ObserverVec, next http.Ha // // See the example for InstrumentHandlerDuration for example usage. func InstrumentHandlerRequestSize(obs prometheus.ObserverVec, next http.Handler, opts ...Option) http.HandlerFunc { - mwOpts := &option{} + hOpts := defaultOptions() for _, o := range opts { - o(mwOpts) + o.apply(hOpts) } code, method := checkLabels(obs) - if code { return func(w http.ResponseWriter, r *http.Request) { d := newDelegator(w, nil) next.ServeHTTP(d, r) size := computeApproximateRequestSize(r) - obs.With(labels(code, method, r.Method, d.Status(), mwOpts.extraMethods...)).Observe(float64(size)) + exemplarObserve( + obs.With(labels(code, method, r.Method, d.Status(), hOpts.extraMethods...)), + float64(size), + hOpts.getExemplarFn(r.Context()), + ) } } return func(w http.ResponseWriter, r *http.Request) { next.ServeHTTP(w, r) size := computeApproximateRequestSize(r) - obs.With(labels(code, method, r.Method, 0, mwOpts.extraMethods...)).Observe(float64(size)) + exemplarObserve( + obs.With(labels(code, method, r.Method, 0, hOpts.extraMethods...)), + float64(size), + hOpts.getExemplarFn(r.Context()), + ) } } @@ -266,9 +272,9 @@ func InstrumentHandlerRequestSize(obs prometheus.ObserverVec, next http.Handler, // // See the example for InstrumentHandlerDuration for example usage. func InstrumentHandlerResponseSize(obs prometheus.ObserverVec, next http.Handler, opts ...Option) http.Handler { - mwOpts := &option{} + hOpts := defaultOptions() for _, o := range opts { - o(mwOpts) + o.apply(hOpts) } code, method := checkLabels(obs) @@ -276,7 +282,11 @@ func InstrumentHandlerResponseSize(obs prometheus.ObserverVec, next http.Handler return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { d := newDelegator(w, nil) next.ServeHTTP(d, r) - obs.With(labels(code, method, r.Method, d.Status(), mwOpts.extraMethods...)).Observe(float64(d.Written())) + exemplarObserve( + obs.With(labels(code, method, r.Method, d.Status(), hOpts.extraMethods...)), + float64(d.Written()), + hOpts.getExemplarFn(r.Context()), + ) }) } @@ -285,7 +295,7 @@ func InstrumentHandlerResponseSize(obs prometheus.ObserverVec, next http.Handler // Collector does not have a Desc or has more than one Desc or its Desc is // invalid. It also panics if the Collector has any non-const, non-curried // labels that are not named "code" or "method". -func checkLabels(c prometheus.Collector) (code bool, method bool) { +func checkLabels(c prometheus.Collector) (code, method bool) { // TODO(beorn7): Remove this hacky way to check for instance labels // once Descriptors can have their dimensionality queried. var ( diff --git a/prometheus/promhttp/instrument_server_test.go b/prometheus/promhttp/instrument_server_test.go index fbd8fa916..421534875 100644 --- a/prometheus/promhttp/instrument_server_test.go +++ b/prometheus/promhttp/instrument_server_test.go @@ -14,6 +14,7 @@ package promhttp import ( + "context" "io" "log" "net/http" @@ -279,7 +280,7 @@ func TestLabels(t *testing.T) { ok: false, }, } - checkLabels := func(labels []string) (gotCode bool, gotMethod bool) { + checkLabels := func(labels []string) (gotCode, gotMethod bool) { for _, label := range labels { switch label { case "code": @@ -321,7 +322,7 @@ func TestLabels(t *testing.T) { } } -func TestMiddlewareAPI(t *testing.T) { +func makeInstrumentedHandler(handler http.HandlerFunc, opts ...Option) (http.Handler, *prometheus.Registry) { reg := prometheus.NewRegistry() inFlightGauge := prometheus.NewGauge(prometheus.GaugeOpts{ @@ -366,25 +367,43 @@ func TestMiddlewareAPI(t *testing.T) { []string{}, ) - handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte("OK")) - }) - reg.MustRegister(inFlightGauge, counter, histVec, responseSize, writeHeaderVec) - chain := InstrumentHandlerInFlight(inFlightGauge, + return InstrumentHandlerInFlight(inFlightGauge, InstrumentHandlerCounter(counter, InstrumentHandlerDuration(histVec, InstrumentHandlerTimeToWriteHeader(writeHeaderVec, - InstrumentHandlerResponseSize(responseSize, handler), - ), - ), - ), - ) + InstrumentHandlerResponseSize(responseSize, handler, opts...), + opts...), + opts...), + opts...), + ), reg +} + +func TestMiddlewareAPI(t *testing.T) { + chain, reg := makeInstrumentedHandler(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("OK")) + }) r, _ := http.NewRequest("GET", "www.example.com", nil) w := httptest.NewRecorder() chain.ServeHTTP(w, r) + + assetMetricAndExemplars(t, reg, 5, nil) +} + +func TestMiddlewareAPI_WithExemplars(t *testing.T) { + exemplar := prometheus.Labels{"traceID": "example situation observed by this metric"} + + chain, reg := makeInstrumentedHandler(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("OK")) + }, WithExemplarFromContext(func(_ context.Context) prometheus.Labels { return exemplar })) + + r, _ := http.NewRequest("GET", "www.example.com", nil) + w := httptest.NewRecorder() + chain.ServeHTTP(w, r) + + assetMetricAndExemplars(t, reg, 5, labelsToLabelPair(exemplar)) } func TestInstrumentTimeToFirstWrite(t *testing.T) { diff --git a/prometheus/promhttp/option.go b/prometheus/promhttp/option.go index 0c88f0298..c590d912c 100644 --- a/prometheus/promhttp/option.go +++ b/prometheus/promhttp/option.go @@ -19,29 +19,40 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -// Option are used to configure a middleware or round tripper. -type Option func(*option) +// Option are used to configure both handler (middleware) or round tripper. +type Option interface { + apply(*options) +} -type option struct { +// options store options for both a handler or round tripper. +type options struct { extraMethods []string - getExemplarFn func(ctx context.Context) prometheus.Labels + getExemplarFn func(requestCtx context.Context) prometheus.Labels +} + +func defaultOptions() *options { + return &options{getExemplarFn: func(ctx context.Context) prometheus.Labels { return nil }} } +type optionApplyFunc func(*options) + +func (o optionApplyFunc) apply(opt *options) { o(opt) } + // WithExtraMethods adds additional HTTP methods to the list of allowed methods. // See https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods for the default list. // // See the example for ExampleInstrumentHandlerWithExtraMethods for example usage. func WithExtraMethods(methods ...string) Option { - return func(o *option) { + return optionApplyFunc(func(o *options) { o.extraMethods = methods - } + }) } -// WithExemplarFromRequestContext adds allows to put a hook to all counter and histogram metrics. -// If the hook function returns non-nil labels, exemplars will be added for that request, otherwise metric will get instrumented -// without exemplar. -func WithExemplarFromRequestContext(getExemplarFn func(ctx context.Context) prometheus.Labels) Option { - return func(o *option) { +// WithExemplarFromContext adds allows to put a hook to all counter and histogram metrics. +// If the hook function returns non-nil labels, exemplars will be added for that request, otherwise metric +// will get instrumented without exemplar. +func WithExemplarFromContext(getExemplarFn func(requestCtx context.Context) prometheus.Labels) Option { + return optionApplyFunc(func(o *options) { o.getExemplarFn = getExemplarFn - } + }) }