Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add per_host option to metrics handler #6279

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
+201 −15
Open

Add per_host option to metrics handler #6279

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
368b156
Add per host config
hussam-almarzoq Apr 28, 2024
c025644
Pass host label when option is enabled
hussam-almarzoq Apr 28, 2024
6bf7874
Test per host enabled
hussam-almarzoq May 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
14 changes: 8 additions & 6 deletions caddyconfig/httpcaddyfile/serveroptions.go
View file
Open in desktop
Expand Up @@ -17,6 +17,7 @@ package httpcaddyfile
import (
"encoding/json"
"fmt"
"strconv"

"github.com/dustin/go-humanize"

Expand Down Expand Up @@ -238,13 +239,14 @@ func unmarshalCaddyfileServerOptions(d *caddyfile.Dispenser) (any, error) {
}

case "metrics":
if d.NextArg() {
return nil, d.ArgErr()
}
if nesting := d.Nesting(); d.NextBlock(nesting) {
return nil, d.ArgErr()
}
serverOpts.Metrics = new(caddyhttp.Metrics)
perHost := false
for nesting := d.Nesting(); d.NextBlock(nesting); {
if d.Val() == "per_host" && d.NextArg() {
perHost, _ = strconv.ParseBool(d.Val())
}
}
serverOpts.Metrics.PerHost = perHost

// TODO: DEPRECATED. (August 2022)
case "protocol":
Expand Down
24 changes: 19 additions & 5 deletions modules/caddyhttp/metrics.go
View file
Open in desktop
Expand Up @@ -15,7 +15,9 @@ import (

// Metrics configures metrics observations.
// EXPERIMENTAL and subject to change or removal.
type Metrics struct{}
type Metrics struct {
PerHost bool
}

var httpMetrics = struct {
init sync.Once
Expand All @@ -30,10 +32,13 @@ var httpMetrics = struct {
init: sync.Once{},
}

func initHTTPMetrics() {
func initHTTPMetrics(perHost bool) {
const ns, sub = "caddy", "http"

basicLabels := []string{"server", "handler"}
if perHost {
basicLabels = append(basicLabels, "host")
}
httpMetrics.requestInFlight = promauto.NewGaugeVec(prometheus.GaugeOpts{
Namespace: ns,
Subsystem: sub,
Expand All @@ -58,6 +63,9 @@ func initHTTPMetrics() {
sizeBuckets := prometheus.ExponentialBuckets(256, 4, 8)

httpLabels := []string{"server", "handler", "code", "method"}
if perHost {
httpLabels = append(httpLabels, "host")
}
httpMetrics.requestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
Namespace: ns,
Subsystem: sub,
Expand Down Expand Up @@ -101,14 +109,15 @@ func serverNameFromContext(ctx context.Context) string {
type metricsInstrumentedHandler struct {
handler string
mh MiddlewareHandler
perHost bool
}

func newMetricsInstrumentedHandler(handler string, mh MiddlewareHandler) *metricsInstrumentedHandler {
func newMetricsInstrumentedHandler(handler string, mh MiddlewareHandler, perHost bool) *metricsInstrumentedHandler {
httpMetrics.init.Do(func() {
initHTTPMetrics()
initHTTPMetrics(perHost)
})

return &metricsInstrumentedHandler{handler, mh}
return &metricsInstrumentedHandler{handler, mh, perHost}
}

func (h *metricsInstrumentedHandler) ServeHTTP(w http.ResponseWriter, r *http.Request, next Handler) error {
Expand All @@ -119,6 +128,11 @@ func (h *metricsInstrumentedHandler) ServeHTTP(w http.ResponseWriter, r *http.Re
// of a panic
statusLabels := prometheus.Labels{"server": server, "handler": h.handler, "method": method, "code": ""}

if h.perHost {
labels["host"] = r.Host
statusLabels["host"] = r.Host
}

inFlight := httpMetrics.requestInFlight.With(labels)
inFlight.Inc()
defer inFlight.Dec()
Expand Down
176 changes: 173 additions & 3 deletions modules/caddyhttp/metrics_test.go
View file
Open in desktop
Expand Up @@ -43,7 +43,7 @@ func TestMetricsInstrumentedHandler(t *testing.T) {
return h.ServeHTTP(w, r)
})

ih := newMetricsInstrumentedHandler("bar", mh)
ih := newMetricsInstrumentedHandler("bar", mh, false)

r := httptest.NewRequest("GET", "/", nil)
w := httptest.NewRecorder()
Expand All @@ -64,7 +64,7 @@ func TestMetricsInstrumentedHandler(t *testing.T) {
mh = middlewareHandlerFunc(func(w http.ResponseWriter, r *http.Request, h Handler) error {
return nil
})
ih = newMetricsInstrumentedHandler("empty", mh)
ih = newMetricsInstrumentedHandler("empty", mh, false)
r = httptest.NewRequest("GET", "/", nil)
w = httptest.NewRecorder()

Expand All @@ -83,7 +83,7 @@ func TestMetricsInstrumentedHandler(t *testing.T) {
return Error(http.StatusTooManyRequests, nil)
})

ih = newMetricsInstrumentedHandler("foo", mh)
ih = newMetricsInstrumentedHandler("foo", mh, false)

r = httptest.NewRequest("GET", "/", nil)
w = httptest.NewRecorder()
Expand Down Expand Up @@ -196,6 +196,176 @@ func TestMetricsInstrumentedHandler(t *testing.T) {
}
}

func TestMetricsInstrumentedHandlerPerHost(t *testing.T) {
handlerErr := errors.New("oh noes")
response := []byte("hello world!")
h := HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
if actual := testutil.ToFloat64(httpMetrics.requestInFlight); actual != 1.0 {
t.Errorf("Not same: expected %#v, but got %#v", 1.0, actual)
}
if handlerErr == nil {
w.Write(response)
}
return handlerErr
})

mh := middlewareHandlerFunc(func(w http.ResponseWriter, r *http.Request, h Handler) error {
return h.ServeHTTP(w, r)
})

ih := newMetricsInstrumentedHandler("bar", mh, true)

r := httptest.NewRequest("GET", "/", nil)
w := httptest.NewRecorder()

if actual := ih.ServeHTTP(w, r, h); actual != handlerErr {
t.Errorf("Not same: expected %#v, but got %#v", handlerErr, actual)
}
if actual := testutil.ToFloat64(httpMetrics.requestInFlight); actual != 0.0 {
t.Errorf("Not same: expected %#v, but got %#v", 0.0, actual)
}

handlerErr = nil
if err := ih.ServeHTTP(w, r, h); err != nil {
t.Errorf("Received unexpected error: %v", err)
}

// an empty handler - no errors, no header written
mh = middlewareHandlerFunc(func(w http.ResponseWriter, r *http.Request, h Handler) error {
return nil
})
ih = newMetricsInstrumentedHandler("empty", mh, true)
r = httptest.NewRequest("GET", "/", nil)
w = httptest.NewRecorder()

if err := ih.ServeHTTP(w, r, h); err != nil {
t.Errorf("Received unexpected error: %v", err)
}
if actual := w.Result().StatusCode; actual != 200 {
t.Errorf("Not same: expected status code %#v, but got %#v", 200, actual)
}
if actual := w.Result().Header; len(actual) != 0 {
t.Errorf("Not empty: expected headers to be empty, but got %#v", actual)
}

// handler returning an error with an HTTP status
mh = middlewareHandlerFunc(func(w http.ResponseWriter, r *http.Request, h Handler) error {
return Error(http.StatusTooManyRequests, nil)
})

ih = newMetricsInstrumentedHandler("foo", mh, true)

r = httptest.NewRequest("GET", "/", nil)
w = httptest.NewRecorder()

if err := ih.ServeHTTP(w, r, nil); err == nil {
t.Errorf("expected error to be propagated")
}

expected := `
# HELP caddy_http_request_duration_seconds Histogram of round-trip request durations.
# TYPE caddy_http_request_duration_seconds histogram
caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="0.005"} 1
caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="0.01"} 1
caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="0.025"} 1
caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="0.05"} 1
caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="0.1"} 1
caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="0.25"} 1
caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="0.5"} 1
caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="1"} 1
caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="2.5"} 1
caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="5"} 1
caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="10"} 1
caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="+Inf"} 1
caddy_http_request_duration_seconds_count{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN"} 1
# HELP caddy_http_request_size_bytes Total size of the request. Includes body
# TYPE caddy_http_request_size_bytes histogram
caddy_http_request_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="256"} 1
caddy_http_request_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="1024"} 1
caddy_http_request_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="4096"} 1
caddy_http_request_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="16384"} 1
caddy_http_request_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="65536"} 1
caddy_http_request_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="262144"} 1
caddy_http_request_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="1.048576e+06"} 1
caddy_http_request_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="4.194304e+06"} 1
caddy_http_request_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="+Inf"} 1
caddy_http_request_size_bytes_sum{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN"} 23
caddy_http_request_size_bytes_count{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN"} 1
caddy_http_request_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="256"} 1
caddy_http_request_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="1024"} 1
caddy_http_request_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="4096"} 1
caddy_http_request_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="16384"} 1
caddy_http_request_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="65536"} 1
caddy_http_request_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="262144"} 1
caddy_http_request_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="1.048576e+06"} 1
caddy_http_request_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="4.194304e+06"} 1
caddy_http_request_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="+Inf"} 1
caddy_http_request_size_bytes_sum{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN"} 23
caddy_http_request_size_bytes_count{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN"} 1
caddy_http_request_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="256"} 1
caddy_http_request_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="1024"} 1
caddy_http_request_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="4096"} 1
caddy_http_request_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="16384"} 1
caddy_http_request_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="65536"} 1
caddy_http_request_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="262144"} 1
caddy_http_request_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="1.048576e+06"} 1
caddy_http_request_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="4.194304e+06"} 1
caddy_http_request_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="+Inf"} 1
caddy_http_request_size_bytes_sum{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN"} 23
caddy_http_request_size_bytes_count{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN"} 1
# HELP caddy_http_response_size_bytes Size of the returned response.
# TYPE caddy_http_response_size_bytes histogram
caddy_http_response_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="256"} 1
caddy_http_response_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="1024"} 1
caddy_http_response_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="4096"} 1
caddy_http_response_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="16384"} 1
caddy_http_response_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="65536"} 1
caddy_http_response_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="262144"} 1
caddy_http_response_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="1.048576e+06"} 1
caddy_http_response_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="4.194304e+06"} 1
caddy_http_response_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="+Inf"} 1
caddy_http_response_size_bytes_sum{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN"} 12
caddy_http_response_size_bytes_count{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN"} 1
caddy_http_response_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="256"} 1
caddy_http_response_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="1024"} 1
caddy_http_response_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="4096"} 1
caddy_http_response_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="16384"} 1
caddy_http_response_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="65536"} 1
caddy_http_response_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="262144"} 1
caddy_http_response_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="1.048576e+06"} 1
caddy_http_response_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="4.194304e+06"} 1
caddy_http_response_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="+Inf"} 1
caddy_http_response_size_bytes_sum{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN"} 0
caddy_http_response_size_bytes_count{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN"} 1
caddy_http_response_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="256"} 1
caddy_http_response_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="1024"} 1
caddy_http_response_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="4096"} 1
caddy_http_response_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="16384"} 1
caddy_http_response_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="65536"} 1
caddy_http_response_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="262144"} 1
caddy_http_response_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="1.048576e+06"} 1
caddy_http_response_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="4.194304e+06"} 1
caddy_http_response_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="+Inf"} 1
caddy_http_response_size_bytes_sum{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN"} 0
caddy_http_response_size_bytes_count{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN"} 1
# HELP caddy_http_request_errors_total Number of requests resulting in middleware errors.
# TYPE caddy_http_request_errors_total counter
caddy_http_request_errors_total{handler="bar",host="example.com",server="UNKNOWN"} 1
caddy_http_request_errors_total{handler="foo",host="example.com",server="UNKNOWN"} 1
`
if err := testutil.GatherAndCompare(prometheus.DefaultGatherer, strings.NewReader(expected),
"caddy_http_request_size_bytes",
"caddy_http_response_size_bytes",
// caddy_http_request_duration_seconds_sum will vary based on how long the test took to run,
// so we check just the _bucket and _count metrics
"caddy_http_request_duration_seconds_bucket",
"caddy_http_request_duration_seconds_count",
"caddy_http_request_errors_total",
); err != nil {
t.Errorf("received unexpected error: %s", err)
}
}

type middlewareHandlerFunc func(http.ResponseWriter, *http.Request, Handler) error

func (f middlewareHandlerFunc) ServeHTTP(w http.ResponseWriter, r *http.Request, h Handler) error {
Expand Down
2 changes: 1 addition & 1 deletion modules/caddyhttp/routes.go
View file
Open in desktop
Expand Up @@ -315,7 +315,7 @@ func wrapMiddleware(_ caddy.Context, mh MiddlewareHandler, metrics *Metrics) Mid
handlerToUse := mh
if metrics != nil {
// wrap the middleware with metrics instrumentation
handlerToUse = newMetricsInstrumentedHandler(caddy.GetModuleName(mh), mh)
handlerToUse = newMetricsInstrumentedHandler(caddy.GetModuleName(mh), mh, metrics.PerHost)
}

return func(next Handler) Handler {
Expand Down