From 9861a46d99d8278915bf38f3d6f3213d5d8793fa Mon Sep 17 00:00:00 2001 From: Dimitar Kostadinov Date: Tue, 5 Sep 2023 17:37:52 +0300 Subject: [PATCH 1/2] Add prometheus proxy related metrics Signed-off-by: Dimitar Kostadinov --- docs/content/about/configuration.md | 8 ++--- metrics/prometheus.go | 3 ++ registry/proxy/proxymetrics.go | 48 ++++++++++++++++++++++++----- 3 files changed, 47 insertions(+), 12 deletions(-) diff --git a/docs/content/about/configuration.md b/docs/content/about/configuration.md index 3f4cb18e32..4107c0b3ae 100644 --- a/docs/content/about/configuration.md +++ b/docs/content/about/configuration.md @@ -829,8 +829,8 @@ Instead, keep the debug endpoint private or enforce authentication for it. The `debug` section takes a single required `addr` parameter, which specifies the `HOST:PORT` on which the debug server should accept connections. -If the registry is configured as a pull-through cache, the `debug` server can be used -to access proxy statistics. These statistics are exposed at `/debug/vars` in JSON format. +If configured, `notification`, `redis`, and `proxy` statistics are exposed +at `/debug/vars` in JSON format. #### `prometheus` @@ -843,8 +843,8 @@ prometheus: The `prometheus` option defines whether the prometheus metrics are enabled, as well as the path to access the metrics. ->**NOTE**: The prometheus metrics do **not** cover pull-through cache statistics. -> Proxy statistics are exposed via `expvar` only. +The prometheus metrics cover `storage`, `notification` and `proxy` statistics. + | Parameter | Required | Description | |-----------|----------|-------------------------------------------------------| diff --git a/metrics/prometheus.go b/metrics/prometheus.go index 91b32b23d9..d84781190d 100644 --- a/metrics/prometheus.go +++ b/metrics/prometheus.go @@ -13,4 +13,7 @@ var ( // NotificationsNamespace is the prometheus namespace of notification related metrics NotificationsNamespace = metrics.NewNamespace(NamespacePrefix, "notifications", nil) + + // ProxyNamespace is the prometheus namespace of proxy related metrics + ProxyNamespace = metrics.NewNamespace(NamespacePrefix, "proxy", nil) ) diff --git a/registry/proxy/proxymetrics.go b/registry/proxy/proxymetrics.go index a767b336f9..6196417365 100644 --- a/registry/proxy/proxymetrics.go +++ b/registry/proxy/proxymetrics.go @@ -3,6 +3,22 @@ package proxy import ( "expvar" "sync/atomic" + + prometheus "github.com/distribution/distribution/v3/metrics" + "github.com/docker/go-metrics" +) + +var ( + // requests is the number of total incoming proxy request received for blob/manifest + requests = prometheus.ProxyNamespace.NewLabeledCounter("requests", "The number of total incoming proxy request received", "type") + // hits is the number of total proxy request hits for blob/manifest + hits = prometheus.ProxyNamespace.NewLabeledCounter("hits", "The number of total proxy request hits", "type") + // hits is the number of total proxy request misses for blob/manifest + misses = prometheus.ProxyNamespace.NewLabeledCounter("misses", "The number of total proxy request misses", "type") + // bytesPulled is the size of total bytes pulled from the upstream for blob/manifest + bytesPulled = prometheus.ProxyNamespace.NewLabeledCounter("bytes_pulled", "The size of total bytes pulled from the upstream", "type") + // bytesPushed is the size of total bytes pushed to the client for blob/manifest + bytesPushed = prometheus.ProxyNamespace.NewLabeledCounter("bytes_pushed", "The size of total bytes pushed to the client", "type") ) // Metrics is used to hold metric counters @@ -21,29 +37,43 @@ type proxyMetricsCollector struct { } // BlobPull tracks metrics about blobs pulled into the cache -func (pmc *proxyMetricsCollector) BlobPull(bytesPulled uint64) { +func (pmc *proxyMetricsCollector) BlobPull(bytes uint64) { atomic.AddUint64(&pmc.blobMetrics.Misses, 1) - atomic.AddUint64(&pmc.blobMetrics.BytesPulled, bytesPulled) + atomic.AddUint64(&pmc.blobMetrics.BytesPulled, bytes) + + misses.WithValues("blob").Inc(1) + bytesPulled.WithValues("blob").Inc(float64(bytes)) } // BlobPush tracks metrics about blobs pushed to clients -func (pmc *proxyMetricsCollector) BlobPush(bytesPushed uint64) { +func (pmc *proxyMetricsCollector) BlobPush(bytes uint64) { atomic.AddUint64(&pmc.blobMetrics.Requests, 1) atomic.AddUint64(&pmc.blobMetrics.Hits, 1) - atomic.AddUint64(&pmc.blobMetrics.BytesPushed, bytesPushed) + atomic.AddUint64(&pmc.blobMetrics.BytesPushed, bytes) + + requests.WithValues("blob").Inc(1) + hits.WithValues("blob").Inc(1) + bytesPushed.WithValues("blob").Inc(float64(bytes)) } // ManifestPull tracks metrics related to Manifests pulled into the cache -func (pmc *proxyMetricsCollector) ManifestPull(bytesPulled uint64) { +func (pmc *proxyMetricsCollector) ManifestPull(bytes uint64) { atomic.AddUint64(&pmc.manifestMetrics.Misses, 1) - atomic.AddUint64(&pmc.manifestMetrics.BytesPulled, bytesPulled) + atomic.AddUint64(&pmc.manifestMetrics.BytesPulled, bytes) + + misses.WithValues("manifest").Inc(1) + bytesPulled.WithValues("manifest").Inc(float64(bytes)) } // ManifestPush tracks metrics about manifests pushed to clients -func (pmc *proxyMetricsCollector) ManifestPush(bytesPushed uint64) { +func (pmc *proxyMetricsCollector) ManifestPush(bytes uint64) { atomic.AddUint64(&pmc.manifestMetrics.Requests, 1) atomic.AddUint64(&pmc.manifestMetrics.Hits, 1) - atomic.AddUint64(&pmc.manifestMetrics.BytesPushed, bytesPushed) + atomic.AddUint64(&pmc.manifestMetrics.BytesPushed, bytes) + + requests.WithValues("manifest").Inc(1) + hits.WithValues("manifest").Inc(1) + bytesPushed.WithValues("manifest").Inc(float64(bytes)) } // proxyMetrics tracks metrics about the proxy cache. This is @@ -70,4 +100,6 @@ func init() { pm.(*expvar.Map).Set("manifests", expvar.Func(func() interface{} { return proxyMetrics.manifestMetrics })) + + metrics.Register(prometheus.ProxyNamespace) } From 2ce5c81f4745e1b64ab07eb66d0500dcad77edeb Mon Sep 17 00:00:00 2001 From: Dimitar Kostadinov Date: Tue, 26 Sep 2023 14:23:05 +0300 Subject: [PATCH 2/2] Address PR review feedback Signed-off-by: Dimitar Kostadinov --- registry/proxy/proxymetrics.go | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/registry/proxy/proxymetrics.go b/registry/proxy/proxymetrics.go index 6196417365..eff8669420 100644 --- a/registry/proxy/proxymetrics.go +++ b/registry/proxy/proxymetrics.go @@ -15,10 +15,10 @@ var ( hits = prometheus.ProxyNamespace.NewLabeledCounter("hits", "The number of total proxy request hits", "type") // hits is the number of total proxy request misses for blob/manifest misses = prometheus.ProxyNamespace.NewLabeledCounter("misses", "The number of total proxy request misses", "type") - // bytesPulled is the size of total bytes pulled from the upstream for blob/manifest - bytesPulled = prometheus.ProxyNamespace.NewLabeledCounter("bytes_pulled", "The size of total bytes pulled from the upstream", "type") - // bytesPushed is the size of total bytes pushed to the client for blob/manifest - bytesPushed = prometheus.ProxyNamespace.NewLabeledCounter("bytes_pushed", "The size of total bytes pushed to the client", "type") + // pulledBytes is the size of total bytes pulled from the upstream for blob/manifest + pulledBytes = prometheus.ProxyNamespace.NewLabeledCounter("pulled_bytes", "The size of total bytes pulled from the upstream", "type") + // pushedBytes is the size of total bytes pushed to the client for blob/manifest + pushedBytes = prometheus.ProxyNamespace.NewLabeledCounter("pushed_bytes", "The size of total bytes pushed to the client", "type") ) // Metrics is used to hold metric counters @@ -37,43 +37,43 @@ type proxyMetricsCollector struct { } // BlobPull tracks metrics about blobs pulled into the cache -func (pmc *proxyMetricsCollector) BlobPull(bytes uint64) { +func (pmc *proxyMetricsCollector) BlobPull(bytesPulled uint64) { atomic.AddUint64(&pmc.blobMetrics.Misses, 1) - atomic.AddUint64(&pmc.blobMetrics.BytesPulled, bytes) + atomic.AddUint64(&pmc.blobMetrics.BytesPulled, bytesPulled) misses.WithValues("blob").Inc(1) - bytesPulled.WithValues("blob").Inc(float64(bytes)) + pulledBytes.WithValues("blob").Inc(float64(bytesPulled)) } // BlobPush tracks metrics about blobs pushed to clients -func (pmc *proxyMetricsCollector) BlobPush(bytes uint64) { +func (pmc *proxyMetricsCollector) BlobPush(bytesPushed uint64) { atomic.AddUint64(&pmc.blobMetrics.Requests, 1) atomic.AddUint64(&pmc.blobMetrics.Hits, 1) - atomic.AddUint64(&pmc.blobMetrics.BytesPushed, bytes) + atomic.AddUint64(&pmc.blobMetrics.BytesPushed, bytesPushed) requests.WithValues("blob").Inc(1) hits.WithValues("blob").Inc(1) - bytesPushed.WithValues("blob").Inc(float64(bytes)) + pushedBytes.WithValues("blob").Inc(float64(bytesPushed)) } // ManifestPull tracks metrics related to Manifests pulled into the cache -func (pmc *proxyMetricsCollector) ManifestPull(bytes uint64) { +func (pmc *proxyMetricsCollector) ManifestPull(bytesPulled uint64) { atomic.AddUint64(&pmc.manifestMetrics.Misses, 1) - atomic.AddUint64(&pmc.manifestMetrics.BytesPulled, bytes) + atomic.AddUint64(&pmc.manifestMetrics.BytesPulled, bytesPulled) misses.WithValues("manifest").Inc(1) - bytesPulled.WithValues("manifest").Inc(float64(bytes)) + pulledBytes.WithValues("manifest").Inc(float64(bytesPulled)) } // ManifestPush tracks metrics about manifests pushed to clients -func (pmc *proxyMetricsCollector) ManifestPush(bytes uint64) { +func (pmc *proxyMetricsCollector) ManifestPush(bytesPushed uint64) { atomic.AddUint64(&pmc.manifestMetrics.Requests, 1) atomic.AddUint64(&pmc.manifestMetrics.Hits, 1) - atomic.AddUint64(&pmc.manifestMetrics.BytesPushed, bytes) + atomic.AddUint64(&pmc.manifestMetrics.BytesPushed, bytesPushed) requests.WithValues("manifest").Inc(1) hits.WithValues("manifest").Inc(1) - bytesPushed.WithValues("manifest").Inc(float64(bytes)) + pushedBytes.WithValues("manifest").Inc(float64(bytesPushed)) } // proxyMetrics tracks metrics about the proxy cache. This is