Add prometheus proxy related metrics (#4047)

docs/content/about/configuration.md

@@ -829,8 +829,8 @@ Instead, keep the debug endpoint private or enforce authentication for it.
 The `debug` section takes a single required `addr` parameter, which specifies
 the `HOST:PORT` on which the debug server should accept connections.
 
-If the registry is configured as a pull-through cache, the `debug` server can be used
-to access proxy statistics. These statistics are exposed at `/debug/vars` in JSON format.
+If configured, `notification`, `redis`, and `proxy` statistics are exposed
+at `/debug/vars` in JSON format.
 
 #### `prometheus`
 
@@ -843,8 +843,8 @@ prometheus:
 The `prometheus` option defines whether the prometheus metrics are enabled, as well
 as the path to access the metrics.
 
->**NOTE**: The prometheus metrics do **not** cover pull-through cache statistics.
-> Proxy statistics are exposed via `expvar` only.
+The prometheus metrics cover `storage`, `notification` and `proxy` statistics.
+
 
 | Parameter | Required | Description                                           |
 |-----------|----------|-------------------------------------------------------|

metrics/prometheus.go

@@ -13,4 +13,7 @@ var (
 
 	// NotificationsNamespace is the prometheus namespace of notification related metrics
 	NotificationsNamespace = metrics.NewNamespace(NamespacePrefix, "notifications", nil)
+
+	// ProxyNamespace is the prometheus namespace of proxy related metrics
+	ProxyNamespace = metrics.NewNamespace(NamespacePrefix, "proxy", nil)
 )

registry/proxy/proxymetrics.go

@@ -3,6 +3,22 @@ package proxy
 import (
 	"expvar"
 	"sync/atomic"
+
+	prometheus "github.com/distribution/distribution/v3/metrics"
+	"github.com/docker/go-metrics"
+)
+
+var (
+	// requests is the number of total incoming proxy request received for blob/manifest
+	requests = prometheus.ProxyNamespace.NewLabeledCounter("requests", "The number of total incoming proxy request received", "type")
+	// hits is the number of total proxy request hits for blob/manifest
+	hits = prometheus.ProxyNamespace.NewLabeledCounter("hits", "The number of total proxy request hits", "type")
+	// hits is the number of total proxy request misses for blob/manifest
+	misses = prometheus.ProxyNamespace.NewLabeledCounter("misses", "The number of total proxy request misses", "type")
+	// pulledBytes is the size of total bytes pulled from the upstream for blob/manifest
+	pulledBytes = prometheus.ProxyNamespace.NewLabeledCounter("pulled_bytes", "The size of total bytes pulled from the upstream", "type")
+	// pushedBytes is the size of total bytes pushed to the client for blob/manifest
+	pushedBytes = prometheus.ProxyNamespace.NewLabeledCounter("pushed_bytes", "The size of total bytes pushed to the client", "type")
 )
 
 // Metrics is used to hold metric counters
@@ -24,26 +40,40 @@ type proxyMetricsCollector struct {
 func (pmc *proxyMetricsCollector) BlobPull(bytesPulled uint64) {
 	atomic.AddUint64(&pmc.blobMetrics.Misses, 1)
 	atomic.AddUint64(&pmc.blobMetrics.BytesPulled, bytesPulled)
+
+	misses.WithValues("blob").Inc(1)
+	pulledBytes.WithValues("blob").Inc(float64(bytesPulled))
 }
 
 // BlobPush tracks metrics about blobs pushed to clients
 func (pmc *proxyMetricsCollector) BlobPush(bytesPushed uint64) {
 	atomic.AddUint64(&pmc.blobMetrics.Requests, 1)
 	atomic.AddUint64(&pmc.blobMetrics.Hits, 1)
 	atomic.AddUint64(&pmc.blobMetrics.BytesPushed, bytesPushed)
+
+	requests.WithValues("blob").Inc(1)
+	hits.WithValues("blob").Inc(1)
+	pushedBytes.WithValues("blob").Inc(float64(bytesPushed))
 }
 
 // ManifestPull tracks metrics related to Manifests pulled into the cache
 func (pmc *proxyMetricsCollector) ManifestPull(bytesPulled uint64) {
 	atomic.AddUint64(&pmc.manifestMetrics.Misses, 1)
 	atomic.AddUint64(&pmc.manifestMetrics.BytesPulled, bytesPulled)
+
+	misses.WithValues("manifest").Inc(1)
+	pulledBytes.WithValues("manifest").Inc(float64(bytesPulled))
 }
 
 // ManifestPush tracks metrics about manifests pushed to clients
 func (pmc *proxyMetricsCollector) ManifestPush(bytesPushed uint64) {
 	atomic.AddUint64(&pmc.manifestMetrics.Requests, 1)
 	atomic.AddUint64(&pmc.manifestMetrics.Hits, 1)
 	atomic.AddUint64(&pmc.manifestMetrics.BytesPushed, bytesPushed)
+
+	requests.WithValues("manifest").Inc(1)
+	hits.WithValues("manifest").Inc(1)
+	pushedBytes.WithValues("manifest").Inc(float64(bytesPushed))
 }
 
 // proxyMetrics tracks metrics about the proxy cache.  This is
@@ -70,4 +100,6 @@ func init() {
 	pm.(*expvar.Map).Set("manifests", expvar.Func(func() interface{} {
 		return proxyMetrics.manifestMetrics
 	}))
+
+	metrics.Register(prometheus.ProxyNamespace)
 }