From d0d127a14e5a1202e7f6d1016cc341842422b5cd Mon Sep 17 00:00:00 2001 From: bwplotka Date: Wed, 3 Aug 2022 13:11:35 +0200 Subject: [PATCH] goCollector: Added rule support for granular metric configuration. Fixes: https://github.com/prometheus/client_golang/issues/1089 Signed-off-by: bwplotka --- prometheus/collectors/go_collector_latest.go | 151 ++++++----- .../collectors/go_collector_latest_test.go | 11 +- prometheus/go_collector.go | 7 +- prometheus/go_collector_latest.go | 248 +++++++++++------- prometheus/go_collector_latest_test.go | 65 +++-- prometheus/internal/go_collector_options.go | 20 ++ 6 files changed, 320 insertions(+), 182 deletions(-) create mode 100644 prometheus/internal/go_collector_options.go diff --git a/prometheus/collectors/go_collector_latest.go b/prometheus/collectors/go_collector_latest.go index 9435e9614..cac72118c 100644 --- a/prometheus/collectors/go_collector_latest.go +++ b/prometheus/collectors/go_collector_latest.go @@ -16,77 +16,102 @@ package collectors -import "github.com/prometheus/client_golang/prometheus" +import ( + "regexp" -//nolint:staticcheck // Ignore SA1019 until v2. -type goOptions = prometheus.GoCollectorOptions -type goOption func(o *goOptions) - -type GoCollectionOption uint32 - -const ( - // GoRuntimeMemStatsCollection represents the metrics represented by runtime.MemStats structure such as - // go_memstats_alloc_bytes - // go_memstats_alloc_bytes_total - // go_memstats_sys_bytes - // go_memstats_lookups_total - // go_memstats_mallocs_total - // go_memstats_frees_total - // go_memstats_heap_alloc_bytes - // go_memstats_heap_sys_bytes - // go_memstats_heap_idle_bytes - // go_memstats_heap_inuse_bytes - // go_memstats_heap_released_bytes - // go_memstats_heap_objects - // go_memstats_stack_inuse_bytes - // go_memstats_stack_sys_bytes - // go_memstats_mspan_inuse_bytes - // go_memstats_mspan_sys_bytes - // go_memstats_mcache_inuse_bytes - // go_memstats_mcache_sys_bytes - // go_memstats_buck_hash_sys_bytes - // go_memstats_gc_sys_bytes - // go_memstats_other_sys_bytes - // go_memstats_next_gc_bytes - // so the metrics known from pre client_golang v1.12.0, except skipped go_memstats_gc_cpu_fraction (see - // https://github.com/prometheus/client_golang/issues/842#issuecomment-861812034 for explanation. - // - // NOTE that this mode represents runtime.MemStats statistics, but they are - // actually implemented using new runtime/metrics package. - // Deprecated: Use GoRuntimeMetricsCollection instead going forward. - GoRuntimeMemStatsCollection GoCollectionOption = 1 << iota - // GoRuntimeMetricsCollection is the new set of metrics represented by runtime/metrics package and follows - // consistent naming. The exposed metric set depends on Go version, but it is controlled against - // unexpected cardinality. This set has overlapping information with GoRuntimeMemStatsCollection, just with - // new names. GoRuntimeMetricsCollection is what is recommended for using going forward. - GoRuntimeMetricsCollection + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/internal" ) -// WithGoCollections allows enabling different collections for Go collector on top of base metrics -// like go_goroutines, go_threads, go_gc_duration_seconds, go_memstats_last_gc_time_seconds, go_info. +// WithGoCollectorMemStatsMetricsDisabled metrics that where representing runtime.MemStats structure such as: +// +// go_memstats_alloc_bytes +// go_memstats_alloc_bytes_total +// go_memstats_sys_bytes +// go_memstats_lookups_total +// go_memstats_mallocs_total +// go_memstats_frees_total +// go_memstats_heap_alloc_bytes +// go_memstats_heap_sys_bytes +// go_memstats_heap_idle_bytes +// go_memstats_heap_inuse_bytes +// go_memstats_heap_released_bytes +// go_memstats_heap_objects +// go_memstats_stack_inuse_bytes +// go_memstats_stack_sys_bytes +// go_memstats_mspan_inuse_bytes +// go_memstats_mspan_sys_bytes +// go_memstats_mcache_inuse_bytes +// go_memstats_mcache_sys_bytes +// go_memstats_buck_hash_sys_bytes +// go_memstats_gc_sys_bytes +// go_memstats_other_sys_bytes +// go_memstats_next_gc_bytes +// +// so the metrics known from pre client_golang v1.12.0, +// +// NOTE(bwplotka): The above represents runtime.MemStats statistics, but they are +// actually implemented using new runtime/metrics package. (except skipped go_memstats_gc_cpu_fraction +// -- see https://github.com/prometheus/client_golang/issues/842#issuecomment-861812034 for explanation). // -// Check GoRuntimeMemStatsCollection and GoRuntimeMetricsCollection for more details. You can use none, -// one or more collections at once. For example: -// WithGoCollections(GoRuntimeMemStatsCollection | GoRuntimeMetricsCollection) means both GoRuntimeMemStatsCollection -// metrics and GoRuntimeMetricsCollection will be exposed. +// Some users might want to disable this on collector level (although you can use scrape relabelling on Prometheus), +// because similar metrics can be now obtained using WithGoCollectorRuntimeMetrics. Note that the semantics of new +// metrics might be different, plus the names can be change over time with different Go version. // -// The current default is GoRuntimeMemStatsCollection, so the compatibility mode with -// client_golang pre v1.12 (move to runtime/metrics). -//nolint:staticcheck // Ignore SA1019 until v2. -func WithGoCollections(flags GoCollectionOption) func(options *prometheus.GoCollectorOptions) { - return func(o *goOptions) { - o.EnabledCollections = uint32(flags) +// NOTE(bwplotka): Changing metric names can be tedious at times as the alerts, recording rules and dashboards have to be adjusted. +// The old metrics are also very useful, with many guides and books written about how to interpret them. +// +// As a result our recommendation would be to stick with MemStats like metrics and enable other runtime/metrics if you are interested +// in advanced insights Go provides. See ExampleGoCollector_WithAdvancedGoMetrics. +func WithGoCollectorMemStatsMetricsDisabled() func(options *internal.GoCollectorOptions) { + return func(o *internal.GoCollectorOptions) { + o.DisableMemStatsLikeMetrics = true } } -// NewGoCollector returns a collector that exports metrics about the current Go -// process using debug.GCStats using runtime/metrics. -func NewGoCollector(opts ...goOption) prometheus.Collector { - //nolint:staticcheck // Ignore SA1019 until v2. - promPkgOpts := make([]func(o *prometheus.GoCollectorOptions), len(opts)) - for i, opt := range opts { - promPkgOpts[i] = opt +// GoRuntimeMetricsRule allow enabling and configuring particular group of runtime/metrics. +type GoRuntimeMetricsRule struct { + // Matcher represents RE2 expression will match the runtime/metrics from https://golang.bg/src/runtime/metrics/description.go + // Use `regexp.MustCompile` or `regexp.Compile` to create this field. + Matcher *regexp.Regexp +} + +// WithGoCollectorRuntimeMetrics allows enabling and configuring particular group of runtime/metrics. +// See the list of metrics https://golang.bg/src/runtime/metrics/description.go (pick the Go version you use there!). +// You can use this option in repeated manner, which will add new rules. The order of rules is important, the last rule +// that matches particular metrics is applied. +func WithGoCollectorRuntimeMetrics(rules ...GoRuntimeMetricsRule) func(options *internal.GoCollectorOptions) { + rs := make([]internal.GoCollectorRule, len(rules)) + for i, r := range rules { + rs[i] = internal.GoCollectorRule{ + Matcher: r.Matcher, + } } + + return func(o *internal.GoCollectorOptions) { + o.DisableMemStatsLikeMetrics = true + } +} + +// WithoutGoCollectorRuntimeMetrics allows disabling group of runtime/metrics that you might have added in WithGoCollectorRuntimeMetrics. +// It behaves similarly to WithGoCollectorRuntimeMetrics just with deny-list semantics. +func WithoutGoCollectorRuntimeMetrics(matchers ...*regexp.Regexp) func(options *internal.GoCollectorOptions) { + rs := make([]internal.GoCollectorRule, len(matchers)) + for i, m := range matchers { + rs[i] = internal.GoCollectorRule{ + Matcher: m, + Deny: true, + } + } + + return func(o *internal.GoCollectorOptions) { + o.RuntimeMetricRules = append(o.RuntimeMetricRules, rs...) + } +} + +// NewGoCollector returns a collector that exports metrics about the current Go +// process using debug.GCStats and runtime/metrics. +func NewGoCollector(opts ...func(o *internal.GoCollectorOptions)) prometheus.Collector { //nolint:staticcheck // Ignore SA1019 until v2. - return prometheus.NewGoCollector(promPkgOpts...) + return prometheus.NewGoCollector(opts...) } diff --git a/prometheus/collectors/go_collector_latest_test.go b/prometheus/collectors/go_collector_latest_test.go index 126864c32..06d9781a1 100644 --- a/prometheus/collectors/go_collector_latest_test.go +++ b/prometheus/collectors/go_collector_latest_test.go @@ -18,6 +18,7 @@ package collectors import ( "encoding/json" + "regexp" "testing" "github.com/prometheus/client_golang/prometheus" @@ -26,7 +27,9 @@ import ( func TestGoCollectorMarshalling(t *testing.T) { reg := prometheus.NewRegistry() reg.MustRegister(NewGoCollector( - WithGoCollections(GoRuntimeMemStatsCollection | GoRuntimeMetricsCollection), + WithGoCollectorRuntimeMetrics(GoRuntimeMetricsRule{ + Matcher: regexp.MustCompile("/.*"), + }), )) result, err := reg.Gather() if err != nil { @@ -37,3 +40,9 @@ func TestGoCollectorMarshalling(t *testing.T) { t.Errorf("json marshalling shoud not fail, %v", err) } } + +func ExampleGoCollector() { +} + +func ExampleGoCollector_WithAdvancedGoMetrics() { +} diff --git a/prometheus/go_collector.go b/prometheus/go_collector.go index 4d792aa29..ba2bf74c0 100644 --- a/prometheus/go_collector.go +++ b/prometheus/go_collector.go @@ -19,6 +19,10 @@ import ( "time" ) +// goRuntimeMemStats provides the metrics initially provided by runtime.ReadMemStats. +// From Go 1.17 those similar (and better) statistics are provided by runtime/metrics, so +// while eval closure works on runtime.MemStats, the struct from Go 1.17+ is +// populated using runtime/metrics. func goRuntimeMemStats() memStatsMetrics { return memStatsMetrics{ { @@ -224,7 +228,7 @@ func newBaseGoCollector() baseGoCollector { "A summary of the pause duration of garbage collection cycles.", nil, nil), gcLastTimeDesc: NewDesc( - memstatNamespace("last_gc_time_seconds"), + "go_memstats_last_gc_time_seconds", "Number of seconds since 1970 of last garbage collection.", nil, nil), goInfoDesc: NewDesc( @@ -269,7 +273,6 @@ func memstatNamespace(s string) string { // memStatsMetrics provide description, evaluator, runtime/metrics name, and // value type for memstat metrics. -// TODO(bwplotka): Remove with end Go 1.16 EOL and replace with runtime/metrics.Description type memStatsMetrics []struct { desc *Desc eval func(*runtime.MemStats) float64 diff --git a/prometheus/go_collector_latest.go b/prometheus/go_collector_latest.go index 68a7a156c..3a2d55e84 100644 --- a/prometheus/go_collector_latest.go +++ b/prometheus/go_collector_latest.go @@ -31,9 +31,11 @@ import ( ) const ( + // constants for strings referenced more than once. goGCHeapTinyAllocsObjects = "/gc/heap/tiny/allocs:objects" goGCHeapAllocsObjects = "/gc/heap/allocs:objects" goGCHeapFreesObjects = "/gc/heap/frees:objects" + goGCHeapFreesBytes = "/gc/heap/frees:bytes" goGCHeapAllocsBytes = "/gc/heap/allocs:bytes" goGCHeapObjects = "/gc/heap/objects:objects" goGCHeapGoalBytes = "/gc/heap/goal:bytes" @@ -53,8 +55,8 @@ const ( goMemoryClassesOtherBytes = "/memory/classes/other:bytes" ) -// runtime/metrics names required for runtimeMemStats like logic. -var rmForMemStats = []string{ +// rmNamesForMemStatsMetrics represents runtime/metrics names required to populate goRuntimeMemStats from like logic. +var rmNamesForMemStatsMetrics = []string{ goGCHeapTinyAllocsObjects, goGCHeapAllocsObjects, goGCHeapFreesObjects, @@ -90,74 +92,90 @@ func bestEffortLookupRM(lookup []string) []metrics.Description { } type goCollector struct { - opt GoCollectorOptions base baseGoCollector // mu protects updates to all fields ensuring a consistent // snapshot is always produced by Collect. mu sync.Mutex - // rm... fields all pertain to the runtime/metrics package. - rmSampleBuf []metrics.Sample - rmSampleMap map[string]*metrics.Sample - rmMetrics []collectorMetric + // Contains all samples that has to retrieved from runtime/metrics (not all of them will be exposed). + sampleBuf []metrics.Sample + // sampleMap allows lookup for MemStats metrics and runtime/metrics histograms for exact sums. + sampleMap map[string]*metrics.Sample + + // rmExposedMetrics represents all runtime/metrics package metrics + // that were configured to be exposed. + rmExposedMetrics []collectorMetric + rmExactSumMapForHist map[string]string // With Go 1.17, the runtime/metrics package was introduced. // From that point on, metric names produced by the runtime/metrics // package could be generated from runtime/metrics names. However, // these differ from the old names for the same values. // - // This field exist to export the same values under the old names + // This field exists to export the same values under the old names // as well. - msMetrics memStatsMetrics + msMetrics memStatsMetrics + msMetricsEnabled bool } -const ( - // Those are not exposed due to need to move Go collector to another package in v2. - // See issue https://github.com/prometheus/client_golang/issues/1030. - goRuntimeMemStatsCollection uint32 = 1 << iota - goRuntimeMetricsCollection -) - -// GoCollectorOptions should not be used be directly by anything, except `collectors` package. -// Use it via collectors package instead. See issue -// https://github.com/prometheus/client_golang/issues/1030. -// -// Deprecated: Use collectors.WithGoCollections -type GoCollectorOptions struct { - // EnabledCollection sets what type of collections collector should expose on top of base collection. - // By default it's goMemStatsCollection | goRuntimeMetricsCollection. - EnabledCollections uint32 +type rmMetricDesc struct { + metrics.Description } -func (c GoCollectorOptions) isEnabled(flag uint32) bool { - return c.EnabledCollections&flag != 0 +func matchRuntimeMetricsRules(rules []internal.GoCollectorRule) []rmMetricDesc { + var descs []rmMetricDesc + for _, d := range metrics.All() { + var ( + deny = true + desc rmMetricDesc + ) + + for _, r := range rules { + if !r.Matcher.MatchString(d.Name) { + continue + } + deny = r.Deny + } + if deny { + continue + } + + desc.Description = d + descs = append(descs, desc) + } + return descs } -const defaultGoCollections = goRuntimeMemStatsCollection +func defaultGoCollectorOptions() internal.GoCollectorOptions { + return internal.GoCollectorOptions{ + RuntimeMetricSumForHist: map[string]string{ + "/gc/heap/allocs-by-size:bytes": goGCHeapAllocsBytes, + "/gc/heap/frees-by-size:bytes": goGCHeapFreesBytes, + }, + RuntimeMetricRules: []internal.GoCollectorRule{ + //{Matcher: regexp.MustCompile("")}, + }, + } +} // NewGoCollector is the obsolete version of collectors.NewGoCollector. // See there for documentation. // // Deprecated: Use collectors.NewGoCollector instead. -func NewGoCollector(opts ...func(o *GoCollectorOptions)) Collector { - opt := GoCollectorOptions{EnabledCollections: defaultGoCollections} +func NewGoCollector(opts ...func(o *internal.GoCollectorOptions)) Collector { + opt := defaultGoCollectorOptions() for _, o := range opts { o(&opt) } - var descriptions []metrics.Description - if opt.isEnabled(goRuntimeMetricsCollection) { - descriptions = metrics.All() - } else if opt.isEnabled(goRuntimeMemStatsCollection) { - descriptions = bestEffortLookupRM(rmForMemStats) - } + exposedDescriptions := matchRuntimeMetricsRules(opt.RuntimeMetricRules) // Collect all histogram samples so that we can get their buckets. // The API guarantees that the buckets are always fixed for the lifetime // of the process. var histograms []metrics.Sample - for _, d := range descriptions { + for _, d := range exposedDescriptions { if d.Kind == metrics.KindFloat64Histogram { histograms = append(histograms, metrics.Sample{Name: d.Name}) } @@ -172,13 +190,14 @@ func NewGoCollector(opts ...func(o *GoCollectorOptions)) Collector { bucketsMap[histograms[i].Name] = histograms[i].Value.Float64Histogram().Buckets } - // Generate a Desc and ValueType for each runtime/metrics metric. - metricSet := make([]collectorMetric, 0, len(descriptions)) - sampleBuf := make([]metrics.Sample, 0, len(descriptions)) - sampleMap := make(map[string]*metrics.Sample, len(descriptions)) - for i := range descriptions { - d := &descriptions[i] - namespace, subsystem, name, ok := internal.RuntimeMetricsToProm(d) + // Generate a collector for each exposed runtime/metrics metric. + metricSet := make([]collectorMetric, 0, len(exposedDescriptions)) + // SampleBuf is used for reading from runtime/metrics. + // We are assuming the largest case to have stable pointers for sampleMap purposes. + sampleBuf := make([]metrics.Sample, 0, len(exposedDescriptions)+len(opt.RuntimeMetricSumForHist)+len(rmNamesForMemStatsMetrics)) + sampleMap := make(map[string]*metrics.Sample, len(exposedDescriptions)) + for _, d := range exposedDescriptions { + namespace, subsystem, name, ok := internal.RuntimeMetricsToProm(&d.Description) if !ok { // Just ignore this metric; we can't do anything with it here. // If a user decides to use the latest version of Go, we don't want @@ -186,19 +205,17 @@ func NewGoCollector(opts ...func(o *GoCollectorOptions)) Collector { continue } - // Set up sample buffer for reading, and a map - // for quick lookup of sample values. sampleBuf = append(sampleBuf, metrics.Sample{Name: d.Name}) sampleMap[d.Name] = &sampleBuf[len(sampleBuf)-1] var m collectorMetric if d.Kind == metrics.KindFloat64Histogram { - _, hasSum := rmExactSumMap[d.Name] + _, hasSum := opt.RuntimeMetricSumForHist[d.Name] unit := d.Name[strings.IndexRune(d.Name, ':')+1:] m = newBatchHistogram( NewDesc( BuildFQName(namespace, subsystem, name), - d.Description, + d.Description.Description, nil, nil, ), @@ -210,30 +227,61 @@ func NewGoCollector(opts ...func(o *GoCollectorOptions)) Collector { Namespace: namespace, Subsystem: subsystem, Name: name, - Help: d.Description, - }) + Help: d.Description.Description, + }, + ) } else { m = NewGauge(GaugeOpts{ Namespace: namespace, Subsystem: subsystem, Name: name, - Help: d.Description, + Help: d.Description.Description, }) } metricSet = append(metricSet, m) } - var msMetrics memStatsMetrics - if opt.isEnabled(goRuntimeMemStatsCollection) { + // Add exact sum metrics to sampleBuf if not added before. + for _, h := range histograms { + sumMetric, ok := opt.RuntimeMetricSumForHist[h.Name] + if !ok { + continue + } + + if _, ok := sampleMap[sumMetric]; ok { + continue + } + sampleBuf = append(sampleBuf, metrics.Sample{Name: sumMetric}) + sampleMap[sumMetric] = &sampleBuf[len(sampleBuf)-1] + } + + var ( + msMetrics memStatsMetrics + msDescriptions []metrics.Description + ) + + if !opt.DisableMemStatsLikeMetrics { msMetrics = goRuntimeMemStats() + msDescriptions = bestEffortLookupRM(rmNamesForMemStatsMetrics) + + // Check if metric was not exposed before and if not, add to sampleBuf. + for _, mdDesc := range msDescriptions { + if _, ok := sampleMap[mdDesc.Name]; ok { + continue + } + sampleBuf = append(sampleBuf, metrics.Sample{Name: mdDesc.Name}) + sampleMap[mdDesc.Name] = &sampleBuf[len(sampleBuf)-1] + } } + return &goCollector{ - opt: opt, - base: newBaseGoCollector(), - rmSampleBuf: sampleBuf, - rmSampleMap: sampleMap, - rmMetrics: metricSet, - msMetrics: msMetrics, + base: newBaseGoCollector(), + sampleBuf: sampleBuf, + sampleMap: sampleMap, + rmExposedMetrics: metricSet, + rmExactSumMapForHist: opt.RuntimeMetricSumForHist, + msMetrics: msMetrics, + msMetricsEnabled: !opt.DisableMemStatsLikeMetrics, } } @@ -243,7 +291,7 @@ func (c *goCollector) Describe(ch chan<- *Desc) { for _, i := range c.msMetrics { ch <- i.desc } - for _, m := range c.rmMetrics { + for _, m := range c.rmExposedMetrics { ch <- m.Desc() } } @@ -253,8 +301,12 @@ func (c *goCollector) Collect(ch chan<- Metric) { // Collect base non-memory metrics. c.base.Collect(ch) + if len(c.sampleBuf) == 0 { + return + } + // Collect must be thread-safe, so prevent concurrent use of - // rmSampleBuf. Just read into rmSampleBuf but write all the data + // sampleBuf elements. Just read into sampleBuf but write all the data // we get into our Metrics or MemStats. // // This lock also ensures that the Metrics we send out are all from @@ -268,44 +320,43 @@ func (c *goCollector) Collect(ch chan<- Metric) { c.mu.Lock() defer c.mu.Unlock() - if len(c.rmSampleBuf) > 0 { - // Populate runtime/metrics sample buffer. - metrics.Read(c.rmSampleBuf) - } - - if c.opt.isEnabled(goRuntimeMetricsCollection) { - // Collect all our metrics from rmSampleBuf. - for i, sample := range c.rmSampleBuf { - // N.B. switch on concrete type because it's significantly more efficient - // than checking for the Counter and Gauge interface implementations. In - // this case, we control all the types here. - switch m := c.rmMetrics[i].(type) { - case *counter: - // Guard against decreases. This should never happen, but a failure - // to do so will result in a panic, which is a harsh consequence for - // a metrics collection bug. - v0, v1 := m.get(), unwrapScalarRMValue(sample.Value) - if v1 > v0 { - m.Add(unwrapScalarRMValue(sample.Value) - m.get()) - } - m.Collect(ch) - case *gauge: - m.Set(unwrapScalarRMValue(sample.Value)) - m.Collect(ch) - case *batchHistogram: - m.update(sample.Value.Float64Histogram(), c.exactSumFor(sample.Name)) - m.Collect(ch) - default: - panic("unexpected metric type") + // Populate runtime/metrics sample buffer. + metrics.Read(c.sampleBuf) + + // Collect all our runtime/metrics user chose to expose from sampleBuf (if any). + for i, metric := range c.rmExposedMetrics { + // We created samples for exposed metrics first in order, so indexes match. + sample := c.sampleBuf[i] + + // N.B. switch on concrete type because it's significantly more efficient + // than checking for the Counter and Gauge interface implementations. In + // this case, we control all the types here. + switch m := metric.(type) { + case *counter: + // Guard against decreases. This should never happen, but a failure + // to do so will result in a panic, which is a harsh consequence for + // a metrics collection bug. + v0, v1 := m.get(), unwrapScalarRMValue(sample.Value) + if v1 > v0 { + m.Add(unwrapScalarRMValue(sample.Value) - m.get()) } + m.Collect(ch) + case *gauge: + m.Set(unwrapScalarRMValue(sample.Value)) + m.Collect(ch) + case *batchHistogram: + m.update(sample.Value.Float64Histogram(), c.exactSumFor(sample.Name)) + m.Collect(ch) + default: + panic("unexpected metric type") } } - // ms is a dummy MemStats that we populate ourselves so that we can - // populate the old metrics from it if goMemStatsCollection is enabled. - if c.opt.isEnabled(goRuntimeMemStatsCollection) { + if c.msMetricsEnabled { + // ms is a dummy MemStats that we populate ourselves so that we can + // populate the old metrics from it if goMemStatsCollection is enabled. var ms runtime.MemStats - memStatsFromRM(&ms, c.rmSampleMap) + memStatsFromRM(&ms, c.sampleMap) for _, i := range c.msMetrics { ch <- MustNewConstMetric(i.desc, i.valType, i.eval(&ms)) } @@ -336,11 +387,6 @@ func unwrapScalarRMValue(v metrics.Value) float64 { } } -var rmExactSumMap = map[string]string{ - "/gc/heap/allocs-by-size:bytes": "/gc/heap/allocs:bytes", - "/gc/heap/frees-by-size:bytes": "/gc/heap/frees:bytes", -} - // exactSumFor takes a runtime/metrics metric name (that is assumed to // be of kind KindFloat64Histogram) and returns its exact sum and whether // its exact sum exists. @@ -348,11 +394,11 @@ var rmExactSumMap = map[string]string{ // The runtime/metrics API for histograms doesn't currently expose exact // sums, but some of the other metrics are in fact exact sums of histograms. func (c *goCollector) exactSumFor(rmName string) float64 { - sumName, ok := rmExactSumMap[rmName] + sumName, ok := c.rmExactSumMapForHist[rmName] if !ok { return 0 } - s, ok := c.rmSampleMap[sumName] + s, ok := c.sampleMap[sumName] if !ok { return 0 } diff --git a/prometheus/go_collector_latest_test.go b/prometheus/go_collector_latest_test.go index 11094c828..11bff61c8 100644 --- a/prometheus/go_collector_latest_test.go +++ b/prometheus/go_collector_latest_test.go @@ -19,6 +19,7 @@ package prometheus import ( "math" "reflect" + "regexp" "runtime" "runtime/metrics" "sync" @@ -30,9 +31,18 @@ import ( ) func TestRmForMemStats(t *testing.T) { - if got, want := len(bestEffortLookupRM(rmForMemStats)), len(rmForMemStats); got != want { + descs := bestEffortLookupRM(rmNamesForMemStatsMetrics) + + if got, want := len(descs), len(rmNamesForMemStatsMetrics); got != want { t.Errorf("got %d, want %d metrics", got, want) } + + for _, d := range descs { + // We don't expect histograms there. + if d.Kind == metrics.KindFloat64Histogram { + t.Errorf("we don't expect to use histograms for MemStats metrics, got %v", d.Name) + } + } } func expectedBaseMetrics() map[string]struct{} { @@ -64,30 +74,43 @@ func addExpectedRuntimeMetrics(metrics map[string]struct{}) map[string]struct{} return metrics } -func TestGoCollector(t *testing.T) { +func TestGoCollector_ExposedMetrics(t *testing.T) { for _, tcase := range []struct { - collections uint32 + opts internal.GoCollectorOptions expectedFQNameSet map[string]struct{} }{ { - collections: 0, + opts: internal.GoCollectorOptions{ + DisableMemStatsLikeMetrics: true, + }, expectedFQNameSet: expectedBaseMetrics(), }, { - collections: goRuntimeMemStatsCollection, + // Default, only MemStats. expectedFQNameSet: addExpectedRuntimeMemStats(expectedBaseMetrics()), }, { - collections: goRuntimeMetricsCollection, + // Get all runtime/metrics without MemStats. + opts: internal.GoCollectorOptions{ + DisableMemStatsLikeMetrics: true, + RuntimeMetricRules: []*internal.GoCollectorRule{ + {Matcher: regexp.MustCompile("/.*")}, + }, + }, expectedFQNameSet: addExpectedRuntimeMetrics(expectedBaseMetrics()), }, { - collections: goRuntimeMemStatsCollection | goRuntimeMetricsCollection, + // Get all runtime/metrics and MemStats. + opts: internal.GoCollectorOptions{ + RuntimeMetricRules: []*internal.GoCollectorRule{ + {Matcher: regexp.MustCompile("/.*")}, + }, + }, expectedFQNameSet: addExpectedRuntimeMemStats(addExpectedRuntimeMetrics(expectedBaseMetrics())), }, } { if ok := t.Run("", func(t *testing.T) { - goMetrics := collectGoMetrics(t, tcase.collections) + goMetrics := collectGoMetrics(t, tcase.opts) goMetricSet := make(map[string]Metric) for _, m := range goMetrics { goMetricSet[m.Desc().fqName] = m @@ -118,7 +141,11 @@ func TestGoCollector(t *testing.T) { var sink interface{} func TestBatchHistogram(t *testing.T) { - goMetrics := collectGoMetrics(t, goRuntimeMetricsCollection) + goMetrics := collectGoMetrics(t, internal.GoCollectorOptions{ + RuntimeMetricRules: []*internal.GoCollectorRule{ + {Matcher: regexp.MustCompile("/.*")}, + }, + }) var mhist Metric for _, m := range goMetrics { @@ -145,7 +172,8 @@ func TestBatchHistogram(t *testing.T) { for i := 0; i < 100; i++ { sink = make([]byte, 128) } - collectGoMetrics(t, defaultGoCollections) + + collectGoMetrics(t, defaultGoCollectorOptions()) for i, v := range hist.counts { if v != countsCopy[i] { t.Error("counts changed during new collection") @@ -194,11 +222,13 @@ func TestBatchHistogram(t *testing.T) { } } -func collectGoMetrics(t *testing.T, enabledCollections uint32) []Metric { +func collectGoMetrics(t *testing.T, opts internal.GoCollectorOptions) []Metric { t.Helper() - c := NewGoCollector(func(o *GoCollectorOptions) { - o.EnabledCollections = enabledCollections + c := NewGoCollector(func(o *internal.GoCollectorOptions) { + o.DisableMemStatsLikeMetrics = opts.DisableMemStatsLikeMetrics + o.RuntimeMetricSumForHist = opts.RuntimeMetricSumForHist + o.RuntimeMetricRules = opts.RuntimeMetricRules }).(*goCollector) // Collect all metrics. @@ -222,7 +252,7 @@ func collectGoMetrics(t *testing.T, enabledCollections uint32) []Metric { func TestMemStatsEquivalence(t *testing.T) { var msReal, msFake runtime.MemStats - descs := bestEffortLookupRM(rmForMemStats) + descs := bestEffortLookupRM(rmNamesForMemStatsMetrics) samples := make([]metrics.Sample, len(descs)) samplesMap := make(map[string]*metrics.Sample) @@ -269,7 +299,12 @@ func TestMemStatsEquivalence(t *testing.T) { } func TestExpectedRuntimeMetrics(t *testing.T) { - goMetrics := collectGoMetrics(t, goRuntimeMetricsCollection) + goMetrics := collectGoMetrics(t, internal.GoCollectorOptions{ + DisableMemStatsLikeMetrics: true, + RuntimeMetricRules: []*internal.GoCollectorRule{ + {Matcher: regexp.MustCompile("/.*")}, + }, + }) goMetricSet := make(map[string]Metric) for _, m := range goMetrics { goMetricSet[m.Desc().fqName] = m diff --git a/prometheus/internal/go_collector_options.go b/prometheus/internal/go_collector_options.go new file mode 100644 index 000000000..1091d4bec --- /dev/null +++ b/prometheus/internal/go_collector_options.go @@ -0,0 +1,20 @@ +package internal + +import "regexp" + +type GoCollectorRule struct { + Matcher *regexp.Regexp + Deny bool + Buckets interface{} +} + +// GoCollectorOptions should not be used be directly by anything, except `collectors` package. +// Use it via collectors package instead. See issue +// https://github.com/prometheus/client_golang/issues/1030. +// +// This is internal, so external users only can use it via `collector.WithGoCollector*` methods +type GoCollectorOptions struct { + DisableMemStatsLikeMetrics bool + RuntimeMetricSumForHist map[string]string + RuntimeMetricRules []GoCollectorRule +}