From b94d7e25292465ac578c2e885e6d61e03a81ec4b Mon Sep 17 00:00:00 2001 From: bwplotka Date: Wed, 3 Aug 2022 13:11:35 +0200 Subject: [PATCH 1/4] goCollector: Added rule support for granular metric configuration. Fixes: https://github.com/prometheus/client_golang/issues/1089 Signed-off-by: bwplotka --- prometheus/collectors/go_collector_latest.go | 151 ++++++----- .../collectors/go_collector_latest_test.go | 11 +- prometheus/go_collector.go | 7 +- prometheus/go_collector_latest.go | 248 +++++++++++------- prometheus/go_collector_latest_test.go | 65 +++-- prometheus/internal/go_collector_options.go | 20 ++ 6 files changed, 320 insertions(+), 182 deletions(-) create mode 100644 prometheus/internal/go_collector_options.go diff --git a/prometheus/collectors/go_collector_latest.go b/prometheus/collectors/go_collector_latest.go index 9435e9614..cac72118c 100644 --- a/prometheus/collectors/go_collector_latest.go +++ b/prometheus/collectors/go_collector_latest.go @@ -16,77 +16,102 @@ package collectors -import "github.com/prometheus/client_golang/prometheus" +import ( + "regexp" -//nolint:staticcheck // Ignore SA1019 until v2. -type goOptions = prometheus.GoCollectorOptions -type goOption func(o *goOptions) - -type GoCollectionOption uint32 - -const ( - // GoRuntimeMemStatsCollection represents the metrics represented by runtime.MemStats structure such as - // go_memstats_alloc_bytes - // go_memstats_alloc_bytes_total - // go_memstats_sys_bytes - // go_memstats_lookups_total - // go_memstats_mallocs_total - // go_memstats_frees_total - // go_memstats_heap_alloc_bytes - // go_memstats_heap_sys_bytes - // go_memstats_heap_idle_bytes - // go_memstats_heap_inuse_bytes - // go_memstats_heap_released_bytes - // go_memstats_heap_objects - // go_memstats_stack_inuse_bytes - // go_memstats_stack_sys_bytes - // go_memstats_mspan_inuse_bytes - // go_memstats_mspan_sys_bytes - // go_memstats_mcache_inuse_bytes - // go_memstats_mcache_sys_bytes - // go_memstats_buck_hash_sys_bytes - // go_memstats_gc_sys_bytes - // go_memstats_other_sys_bytes - // go_memstats_next_gc_bytes - // so the metrics known from pre client_golang v1.12.0, except skipped go_memstats_gc_cpu_fraction (see - // https://github.com/prometheus/client_golang/issues/842#issuecomment-861812034 for explanation. - // - // NOTE that this mode represents runtime.MemStats statistics, but they are - // actually implemented using new runtime/metrics package. - // Deprecated: Use GoRuntimeMetricsCollection instead going forward. - GoRuntimeMemStatsCollection GoCollectionOption = 1 << iota - // GoRuntimeMetricsCollection is the new set of metrics represented by runtime/metrics package and follows - // consistent naming. The exposed metric set depends on Go version, but it is controlled against - // unexpected cardinality. This set has overlapping information with GoRuntimeMemStatsCollection, just with - // new names. GoRuntimeMetricsCollection is what is recommended for using going forward. - GoRuntimeMetricsCollection + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/internal" ) -// WithGoCollections allows enabling different collections for Go collector on top of base metrics -// like go_goroutines, go_threads, go_gc_duration_seconds, go_memstats_last_gc_time_seconds, go_info. +// WithGoCollectorMemStatsMetricsDisabled metrics that where representing runtime.MemStats structure such as: +// +// go_memstats_alloc_bytes +// go_memstats_alloc_bytes_total +// go_memstats_sys_bytes +// go_memstats_lookups_total +// go_memstats_mallocs_total +// go_memstats_frees_total +// go_memstats_heap_alloc_bytes +// go_memstats_heap_sys_bytes +// go_memstats_heap_idle_bytes +// go_memstats_heap_inuse_bytes +// go_memstats_heap_released_bytes +// go_memstats_heap_objects +// go_memstats_stack_inuse_bytes +// go_memstats_stack_sys_bytes +// go_memstats_mspan_inuse_bytes +// go_memstats_mspan_sys_bytes +// go_memstats_mcache_inuse_bytes +// go_memstats_mcache_sys_bytes +// go_memstats_buck_hash_sys_bytes +// go_memstats_gc_sys_bytes +// go_memstats_other_sys_bytes +// go_memstats_next_gc_bytes +// +// so the metrics known from pre client_golang v1.12.0, +// +// NOTE(bwplotka): The above represents runtime.MemStats statistics, but they are +// actually implemented using new runtime/metrics package. (except skipped go_memstats_gc_cpu_fraction +// -- see https://github.com/prometheus/client_golang/issues/842#issuecomment-861812034 for explanation). // -// Check GoRuntimeMemStatsCollection and GoRuntimeMetricsCollection for more details. You can use none, -// one or more collections at once. For example: -// WithGoCollections(GoRuntimeMemStatsCollection | GoRuntimeMetricsCollection) means both GoRuntimeMemStatsCollection -// metrics and GoRuntimeMetricsCollection will be exposed. +// Some users might want to disable this on collector level (although you can use scrape relabelling on Prometheus), +// because similar metrics can be now obtained using WithGoCollectorRuntimeMetrics. Note that the semantics of new +// metrics might be different, plus the names can be change over time with different Go version. // -// The current default is GoRuntimeMemStatsCollection, so the compatibility mode with -// client_golang pre v1.12 (move to runtime/metrics). -//nolint:staticcheck // Ignore SA1019 until v2. -func WithGoCollections(flags GoCollectionOption) func(options *prometheus.GoCollectorOptions) { - return func(o *goOptions) { - o.EnabledCollections = uint32(flags) +// NOTE(bwplotka): Changing metric names can be tedious at times as the alerts, recording rules and dashboards have to be adjusted. +// The old metrics are also very useful, with many guides and books written about how to interpret them. +// +// As a result our recommendation would be to stick with MemStats like metrics and enable other runtime/metrics if you are interested +// in advanced insights Go provides. See ExampleGoCollector_WithAdvancedGoMetrics. +func WithGoCollectorMemStatsMetricsDisabled() func(options *internal.GoCollectorOptions) { + return func(o *internal.GoCollectorOptions) { + o.DisableMemStatsLikeMetrics = true } } -// NewGoCollector returns a collector that exports metrics about the current Go -// process using debug.GCStats using runtime/metrics. -func NewGoCollector(opts ...goOption) prometheus.Collector { - //nolint:staticcheck // Ignore SA1019 until v2. - promPkgOpts := make([]func(o *prometheus.GoCollectorOptions), len(opts)) - for i, opt := range opts { - promPkgOpts[i] = opt +// GoRuntimeMetricsRule allow enabling and configuring particular group of runtime/metrics. +type GoRuntimeMetricsRule struct { + // Matcher represents RE2 expression will match the runtime/metrics from https://golang.bg/src/runtime/metrics/description.go + // Use `regexp.MustCompile` or `regexp.Compile` to create this field. + Matcher *regexp.Regexp +} + +// WithGoCollectorRuntimeMetrics allows enabling and configuring particular group of runtime/metrics. +// See the list of metrics https://golang.bg/src/runtime/metrics/description.go (pick the Go version you use there!). +// You can use this option in repeated manner, which will add new rules. The order of rules is important, the last rule +// that matches particular metrics is applied. +func WithGoCollectorRuntimeMetrics(rules ...GoRuntimeMetricsRule) func(options *internal.GoCollectorOptions) { + rs := make([]internal.GoCollectorRule, len(rules)) + for i, r := range rules { + rs[i] = internal.GoCollectorRule{ + Matcher: r.Matcher, + } } + + return func(o *internal.GoCollectorOptions) { + o.DisableMemStatsLikeMetrics = true + } +} + +// WithoutGoCollectorRuntimeMetrics allows disabling group of runtime/metrics that you might have added in WithGoCollectorRuntimeMetrics. +// It behaves similarly to WithGoCollectorRuntimeMetrics just with deny-list semantics. +func WithoutGoCollectorRuntimeMetrics(matchers ...*regexp.Regexp) func(options *internal.GoCollectorOptions) { + rs := make([]internal.GoCollectorRule, len(matchers)) + for i, m := range matchers { + rs[i] = internal.GoCollectorRule{ + Matcher: m, + Deny: true, + } + } + + return func(o *internal.GoCollectorOptions) { + o.RuntimeMetricRules = append(o.RuntimeMetricRules, rs...) + } +} + +// NewGoCollector returns a collector that exports metrics about the current Go +// process using debug.GCStats and runtime/metrics. +func NewGoCollector(opts ...func(o *internal.GoCollectorOptions)) prometheus.Collector { //nolint:staticcheck // Ignore SA1019 until v2. - return prometheus.NewGoCollector(promPkgOpts...) + return prometheus.NewGoCollector(opts...) } diff --git a/prometheus/collectors/go_collector_latest_test.go b/prometheus/collectors/go_collector_latest_test.go index 126864c32..06d9781a1 100644 --- a/prometheus/collectors/go_collector_latest_test.go +++ b/prometheus/collectors/go_collector_latest_test.go @@ -18,6 +18,7 @@ package collectors import ( "encoding/json" + "regexp" "testing" "github.com/prometheus/client_golang/prometheus" @@ -26,7 +27,9 @@ import ( func TestGoCollectorMarshalling(t *testing.T) { reg := prometheus.NewRegistry() reg.MustRegister(NewGoCollector( - WithGoCollections(GoRuntimeMemStatsCollection | GoRuntimeMetricsCollection), + WithGoCollectorRuntimeMetrics(GoRuntimeMetricsRule{ + Matcher: regexp.MustCompile("/.*"), + }), )) result, err := reg.Gather() if err != nil { @@ -37,3 +40,9 @@ func TestGoCollectorMarshalling(t *testing.T) { t.Errorf("json marshalling shoud not fail, %v", err) } } + +func ExampleGoCollector() { +} + +func ExampleGoCollector_WithAdvancedGoMetrics() { +} diff --git a/prometheus/go_collector.go b/prometheus/go_collector.go index 5fd730043..ad9a71a5e 100644 --- a/prometheus/go_collector.go +++ b/prometheus/go_collector.go @@ -19,6 +19,10 @@ import ( "time" ) +// goRuntimeMemStats provides the metrics initially provided by runtime.ReadMemStats. +// From Go 1.17 those similar (and better) statistics are provided by runtime/metrics, so +// while eval closure works on runtime.MemStats, the struct from Go 1.17+ is +// populated using runtime/metrics. func goRuntimeMemStats() memStatsMetrics { return memStatsMetrics{ { @@ -224,7 +228,7 @@ func newBaseGoCollector() baseGoCollector { "A summary of the pause duration of garbage collection cycles.", nil, nil), gcLastTimeDesc: NewDesc( - memstatNamespace("last_gc_time_seconds"), + "go_memstats_last_gc_time_seconds", "Number of seconds since 1970 of last garbage collection.", nil, nil), goInfoDesc: NewDesc( @@ -270,7 +274,6 @@ func memstatNamespace(s string) string { // memStatsMetrics provide description, evaluator, runtime/metrics name, and // value type for memstat metrics. -// TODO(bwplotka): Remove with end Go 1.16 EOL and replace with runtime/metrics.Description type memStatsMetrics []struct { desc *Desc eval func(*runtime.MemStats) float64 diff --git a/prometheus/go_collector_latest.go b/prometheus/go_collector_latest.go index 68a7a156c..3a2d55e84 100644 --- a/prometheus/go_collector_latest.go +++ b/prometheus/go_collector_latest.go @@ -31,9 +31,11 @@ import ( ) const ( + // constants for strings referenced more than once. goGCHeapTinyAllocsObjects = "/gc/heap/tiny/allocs:objects" goGCHeapAllocsObjects = "/gc/heap/allocs:objects" goGCHeapFreesObjects = "/gc/heap/frees:objects" + goGCHeapFreesBytes = "/gc/heap/frees:bytes" goGCHeapAllocsBytes = "/gc/heap/allocs:bytes" goGCHeapObjects = "/gc/heap/objects:objects" goGCHeapGoalBytes = "/gc/heap/goal:bytes" @@ -53,8 +55,8 @@ const ( goMemoryClassesOtherBytes = "/memory/classes/other:bytes" ) -// runtime/metrics names required for runtimeMemStats like logic. -var rmForMemStats = []string{ +// rmNamesForMemStatsMetrics represents runtime/metrics names required to populate goRuntimeMemStats from like logic. +var rmNamesForMemStatsMetrics = []string{ goGCHeapTinyAllocsObjects, goGCHeapAllocsObjects, goGCHeapFreesObjects, @@ -90,74 +92,90 @@ func bestEffortLookupRM(lookup []string) []metrics.Description { } type goCollector struct { - opt GoCollectorOptions base baseGoCollector // mu protects updates to all fields ensuring a consistent // snapshot is always produced by Collect. mu sync.Mutex - // rm... fields all pertain to the runtime/metrics package. - rmSampleBuf []metrics.Sample - rmSampleMap map[string]*metrics.Sample - rmMetrics []collectorMetric + // Contains all samples that has to retrieved from runtime/metrics (not all of them will be exposed). + sampleBuf []metrics.Sample + // sampleMap allows lookup for MemStats metrics and runtime/metrics histograms for exact sums. + sampleMap map[string]*metrics.Sample + + // rmExposedMetrics represents all runtime/metrics package metrics + // that were configured to be exposed. + rmExposedMetrics []collectorMetric + rmExactSumMapForHist map[string]string // With Go 1.17, the runtime/metrics package was introduced. // From that point on, metric names produced by the runtime/metrics // package could be generated from runtime/metrics names. However, // these differ from the old names for the same values. // - // This field exist to export the same values under the old names + // This field exists to export the same values under the old names // as well. - msMetrics memStatsMetrics + msMetrics memStatsMetrics + msMetricsEnabled bool } -const ( - // Those are not exposed due to need to move Go collector to another package in v2. - // See issue https://github.com/prometheus/client_golang/issues/1030. - goRuntimeMemStatsCollection uint32 = 1 << iota - goRuntimeMetricsCollection -) - -// GoCollectorOptions should not be used be directly by anything, except `collectors` package. -// Use it via collectors package instead. See issue -// https://github.com/prometheus/client_golang/issues/1030. -// -// Deprecated: Use collectors.WithGoCollections -type GoCollectorOptions struct { - // EnabledCollection sets what type of collections collector should expose on top of base collection. - // By default it's goMemStatsCollection | goRuntimeMetricsCollection. - EnabledCollections uint32 +type rmMetricDesc struct { + metrics.Description } -func (c GoCollectorOptions) isEnabled(flag uint32) bool { - return c.EnabledCollections&flag != 0 +func matchRuntimeMetricsRules(rules []internal.GoCollectorRule) []rmMetricDesc { + var descs []rmMetricDesc + for _, d := range metrics.All() { + var ( + deny = true + desc rmMetricDesc + ) + + for _, r := range rules { + if !r.Matcher.MatchString(d.Name) { + continue + } + deny = r.Deny + } + if deny { + continue + } + + desc.Description = d + descs = append(descs, desc) + } + return descs } -const defaultGoCollections = goRuntimeMemStatsCollection +func defaultGoCollectorOptions() internal.GoCollectorOptions { + return internal.GoCollectorOptions{ + RuntimeMetricSumForHist: map[string]string{ + "/gc/heap/allocs-by-size:bytes": goGCHeapAllocsBytes, + "/gc/heap/frees-by-size:bytes": goGCHeapFreesBytes, + }, + RuntimeMetricRules: []internal.GoCollectorRule{ + //{Matcher: regexp.MustCompile("")}, + }, + } +} // NewGoCollector is the obsolete version of collectors.NewGoCollector. // See there for documentation. // // Deprecated: Use collectors.NewGoCollector instead. -func NewGoCollector(opts ...func(o *GoCollectorOptions)) Collector { - opt := GoCollectorOptions{EnabledCollections: defaultGoCollections} +func NewGoCollector(opts ...func(o *internal.GoCollectorOptions)) Collector { + opt := defaultGoCollectorOptions() for _, o := range opts { o(&opt) } - var descriptions []metrics.Description - if opt.isEnabled(goRuntimeMetricsCollection) { - descriptions = metrics.All() - } else if opt.isEnabled(goRuntimeMemStatsCollection) { - descriptions = bestEffortLookupRM(rmForMemStats) - } + exposedDescriptions := matchRuntimeMetricsRules(opt.RuntimeMetricRules) // Collect all histogram samples so that we can get their buckets. // The API guarantees that the buckets are always fixed for the lifetime // of the process. var histograms []metrics.Sample - for _, d := range descriptions { + for _, d := range exposedDescriptions { if d.Kind == metrics.KindFloat64Histogram { histograms = append(histograms, metrics.Sample{Name: d.Name}) } @@ -172,13 +190,14 @@ func NewGoCollector(opts ...func(o *GoCollectorOptions)) Collector { bucketsMap[histograms[i].Name] = histograms[i].Value.Float64Histogram().Buckets } - // Generate a Desc and ValueType for each runtime/metrics metric. - metricSet := make([]collectorMetric, 0, len(descriptions)) - sampleBuf := make([]metrics.Sample, 0, len(descriptions)) - sampleMap := make(map[string]*metrics.Sample, len(descriptions)) - for i := range descriptions { - d := &descriptions[i] - namespace, subsystem, name, ok := internal.RuntimeMetricsToProm(d) + // Generate a collector for each exposed runtime/metrics metric. + metricSet := make([]collectorMetric, 0, len(exposedDescriptions)) + // SampleBuf is used for reading from runtime/metrics. + // We are assuming the largest case to have stable pointers for sampleMap purposes. + sampleBuf := make([]metrics.Sample, 0, len(exposedDescriptions)+len(opt.RuntimeMetricSumForHist)+len(rmNamesForMemStatsMetrics)) + sampleMap := make(map[string]*metrics.Sample, len(exposedDescriptions)) + for _, d := range exposedDescriptions { + namespace, subsystem, name, ok := internal.RuntimeMetricsToProm(&d.Description) if !ok { // Just ignore this metric; we can't do anything with it here. // If a user decides to use the latest version of Go, we don't want @@ -186,19 +205,17 @@ func NewGoCollector(opts ...func(o *GoCollectorOptions)) Collector { continue } - // Set up sample buffer for reading, and a map - // for quick lookup of sample values. sampleBuf = append(sampleBuf, metrics.Sample{Name: d.Name}) sampleMap[d.Name] = &sampleBuf[len(sampleBuf)-1] var m collectorMetric if d.Kind == metrics.KindFloat64Histogram { - _, hasSum := rmExactSumMap[d.Name] + _, hasSum := opt.RuntimeMetricSumForHist[d.Name] unit := d.Name[strings.IndexRune(d.Name, ':')+1:] m = newBatchHistogram( NewDesc( BuildFQName(namespace, subsystem, name), - d.Description, + d.Description.Description, nil, nil, ), @@ -210,30 +227,61 @@ func NewGoCollector(opts ...func(o *GoCollectorOptions)) Collector { Namespace: namespace, Subsystem: subsystem, Name: name, - Help: d.Description, - }) + Help: d.Description.Description, + }, + ) } else { m = NewGauge(GaugeOpts{ Namespace: namespace, Subsystem: subsystem, Name: name, - Help: d.Description, + Help: d.Description.Description, }) } metricSet = append(metricSet, m) } - var msMetrics memStatsMetrics - if opt.isEnabled(goRuntimeMemStatsCollection) { + // Add exact sum metrics to sampleBuf if not added before. + for _, h := range histograms { + sumMetric, ok := opt.RuntimeMetricSumForHist[h.Name] + if !ok { + continue + } + + if _, ok := sampleMap[sumMetric]; ok { + continue + } + sampleBuf = append(sampleBuf, metrics.Sample{Name: sumMetric}) + sampleMap[sumMetric] = &sampleBuf[len(sampleBuf)-1] + } + + var ( + msMetrics memStatsMetrics + msDescriptions []metrics.Description + ) + + if !opt.DisableMemStatsLikeMetrics { msMetrics = goRuntimeMemStats() + msDescriptions = bestEffortLookupRM(rmNamesForMemStatsMetrics) + + // Check if metric was not exposed before and if not, add to sampleBuf. + for _, mdDesc := range msDescriptions { + if _, ok := sampleMap[mdDesc.Name]; ok { + continue + } + sampleBuf = append(sampleBuf, metrics.Sample{Name: mdDesc.Name}) + sampleMap[mdDesc.Name] = &sampleBuf[len(sampleBuf)-1] + } } + return &goCollector{ - opt: opt, - base: newBaseGoCollector(), - rmSampleBuf: sampleBuf, - rmSampleMap: sampleMap, - rmMetrics: metricSet, - msMetrics: msMetrics, + base: newBaseGoCollector(), + sampleBuf: sampleBuf, + sampleMap: sampleMap, + rmExposedMetrics: metricSet, + rmExactSumMapForHist: opt.RuntimeMetricSumForHist, + msMetrics: msMetrics, + msMetricsEnabled: !opt.DisableMemStatsLikeMetrics, } } @@ -243,7 +291,7 @@ func (c *goCollector) Describe(ch chan<- *Desc) { for _, i := range c.msMetrics { ch <- i.desc } - for _, m := range c.rmMetrics { + for _, m := range c.rmExposedMetrics { ch <- m.Desc() } } @@ -253,8 +301,12 @@ func (c *goCollector) Collect(ch chan<- Metric) { // Collect base non-memory metrics. c.base.Collect(ch) + if len(c.sampleBuf) == 0 { + return + } + // Collect must be thread-safe, so prevent concurrent use of - // rmSampleBuf. Just read into rmSampleBuf but write all the data + // sampleBuf elements. Just read into sampleBuf but write all the data // we get into our Metrics or MemStats. // // This lock also ensures that the Metrics we send out are all from @@ -268,44 +320,43 @@ func (c *goCollector) Collect(ch chan<- Metric) { c.mu.Lock() defer c.mu.Unlock() - if len(c.rmSampleBuf) > 0 { - // Populate runtime/metrics sample buffer. - metrics.Read(c.rmSampleBuf) - } - - if c.opt.isEnabled(goRuntimeMetricsCollection) { - // Collect all our metrics from rmSampleBuf. - for i, sample := range c.rmSampleBuf { - // N.B. switch on concrete type because it's significantly more efficient - // than checking for the Counter and Gauge interface implementations. In - // this case, we control all the types here. - switch m := c.rmMetrics[i].(type) { - case *counter: - // Guard against decreases. This should never happen, but a failure - // to do so will result in a panic, which is a harsh consequence for - // a metrics collection bug. - v0, v1 := m.get(), unwrapScalarRMValue(sample.Value) - if v1 > v0 { - m.Add(unwrapScalarRMValue(sample.Value) - m.get()) - } - m.Collect(ch) - case *gauge: - m.Set(unwrapScalarRMValue(sample.Value)) - m.Collect(ch) - case *batchHistogram: - m.update(sample.Value.Float64Histogram(), c.exactSumFor(sample.Name)) - m.Collect(ch) - default: - panic("unexpected metric type") + // Populate runtime/metrics sample buffer. + metrics.Read(c.sampleBuf) + + // Collect all our runtime/metrics user chose to expose from sampleBuf (if any). + for i, metric := range c.rmExposedMetrics { + // We created samples for exposed metrics first in order, so indexes match. + sample := c.sampleBuf[i] + + // N.B. switch on concrete type because it's significantly more efficient + // than checking for the Counter and Gauge interface implementations. In + // this case, we control all the types here. + switch m := metric.(type) { + case *counter: + // Guard against decreases. This should never happen, but a failure + // to do so will result in a panic, which is a harsh consequence for + // a metrics collection bug. + v0, v1 := m.get(), unwrapScalarRMValue(sample.Value) + if v1 > v0 { + m.Add(unwrapScalarRMValue(sample.Value) - m.get()) } + m.Collect(ch) + case *gauge: + m.Set(unwrapScalarRMValue(sample.Value)) + m.Collect(ch) + case *batchHistogram: + m.update(sample.Value.Float64Histogram(), c.exactSumFor(sample.Name)) + m.Collect(ch) + default: + panic("unexpected metric type") } } - // ms is a dummy MemStats that we populate ourselves so that we can - // populate the old metrics from it if goMemStatsCollection is enabled. - if c.opt.isEnabled(goRuntimeMemStatsCollection) { + if c.msMetricsEnabled { + // ms is a dummy MemStats that we populate ourselves so that we can + // populate the old metrics from it if goMemStatsCollection is enabled. var ms runtime.MemStats - memStatsFromRM(&ms, c.rmSampleMap) + memStatsFromRM(&ms, c.sampleMap) for _, i := range c.msMetrics { ch <- MustNewConstMetric(i.desc, i.valType, i.eval(&ms)) } @@ -336,11 +387,6 @@ func unwrapScalarRMValue(v metrics.Value) float64 { } } -var rmExactSumMap = map[string]string{ - "/gc/heap/allocs-by-size:bytes": "/gc/heap/allocs:bytes", - "/gc/heap/frees-by-size:bytes": "/gc/heap/frees:bytes", -} - // exactSumFor takes a runtime/metrics metric name (that is assumed to // be of kind KindFloat64Histogram) and returns its exact sum and whether // its exact sum exists. @@ -348,11 +394,11 @@ var rmExactSumMap = map[string]string{ // The runtime/metrics API for histograms doesn't currently expose exact // sums, but some of the other metrics are in fact exact sums of histograms. func (c *goCollector) exactSumFor(rmName string) float64 { - sumName, ok := rmExactSumMap[rmName] + sumName, ok := c.rmExactSumMapForHist[rmName] if !ok { return 0 } - s, ok := c.rmSampleMap[sumName] + s, ok := c.sampleMap[sumName] if !ok { return 0 } diff --git a/prometheus/go_collector_latest_test.go b/prometheus/go_collector_latest_test.go index df18d5d25..d64120cef 100644 --- a/prometheus/go_collector_latest_test.go +++ b/prometheus/go_collector_latest_test.go @@ -19,6 +19,7 @@ package prometheus import ( "math" "reflect" + "regexp" "runtime" "runtime/metrics" "sync" @@ -30,9 +31,18 @@ import ( ) func TestRmForMemStats(t *testing.T) { - if got, want := len(bestEffortLookupRM(rmForMemStats)), len(rmForMemStats); got != want { + descs := bestEffortLookupRM(rmNamesForMemStatsMetrics) + + if got, want := len(descs), len(rmNamesForMemStatsMetrics); got != want { t.Errorf("got %d, want %d metrics", got, want) } + + for _, d := range descs { + // We don't expect histograms there. + if d.Kind == metrics.KindFloat64Histogram { + t.Errorf("we don't expect to use histograms for MemStats metrics, got %v", d.Name) + } + } } func expectedBaseMetrics() map[string]struct{} { @@ -64,30 +74,43 @@ func addExpectedRuntimeMetrics(metrics map[string]struct{}) map[string]struct{} return metrics } -func TestGoCollector(t *testing.T) { +func TestGoCollector_ExposedMetrics(t *testing.T) { for _, tcase := range []struct { - collections uint32 + opts internal.GoCollectorOptions expectedFQNameSet map[string]struct{} }{ { - collections: 0, + opts: internal.GoCollectorOptions{ + DisableMemStatsLikeMetrics: true, + }, expectedFQNameSet: expectedBaseMetrics(), }, { - collections: goRuntimeMemStatsCollection, + // Default, only MemStats. expectedFQNameSet: addExpectedRuntimeMemStats(expectedBaseMetrics()), }, { - collections: goRuntimeMetricsCollection, + // Get all runtime/metrics without MemStats. + opts: internal.GoCollectorOptions{ + DisableMemStatsLikeMetrics: true, + RuntimeMetricRules: []internal.GoCollectorRule{ + {Matcher: regexp.MustCompile("/.*")}, + }, + }, expectedFQNameSet: addExpectedRuntimeMetrics(expectedBaseMetrics()), }, { - collections: goRuntimeMemStatsCollection | goRuntimeMetricsCollection, + // Get all runtime/metrics and MemStats. + opts: internal.GoCollectorOptions{ + RuntimeMetricRules: []internal.GoCollectorRule{ + {Matcher: regexp.MustCompile("/.*")}, + }, + }, expectedFQNameSet: addExpectedRuntimeMemStats(addExpectedRuntimeMetrics(expectedBaseMetrics())), }, } { if ok := t.Run("", func(t *testing.T) { - goMetrics := collectGoMetrics(t, tcase.collections) + goMetrics := collectGoMetrics(t, tcase.opts) goMetricSet := make(map[string]Metric) for _, m := range goMetrics { goMetricSet[m.Desc().fqName] = m @@ -118,7 +141,11 @@ func TestGoCollector(t *testing.T) { var sink interface{} func TestBatchHistogram(t *testing.T) { - goMetrics := collectGoMetrics(t, goRuntimeMetricsCollection) + goMetrics := collectGoMetrics(t, internal.GoCollectorOptions{ + RuntimeMetricRules: []internal.GoCollectorRule{ + {Matcher: regexp.MustCompile("/.*")}, + }, + }) var mhist Metric for _, m := range goMetrics { @@ -145,7 +172,8 @@ func TestBatchHistogram(t *testing.T) { for i := 0; i < 100; i++ { sink = make([]byte, 128) } - collectGoMetrics(t, defaultGoCollections) + + collectGoMetrics(t, defaultGoCollectorOptions()) for i, v := range hist.counts { if v != countsCopy[i] { t.Error("counts changed during new collection") @@ -194,11 +222,13 @@ func TestBatchHistogram(t *testing.T) { } } -func collectGoMetrics(t *testing.T, enabledCollections uint32) []Metric { +func collectGoMetrics(t *testing.T, opts internal.GoCollectorOptions) []Metric { t.Helper() - c := NewGoCollector(func(o *GoCollectorOptions) { - o.EnabledCollections = enabledCollections + c := NewGoCollector(func(o *internal.GoCollectorOptions) { + o.DisableMemStatsLikeMetrics = opts.DisableMemStatsLikeMetrics + o.RuntimeMetricSumForHist = opts.RuntimeMetricSumForHist + o.RuntimeMetricRules = opts.RuntimeMetricRules }).(*goCollector) // Collect all metrics. @@ -222,7 +252,7 @@ func collectGoMetrics(t *testing.T, enabledCollections uint32) []Metric { func TestMemStatsEquivalence(t *testing.T) { var msReal, msFake runtime.MemStats - descs := bestEffortLookupRM(rmForMemStats) + descs := bestEffortLookupRM(rmNamesForMemStatsMetrics) samples := make([]metrics.Sample, len(descs)) samplesMap := make(map[string]*metrics.Sample) @@ -269,7 +299,12 @@ func TestMemStatsEquivalence(t *testing.T) { } func TestExpectedRuntimeMetrics(t *testing.T) { - goMetrics := collectGoMetrics(t, goRuntimeMemStatsCollection|goRuntimeMetricsCollection) + goMetrics := collectGoMetrics(t, internal.GoCollectorOptions{ + DisableMemStatsLikeMetrics: true, + RuntimeMetricRules: []internal.GoCollectorRule{ + {Matcher: regexp.MustCompile("/.*")}, + }, + }) goMetricSet := make(map[string]Metric) for _, m := range goMetrics { goMetricSet[m.Desc().fqName] = m diff --git a/prometheus/internal/go_collector_options.go b/prometheus/internal/go_collector_options.go new file mode 100644 index 000000000..1091d4bec --- /dev/null +++ b/prometheus/internal/go_collector_options.go @@ -0,0 +1,20 @@ +package internal + +import "regexp" + +type GoCollectorRule struct { + Matcher *regexp.Regexp + Deny bool + Buckets interface{} +} + +// GoCollectorOptions should not be used be directly by anything, except `collectors` package. +// Use it via collectors package instead. See issue +// https://github.com/prometheus/client_golang/issues/1030. +// +// This is internal, so external users only can use it via `collector.WithGoCollector*` methods +type GoCollectorOptions struct { + DisableMemStatsLikeMetrics bool + RuntimeMetricSumForHist map[string]string + RuntimeMetricRules []GoCollectorRule +} From 881c60842d7247b146eceef083ef7880ecf662ed Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Fri, 5 Aug 2022 17:31:21 +0200 Subject: [PATCH 2/4] Added compatibility mode with old options. (#1107) * Added compatibility mode with old options. Signed-off-by: bwplotka * Copyright header. Signed-off-by: bwplotka --- prometheus/collectors/go_collector_latest.go | 30 +++++++++++++++++++- prometheus/internal/go_collector_options.go | 13 +++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/prometheus/collectors/go_collector_latest.go b/prometheus/collectors/go_collector_latest.go index cac72118c..10756e030 100644 --- a/prometheus/collectors/go_collector_latest.go +++ b/prometheus/collectors/go_collector_latest.go @@ -109,8 +109,36 @@ func WithoutGoCollectorRuntimeMetrics(matchers ...*regexp.Regexp) func(options * } } +// GoCollectionOption represents Go collection option flag. +// Deprecated. +type GoCollectionOption uint32 + +const ( + // GoRuntimeMemStatsCollection represents the metrics represented by runtime.MemStats structure. + // Deprecated. Use WithGoCollectorMemStatsMetricsDisabled() function to disable those metrics in the collector. + GoRuntimeMemStatsCollection GoCollectionOption = 1 << iota + // GoRuntimeMetricsCollection is the new set of metrics represented by runtime/metrics package. + // Deprecated. Use WithGoCollectorRuntimeMetrics(GoRuntimeMetricsRule{Matcher: regexp.MustCompile("/.*")}) + // function to enable those metrics in the collector. + GoRuntimeMetricsCollection +) + +// WithGoCollections allows enabling different collections for Go collector on top of base metrics. +// Deprecated. Use WithGoCollectorRuntimeMetrics() and WithGoCollectorMemStatsMetricsDisabled() instead to control metrics. +func WithGoCollections(flags GoCollectionOption) func(options *internal.GoCollectorOptions) { + return func(options *internal.GoCollectorOptions) { + if flags&GoRuntimeMemStatsCollection == 0 { + WithGoCollectorMemStatsMetricsDisabled()(options) + } + + if flags&GoRuntimeMetricsCollection != 0 { + WithGoCollectorRuntimeMetrics(GoRuntimeMetricsRule{Matcher: regexp.MustCompile("/.*")})(options) + } + } +} + // NewGoCollector returns a collector that exports metrics about the current Go -// process using debug.GCStats and runtime/metrics. +// process using debug.GCStats (base metrics) and runtime/metrics (both in MemStats style and new ones). func NewGoCollector(opts ...func(o *internal.GoCollectorOptions)) prometheus.Collector { //nolint:staticcheck // Ignore SA1019 until v2. return prometheus.NewGoCollector(opts...) diff --git a/prometheus/internal/go_collector_options.go b/prometheus/internal/go_collector_options.go index 1091d4bec..8fd32fcb7 100644 --- a/prometheus/internal/go_collector_options.go +++ b/prometheus/internal/go_collector_options.go @@ -1,3 +1,16 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package internal import "regexp" From c0a21ca21b0618b4650dd983bc37b19cff413f33 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Fri, 5 Aug 2022 18:24:50 +0200 Subject: [PATCH 3/4] Remove bucket option for now. (#1108) Signed-off-by: bwplotka --- prometheus/collectors/go_collector_latest.go | 1 + prometheus/internal/go_collector_options.go | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/prometheus/collectors/go_collector_latest.go b/prometheus/collectors/go_collector_latest.go index 10756e030..232601002 100644 --- a/prometheus/collectors/go_collector_latest.go +++ b/prometheus/collectors/go_collector_latest.go @@ -70,6 +70,7 @@ func WithGoCollectorMemStatsMetricsDisabled() func(options *internal.GoCollector } // GoRuntimeMetricsRule allow enabling and configuring particular group of runtime/metrics. +// TODO(bwplotka): Consider adding ability to adjust buckets. type GoRuntimeMetricsRule struct { // Matcher represents RE2 expression will match the runtime/metrics from https://golang.bg/src/runtime/metrics/description.go // Use `regexp.MustCompile` or `regexp.Compile` to create this field. diff --git a/prometheus/internal/go_collector_options.go b/prometheus/internal/go_collector_options.go index 8fd32fcb7..723b45d64 100644 --- a/prometheus/internal/go_collector_options.go +++ b/prometheus/internal/go_collector_options.go @@ -18,7 +18,6 @@ import "regexp" type GoCollectorRule struct { Matcher *regexp.Regexp Deny bool - Buckets interface{} } // GoCollectorOptions should not be used be directly by anything, except `collectors` package. From 434aec433fcfc87ea6188661a7689f1612fd201d Mon Sep 17 00:00:00 2001 From: Kemal Akkoyun Date: Fri, 5 Aug 2022 19:31:36 +0200 Subject: [PATCH 4/4] collectors/GoCollector: Add tests and examples (#1109) * Add tests and examples Signed-off-by: Kemal Akkoyun * Add docs for the presets Signed-off-by: Kemal Akkoyun --- prometheus/collector.go | 6 +- prometheus/collectors/go_collector_latest.go | 18 +- .../collectors/go_collector_latest_test.go | 252 ++++++++++++++++++ 3 files changed, 271 insertions(+), 5 deletions(-) diff --git a/prometheus/collector.go b/prometheus/collector.go index ac1ca3cf5..cf05079fb 100644 --- a/prometheus/collector.go +++ b/prometheus/collector.go @@ -69,9 +69,9 @@ type Collector interface { // If a Collector collects the same metrics throughout its lifetime, its // Describe method can simply be implemented as: // -// func (c customCollector) Describe(ch chan<- *Desc) { -// DescribeByCollect(c, ch) -// } +// func (c customCollector) Describe(ch chan<- *Desc) { +// DescribeByCollect(c, ch) +// } // // However, this will not work if the metrics collected change dynamically over // the lifetime of the Collector in a way that their combined set of descriptors diff --git a/prometheus/collectors/go_collector_latest.go b/prometheus/collectors/go_collector_latest.go index 232601002..246c5ea94 100644 --- a/prometheus/collectors/go_collector_latest.go +++ b/prometheus/collectors/go_collector_latest.go @@ -23,7 +23,21 @@ import ( "github.com/prometheus/client_golang/prometheus/internal" ) -// WithGoCollectorMemStatsMetricsDisabled metrics that where representing runtime.MemStats structure such as: +var ( + // MetricsAll allows all the metrics to be collected from Go runtime. + MetricsAll = GoRuntimeMetricsRule{regexp.MustCompile("/.*")} + // MetricsGC allows only GC metrics to be collected from Go runtime. + // e.g. go_gc_cycles_automatic_gc_cycles_total + MetricsGC = GoRuntimeMetricsRule{regexp.MustCompile(`^/gc/.*`)} + // MetricsMemory allows only memory metrics to be collected from Go runtime. + // e.g. go_memory_classes_heap_free_bytes + MetricsMemory = GoRuntimeMetricsRule{regexp.MustCompile(`^/memory/.*`)} + // MetricsScheduler allows only scheduler metrics to be collected from Go runtime. + // e.g. go_sched_goroutines_goroutines + MetricsScheduler = GoRuntimeMetricsRule{regexp.MustCompile(`^/sched/.*`)} +) + +// WithGoCollectorMemStatsMetricsDisabled disables metrics that is gathered in runtime.MemStats structure such as: // // go_memstats_alloc_bytes // go_memstats_alloc_bytes_total @@ -90,7 +104,7 @@ func WithGoCollectorRuntimeMetrics(rules ...GoRuntimeMetricsRule) func(options * } return func(o *internal.GoCollectorOptions) { - o.DisableMemStatsLikeMetrics = true + o.RuntimeMetricRules = append(o.RuntimeMetricRules, rs...) } } diff --git a/prometheus/collectors/go_collector_latest_test.go b/prometheus/collectors/go_collector_latest_test.go index 06d9781a1..96cdb7183 100644 --- a/prometheus/collectors/go_collector_latest_test.go +++ b/prometheus/collectors/go_collector_latest_test.go @@ -18,12 +18,25 @@ package collectors import ( "encoding/json" + "log" + "net/http" + "reflect" "regexp" + "sort" "testing" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" ) +var baseMetrics = []string{ + "go_gc_duration_seconds", + "go_goroutines", + "go_info", + "go_memstats_last_gc_time_seconds", + "go_threads", +} + func TestGoCollectorMarshalling(t *testing.T) { reg := prometheus.NewRegistry() reg.MustRegister(NewGoCollector( @@ -41,8 +54,247 @@ func TestGoCollectorMarshalling(t *testing.T) { } } +func TestWithGoCollectorMemStatsMetricsDisabled(t *testing.T) { + reg := prometheus.NewRegistry() + reg.MustRegister(NewGoCollector( + WithGoCollectorMemStatsMetricsDisabled(), + )) + result, err := reg.Gather() + if err != nil { + t.Fatal(err) + } + + got := []string{} + for _, r := range result { + got = append(got, r.GetName()) + } + + if !reflect.DeepEqual(got, baseMetrics) { + t.Errorf("got %v, want %v", got, baseMetrics) + } +} + +func TestGoCollectorAllowList(t *testing.T) { + for _, test := range []struct { + name string + rules []GoRuntimeMetricsRule + expected []string + }{ + { + name: "Without any rules", + rules: nil, + expected: baseMetrics, + }, + { + name: "allow all", + rules: []GoRuntimeMetricsRule{MetricsAll}, + expected: withBaseMetrics([]string{ + "go_gc_cycles_automatic_gc_cycles_total", + "go_gc_cycles_forced_gc_cycles_total", + "go_gc_cycles_total_gc_cycles_total", + "go_gc_heap_allocs_by_size_bytes", + "go_gc_heap_allocs_bytes_total", + "go_gc_heap_allocs_objects_total", + "go_gc_heap_frees_by_size_bytes", + "go_gc_heap_frees_bytes_total", + "go_gc_heap_frees_objects_total", + "go_gc_heap_goal_bytes", + "go_gc_heap_objects_objects", + "go_gc_heap_tiny_allocs_objects_total", + "go_gc_pauses_seconds", + "go_memory_classes_heap_free_bytes", + "go_memory_classes_heap_objects_bytes", + "go_memory_classes_heap_released_bytes", + "go_memory_classes_heap_stacks_bytes", + "go_memory_classes_heap_unused_bytes", + "go_memory_classes_metadata_mcache_free_bytes", + "go_memory_classes_metadata_mcache_inuse_bytes", + "go_memory_classes_metadata_mspan_free_bytes", + "go_memory_classes_metadata_mspan_inuse_bytes", + "go_memory_classes_metadata_other_bytes", + "go_memory_classes_os_stacks_bytes", + "go_memory_classes_other_bytes", + "go_memory_classes_profiling_buckets_bytes", + "go_memory_classes_total_bytes", + "go_sched_goroutines_goroutines", + "go_sched_latencies_seconds", + }), + }, + { + name: "allow GC", + rules: []GoRuntimeMetricsRule{MetricsGC}, + expected: withBaseMetrics([]string{ + "go_gc_cycles_automatic_gc_cycles_total", + "go_gc_cycles_forced_gc_cycles_total", + "go_gc_cycles_total_gc_cycles_total", + "go_gc_heap_allocs_by_size_bytes", + "go_gc_heap_allocs_bytes_total", + "go_gc_heap_allocs_objects_total", + "go_gc_heap_frees_by_size_bytes", + "go_gc_heap_frees_bytes_total", + "go_gc_heap_frees_objects_total", + "go_gc_heap_goal_bytes", + "go_gc_heap_objects_objects", + "go_gc_heap_tiny_allocs_objects_total", + "go_gc_pauses_seconds", + }), + }, + { + name: "allow Memory", + rules: []GoRuntimeMetricsRule{MetricsMemory}, + expected: withBaseMetrics([]string{ + "go_memory_classes_heap_free_bytes", + "go_memory_classes_heap_objects_bytes", + "go_memory_classes_heap_released_bytes", + "go_memory_classes_heap_stacks_bytes", + "go_memory_classes_heap_unused_bytes", + "go_memory_classes_metadata_mcache_free_bytes", + "go_memory_classes_metadata_mcache_inuse_bytes", + "go_memory_classes_metadata_mspan_free_bytes", + "go_memory_classes_metadata_mspan_inuse_bytes", + "go_memory_classes_metadata_other_bytes", + "go_memory_classes_os_stacks_bytes", + "go_memory_classes_other_bytes", + "go_memory_classes_profiling_buckets_bytes", + "go_memory_classes_total_bytes", + }), + }, + { + name: "allow Scheduler", + rules: []GoRuntimeMetricsRule{MetricsScheduler}, + expected: []string{ + "go_gc_duration_seconds", + "go_goroutines", + "go_info", + "go_memstats_last_gc_time_seconds", + "go_sched_goroutines_goroutines", + "go_sched_latencies_seconds", + "go_threads", + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + reg := prometheus.NewRegistry() + reg.MustRegister(NewGoCollector( + WithGoCollectorMemStatsMetricsDisabled(), + WithGoCollectorRuntimeMetrics(test.rules...), + )) + result, err := reg.Gather() + if err != nil { + t.Fatal(err) + } + + got := []string{} + for _, r := range result { + got = append(got, r.GetName()) + } + + if !reflect.DeepEqual(got, test.expected) { + t.Errorf("got %v, want %v", got, test.expected) + } + }) + } +} + +func withBaseMetrics(metricNames []string) []string { + metricNames = append(metricNames, baseMetrics...) + sort.Strings(metricNames) + return metricNames +} + +func TestGoCollectorDenyList(t *testing.T) { + for _, test := range []struct { + name string + matchers []*regexp.Regexp + expected []string + }{ + { + name: "Without any matchers", + matchers: nil, + expected: baseMetrics, + }, + { + name: "deny all", + matchers: []*regexp.Regexp{regexp.MustCompile("/.*")}, + expected: baseMetrics, + }, + { + name: "deny gc and scheduler latency", + matchers: []*regexp.Regexp{ + regexp.MustCompile("^/gc/.*"), + regexp.MustCompile("^/sched/latencies:.*"), + }, + expected: baseMetrics, + }, + } { + t.Run(test.name, func(t *testing.T) { + reg := prometheus.NewRegistry() + reg.MustRegister(NewGoCollector( + WithGoCollectorMemStatsMetricsDisabled(), + WithoutGoCollectorRuntimeMetrics(test.matchers...), + )) + result, err := reg.Gather() + if err != nil { + t.Fatal(err) + } + + got := []string{} + for _, r := range result { + got = append(got, r.GetName()) + } + + if !reflect.DeepEqual(got, test.expected) { + t.Errorf("got %v, want %v", got, test.expected) + } + }) + } +} + func ExampleGoCollector() { + reg := prometheus.NewRegistry() + + // Register the GoCollector with the default options. Only the base metrics will be enabled. + reg.MustRegister(NewGoCollector()) + + http.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{})) + log.Fatal(http.ListenAndServe(":8080", nil)) } func ExampleGoCollector_WithAdvancedGoMetrics() { + reg := prometheus.NewRegistry() + + // Enable Go metrics with pre-defined rules. Or your custom rules. + reg.MustRegister( + NewGoCollector( + WithGoCollectorMemStatsMetricsDisabled(), + WithGoCollectorRuntimeMetrics( + MetricsScheduler, + MetricsMemory, + GoRuntimeMetricsRule{ + Matcher: regexp.MustCompile("^/mycustomrule.*"), + }, + ), + WithoutGoCollectorRuntimeMetrics(regexp.MustCompile("^/gc/.*")), + )) + + http.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{})) + log.Fatal(http.ListenAndServe(":8080", nil)) +} + +func ExampleGoCollector_DefaultRegister() { + // Unregister the default GoCollector. + prometheus.Unregister(NewGoCollector()) + + // Register the default GoCollector with a custom config. + prometheus.MustRegister(NewGoCollector(WithGoCollectorRuntimeMetrics( + MetricsScheduler, + MetricsGC, + GoRuntimeMetricsRule{ + Matcher: regexp.MustCompile("^/mycustomrule.*"), + }, + ), + )) + + http.Handle("/metrics", promhttp.Handler()) + log.Fatal(http.ListenAndServe(":8080", nil)) }