Skip to content

Commit

Permalink
goCollector: Added rule support for granular metric configuration.
Browse files Browse the repository at this point in the history 
Fixes: #1089

Signed-off-by: bwplotka <bwplotka@gmail.com>
  • Loading branch information
@bwplotka
bwplotka committed Aug 5, 2022
1 parent d44fbbe commit b94d7e2
Show file tree
Hide file tree
Showing 6 changed files with 320 additions and 182 deletions.
151 changes: 88 additions & 63 deletions prometheus/collectors/go_collector_latest.go
View file
Expand Up @@ -16,77 +16,102 @@

package collectors

import "github.com/prometheus/client_golang/prometheus"
import (
"regexp"

//nolint:staticcheck // Ignore SA1019 until v2.
type goOptions = prometheus.GoCollectorOptions
type goOption func(o *goOptions)

type GoCollectionOption uint32

const (
// GoRuntimeMemStatsCollection represents the metrics represented by runtime.MemStats structure such as
// go_memstats_alloc_bytes
// go_memstats_alloc_bytes_total
// go_memstats_sys_bytes
// go_memstats_lookups_total
// go_memstats_mallocs_total
// go_memstats_frees_total
// go_memstats_heap_alloc_bytes
// go_memstats_heap_sys_bytes
// go_memstats_heap_idle_bytes
// go_memstats_heap_inuse_bytes
// go_memstats_heap_released_bytes
// go_memstats_heap_objects
// go_memstats_stack_inuse_bytes
// go_memstats_stack_sys_bytes
// go_memstats_mspan_inuse_bytes
// go_memstats_mspan_sys_bytes
// go_memstats_mcache_inuse_bytes
// go_memstats_mcache_sys_bytes
// go_memstats_buck_hash_sys_bytes
// go_memstats_gc_sys_bytes
// go_memstats_other_sys_bytes
// go_memstats_next_gc_bytes
// so the metrics known from pre client_golang v1.12.0, except skipped go_memstats_gc_cpu_fraction (see
// https://github.com/prometheus/client_golang/issues/842#issuecomment-861812034 for explanation.
//
// NOTE that this mode represents runtime.MemStats statistics, but they are
// actually implemented using new runtime/metrics package.
// Deprecated: Use GoRuntimeMetricsCollection instead going forward.
GoRuntimeMemStatsCollection GoCollectionOption = 1 << iota
// GoRuntimeMetricsCollection is the new set of metrics represented by runtime/metrics package and follows
// consistent naming. The exposed metric set depends on Go version, but it is controlled against
// unexpected cardinality. This set has overlapping information with GoRuntimeMemStatsCollection, just with
// new names. GoRuntimeMetricsCollection is what is recommended for using going forward.
GoRuntimeMetricsCollection
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/internal"
)

// WithGoCollections allows enabling different collections for Go collector on top of base metrics
// like go_goroutines, go_threads, go_gc_duration_seconds, go_memstats_last_gc_time_seconds, go_info.
// WithGoCollectorMemStatsMetricsDisabled metrics that where representing runtime.MemStats structure such as:
//
// go_memstats_alloc_bytes
// go_memstats_alloc_bytes_total
// go_memstats_sys_bytes
// go_memstats_lookups_total
// go_memstats_mallocs_total
// go_memstats_frees_total
// go_memstats_heap_alloc_bytes
// go_memstats_heap_sys_bytes
// go_memstats_heap_idle_bytes
// go_memstats_heap_inuse_bytes
// go_memstats_heap_released_bytes
// go_memstats_heap_objects
// go_memstats_stack_inuse_bytes
// go_memstats_stack_sys_bytes
// go_memstats_mspan_inuse_bytes
// go_memstats_mspan_sys_bytes
// go_memstats_mcache_inuse_bytes
// go_memstats_mcache_sys_bytes
// go_memstats_buck_hash_sys_bytes
// go_memstats_gc_sys_bytes
// go_memstats_other_sys_bytes
// go_memstats_next_gc_bytes
//
// so the metrics known from pre client_golang v1.12.0,
//
// NOTE(bwplotka): The above represents runtime.MemStats statistics, but they are
// actually implemented using new runtime/metrics package. (except skipped go_memstats_gc_cpu_fraction
// -- see https://github.com/prometheus/client_golang/issues/842#issuecomment-861812034 for explanation).
//
// Check GoRuntimeMemStatsCollection and GoRuntimeMetricsCollection for more details. You can use none,
// one or more collections at once. For example:
// WithGoCollections(GoRuntimeMemStatsCollection | GoRuntimeMetricsCollection) means both GoRuntimeMemStatsCollection
// metrics and GoRuntimeMetricsCollection will be exposed.
// Some users might want to disable this on collector level (although you can use scrape relabelling on Prometheus),
// because similar metrics can be now obtained using WithGoCollectorRuntimeMetrics. Note that the semantics of new
// metrics might be different, plus the names can be change over time with different Go version.
//
// The current default is GoRuntimeMemStatsCollection, so the compatibility mode with
// client_golang pre v1.12 (move to runtime/metrics).
//nolint:staticcheck // Ignore SA1019 until v2.
func WithGoCollections(flags GoCollectionOption) func(options *prometheus.GoCollectorOptions) {
return func(o *goOptions) {
o.EnabledCollections = uint32(flags)
// NOTE(bwplotka): Changing metric names can be tedious at times as the alerts, recording rules and dashboards have to be adjusted.
// The old metrics are also very useful, with many guides and books written about how to interpret them.
//
// As a result our recommendation would be to stick with MemStats like metrics and enable other runtime/metrics if you are interested
// in advanced insights Go provides. See ExampleGoCollector_WithAdvancedGoMetrics.
func WithGoCollectorMemStatsMetricsDisabled() func(options *internal.GoCollectorOptions) {
return func(o *internal.GoCollectorOptions) {
o.DisableMemStatsLikeMetrics = true
}
}

// NewGoCollector returns a collector that exports metrics about the current Go
// process using debug.GCStats using runtime/metrics.
func NewGoCollector(opts ...goOption) prometheus.Collector {
//nolint:staticcheck // Ignore SA1019 until v2.
promPkgOpts := make([]func(o *prometheus.GoCollectorOptions), len(opts))
for i, opt := range opts {
promPkgOpts[i] = opt
// GoRuntimeMetricsRule allow enabling and configuring particular group of runtime/metrics.
type GoRuntimeMetricsRule struct {
// Matcher represents RE2 expression will match the runtime/metrics from https://golang.bg/src/runtime/metrics/description.go
// Use `regexp.MustCompile` or `regexp.Compile` to create this field.
Matcher *regexp.Regexp
}

// WithGoCollectorRuntimeMetrics allows enabling and configuring particular group of runtime/metrics.
// See the list of metrics https://golang.bg/src/runtime/metrics/description.go (pick the Go version you use there!).
// You can use this option in repeated manner, which will add new rules. The order of rules is important, the last rule
// that matches particular metrics is applied.
func WithGoCollectorRuntimeMetrics(rules ...GoRuntimeMetricsRule) func(options *internal.GoCollectorOptions) {
rs := make([]internal.GoCollectorRule, len(rules))
for i, r := range rules {
rs[i] = internal.GoCollectorRule{
Matcher: r.Matcher,
}
}

return func(o *internal.GoCollectorOptions) {
o.DisableMemStatsLikeMetrics = true
}
}

// WithoutGoCollectorRuntimeMetrics allows disabling group of runtime/metrics that you might have added in WithGoCollectorRuntimeMetrics.
// It behaves similarly to WithGoCollectorRuntimeMetrics just with deny-list semantics.
func WithoutGoCollectorRuntimeMetrics(matchers ...*regexp.Regexp) func(options *internal.GoCollectorOptions) {
rs := make([]internal.GoCollectorRule, len(matchers))
for i, m := range matchers {
rs[i] = internal.GoCollectorRule{
Matcher: m,
Deny: true,
}
}

return func(o *internal.GoCollectorOptions) {
o.RuntimeMetricRules = append(o.RuntimeMetricRules, rs...)
}
}

// NewGoCollector returns a collector that exports metrics about the current Go
// process using debug.GCStats and runtime/metrics.
func NewGoCollector(opts ...func(o *internal.GoCollectorOptions)) prometheus.Collector {
//nolint:staticcheck // Ignore SA1019 until v2.
return prometheus.NewGoCollector(promPkgOpts...)
return prometheus.NewGoCollector(opts...)
}
11 changes: 10 additions & 1 deletion prometheus/collectors/go_collector_latest_test.go
View file
Expand Up @@ -18,6 +18,7 @@ package collectors

import (
"encoding/json"
"regexp"
"testing"

"github.com/prometheus/client_golang/prometheus"
Expand All @@ -26,7 +27,9 @@ import (
func TestGoCollectorMarshalling(t *testing.T) {
reg := prometheus.NewRegistry()
reg.MustRegister(NewGoCollector(
WithGoCollections(GoRuntimeMemStatsCollection | GoRuntimeMetricsCollection),
WithGoCollectorRuntimeMetrics(GoRuntimeMetricsRule{
Matcher: regexp.MustCompile("/.*"),
}),
))
result, err := reg.Gather()
if err != nil {
Expand All @@ -37,3 +40,9 @@ func TestGoCollectorMarshalling(t *testing.T) {
t.Errorf("json marshalling shoud not fail, %v", err)
}
}

func ExampleGoCollector() {
}

func ExampleGoCollector_WithAdvancedGoMetrics() {
}
7 changes: 5 additions & 2 deletions prometheus/go_collector.go
View file
Expand Up @@ -19,6 +19,10 @@ import (
"time"
)

// goRuntimeMemStats provides the metrics initially provided by runtime.ReadMemStats.
// From Go 1.17 those similar (and better) statistics are provided by runtime/metrics, so
// while eval closure works on runtime.MemStats, the struct from Go 1.17+ is
// populated using runtime/metrics.
func goRuntimeMemStats() memStatsMetrics {
return memStatsMetrics{
{
Expand Down Expand Up @@ -224,7 +228,7 @@ func newBaseGoCollector() baseGoCollector {
"A summary of the pause duration of garbage collection cycles.",
nil, nil),
gcLastTimeDesc: NewDesc(
memstatNamespace("last_gc_time_seconds"),
"go_memstats_last_gc_time_seconds",
"Number of seconds since 1970 of last garbage collection.",
nil, nil),
goInfoDesc: NewDesc(
Expand Down Expand Up @@ -270,7 +274,6 @@ func memstatNamespace(s string) string {

// memStatsMetrics provide description, evaluator, runtime/metrics name, and
// value type for memstat metrics.
// TODO(bwplotka): Remove with end Go 1.16 EOL and replace with runtime/metrics.Description
type memStatsMetrics []struct {
desc *Desc
eval func(*runtime.MemStats) float64
Expand Down

0 comments on commit b94d7e2

Please sign in to comment.