Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Metrics wrapper that adds the cluster name as a label. (#8961)
- Loading branch information
Mark Gritter
committed
May 13, 2020
1 parent
06f131e
commit d5b1d5d
Showing
5 changed files
with
201 additions
and
10 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,59 @@ | ||
| package metricsutil | ||
|
|
||
| import ( | ||
| "time" | ||
|
|
||
| metrics "github.com/armon/go-metrics" | ||
| ) | ||
|
|
||
| // ClusterMetricSink serves as a shim around go-metrics | ||
| // and inserts a "cluster" label. | ||
| // | ||
| // It also provides a mechanism to limit the cardinality of the labels on a gauge | ||
| // (at each reporting interval, which isn't sufficient if there is variability in which | ||
| // labels are the top N) and a backoff mechanism for gauge computation. | ||
| type ClusterMetricSink struct { | ||
| // ClusterName is either the cluster ID, or a name provided | ||
| // in the telemetry configuration stanza. | ||
| ClusterName string | ||
|
|
||
| MaxGaugeCardinality int | ||
| GaugeInterval time.Duration | ||
|
|
||
| // Sink is the go-metrics sink to send to | ||
| Sink metrics.MetricSink | ||
| } | ||
|
|
||
| // Convenience alias | ||
| type Label = metrics.Label | ||
|
|
||
| func (m *ClusterMetricSink) SetGaugeWithLabels(key []string, val float32, labels []Label) { | ||
| m.Sink.SetGaugeWithLabels(key, val, | ||
| append(labels, Label{"cluster", m.ClusterName})) | ||
| } | ||
|
|
||
| func (m *ClusterMetricSink) IncrCounterWithLabels(key []string, val float32, labels []Label) { | ||
| m.Sink.IncrCounterWithLabels(key, val, | ||
| append(labels, Label{"cluster", m.ClusterName})) | ||
| } | ||
|
|
||
| func (m *ClusterMetricSink) AddSampleWithLabels(key []string, val float32, labels []Label) { | ||
| m.Sink.AddSampleWithLabels(key, val, | ||
| append(labels, Label{"cluster", m.ClusterName})) | ||
| } | ||
|
|
||
| // BlackholeSink is a default suitable for use in unit tests. | ||
| func BlackholeSink() *ClusterMetricSink { | ||
| return &ClusterMetricSink{ | ||
| ClusterName: "", | ||
| Sink: &metrics.BlackholeSink{}, | ||
| } | ||
| } | ||
|
|
||
| // SetDefaultClusterName changes the cluster name from its default value, | ||
| // if it has not previously been configured. | ||
| func (m *ClusterMetricSink) SetDefaultClusterName(clusterName string) { | ||
| if m.ClusterName == "" { | ||
| m.ClusterName = clusterName | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,102 @@ | ||
| package metricsutil | ||
|
|
||
| import ( | ||
| "testing" | ||
| "time" | ||
|
|
||
| "github.com/armon/go-metrics" | ||
| ) | ||
|
|
||
| func isLabelPresent(toFind Label, ls []Label) bool { | ||
| for _, l := range ls { | ||
| if l == toFind { | ||
| return true | ||
| } | ||
| } | ||
| return false | ||
| } | ||
|
|
||
| func TestClusterLabelPresent(t *testing.T) { | ||
| testClusterName := "test-cluster" | ||
|
|
||
| // Use a ridiculously long time to minimize the chance | ||
| // that we have to deal with more than one interval. | ||
| // InMemSink rounds down to an interval boundary rather than | ||
| // starting one at the time of initialization. | ||
| inmemSink := metrics.NewInmemSink( | ||
| 1000000*time.Hour, | ||
| 2000000*time.Hour) | ||
| clusterSink := &ClusterMetricSink{ | ||
| ClusterName: testClusterName, | ||
| Sink: inmemSink, | ||
| } | ||
|
|
||
| key1 := []string{"aaa", "bbb"} | ||
| key2 := []string{"ccc", "ddd"} | ||
| key3 := []string{"eee", "fff"} | ||
| labels1 := []Label{{"dim1", "val1"}} | ||
| labels2 := []Label{{"dim2", "val2"}} | ||
| labels3 := []Label{{"dim3", "val3"}} | ||
| clusterLabel := Label{"cluster", testClusterName} | ||
| expectedKey1 := "aaa.bbb;dim1=val1;cluster=" + testClusterName | ||
| expectedKey2 := "ccc.ddd;dim2=val2;cluster=" + testClusterName | ||
| expectedKey3 := "eee.fff;dim3=val3;cluster=" + testClusterName | ||
|
|
||
| clusterSink.SetGaugeWithLabels(key1, 1.0, labels1) | ||
| clusterSink.IncrCounterWithLabels(key2, 2.0, labels2) | ||
| clusterSink.AddSampleWithLabels(key3, 3.0, labels3) | ||
|
|
||
| intervals := inmemSink.Data() | ||
| // If we start very close to the end of an interval, then our metrics might be | ||
| // split across two different buckets. We won't write the code to try to handle that. | ||
| // 100000-hours = at most once every 4167 days | ||
| if len(intervals) > 1 { | ||
| t.Skip("Detected interval crossing.") | ||
| } | ||
|
|
||
| // Check Gauge | ||
| g, ok := intervals[0].Gauges[expectedKey1] | ||
| if !ok { | ||
| t.Fatal("Key", expectedKey1, "not found in map", intervals[0].Gauges) | ||
| } | ||
| if g.Value != 1.0 { | ||
| t.Error("Gauge value", g.Value, "does not match", 1.0) | ||
| } | ||
| if !isLabelPresent(labels1[0], g.Labels) { | ||
| t.Error("Gauge label", g.Labels, "does not include", labels1) | ||
| } | ||
| if !isLabelPresent(clusterLabel, g.Labels) { | ||
| t.Error("Gauge label", g.Labels, "does not include", clusterLabel) | ||
| } | ||
|
|
||
| // Check Counter | ||
| c, ok := intervals[0].Counters[expectedKey2] | ||
| if !ok { | ||
| t.Fatal("Key", expectedKey2, "not found in map", intervals[0].Counters) | ||
| } | ||
| if c.Sum != 2.0 { | ||
| t.Error("Counter value", c.Sum, "does not match", 2.0) | ||
| } | ||
| if !isLabelPresent(labels2[0], c.Labels) { | ||
| t.Error("Counter label", c.Labels, "does not include", labels2) | ||
| } | ||
| if !isLabelPresent(clusterLabel, c.Labels) { | ||
| t.Error("Counter label", c.Labels, "does not include", clusterLabel) | ||
| } | ||
|
|
||
| // Check Sample | ||
| s, ok := intervals[0].Samples[expectedKey3] | ||
| if !ok { | ||
| t.Fatal("Key", expectedKey3, "not found in map", intervals[0].Samples) | ||
| } | ||
| if s.Sum != 3.0 { | ||
| t.Error("Sample value", s.Sum, "does not match", 3.0) | ||
| } | ||
| if !isLabelPresent(labels3[0], s.Labels) { | ||
| t.Error("Sample label", s.Labels, "does not include", labels3) | ||
| } | ||
| if !isLabelPresent(clusterLabel, s.Labels) { | ||
| t.Error("Sample label", s.Labels, "does not include", clusterLabel) | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters