-
Notifications
You must be signed in to change notification settings - Fork 1.1k
/
go_collector_latest_test.go
340 lines (304 loc) · 9.67 KB
/
go_collector_latest_test.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
// Copyright 2021 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build go1.17
// +build go1.17
package prometheus
import (
"math"
"reflect"
"runtime"
"runtime/metrics"
"sync"
"testing"
"github.com/prometheus/client_golang/prometheus/internal"
dto "github.com/prometheus/client_model/go"
)
func TestGoCollectorRuntimeMetrics(t *testing.T) {
metrics := collectGoMetrics(t)
msChecklist := make(map[string]bool)
for _, m := range goRuntimeMemStats() {
msChecklist[m.desc.fqName] = false
}
if len(metrics) == 0 {
t.Fatal("no metrics created by Collect")
}
// Check a few specific metrics.
//
// Checking them all is somewhat pointless because the runtime/metrics
// metrics are going to shift underneath us. Also if we try to check
// against the runtime/metrics package in an automated fashion we're kind
// of missing the point, because we have to do all the same work the code
// has to do to perform the translation. Same for supporting old metric
// names (the best we can do here is make sure they're all accounted for).
var sysBytes, allocs float64
for _, m := range metrics {
name := m.Desc().fqName
switch name {
case "go_memory_classes_total_bytes":
checkMemoryMetric(t, m, &sysBytes)
case "go_sys_bytes":
checkMemoryMetric(t, m, &sysBytes)
case "go_gc_heap_allocs_bytes_total":
checkMemoryMetric(t, m, &allocs)
case "go_alloc_bytes_total":
checkMemoryMetric(t, m, &allocs)
}
if present, ok := msChecklist[name]; ok {
if present {
t.Errorf("memstats metric %s found more than once", name)
}
msChecklist[name] = true
}
}
for name := range msChecklist {
if present := msChecklist[name]; !present {
t.Errorf("memstats metric %s not collected", name)
}
}
}
func checkMemoryMetric(t *testing.T, m Metric, expValue *float64) {
t.Helper()
pb := &dto.Metric{}
m.Write(pb)
var value float64
if g := pb.GetGauge(); g != nil {
value = g.GetValue()
} else {
value = pb.GetCounter().GetValue()
}
if value <= 0 {
t.Error("bad value for total memory")
}
if *expValue == 0 {
*expValue = value
} else if value != *expValue {
t.Errorf("legacy metric and runtime/metrics metric do not match: want %d, got %d", int64(*expValue), int64(value))
}
}
var sink interface{}
func TestBatchHistogram(t *testing.T) {
goMetrics := collectGoMetrics(t)
var mhist Metric
for _, m := range goMetrics {
if m.Desc().fqName == "go_gc_heap_allocs_by_size_bytes_total" {
mhist = m
break
}
}
if mhist == nil {
t.Fatal("failed to find metric to test")
}
hist, ok := mhist.(*batchHistogram)
if !ok {
t.Fatal("found metric is not a runtime/metrics histogram")
}
// Make a bunch of allocations then do another collection.
//
// The runtime/metrics API tries to reuse memory where possible,
// so make sure that we didn't hang on to any of that memory in
// hist.
countsCopy := make([]uint64, len(hist.counts))
copy(countsCopy, hist.counts)
for i := 0; i < 100; i++ {
sink = make([]byte, 128)
}
collectGoMetrics(t)
for i, v := range hist.counts {
if v != countsCopy[i] {
t.Error("counts changed during new collection")
break
}
}
// Get the runtime/metrics copy.
s := []metrics.Sample{
{Name: "/gc/heap/allocs-by-size:bytes"},
}
metrics.Read(s)
rmHist := s[0].Value.Float64Histogram()
wantBuckets := internal.RuntimeMetricsBucketsForUnit(rmHist.Buckets, "bytes")
// runtime/metrics histograms always have a +Inf bucket and are lower
// bound inclusive. In contrast, we have an implicit +Inf bucket and
// are upper bound inclusive, so we can chop off the first bucket
// (since the conversion to upper bound inclusive will shift all buckets
// down one index) and the +Inf for the last bucket.
wantBuckets = wantBuckets[1 : len(wantBuckets)-1]
// Check to make sure the output proto makes sense.
pb := &dto.Metric{}
hist.Write(pb)
if math.IsInf(pb.Histogram.Bucket[len(pb.Histogram.Bucket)-1].GetUpperBound(), +1) {
t.Errorf("found +Inf bucket")
}
if got := len(pb.Histogram.Bucket); got != len(wantBuckets) {
t.Errorf("got %d buckets in protobuf, want %d", got, len(wantBuckets))
}
for i, bucket := range pb.Histogram.Bucket {
// runtime/metrics histograms are lower-bound inclusive, but we're
// upper-bound inclusive. So just make sure the new inclusive upper
// bound is somewhere close by (in some cases it's equal).
wantBound := wantBuckets[i]
if gotBound := *bucket.UpperBound; (wantBound-gotBound)/wantBound > 0.001 {
t.Errorf("got bound %f, want within 0.1%% of %f", gotBound, wantBound)
}
// Make sure counts are cumulative. Because of the consistency guarantees
// made by the runtime/metrics package, we're really not guaranteed to get
// anything even remotely the same here.
if i > 0 && *bucket.CumulativeCount < *pb.Histogram.Bucket[i-1].CumulativeCount {
t.Error("cumulative counts are non-monotonic")
}
}
}
func collectGoMetrics(t *testing.T) []Metric {
t.Helper()
c := NewGoCollector().(*goCollector)
// Collect all metrics.
ch := make(chan Metric)
var wg sync.WaitGroup
var metrics []Metric
wg.Add(1)
go func() {
defer wg.Done()
for metric := range ch {
metrics = append(metrics, metric)
}
}()
c.Collect(ch)
close(ch)
wg.Wait()
return metrics
}
func TestMemStatsEquivalence(t *testing.T) {
var msReal, msFake runtime.MemStats
descs := metrics.All()
samples := make([]metrics.Sample, len(descs))
samplesMap := make(map[string]*metrics.Sample)
for i := range descs {
samples[i].Name = descs[i].Name
samplesMap[descs[i].Name] = &samples[i]
}
// Force a GC cycle to try to reach a clean slate.
runtime.GC()
// Populate msReal.
runtime.ReadMemStats(&msReal)
// Populate msFake.
metrics.Read(samples)
memStatsFromRM(&msFake, samplesMap)
// Iterate over them and make sure they're somewhat close.
msRealValue := reflect.ValueOf(msReal)
msFakeValue := reflect.ValueOf(msFake)
typ := msRealValue.Type()
for i := 0; i < msRealValue.NumField(); i++ {
fr := msRealValue.Field(i)
ff := msFakeValue.Field(i)
switch typ.Kind() {
case reflect.Uint64:
// N.B. Almost all fields of MemStats are uint64s.
vr := fr.Interface().(uint64)
vf := ff.Interface().(uint64)
if float64(vr-vf)/float64(vf) > 0.05 {
t.Errorf("wrong value for %s: got %d, want %d", typ.Field(i).Name, vf, vr)
}
}
}
}
func TestExpectedRuntimeMetrics(t *testing.T) {
goMetrics := collectGoMetrics(t)
goMetricSet := make(map[string]Metric)
for _, m := range goMetrics {
goMetricSet[m.Desc().fqName] = m
}
descs := metrics.All()
rmSet := make(map[string]struct{})
// Iterate over runtime-reported descriptions to find new metrics.
for i := range descs {
rmName := descs[i].Name
rmSet[rmName] = struct{}{}
expFQName, ok := expectedRuntimeMetrics[rmName]
if !ok {
t.Errorf("found new runtime/metrics metric %s", rmName)
_, _, _, ok := internal.RuntimeMetricsToProm(&descs[i])
if !ok {
t.Errorf("new metric has name that can't be converted, or has an unsupported Kind")
}
continue
}
_, ok = goMetricSet[expFQName]
if !ok {
t.Errorf("existing runtime/metrics metric %s (expected fq name %s) not collected", rmName, expFQName)
continue
}
}
// Now iterate over the expected metrics and look for removals.
cardinality := 0
for rmName, fqName := range expectedRuntimeMetrics {
if _, ok := rmSet[rmName]; !ok {
t.Errorf("runtime/metrics metric %s removed", rmName)
continue
}
if _, ok := goMetricSet[fqName]; !ok {
t.Errorf("runtime/metrics metric %s not appearing under expected name %s", rmName, fqName)
continue
}
// While we're at it, check to make sure expected cardinality lines
// up, but at the point of the protobuf write to get as close to the
// real deal as possible.
//
// Note that we filter out non-runtime/metrics metrics here, because
// those are manually managed.
var m dto.Metric
if err := goMetricSet[fqName].Write(&m); err != nil {
t.Errorf("writing metric %s: %v", fqName, err)
continue
}
// N.B. These are the only fields populated by runtime/metrics metrics specifically.
// Other fields are populated by e.g. GCStats metrics.
switch {
case m.Counter != nil:
fallthrough
case m.Gauge != nil:
cardinality++
case m.Histogram != nil:
cardinality += len(m.Histogram.Bucket) + 3 // + sum, count, and +inf
default:
t.Errorf("unexpected protobuf structure for metric %s", fqName)
}
}
if t.Failed() {
t.Log("a new Go version may have been detected, please run")
t.Log("\tgo run gen_go_collector_metrics_set.go go1.X")
t.Log("where X is the Go version you are currently using")
}
expectCardinality := expectedRuntimeMetricsCardinality
if cardinality != expectCardinality {
t.Errorf("unexpected cardinality for runtime/metrics metrics: got %d, want %d", cardinality, expectCardinality)
}
}
func TestGoCollectorConcurrency(t *testing.T) {
c := NewGoCollector().(*goCollector)
// Set up multiple goroutines to Collect from the
// same GoCollector. In race mode with GOMAXPROCS > 1,
// this test should fail often if Collect is not
// concurrent-safe.
for i := 0; i < 4; i++ {
go func() {
ch := make(chan Metric)
go func() {
// Drain all metrics received until the
// channel is closed.
for range ch {
}
}()
c.Collect(ch)
close(ch)
}()
}
}