internal/traceprof/traceproftest: fix flaxy endpoints/hotspots test (#…

…1145)

* internal/traceprof/traceprof: fix flaxy endpoints/hotspots test

TestEndpointsAndCodeHotspots fails occasionally in CI. Change the retry
logic so that every required label is checked for in testCommon before
ending the retry loop.

* internal/traceprof/traceproftest: identify reason for test failure

If TestEndpointsAndCodeHotspots fails after too many attempts, show
which condition wasn't met after all the attempts.

internal/traceprof/traceproftest/traceprof_test.go

@@ -42,29 +42,41 @@ func TestEndpointsAndCodeHotspots(t *testing.T) {
 
 		// Rerun test a few times with doubled duration until it passes to avoid
 		// flaky behavior in CI.
-		for attempt := 1; ; attempt++ {
+		var (
+			prof *CPUProfile
+			res  *pb.WorkRes
+		)
+		for attempt := 1; attempt <= 5; attempt++ {
 			app := c.Start(t)
 			defer app.Stop(t)
 
-			res := app.WorkRequest(t, req)
-			prof := app.CPUProfile(t)
+			res = app.WorkRequest(t, req)
+			prof = app.CPUProfile(t)
 
 			notEnoughSamples := (prof.Duration() < minCPUDuration) ||
 				(prof.LabelsDuration(CustomLabels) < minCPUDuration) ||
-				(c.CodeHotspots && prof.LabelDuration(traceprof.SpanID, "*") < minCPUDuration) ||
-				(c.AppType != Direct && c.Endpoints && prof.LabelDuration(traceprof.TraceEndpoint, "*") < minCPUDuration)
-			if attempt <= 5 && notEnoughSamples {
+				(c.CodeHotspots && prof.LabelsDuration(map[string]string{traceprof.LocalRootSpanID: res.LocalRootSpanId, traceprof.SpanID: res.SpanId}) < minCPUDuration) ||
+				(c.Endpoints && prof.LabelDuration(traceprof.TraceEndpoint, c.AppType.Endpoint()) < minCPUDuration)
+			if notEnoughSamples {
 				req.CpuDuration *= 2
 				req.SqlDuration *= 2
 				t.Logf("attempt %d: not enough cpu samples, doubling duration", attempt)
 				continue
 			}
 			require.True(t, ValidSpanID(res.SpanId))
 			require.True(t, ValidSpanID(res.LocalRootSpanId))
-			require.GreaterOrEqual(t, prof.Duration(), minCPUDuration)
-			require.GreaterOrEqual(t, prof.LabelsDuration(CustomLabels), minCPUDuration)
 			return res, prof
 		}
+		// Failed after 5 attempts, identify which condition wasn't met
+		require.GreaterOrEqual(t, prof.Duration(), minCPUDuration)
+		require.GreaterOrEqual(t, prof.LabelsDuration(CustomLabels), minCPUDuration)
+		if c.Endpoints {
+			require.GreaterOrEqual(t, prof.LabelDuration(traceprof.TraceEndpoint, c.AppType.Endpoint()), minCPUDuration)
+		}
+		if c.CodeHotspots {
+			require.GreaterOrEqual(t, prof.LabelsDuration(map[string]string{traceprof.LocalRootSpanID: res.LocalRootSpanId, traceprof.SpanID: res.SpanId}), minCPUDuration)
+		}
+		return nil, nil
 	}
 
 	for _, appType := range []testAppType{Direct, HTTP, GRPC} {