OCPBUGS-13153: Limit the value of GOMAXPROCS on node-exporter to 4.

On nodes with multiple CPU cores, this should help avoid lock contentions
without having any side effects, see the ticket for more details.

node-exporter versions that have "--runtime.gomaxprocs" can still
override this: the flag has precedence over this automatic setting.

Make the entrypoint set the env var before runnning the process.
Get the CPU count from /proc/cpuinfo without taking CPU affinity (via cpuset)
into account as the container's CPU requests is not an integer, thus no
affinity is applied.

Signed-off-by: Ayoub Mrini <amrini@redhat.com>

assets/node-exporter/daemonset.yaml

@@ -41,6 +41,19 @@ spec:
         - --collector.cpu.info
         - --collector.textfile.directory=/var/node_exporter/textfile
         - --no-collector.btrfs
+        command:
+        - /bin/sh
+        - -c
+        - |
+          export GOMAXPROCS=4
+          # We don't take CPU affinity into account as the container dones't have integer CPU requests.
+          # In case of error, fallback to the default value.
+          NUM_CPUS=$(grep -c '^processor' "/proc/cpuinfo" 2>/dev/null || echo "0")
+          if [ "$NUM_CPUS" -lt "$GOMAXPROCS" ]; then
+            export GOMAXPROCS="$NUM_CPUS"
+          fi
+          echo "Set GOMAXPROCS=$GOMAXPROCS"
+          exec /bin/node_exporter "$0" "$@"
         image: quay.io/prometheus/node-exporter:v1.5.0
         name: node-exporter
         resources:

jsonnet/components/node-exporter.libsonnet

@@ -214,6 +214,21 @@ function(params)
                               '--collector.textfile.directory=' + textfileDir,
                               '--no-collector.btrfs',
                             ],
+                      command: [
+                        '/bin/sh',
+                        '-c',
+                        |||
+                          export GOMAXPROCS=4
+                          # We don't take CPU affinity into account as the container dones't have integer CPU requests.
+                          # In case of error, fallback to the default value.
+                          NUM_CPUS=$(grep -c '^processor' "/proc/cpuinfo" 2>/dev/null || echo "0")
+                          if [ "$NUM_CPUS" -lt "$GOMAXPROCS" ]; then
+                            export GOMAXPROCS="$NUM_CPUS"
+                          fi
+                          echo "Set GOMAXPROCS=$GOMAXPROCS"
+                          exec /bin/node_exporter "$0" "$@"
+                        |||,
+                      ],
                       terminationMessagePolicy: 'FallbackToLogsOnError',
                       volumeMounts+: [{
                         mountPath: textfileDir,

test/e2e/node_exporter_test.go

@@ -257,3 +257,17 @@ nodeExporter:
 	}
 
 }
+
+func TestNodeExporterGoMaxProcs(t *testing.T) {
+	t.Run("limited GOMAXPROCS", func(st *testing.T) {
+		f.PrometheusK8sClient.WaitForQueryReturn(
+			t, 5*time.Minute, `max(go_sched_gomaxprocs_threads{job="node-exporter"})`,
+			func(v float64) error {
+				if v > 4 {
+					return fmt.Errorf(`expecting max(go_sched_gomaxprocs_threads{job="node-exporter"}) <= 4 but got %v.`, v)
+				}
+				return nil
+			},
+		)
+	})
+}