From 95819e7a1f2fca5e50101f3652ba61abaa623d31 Mon Sep 17 00:00:00 2001 From: Ayoub Mrini Date: Mon, 12 Jun 2023 00:43:08 +0200 Subject: [PATCH] OCPBUGS-13153: Limit the value of GOMAXPROCS on node-exporter to 4. On nodes with multiple CPU cores, this should help avoid lock contentions without having any side effects, see the ticket for more details. node-exporter versions that have "--runtime.gomaxprocs" can still override this: the flag has precedence over this automatic setting. Make the entrypoint set the env var before runnning the process. Get the CPU count from /proc/cpuinfo without taking CPU affinity (via cpuset) into account as the container's CPU requests is not an integer, thus no affinity is applied. Signed-off-by: Ayoub Mrini --- assets/node-exporter/daemonset.yaml | 13 +++++++++++++ jsonnet/components/node-exporter.libsonnet | 15 +++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/assets/node-exporter/daemonset.yaml b/assets/node-exporter/daemonset.yaml index 65a5e3d334..c3a2b0daa2 100644 --- a/assets/node-exporter/daemonset.yaml +++ b/assets/node-exporter/daemonset.yaml @@ -41,6 +41,19 @@ spec: - --collector.cpu.info - --collector.textfile.directory=/var/node_exporter/textfile - --no-collector.btrfs + command: + - /bin/sh + - -c + - | + export GOMAXPROCS=4 + # We don't take CPU affinity into account as the container dones't have integer CPU requests. + # In case of error, fallback to the default value. + NUM_CPUS=$(grep -c '^processor' "/proc/cpuinfo" 2>/dev/null || echo "0") + if [ "$NUM_CPUS" -lt "$GOMAXPROCS" ]; then + export GOMAXPROCS="$NUM_CPUS" + fi + echo "Set GOMAXPROCS=$GOMAXPROCS" + exec /bin/node_exporter "$0" "$@" image: quay.io/prometheus/node-exporter:v1.6.0 name: node-exporter resources: diff --git a/jsonnet/components/node-exporter.libsonnet b/jsonnet/components/node-exporter.libsonnet index 483fd073ca..31a264b232 100644 --- a/jsonnet/components/node-exporter.libsonnet +++ b/jsonnet/components/node-exporter.libsonnet @@ -214,6 +214,21 @@ function(params) '--collector.textfile.directory=' + textfileDir, '--no-collector.btrfs', ], + command: [ + '/bin/sh', + '-c', + ||| + export GOMAXPROCS=4 + # We don't take CPU affinity into account as the container dones't have integer CPU requests. + # In case of error, fallback to the default value. + NUM_CPUS=$(grep -c '^processor' "/proc/cpuinfo" 2>/dev/null || echo "0") + if [ "$NUM_CPUS" -lt "$GOMAXPROCS" ]; then + export GOMAXPROCS="$NUM_CPUS" + fi + echo "Set GOMAXPROCS=$GOMAXPROCS" + exec /bin/node_exporter "$0" "$@" + |||, + ], terminationMessagePolicy: 'FallbackToLogsOnError', volumeMounts+: [{ mountPath: textfileDir,