e2e integration on cgroupv2

Fix a few of the integration tests to be cgroupv2 compatible.

Signed-off-by: David Porter <porterdavid@google.com>

build/integration.sh

@@ -56,6 +56,22 @@ while [ "$(curl -Gs http://localhost:8080/healthz)" != "ok" ]; do
   sleep 1
 done
 
+if [[ "${DOCKER_IN_DOCKER_ENABLED:-}" == "true" ]]; then
+  # see https://github.com/moby/moby/blob/master/hack/dind
+  # cgroup v2: enable nesting
+  if [ -f /sys/fs/cgroup/cgroup.controllers ]; then
+    echo ">> configuring cgroupsv2 for docker in docker..."
+    # move the processes from the root group to the /init group,
+    # otherwise writing subtree_control fails with EBUSY.
+    # An error during moving non-existent process (i.e., "cat") is ignored.
+    mkdir -p /sys/fs/cgroup/init
+    xargs -rn1 < /sys/fs/cgroup/cgroup.procs > /sys/fs/cgroup/init/cgroup.procs || :
+    # enable controllers
+    sed -e 's/ / +/g' -e 's/^/+/' < /sys/fs/cgroup/cgroup.controllers \
+      > /sys/fs/cgroup/cgroup.subtree_control
+  fi
+fi
+
 echo ">> running integration tests against local cAdvisor"
 ./api.test --vmodule=*=2 -test.v
 ./healthz.test --vmodule=*=2 -test.v

integration/tests/api/docker_test.go

@@ -25,6 +25,7 @@ import (
 	v2 "github.com/google/cadvisor/info/v2"
 	"github.com/google/cadvisor/integration/framework"
 
+	"github.com/opencontainers/runc/libcontainer/cgroups"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -211,7 +212,15 @@ func TestDockerContainerSpec(t *testing.T) {
 	assert := assert.New(t)
 
 	assert.True(containerInfo.Spec.HasCpu, "CPU should be isolated")
-	assert.Equal(cpuShares, containerInfo.Spec.Cpu.Limit, "Container should have %d shares, has %d", cpuShares, containerInfo.Spec.Cpu.Limit)
+	if cgroups.IsCgroup2UnifiedMode() {
+		// see https://github.com/google/cadvisor/blob/24e7a9883d12f944fd4403861707f4bafcaf4f3d/container/common/helpers.go#L249-L260
+		// (Shares are rounded slighly on cgroupv2x1)
+		var cgroupV2Shares uint64 = 2046
+		assert.Equal(cgroupV2Shares, containerInfo.Spec.Cpu.Limit, "Container should have %d shares, has %d", cgroupV2Shares, containerInfo.Spec.Cpu.Limit)
+	} else {
+		assert.Equal(cpuShares, containerInfo.Spec.Cpu.Limit, "Container should have %d shares, has %d", cpuShares, containerInfo.Spec.Cpu.Limit)
+	}
+
 	assert.Equal(cpuMask, containerInfo.Spec.Cpu.Mask, "Cpu mask should be %q, but is %q", cpuMask, containerInfo.Spec.Cpu.Mask)
 	assert.True(containerInfo.Spec.HasMemory, "Memory should be isolated")
 	assert.Equal(memoryLimit, containerInfo.Spec.Memory.Limit, "Container should have memory limit of %d, has %d", memoryLimit, containerInfo.Spec.Memory.Limit)

integration/tests/api/machinestats_test.go

@@ -22,6 +22,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/google/cadvisor/integration/framework"
+	"github.com/opencontainers/runc/libcontainer/cgroups"
 )
 
 func TestMachineStatsIsReturned(t *testing.T) {
@@ -37,7 +38,11 @@ func TestMachineStatsIsReturned(t *testing.T) {
 	for _, stat := range machineStats {
 		as.NotEqual(stat.Timestamp, time.Time{})
 		as.True(stat.Cpu.Usage.Total > 0)
-		as.True(len(stat.Cpu.Usage.PerCpu) > 0)
+		// PerCPU CPU usage is not supported in cgroupv2 (cpuacct.usage_percpu)
+		// https://github.com/google/cadvisor/issues/3065
+		if !cgroups.IsCgroup2UnifiedMode() {
+			as.True(len(stat.Cpu.Usage.PerCpu) > 0)
+		}
 		if stat.CpuInst != nil {
 			as.True(stat.CpuInst.Usage.Total > 0)
 		}

integration/tests/api/test_utils.go

@@ -20,6 +20,7 @@ import (
 
 	info "github.com/google/cadvisor/info/v1"
 
+	"github.com/opencontainers/runc/libcontainer/cgroups"
 	"github.com/stretchr/testify/assert"
 	"k8s.io/klog/v2"
 )
@@ -46,12 +47,19 @@ func checkCPUStats(t *testing.T, stat info.CpuStats) {
 	assert := assert.New(t)
 
 	assert.NotEqual(0, stat.Usage.Total, "Total CPU usage should not be zero")
-	assert.NotEmpty(stat.Usage.PerCpu, "Per-core usage should not be empty")
-	totalUsage := uint64(0)
-	for _, usage := range stat.Usage.PerCpu {
-		totalUsage += usage
+
+	// PerCPU CPU usage is not supported in cgroupv2 (cpuacct.usage_percpu)
+	// https://github.com/google/cadvisor/issues/3065
+	if !cgroups.IsCgroup2UnifiedMode() {
+		assert.NotEmpty(stat.Usage.PerCpu, "Per-core usage should not be empty")
+
+		totalUsage := uint64(0)
+		for _, usage := range stat.Usage.PerCpu {
+			totalUsage += usage
+		}
+		inDelta(t, stat.Usage.Total, totalUsage, uint64((5 * time.Millisecond).Nanoseconds()), "Per-core CPU usage")
 	}
-	inDelta(t, stat.Usage.Total, totalUsage, uint64((5 * time.Millisecond).Nanoseconds()), "Per-core CPU usage")
+
 	inDelta(t, stat.Usage.Total, stat.Usage.User+stat.Usage.System, uint64((500 * time.Millisecond).Nanoseconds()), "User + system CPU usage")
 	// TODO(rjnagal): Add verification for cpu load.
 }