Support running integration tests on cgroupv2 hosts

build/integration.sh

@@ -72,6 +72,22 @@ while [ "$(curl -Gs http://localhost:8080/healthz)" != "ok" ]; do
   sleep 1
 done
 
+if [[ "${DOCKER_IN_DOCKER_ENABLED:-}" == "true" ]]; then
+  # see https://github.com/moby/moby/blob/master/hack/dind
+  # cgroup v2: enable nesting
+  if [ -f /sys/fs/cgroup/cgroup.controllers ]; then
+    echo ">> configuring cgroupsv2 for docker in docker..."
+    # move the processes from the root group to the /init group,
+    # otherwise writing subtree_control fails with EBUSY.
+    # An error during moving non-existent process (i.e., "cat") is ignored.
+    mkdir -p /sys/fs/cgroup/init
+    xargs -rn1 < /sys/fs/cgroup/cgroup.procs > /sys/fs/cgroup/init/cgroup.procs || :
+    # enable controllers
+    sed -e 's/ / +/g' -e 's/^/+/' < /sys/fs/cgroup/cgroup.controllers \
+      > /sys/fs/cgroup/cgroup.subtree_control
+  fi
+fi
+
 echo ">> running integration tests against local cAdvisor"
 if ! [ -f ./api.test ] || ! [ -f ./healthz.test ]; then
   echo You must compile the ./api.test binary and ./healthz.test binary before

container/common/helpers.go

@@ -210,7 +210,8 @@ func getSpecInternal(cgroupPaths map[string]string, machineInfoFactory info.Mach
 	if cgroup2UnifiedMode {
 		ioControllerName = "io"
 	}
-	if blkioRoot, ok := cgroupPaths[ioControllerName]; ok && utils.FileExists(blkioRoot) {
+
+	if blkioRoot, ok := GetControllerPath(cgroupPaths, ioControllerName, cgroup2UnifiedMode); ok && utils.FileExists(blkioRoot) {
 		spec.HasDiskIo = true
 	}
 

container/common/helpers_test.go

@@ -165,7 +165,7 @@ func TestGetSpecCgroupV2(t *testing.T) {
 	assert.EqualValues(t, spec.Processes.Limit, 1027)
 
 	assert.False(t, spec.HasHugetlb)
-	assert.False(t, spec.HasDiskIo)
+	assert.True(t, spec.HasDiskIo)
 }
 
 func TestGetSpecCgroupV2Max(t *testing.T) {

integration/tests/api/docker_test.go

@@ -25,6 +25,7 @@ import (
 	v2 "github.com/google/cadvisor/info/v2"
 	"github.com/google/cadvisor/integration/framework"
 
+	"github.com/opencontainers/runc/libcontainer/cgroups"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -211,7 +212,20 @@ func TestDockerContainerSpec(t *testing.T) {
 	assert := assert.New(t)
 
 	assert.True(containerInfo.Spec.HasCpu, "CPU should be isolated")
-	assert.Equal(cpuShares, containerInfo.Spec.Cpu.Limit, "Container should have %d shares, has %d", cpuShares, containerInfo.Spec.Cpu.Limit)
+	if cgroups.IsCgroup2UnifiedMode() {
+		// cpu shares are rounded slighly on cgroupv2 due to conversion between cgroupv1 (cpu.shares) and cgroupv2 (cpu.weight)
+		// When container is created via docker, runc will convert cpu shares to cpu.weight https://github.com/opencontainers/runc/blob/d11f4d756e85ece5cdba8bb69f8bd4db3cdcbeab/libcontainer/cgroups/utils.go#L423-L428
+		// And cAdvisor will convert cpu.weight back to cpu shares in https://github.com/google/cadvisor/blob/24e7a9883d12f944fd4403861707f4bafcaf4f3d/container/common/helpers.go#L249-L260
+		// Worked example:
+		// cpuShares = 2048 (input to docker --cpu-shares)
+		// cpuWeight = int((1 + ((cpuShares-2)*9999)/262142))=79 (conversion done by runc)
+		// cpuWeight back to cpuShares = int(2 + ((cpuWeight-1)*262142)/9999)= 2046
+		var cgroupV2Shares uint64 = 2046
+		assert.Equal(cgroupV2Shares, containerInfo.Spec.Cpu.Limit, "Container should have %d shares, has %d", cgroupV2Shares, containerInfo.Spec.Cpu.Limit)
+	} else {
+		assert.Equal(cpuShares, containerInfo.Spec.Cpu.Limit, "Container should have %d shares, has %d", cpuShares, containerInfo.Spec.Cpu.Limit)
+	}
+
 	assert.Equal(cpuMask, containerInfo.Spec.Cpu.Mask, "Cpu mask should be %q, but is %q", cpuMask, containerInfo.Spec.Cpu.Mask)
 	assert.True(containerInfo.Spec.HasMemory, "Memory should be isolated")
 	assert.Equal(memoryLimit, containerInfo.Spec.Memory.Limit, "Container should have memory limit of %d, has %d", memoryLimit, containerInfo.Spec.Memory.Limit)

integration/tests/api/machinestats_test.go

@@ -22,6 +22,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/google/cadvisor/integration/framework"
+	"github.com/opencontainers/runc/libcontainer/cgroups"
 )
 
 func TestMachineStatsIsReturned(t *testing.T) {
@@ -37,7 +38,11 @@ func TestMachineStatsIsReturned(t *testing.T) {
 	for _, stat := range machineStats {
 		as.NotEqual(stat.Timestamp, time.Time{})
 		as.True(stat.Cpu.Usage.Total > 0)
-		as.True(len(stat.Cpu.Usage.PerCpu) > 0)
+		// PerCPU CPU usage is not supported in cgroupv2 (cpuacct.usage_percpu)
+		// https://github.com/google/cadvisor/issues/3065
+		if !cgroups.IsCgroup2UnifiedMode() {
+			as.True(len(stat.Cpu.Usage.PerCpu) > 0)
+		}
 		if stat.CpuInst != nil {
 			as.True(stat.CpuInst.Usage.Total > 0)
 		}

integration/tests/api/test_utils.go

@@ -20,6 +20,7 @@ import (
 
 	info "github.com/google/cadvisor/info/v1"
 
+	"github.com/opencontainers/runc/libcontainer/cgroups"
 	"github.com/stretchr/testify/assert"
 	"k8s.io/klog/v2"
 )
@@ -46,12 +47,19 @@ func checkCPUStats(t *testing.T, stat info.CpuStats) {
 	assert := assert.New(t)
 
 	assert.NotEqual(0, stat.Usage.Total, "Total CPU usage should not be zero")
-	assert.NotEmpty(stat.Usage.PerCpu, "Per-core usage should not be empty")
-	totalUsage := uint64(0)
-	for _, usage := range stat.Usage.PerCpu {
-		totalUsage += usage
+
+	// PerCPU CPU usage is not supported in cgroupv2 (cpuacct.usage_percpu)
+	// https://github.com/google/cadvisor/issues/3065
+	if !cgroups.IsCgroup2UnifiedMode() {
+		assert.NotEmpty(stat.Usage.PerCpu, "Per-core usage should not be empty")
+
+		totalUsage := uint64(0)
+		for _, usage := range stat.Usage.PerCpu {
+			totalUsage += usage
+		}
+		inDelta(t, stat.Usage.Total, totalUsage, uint64((5 * time.Millisecond).Nanoseconds()), "Per-core CPU usage")
 	}
-	inDelta(t, stat.Usage.Total, totalUsage, uint64((5 * time.Millisecond).Nanoseconds()), "Per-core CPU usage")
+
 	inDelta(t, stat.Usage.Total, stat.Usage.User+stat.Usage.System, uint64((500 * time.Millisecond).Nanoseconds()), "User + system CPU usage")
 	// TODO(rjnagal): Add verification for cpu load.
 }