From cbc72894220857d5ab25df6ebd9306e790f2db23 Mon Sep 17 00:00:00 2001 From: Olivia Trewin Date: Tue, 27 Sep 2022 18:43:48 -0500 Subject: [PATCH] Support reading cpu.max from cgroups v2 --- fixtures/cgroups2/cgroups/ceil/cpu.max | 1 + fixtures/cgroups2/cgroups/good/cpu.max | 1 + fixtures/cgroups2/cgroups/zero-period/cpu.max | 1 + fixtures/cgroups2/proc/cgroups/cgroup | 2 + fixtures/cgroups2/proc/cgroups/cgroup_multi | 3 + fixtures/cgroups2/proc/cgroups/mountinfo | 5 + src/linux.rs | 402 +++++++++++++----- 7 files changed, 300 insertions(+), 115 deletions(-) create mode 100644 fixtures/cgroups2/cgroups/ceil/cpu.max create mode 100644 fixtures/cgroups2/cgroups/good/cpu.max create mode 100644 fixtures/cgroups2/cgroups/zero-period/cpu.max create mode 100644 fixtures/cgroups2/proc/cgroups/cgroup create mode 100644 fixtures/cgroups2/proc/cgroups/cgroup_multi create mode 100644 fixtures/cgroups2/proc/cgroups/mountinfo diff --git a/fixtures/cgroups2/cgroups/ceil/cpu.max b/fixtures/cgroups2/cgroups/ceil/cpu.max new file mode 100644 index 0000000..833a8f2 --- /dev/null +++ b/fixtures/cgroups2/cgroups/ceil/cpu.max @@ -0,0 +1 @@ +150000 100000 diff --git a/fixtures/cgroups2/cgroups/good/cpu.max b/fixtures/cgroups2/cgroups/good/cpu.max new file mode 100644 index 0000000..e469067 --- /dev/null +++ b/fixtures/cgroups2/cgroups/good/cpu.max @@ -0,0 +1 @@ +600000 100000 diff --git a/fixtures/cgroups2/cgroups/zero-period/cpu.max b/fixtures/cgroups2/cgroups/zero-period/cpu.max new file mode 100644 index 0000000..24e757f --- /dev/null +++ b/fixtures/cgroups2/cgroups/zero-period/cpu.max @@ -0,0 +1 @@ +600000 0 diff --git a/fixtures/cgroups2/proc/cgroups/cgroup b/fixtures/cgroups2/proc/cgroups/cgroup new file mode 100644 index 0000000..35b49db --- /dev/null +++ b/fixtures/cgroups2/proc/cgroups/cgroup @@ -0,0 +1,2 @@ +12::/ +3::/user.slice diff --git a/fixtures/cgroups2/proc/cgroups/cgroup_multi b/fixtures/cgroups2/proc/cgroups/cgroup_multi new file mode 100644 index 0000000..1a9282a --- /dev/null +++ b/fixtures/cgroups2/proc/cgroups/cgroup_multi @@ -0,0 +1,3 @@ +12::/ +11:cpu,cpuacct:/ +3::/user.slice diff --git a/fixtures/cgroups2/proc/cgroups/mountinfo b/fixtures/cgroups2/proc/cgroups/mountinfo new file mode 100644 index 0000000..da36e41 --- /dev/null +++ b/fixtures/cgroups2/proc/cgroups/mountinfo @@ -0,0 +1,5 @@ +1 0 8:1 / / rw,noatime shared:1 - ext4 /dev/sda1 rw,errors=remount-ro,data=reordered +2 1 0:1 / /dev rw,relatime shared:2 - devtmpfs udev rw,size=10240k,nr_inodes=16487629,mode=755 +3 1 0:2 / /proc rw,nosuid,nodev,noexec,relatime shared:3 - proc proc rw +4 1 0:3 / /sys rw,nosuid,nodev,noexec,relatime shared:4 - sysfs sysfs rw +5 4 0:4 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime shared:5 - cgroup2 cgroup2 rw,nsdelegate,memory_recursiveprot diff --git a/src/linux.rs b/src/linux.rs index 7a85633..295c925 100644 --- a/src/linux.rs +++ b/src/linux.rs @@ -149,27 +149,36 @@ where P2: AsRef, { let subsys = some!(Subsys::load_cpu(cgroup_proc)); - let mntinfo = some!(MountInfo::load_cpu(mountinfo_proc)); + let mntinfo = some!(MountInfo::load_cpu(mountinfo_proc, subsys.version)); let cgroup = some!(Cgroup::translate(mntinfo, subsys)); cgroup.cpu_quota() } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum CgroupVersion { + V1, + V2, +} + struct Cgroup { + version: CgroupVersion, base: PathBuf, } struct MountInfo { + version: CgroupVersion, root: String, mount_point: String, } struct Subsys { + version: CgroupVersion, base: String, } impl Cgroup { - fn new(dir: PathBuf) -> Cgroup { - Cgroup { base: dir } + fn new(version: CgroupVersion, dir: PathBuf) -> Cgroup { + Cgroup { version: version, base: dir } } fn translate(mntinfo: MountInfo, subsys: Subsys) -> Option { @@ -186,12 +195,14 @@ impl Cgroup { // join(mp.MountPoint, relPath) let mut path = PathBuf::from(mntinfo.mount_point); path.push(rel_from_root); - Some(Cgroup::new(path)) + Some(Cgroup::new(mntinfo.version, path)) } fn cpu_quota(&self) -> Option { - let quota_us = some!(self.quota_us()); - let period_us = some!(self.period_us()); + let (quota_us, period_us) = match self.version { + CgroupVersion::V1 => (some!(self.quota_us()), some!(self.period_us())), + CgroupVersion::V2 => some!(self.max()), + }; // protect against dividing by zero if period_us == 0 { @@ -212,25 +223,41 @@ impl Cgroup { self.param("cpu.cfs_period_us") } + fn max(&self) -> Option<(usize, usize)> { + let max = some!(self.raw_param("cpu.max")); + let mut max = some!(max.lines().next()).split(' '); + + let quota = some!(max.next().and_then(|quota| quota.parse().ok())); + let period = some!(max.next().and_then(|period| period.parse().ok())); + + Some((quota, period)) + } + fn param(&self, param: &str) -> Option { + let buf = some!(self.raw_param(param)); + + buf.trim().parse().ok() + } + + fn raw_param(&self, param: &str) -> Option { let mut file = some!(File::open(self.base.join(param)).ok()); let mut buf = String::new(); some!(file.read_to_string(&mut buf).ok()); - buf.trim().parse().ok() + Some(buf) } } impl MountInfo { - fn load_cpu>(proc_path: P) -> Option { + fn load_cpu>(proc_path: P, version: CgroupVersion) -> Option { let file = some!(File::open(proc_path).ok()); let file = BufReader::new(file); file.lines() .filter_map(|result| result.ok()) .filter_map(MountInfo::parse_line) - .next() + .find(|mount_info| mount_info.version == version) } fn parse_line(line: String) -> Option { @@ -252,19 +279,25 @@ impl MountInfo { }; // 7 5 0:6 / /sys/fs/cgroup/cpu,cpuacct rw,nosuid,nodev,noexec,relatime shared:7 - cgroup rw,cpu,cpuacct - if fields.next() != Some("cgroup") { - return None; - } + let version = match fields.next() { + Some("cgroup") => CgroupVersion::V1, + Some("cgroup2") => CgroupVersion::V2, + _ => return None, + }; - // 7 5 0:6 / /sys/fs/cgroup/cpu,cpuacct rw,nosuid,nodev,noexec,relatime shared:7 - cgroup cgroup - let super_opts = some!(fields.nth(1)); + // cgroups2 only has a single mount point + if version == CgroupVersion::V1 { + // 7 5 0:6 / /sys/fs/cgroup/cpu,cpuacct rw,nosuid,nodev,noexec,relatime shared:7 - cgroup cgroup + let super_opts = some!(fields.nth(1)); - // We only care about the 'cpu' option - if !super_opts.split(',').any(|opt| opt == "cpu") { - return None; + // We only care about the 'cpu' option + if !super_opts.split(',').any(|opt| opt == "cpu") { + return None; + } } Some(MountInfo { + version: version, root: mnt_root.to_owned(), mount_point: mnt_point.to_owned(), }) @@ -279,7 +312,14 @@ impl Subsys { file.lines() .filter_map(|result| result.ok()) .filter_map(Subsys::parse_line) - .next() + .fold(None, |previous, line| { + // already-found v1 trumps v2 since it explicitly specifies its controllers + if previous.is_some() && line.version == CgroupVersion::V2 { + return previous; + } + + Some(line) + }) } fn parse_line(line: String) -> Option { @@ -289,11 +329,18 @@ impl Subsys { let sub_systems = some!(fields.nth(1)); - if !sub_systems.split(',').any(|sub| sub == "cpu") { + let version = if sub_systems.is_empty() { + CgroupVersion::V2 + } else { + CgroupVersion::V1 + }; + + if version == CgroupVersion::V1 && !sub_systems.split(',').any(|sub| sub == "cpu") { return None; } fields.next().map(|path| Subsys { + version: version, base: path.to_owned(), }) } @@ -301,123 +348,248 @@ impl Subsys { #[cfg(test)] mod tests { - use super::{Cgroup, MountInfo, Subsys}; - use std::path::{Path, PathBuf}; + mod v1 { + use super::super::{Cgroup, CgroupVersion, MountInfo, Subsys}; + use std::path::{Path, PathBuf}; - // `static_in_const` feature is not stable in Rust 1.13. - static FIXTURES_PROC: &'static str = "fixtures/cgroups/proc/cgroups"; + // `static_in_const` feature is not stable in Rust 1.13. + static FIXTURES_PROC: &'static str = "fixtures/cgroups/proc/cgroups"; - static FIXTURES_CGROUPS: &'static str = "fixtures/cgroups/cgroups"; + static FIXTURES_CGROUPS: &'static str = "fixtures/cgroups/cgroups"; - macro_rules! join { - ($base:expr, $($path:expr),+) => ({ - Path::new($base) - $(.join($path))+ - }) - } + macro_rules! join { + ($base:expr, $($path:expr),+) => ({ + Path::new($base) + $(.join($path))+ + }) + } - #[test] - fn test_load_mountinfo() { - // test only one optional fields - let path = join!(FIXTURES_PROC, "mountinfo"); + #[test] + fn test_load_mountinfo() { + // test only one optional fields + let path = join!(FIXTURES_PROC, "mountinfo"); - let mnt_info = MountInfo::load_cpu(path).unwrap(); + let mnt_info = MountInfo::load_cpu(path, CgroupVersion::V1).unwrap(); - assert_eq!(mnt_info.root, "/"); - assert_eq!(mnt_info.mount_point, "/sys/fs/cgroup/cpu,cpuacct"); + assert_eq!(mnt_info.root, "/"); + assert_eq!(mnt_info.mount_point, "/sys/fs/cgroup/cpu,cpuacct"); - // test zero optional field - let path = join!(FIXTURES_PROC, "mountinfo_zero_opt"); + // test zero optional field + let path = join!(FIXTURES_PROC, "mountinfo_zero_opt"); - let mnt_info = MountInfo::load_cpu(path).unwrap(); + let mnt_info = MountInfo::load_cpu(path, CgroupVersion::V1).unwrap(); - assert_eq!(mnt_info.root, "/"); - assert_eq!(mnt_info.mount_point, "/sys/fs/cgroup/cpu,cpuacct"); + assert_eq!(mnt_info.root, "/"); + assert_eq!(mnt_info.mount_point, "/sys/fs/cgroup/cpu,cpuacct"); - // test multi optional fields - let path = join!(FIXTURES_PROC, "mountinfo_multi_opt"); + // test multi optional fields + let path = join!(FIXTURES_PROC, "mountinfo_multi_opt"); - let mnt_info = MountInfo::load_cpu(path).unwrap(); + let mnt_info = MountInfo::load_cpu(path, CgroupVersion::V1).unwrap(); - assert_eq!(mnt_info.root, "/"); - assert_eq!(mnt_info.mount_point, "/sys/fs/cgroup/cpu,cpuacct"); - } + assert_eq!(mnt_info.root, "/"); + assert_eq!(mnt_info.mount_point, "/sys/fs/cgroup/cpu,cpuacct"); + } - #[test] - fn test_load_subsys() { - let path = join!(FIXTURES_PROC, "cgroup"); + #[test] + fn test_load_subsys() { + let path = join!(FIXTURES_PROC, "cgroup"); - let subsys = Subsys::load_cpu(path).unwrap(); + let subsys = Subsys::load_cpu(path).unwrap(); - assert_eq!(subsys.base, "/"); - } + assert_eq!(subsys.base, "/"); + assert_eq!(subsys.version, CgroupVersion::V1); + } - #[test] - fn test_cgroup_mount() { - let cases = &[ - ("/", "/sys/fs/cgroup/cpu", "/", Some("/sys/fs/cgroup/cpu")), - ( - "/docker/01abcd", - "/sys/fs/cgroup/cpu", - "/docker/01abcd", - Some("/sys/fs/cgroup/cpu"), - ), - ( - "/docker/01abcd", - "/sys/fs/cgroup/cpu", - "/docker/01abcd/", - Some("/sys/fs/cgroup/cpu"), - ), - ( - "/docker/01abcd", - "/sys/fs/cgroup/cpu", - "/docker/01abcd/large", - Some("/sys/fs/cgroup/cpu/large"), - ), - // fails - ("/docker/01abcd", "/sys/fs/cgroup/cpu", "/", None), - ("/docker/01abcd", "/sys/fs/cgroup/cpu", "/docker", None), - ("/docker/01abcd", "/sys/fs/cgroup/cpu", "/elsewhere", None), - ( - "/docker/01abcd", - "/sys/fs/cgroup/cpu", - "/docker/01abcd-other-dir", - None, - ), - ]; - - for &(root, mount_point, subsys, expected) in cases.iter() { - let mnt_info = MountInfo { - root: root.into(), - mount_point: mount_point.into(), - }; - let subsys = Subsys { - base: subsys.into(), - }; + #[test] + fn test_cgroup_mount() { + let cases = &[ + ("/", "/sys/fs/cgroup/cpu", "/", Some("/sys/fs/cgroup/cpu")), + ( + "/docker/01abcd", + "/sys/fs/cgroup/cpu", + "/docker/01abcd", + Some("/sys/fs/cgroup/cpu"), + ), + ( + "/docker/01abcd", + "/sys/fs/cgroup/cpu", + "/docker/01abcd/", + Some("/sys/fs/cgroup/cpu"), + ), + ( + "/docker/01abcd", + "/sys/fs/cgroup/cpu", + "/docker/01abcd/large", + Some("/sys/fs/cgroup/cpu/large"), + ), + // fails + ("/docker/01abcd", "/sys/fs/cgroup/cpu", "/", None), + ("/docker/01abcd", "/sys/fs/cgroup/cpu", "/docker", None), + ("/docker/01abcd", "/sys/fs/cgroup/cpu", "/elsewhere", None), + ( + "/docker/01abcd", + "/sys/fs/cgroup/cpu", + "/docker/01abcd-other-dir", + None, + ), + ]; + + for &(root, mount_point, subsys, expected) in cases.iter() { + let mnt_info = MountInfo { + version: CgroupVersion::V1, + root: root.into(), + mount_point: mount_point.into(), + }; + let subsys = Subsys { + version: CgroupVersion::V1, + base: subsys.into(), + }; + + let actual = Cgroup::translate(mnt_info, subsys).map(|c| c.base); + let expected = expected.map(PathBuf::from); + assert_eq!(actual, expected); + } + } - let actual = Cgroup::translate(mnt_info, subsys).map(|c| c.base); - let expected = expected.map(PathBuf::from); - assert_eq!(actual, expected); + #[test] + fn test_cgroup_cpu_quota() { + let cgroup = Cgroup::new(CgroupVersion::V1, join!(FIXTURES_CGROUPS, "good")); + assert_eq!(cgroup.cpu_quota(), Some(6)); } - } - #[test] - fn test_cgroup_cpu_quota() { - let cgroup = Cgroup::new(join!(FIXTURES_CGROUPS, "good")); - assert_eq!(cgroup.cpu_quota(), Some(6)); - } + #[test] + fn test_cgroup_cpu_quota_divide_by_zero() { + let cgroup = Cgroup::new(CgroupVersion::V1, join!(FIXTURES_CGROUPS, "zero-period")); + assert!(cgroup.quota_us().is_some()); + assert_eq!(cgroup.period_us(), Some(0)); + assert_eq!(cgroup.cpu_quota(), None); + } - #[test] - fn test_cgroup_cpu_quota_divide_by_zero() { - let cgroup = Cgroup::new(join!(FIXTURES_CGROUPS, "zero-period")); - assert!(cgroup.quota_us().is_some()); - assert_eq!(cgroup.period_us(), Some(0)); - assert_eq!(cgroup.cpu_quota(), None); + #[test] + fn test_cgroup_cpu_quota_ceil() { + let cgroup = Cgroup::new(CgroupVersion::V1, join!(FIXTURES_CGROUPS, "ceil")); + assert_eq!(cgroup.cpu_quota(), Some(2)); + } } - #[test] - fn test_cgroup_cpu_quota_ceil() { - let cgroup = Cgroup::new(join!(FIXTURES_CGROUPS, "ceil")); - assert_eq!(cgroup.cpu_quota(), Some(2)); + mod v2 { + use super::super::{Cgroup, CgroupVersion, MountInfo, Subsys}; + use std::path::{Path, PathBuf}; + + // `static_in_const` feature is not stable in Rust 1.13. + static FIXTURES_PROC: &'static str = "fixtures/cgroups2/proc/cgroups"; + + static FIXTURES_CGROUPS: &'static str = "fixtures/cgroups2/cgroups"; + + macro_rules! join { + ($base:expr, $($path:expr),+) => ({ + Path::new($base) + $(.join($path))+ + }) + } + + #[test] + fn test_load_mountinfo() { + // test only one optional fields + let path = join!(FIXTURES_PROC, "mountinfo"); + + let mnt_info = MountInfo::load_cpu(path, CgroupVersion::V2).unwrap(); + + assert_eq!(mnt_info.root, "/"); + assert_eq!(mnt_info.mount_point, "/sys/fs/cgroup"); + } + + #[test] + fn test_load_subsys() { + let path = join!(FIXTURES_PROC, "cgroup"); + + let subsys = Subsys::load_cpu(path).unwrap(); + + assert_eq!(subsys.base, "/"); + assert_eq!(subsys.version, CgroupVersion::V2); + } + + #[test] + fn test_load_subsys_multi() { + let path = join!(FIXTURES_PROC, "cgroup_multi"); + + let subsys = Subsys::load_cpu(path).unwrap(); + + assert_eq!(subsys.base, "/"); + assert_eq!(subsys.version, CgroupVersion::V1); + } + + #[test] + fn test_cgroup_mount() { + let cases = &[ + ("/", "/sys/fs/cgroup/cpu", "/", Some("/sys/fs/cgroup/cpu")), + ( + "/docker/01abcd", + "/sys/fs/cgroup/cpu", + "/docker/01abcd", + Some("/sys/fs/cgroup/cpu"), + ), + ( + "/docker/01abcd", + "/sys/fs/cgroup/cpu", + "/docker/01abcd/", + Some("/sys/fs/cgroup/cpu"), + ), + ( + "/docker/01abcd", + "/sys/fs/cgroup/cpu", + "/docker/01abcd/large", + Some("/sys/fs/cgroup/cpu/large"), + ), + // fails + ("/docker/01abcd", "/sys/fs/cgroup/cpu", "/", None), + ("/docker/01abcd", "/sys/fs/cgroup/cpu", "/docker", None), + ("/docker/01abcd", "/sys/fs/cgroup/cpu", "/elsewhere", None), + ( + "/docker/01abcd", + "/sys/fs/cgroup/cpu", + "/docker/01abcd-other-dir", + None, + ), + ]; + + for &(root, mount_point, subsys, expected) in cases.iter() { + let mnt_info = MountInfo { + version: CgroupVersion::V1, + root: root.into(), + mount_point: mount_point.into(), + }; + let subsys = Subsys { + version: CgroupVersion::V1, + base: subsys.into(), + }; + + let actual = Cgroup::translate(mnt_info, subsys).map(|c| c.base); + let expected = expected.map(PathBuf::from); + assert_eq!(actual, expected); + } + } + + #[test] + fn test_cgroup_cpu_quota() { + let cgroup = Cgroup::new(CgroupVersion::V2, join!(FIXTURES_CGROUPS, "good")); + assert_eq!(cgroup.cpu_quota(), Some(6)); + } + + #[test] + fn test_cgroup_cpu_quota_divide_by_zero() { + let cgroup = Cgroup::new(CgroupVersion::V2, join!(FIXTURES_CGROUPS, "zero-period")); + let period = cgroup.max().map(|max| max.1); + + assert_eq!(period, Some(0)); + assert_eq!(cgroup.cpu_quota(), None); + } + + #[test] + fn test_cgroup_cpu_quota_ceil() { + let cgroup = Cgroup::new(CgroupVersion::V2, join!(FIXTURES_CGROUPS, "ceil")); + assert_eq!(cgroup.cpu_quota(), Some(2)); + } } }