Add support for cgroup v2 (#18)

elba-docker · Jan 9, 2022 · 60ae5fa · 60ae5fa
1 parent bf0349a
commit 60ae5fa
Show file tree

Hide file tree

Showing 17 changed files with 1,002 additions and 245 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -9,13 +9,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ---
 
-## [1.4.0](https://github.com/elba-docker/radvisor/compare/v1.3.0...v1.4.0) - 2021-02-02
+## [1.4.0](https://github.com/elba-docker/radvisor/compare/v1.3.0...v1.4.0) - 2022-01-09
 
 [![v1.4.0](https://img.shields.io/badge/release-v1.4.0-2bab64)](https://github.com/elba-docker/radvisor/releases/tag/v1.4.0)
 
 ### Added
 
 - Changed license from the MIT License to the GNU General Public License v3.0
+- Support for [cgroup v2](https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html) was added to the Docker container statistics collection.
+  - **Motivation**: Cgroup v2 handles accounting for writeback I/O better than cgroup v1, and is slowly replacing cgroup v1 as the more modern system.
+  - **Support**: Docker added support for Cgroup v2 in its 20.10 release, and most low-level container runtimes added support for it by the end of 2021. Additionally Cgroup v2 is the default-mounted cgroup version in Ubuntu starting in 21.10, and in most other popular distros starting around 2021.
+  - **Changes**:
+    - The schema for the CSV data in the log-file is different in cgroup v1 and cgroup v2, since the kernel exposes different statistics for each.
+    - The `CollectorType` log metadata field was added to distinguish whether a log file contains statistics from `cgroup_v1` or `cgroup_v2`
+    - `Cgroup` and `CgroupDriver` log metadata fields were moved under `CollectorMetadata`
+    - (internal) A new abstraction was introduced, `Collector`, which defines a trait that is used to collect resource utilization statistics for a running target. Both `cgroup_v1::Collector` and `cgroup_v2::Collector` implement this trait.
 
 ---
 

diff --git a/README.md b/README.md
@@ -10,14 +10,14 @@
 
 ### 🐋 Docker
 
-##### `/var/log/radvisor/stats/c0cd2077ec95e1b340e85c2...b_1585108344.log`
+##### `/var/log/radvisor/stats/7762ff15c99a2d238f4d26c...1_1641734705.log`
 
 ```yaml
 ---
-Version: 1.3.1
+Version: 1.4.0
 Provider: docker
 Metadata:
-  Created: "2020-10-11T04:22:18Z"
+  Created: "2022-01-09T13:25:04Z"
   Command: 'bash -c ''sleep 2s; apt-get update; sleep 2s; DEBIAN_FRONTEND=noninteractive apt-get install -y stress wget; sleep 2s; dd if=/dev/zero of=/tmp/file1 bs=512M count=1 oflag=direct; sleep 2s; stress --cpu 8 --io 4 --vm 4 --vm-bytes 1024M --timeout 10s; sleep 2s; wget "http://ipv4.download.thinkbroadband.com/10MB.zip"; sleep 2s'''
   Id: 7762ff15c99a2d238f4d26c22b5eda5b97ebc03bd0a711693104dcb6f71fe411
   Image: ubuntu
@@ -28,54 +28,38 @@ Metadata:
   Status: Up Less than a second
   SizeRw: ~
   SizeRootFs: ~
-PerfTable:
-  Delimiter: ","
-  Columns:
-    cpu.usage.percpu:
-      Type: int
-      Count: 32
-    read:
-      Type: epoch19
+PerfTable: # ...
 System:
   OsType: Linux
   OsRelease: 4.15.0
-  Distribution:
-    Id: ubuntu
-    IdLike: debian
-    Name: Ubuntu
-    PrettyName: Ubuntu 18.04.1 LTS
-    Version: 18.04.1 LTS (Bionic Beaver)
-    VersionId: "18.04"
-    VersionCodename: bionic
-    CpeName: ~
-    BuildId: ~
-    Variant: ~
-    VariantId: ~
+  Distribution: # ...
   MemoryTotal: 65870408
   SwapTotal: 3145724
   Hostname: node-0.sandbox.infosphere.emulab.net
   CpuCount: 32
   CpuOnlineCount: 4
   CpuSpeed: 1279
-Cgroup: system.slice/docker-7762ff15c99a2d238f4d26c22b5eda5b97ebc03bd0a711693104dcb6f71fe411.scope
-CgroupDriver: systemd
-PolledAt: 1602390140142271945
-InitializedAt: 1602390140157676566
+CollectorType: cgroup_v2
+CollectorMetadata:
+  Cgroup: system.slice/docker-7762ff15c99a2d238f4d26c22b5eda5b97ebc03bd0a711693104dcb6f71fe411.scope
+  CgroupDriver: systemd
+PolledAt: 1641734740142271945
+InitializedAt: 1641734740157676566
 ---
-read,pids.current,pids.max,cpu.usage.total,cpu.usage.system,cpu.usage.user,cpu.usage.percpu,cpu.stat.user,cpu.stat.system,cpu.throttling.periods,cpu.throttling.throttled.count,cpu.throttling.throttled.time,memory.usage.current,memory.usage.max,memory.limit.hard,memory.limit.soft,memory.failcnt,memory.hierarchical_limit.memory,memory.hierarchical_limit.memoryswap,memory.cache,memory.rss.all,memory.rss.huge,memory.mapped,memory.swap,memory.paged.in,memory.paged.out,memory.fault.total,memory.fault.major,memory.anon.inactive,memory.anon.active,memory.file.inactive,memory.file.active,memory.unevictable,blkio.time,blkio.sectors,blkio.service.bytes.read,blkio.service.bytes.write,blkio.service.bytes.sync,blkio.service.bytes.async,blkio.service.ios.read,blkio.service.ios.write,blkio.service.ios.sync,blkio.service.ios.async,blkio.service.time.read,blkio.service.time.write,blkio.service.time.sync,blkio.service.time.async,blkio.queued.read,blkio.queued.write,blkio.queued.sync,blkio.queued.async,blkio.wait.read,blkio.wait.write,blkio.wait.sync,blkio.wait.async,blkio.merged.read,blkio.merged.write,blkio.merged.sync,blkio.merged.async,blkio.throttle.service.bytes.read,blkio.throttle.service.bytes.write,blkio.throttle.service.bytes.sync,blkio.throttle.service.bytes.async,blkio.throttle.service.ios.read,blkio.throttle.service.ios.write,blkio.throttle.service.ios.sync,blkio.throttle.service.ios.async,blkio.bfq.service.bytes.read,blkio.bfq.service.bytes.write,blkio.bfq.service.bytes.sync,blkio.bfq.service.bytes.async,blkio.bfq.service.ios.read,blkio.bfq.service.ios.write,blkio.bfq.service.ios.sync,blkio.bfq.service.ios.async
-1602390175053135973,18,4915,45675783181,0,45675783181,9719044209 12310201631 11027849186 12618688155 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0,2925,1668,0,0,0,2802823168,3667771392,9223372036854771712,9223372036854771712,0,9223372036854771712,,35323904,2754781184,0,28672,,6837817,6156636,6854722,0,0,2754711552,7380992,27942912,0,2273087370,1306336,0,668844032,662777856,6066176,0,331937,331753,184,0,68057100860,68011971780,45129080,0,0,0,0,0,222907407415,222860666999,46740416,0,32,0,32,0,668844032,662777856,6066176,0,331937,331753,184,,,,,,,,
-1602390175103189646,18,4915,45876609757,0,45876610443,9767491855 12362201213 11076227542 12670689833 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0,2938,1676,0,0,0,2968367104,3667771392,9223372036854771712,9223372036854771712,0,9223372036854771712,,35323904,2920067072,0,28672,,6878171,6156636,6895076,0,0,2919976960,7380992,27942912,0,2273087370,1306336,0,668844032,662777856,6066176,0,333749,333565,184,0,68057100860,68011971780,45129080,0,0,0,0,0,222907407415,222860666999,46740416,0,32,0,32,0,668844032,662777856,6066176,0,333750,333566,184,,,,,,,,
+read,pids.current,pids.max,cpu.stat/usage_usec,cpu.stat/system_usec,cpu.stat/user_usec,cpu.stat/nr_periods,cpu.stat/nr_throttled,cpu.stat/throttled_usec,memory.current,memory.high,memory.max,memory.stat/anon,memory.stat/file,memory.stat/kernel_stack,memory.stat/pagetables,memory.stat/percpu,memory.stat/sock,memory.stat/shmem,memory.stat/file_mapped,memory.stat/file_dirty,memory.stat/file_writeback,memory.stat/swapcached,memory.stat/inactive_anon,memory.stat/active_anon,memory.stat/inactive_file,memory.stat/active_file,memory.stat/unevictable,memory.stat/pgfault,memory.stat/pgmajfault,io.stat/rbytes,io.stat/wbytes,io.stat/rios,io.stat/wios,io.stat/dbytes,io.stat/dios
+1641734705052508079,1,28989,58688,40630,18057,0,0,0,5558272,max,max,405504,3514368,49152,0,0,0,0,2838528,0,0,0,270336,0,1486848,2027520,0,1650,0,3891200,0,58,0,0,0
+# ...
 ```
 
-More information about what each column represents can be found in the [docs page](https://github.com/elba-docker/radvisor/blob/master/docs/collecting.md).
+More information about what each column represents can be found in the [docs pages](https://github.com/elba-docker/radvisor/blob/master/docs/collecting_cgroup_v2.md) (for information about the columns ouputted when `CollectorType: cgroup_v1`, see [this page instead](https://github.com/elba-docker/radvisor/blob/master/docs/collecting.md)).
 
 ### ⚓ Kubernetes
 
 ##### `/var/log/radvisor/stats/9f0b1893-15e7-4...c_1585470948.log.log`
 
 ```yaml
 ---
-Version: 1.3.1
+Version: 1.4.0
 Provider: kubernetes
 Metadata:
   Uid: 9f0b1893-15e7-442a-966a-b0d19a35fc1c
@@ -91,43 +75,27 @@ Metadata:
   Phase: Running
   QosClass: BestEffort
   StartedAt: "2020-03-29T04:32:36Z"
-PerfTable:
-  Delimiter: ","
-  Columns:
-    cpu.usage.percpu:
-      Type: int
-      Count: 32
-    read:
-      Type: epoch19
+PerfTable: # ...
 System:
   OsType: Linux
   OsRelease: 4.15.0
-  Distribution:
-    Id: ubuntu
-    IdLike: debian
-    Name: Ubuntu
-    PrettyName: Ubuntu 18.04.1 LTS
-    Version: 18.04.1 LTS (Bionic Beaver)
-    VersionId: "18.04"
-    VersionCodename: bionic
-    CpeName: ~
-    BuildId: ~
-    Variant: ~
-    VariantId: ~
+  Distribution: # ...
   MemoryTotal: 65870408
   SwapTotal: 3145724
   Hostname: node-0.sandbox.infosphere.emulab.net
   CpuCount: 32
   CpuOnlineCount: 32
   CpuSpeed: 1198
 PolledAt: 1585470948008442929
-Cgroup: /kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod9f0b1893_15e7_442a_966a_b0d19a35fc1c.slice
-CgroupDriver: systemd
+CollectorType: cgroup_v1
+CollectorMetadata:
+  Cgroup: /kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod9f0b1893_15e7_442a_966a_b0d19a35fc1c.slice
+  CgroupDriver: systemd
 InitializedAt: 1585470948030565581
 ---
 read,pids.current,pids.max,cpu.usage.total,cpu.usage.system,cpu.usage.user,cpu.usage.percpu,cpu.stat.user,cpu.stat.system,cpu.throttling.periods,cpu.throttling.throttled.count,cpu.throttling.throttled.time,memory.usage.current,memory.usage.max,memory.limit.hard,memory.limit.soft,memory.failcnt,memory.hierarchical_limit.memory,memory.hierarchical_limit.memoryswap,memory.cache,memory.rss.all,memory.rss.huge,memory.mapped,memory.swap,memory.paged.in,memory.paged.out,memory.fault.total,memory.fault.major,memory.anon.inactive,memory.anon.active,memory.file.inactive,memory.file.active,memory.unevictable,blkio.time,blkio.sectors,blkio.service.bytes.read,blkio.service.bytes.write,blkio.service.bytes.sync,blkio.service.bytes.async,blkio.service.ios.read,blkio.service.ios.write,blkio.service.ios.sync,blkio.service.ios.async,blkio.service.time.read,blkio.service.time.write,blkio.service.time.sync,blkio.service.time.async,blkio.queued.read,blkio.queued.write,blkio.queued.sync,blkio.queued.async,blkio.wait.read,blkio.wait.write,blkio.wait.sync,blkio.wait.async,blkio.merged.read,blkio.merged.write,blkio.merged.sync,blkio.merged.async,blkio.throttle.service.bytes.read,blkio.throttle.service.bytes.write,blkio.throttle.service.bytes.sync,blkio.throttle.service.bytes.async,blkio.throttle.service.ios.read,blkio.throttle.service.ios.write,blkio.throttle.service.ios.sync,blkio.throttle.service.ios.async,blkio.bfq.service.bytes.read,blkio.bfq.service.bytes.write,blkio.bfq.service.bytes.sync,blkio.bfq.service.bytes.async,blkio.bfq.service.ios.read,blkio.bfq.service.ios.write,blkio.bfq.service.ios.sync,blkio.bfq.service.ios.async
 1602390175053135973,18,4915,45675783181,0,45675783181,9719044209 12310201631 11027849186 12618688155 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0,2925,1668,0,0,0,2802823168,3667771392,9223372036854771712,9223372036854771712,0,9223372036854771712,,35323904,2754781184,0,28672,,6837817,6156636,6854722,0,0,2754711552,7380992,27942912,0,2273087370,1306336,0,668844032,662777856,6066176,0,331937,331753,184,0,68057100860,68011971780,45129080,0,0,0,0,0,222907407415,222860666999,46740416,0,32,0,32,0,668844032,662777856,6066176,0,331937,331753,184,,,,,,,,
-1602390175103189646,18,4915,45876609757,0,45876610443,9767491855 12362201213 11076227542 12670689833 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0,2938,1676,0,0,0,2968367104,3667771392,9223372036854771712,9223372036854771712,0,9223372036854771712,,35323904,2920067072,0,28672,,6878171,6156636,6895076,0,0,2919976960,7380992,27942912,0,2273087370,1306336,0,668844032,662777856,6066176,0,333749,333565,184,0,68057100860,68011971780,45129080,0,0,0,0,0,222907407415,222860666999,46740416,0,32,0,32,0,668844032,662777856,6066176,0,333750,333566,184,,,,,,,,
+# ...
 ```
 
 ## 📜 Runtime Options
@@ -136,7 +104,7 @@ Many of the specific details of collection can be controlled via the command lin
 
 ```console
 $ radvisor help
-radvisor 1.3.1
+radvisor 1.4.0
 Joseph Azevedo <[email protected]>, Bhanu Garg <[email protected]>
 Monitors container resource utilization with high granularity and low overhead
 

diff --git a/docs/collecting.md b/docs/collecting.md
@@ -1,5 +1,7 @@
 # Runtime Statistics Collection - cgroup v1
 
+> **Note**: this document contains information about the cgroup v1 collector implementation. For information about the statistics collection mechanisms used with cgroup v2, see collecting_cgroup_v2.md.
+
 Runtime statistics for each Docker container are taken from the virtual files for each container's cgroup, located at `/sys/fs/cgroup/<subsystem>/docker/<container id>/file`.
 
 More information is available at the Docker wiki: [Runtime metrics](https://docs.docker.com/config/containers/runmetrics/).

diff --git a/docs/collecting_cgroup_v2.md b/docs/collecting_cgroup_v2.md
@@ -0,0 +1,55 @@
+# Runtime Statistics Collection - cgroup v2
+
+> **Note**: this document contains information about the cgroup v2 collector implementation. For information about the statistics collection mechanisms used with cgroup v1, see collecting.md.
+
+Docker prepares individual cgroups for each container, and these are mounted (by default) at `/sys/fs/cgroup/system.slice/docker-<container id>.scope` when using systemd as the cgroup driver.
+
+As with cgroup v1, network transfer amounts is out-of-scope of this tool (even for cgroup v2), since instrumenting network utilization requires an entirely different mechanism than the one used for block (disk) I/O, CPU, and memory.
+
+## Statistics collected
+
+The following fields are collected for each log line in the target log files:
+
+- `read`
+- `pids.current`
+- `pids.max`
+- `cpu.stat/usage_usec`
+- `cpu.stat/system_usec`
+- `cpu.stat/user_usec`
+- `cpu.stat/nr_periods`
+- `cpu.stat/nr_throttled`
+- `cpu.stat/throttled_usec`
+- `memory.current`
+- `memory.high`
+- `memory.max`
+- `memory.stat/anon`
+- `memory.stat/file`
+- `memory.stat/kernel_stack`
+- `memory.stat/pagetables`
+- `memory.stat/percpu`
+- `memory.stat/sock`
+- `memory.stat/shmem`
+- `memory.stat/file_mapped`
+- `memory.stat/file_dirty`
+- `memory.stat/file_writeback`
+- `memory.stat/swapcached`
+- `memory.stat/inactive_anon`
+- `memory.stat/active_anon`
+- `memory.stat/inactive_file`
+- `memory.stat/active_file`
+- `memory.stat/unevictable`
+- `memory.stat/pgfault`
+- `memory.stat/pgmajfault`
+- `io.stat/rbytes`
+- `io.stat/wbytes`
+- `io.stat/rios`
+- `io.stat/wios`
+- `io.stat/dbytes`
+- `io.stat/dios`
+
+Most of these fields are straightforward, as they directly correspond to a field in a cgroup accounting file (when in the format of `<file>/<field>`, such as `cup.stat/usage_usec`). Alternatively, some fields come from cgroup accounting files that contain a single field, such as `pids.current` and `pids.max`. Information about what these fields specifically mean can be found in the [documentation for cgroup v2](https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html).
+
+The only fields that require discussion are:
+
+- `read` - this is the timestamp of the log line, as a nanosecond Unix timestamp
+- `io.stat/*` - these fields all come from the `io.stat` file, except the valuses are added together among all devices to produce a single value for each field.
diff --git a/man/radvisor-run-docker.1.md b/man/radvisor-run-docker.1.md
@@ -14,7 +14,7 @@ DESCRIPTION
 ===========
 
 **radvisor run docker** runs a collection thread that writes resource statistics to
-output CSV files using configurable intervals. While running, it collects statistics for containers by polling the docker daemon to get a list of active running containers (every 1s by default) and using their cgroups to read information on their system resource utilization.
+output CSV files using configurable intervals. While running, it collects statistics for containers by polling the docker daemon to get a list of active running containers (every 1s by default) and using their cgroups to read information on their system resource utilization. This works whether the host has enabled cgroup v1 or cgroup v2, though the individual fields collected will be different.
 
 Likely needs to be run as root.
 

diff --git a/man/radvisor-run-kubernetes.1.md b/man/radvisor-run-kubernetes.1.md
@@ -14,7 +14,7 @@ DESCRIPTION
 ===========
 
 **radvisor run kubernetes** runs a collection thread that writes resource statistics to
-output CSV files using configurable intervals. While running, it collects statistics for Kubernetes pods, polling the Kubernetes API server to get a list of all active running pods that have been scheduled on the current machine's node, using the cgroup for each pod.
+output CSV files using configurable intervals. While running, it collects statistics for Kubernetes pods, polling the Kubernetes API server to get a list of all active running pods that have been scheduled on the current machine's node, using the cgroup for each pod. Note that the Kubernetes command only supports cgroup v1.
 
 Needs to be a part of an active cluster and needs to be able to find the Kubernetes config file (or specified using **\--kube-config**).
 

diff --git a/man/radvisor.1.md b/man/radvisor.1.md
@@ -21,7 +21,7 @@ Originally developed in Rust as a custom tool to help detect and analyze millibo
 rAdvisor runs by polling the target *provider* (either the local Docker daemon or the Kubernetes API server)
 every 1 second to get a list of active, running containers/pods.
 From this list, rAdvisor runs a collection thread every 50ms to get resource utilization data for each active target
-using Linux [`cgroups`](https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/6/html/resource_management_guide/ch01),
+using Linux [`cgroups`](https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/6/html/resource_management_guide/ch01) (both v1 and v2),
 outputting the resultant logs in `/var/log/radvisor/stats`.
 
 The primary command is `radvisor run`, which has its own man page at **radvisor-run(1)**.

diff --git a/src/collection/collectors/all.rs b/src/collection/collectors/all.rs
@@ -1,41 +1,47 @@
 use crate::collection::buffers::WorkingBuffers;
-use crate::collection::collectors::{cgroup_v1, Collector, StatWriter};
+use crate::collection::collectors::{cgroup_v1, cgroup_v2, Collector, StatWriter};
 use crate::collection::perf_table::TableMetadata;
 use crate::shared::CollectionMethod;
 use anyhow::Error;
 
 pub enum CollectorImpl {
     CgroupV1(cgroup_v1::Collector),
+    CgroupV2(cgroup_v2::Collector),
 }
 
 impl Collector for CollectorImpl {
     fn metadata(&mut self) -> Option<serde_yaml::Value> {
         match self {
             Self::CgroupV1(v1) => v1.metadata(),
+            Self::CgroupV2(v2) => v2.metadata(),
         }
     }
 
     fn table_metadata(&mut self) -> TableMetadata {
         match self {
             Self::CgroupV1(v1) => v1.table_metadata(),
+            Self::CgroupV2(v2) => v2.table_metadata(),
         }
     }
 
     fn get_type(&self) -> &'static str {
         match self {
             Self::CgroupV1(v1) => v1.get_type(),
+            Self::CgroupV2(v2) => v2.get_type(),
         }
     }
 
     fn init(&mut self) -> Result<(), Error> {
         match self {
             Self::CgroupV1(v1) => v1.init(),
+            Self::CgroupV2(v2) => v2.init(),
         }
     }
 
     fn write_header(&mut self, writer: &mut StatWriter) -> Result<(), csv::Error> {
         match self {
             Self::CgroupV1(v1) => v1.write_header(writer),
+            Self::CgroupV2(v2) => v2.write_header(writer),
         }
     }
 
@@ -46,6 +52,7 @@ impl Collector for CollectorImpl {
     ) -> Result<(), csv::Error> {
         match self {
             Self::CgroupV1(v1) => v1.collect(writer, working_buffers),
+            Self::CgroupV2(v2) => v2.collect(writer, working_buffers),
         }
     }
 }
@@ -56,6 +63,9 @@ impl From<CollectionMethod> for CollectorImpl {
             CollectionMethod::LinuxCgroupV1(path) => {
                 Self::CgroupV1(cgroup_v1::Collector::new(path))
             },
+            CollectionMethod::LinuxCgroupV2(path) => {
+                Self::CgroupV2(cgroup_v2::Collector::new(path))
+            },
         }
     }
 }