Merge pull request #203 from navidys/container_size

cached scraping of container size
containers · Mar 11, 2024 · cfc43a4 · cfc43a4
2 parents c2c3a3d + cbc6c09
commit cfc43a4
Show file tree

Hide file tree

Showing 10 changed files with 336 additions and 136 deletions.
diff --git a/README.md b/README.md
@@ -24,6 +24,7 @@ Usage:
   prometheus-podman-exporter [flags]
 
 Flags:
+  -t, --collector.cache_duration int          Duration (seconds) to retrieve container, size and refresh the cache (default 3600)
   -a, --collector.enable-all                  Enable all collectors by default.
   -i, --collector.image                       Enable image collector.
   -n, --collector.network                     Enable network collector.
@@ -76,8 +77,6 @@ The table below list all existing collector and their description.
 # HELP podman_container_info Container information.
 # TYPE podman_container_info gauge
 podman_container_info{id="19286a13dc23",image="docker.io/library/sonarqube:latest",name="sonar01",pod_id="",pod_name="",ports="0.0.0.0:9000->9000/tcp"} 1
-podman_container_info{id="22e3d69be889",image="localhost/podman-pause:4.1.0-1651853754",name="959a0a3530db-infra",pod_id="959a0a3530db",pod_name="pod02",ports=""} 1
-podman_container_info{id="390ac740fa80",image="localhost/podman-pause:4.1.0-1651853754",name="d05cda23085a-infra",pod_id="d05cda23085a",pod_name="pod03",ports=""} 1
 podman_container_info{id="482113b805f7",image="docker.io/library/httpd:latest",name="web_server",pod_id="",pod_name="",ports="0.0.0.0:8000->80/tcp"} 1
 podman_container_info{id="642490688d9c",image="docker.io/grafana/grafana:latest",name="grafana",pod_id="",pod_name="",ports="0.0.0.0:3000->3000/tcp"} 1
 podman_container_info{id="ad36e85960a1",image="docker.io/library/busybox:latest",name="busybox01",pod_id="3e8bae64e9af",pod_name="pod01",ports=""} 1
@@ -86,8 +85,6 @@ podman_container_info{id="dda983cc3ecf",image="localhost/podman-pause:4.1.0-1651
 # HELP podman_container_state Container current state (-1=unknown,0=created,1=initialized,2=running,3=stopped,4=paused,5=exited,6=removing,7=stopping).
 # TYPE podman_container_state gauge
 podman_container_state{id="19286a13dc23",pod_id="",pod_name=""} 2
-podman_container_state{id="22e3d69be889",pod_id="959a0a3530db",pod_name="pod02"} 0
-podman_container_state{id="390ac740fa80",pod_id="d05cda23085a",pod_name="pod03"} 5
 podman_container_state{id="482113b805f7",pod_id="",pod_name=""} 4
 podman_container_state{id="642490688d9c",pod_id="",pod_name=""} 2
 podman_container_state{id="ad36e85960a1",pod_id="3e8bae64e9af",pod_name="pod01"} 5
@@ -96,8 +93,6 @@ podman_container_state{id="dda983cc3ecf",pod_id="3e8bae64e9af",pod_name="pod01"}
 # HELP podman_container_block_input_total Container block input.
 # TYPE podman_container_block_input_total counter
 podman_container_block_input_total{id="19286a13dc23",pod_id="",pod_name=""} 49152
-podman_container_block_input_total{id="22e3d69be889",pod_id="959a0a3530db",pod_name="pod02"} 0
-podman_container_block_input_total{id="390ac740fa80",pod_id="d05cda23085a",pod_name="pod03"} 0
 podman_container_block_input_total{id="482113b805f7",pod_id="",pod_name=""} 0
 podman_container_block_input_total{id="642490688d9c",pod_id="",pod_name=""} 1.41533184e+08
 podman_container_block_input_total{id="ad36e85960a1",pod_id="3e8bae64e9af",pod_name="pod01"} 0
@@ -106,8 +101,6 @@ podman_container_block_input_total{id="dda983cc3ecf",pod_id="3e8bae64e9af",pod_n
 # HELP podman_container_block_output_total Container block output.
 # TYPE podman_container_block_output_total counter
 podman_container_block_output_total{id="19286a13dc23",pod_id="",pod_name=""} 1.790976e+06
-podman_container_block_output_total{id="22e3d69be889",pod_id="959a0a3530db",pod_name="pod02"} 0
-podman_container_block_output_total{id="390ac740fa80",pod_id="d05cda23085a",pod_name="pod03"} 0
 podman_container_block_output_total{id="482113b805f7",pod_id="",pod_name=""} 8192
 podman_container_block_output_total{id="642490688d9c",pod_id="",pod_name=""} 4.69248e+07
 podman_container_block_output_total{id="ad36e85960a1",pod_id="3e8bae64e9af",pod_name="pod01"} 0
@@ -116,8 +109,6 @@ podman_container_block_output_total{id="dda983cc3ecf",pod_id="3e8bae64e9af",pod_
 # HELP podman_container_cpu_seconds_total total CPU time spent for container in seconds.
 # TYPE podman_container_cpu_seconds_total counter
 podman_container_cpu_seconds_total{id="19286a13dc23",pod_id="",pod_name=""} 83.231904
-podman_container_cpu_seconds_total{id="22e3d69be889",pod_id="959a0a3530db",pod_name="pod02"} 0
-podman_container_cpu_seconds_total{id="390ac740fa80",pod_id="d05cda23085a",pod_name="pod03"} 0
 podman_container_cpu_seconds_total{id="482113b805f7",pod_id="",pod_name=""} 0.069712
 podman_container_cpu_seconds_total{id="642490688d9c",pod_id="",pod_name=""} 3.028685
 podman_container_cpu_seconds_total{id="ad36e85960a1",pod_id="3e8bae64e9af",pod_name="pod01"} 0
@@ -126,8 +117,6 @@ podman_container_cpu_seconds_total{id="dda983cc3ecf",pod_id="3e8bae64e9af",pod_n
 # HELP podman_container_cpu_system_seconds_total total system CPU time spent for container in seconds.
 # TYPE podman_container_cpu_system_seconds_total counter
 podman_container_cpu_system_seconds_total{id="19286a13dc23",pod_id="",pod_name=""} 0.007993418
-podman_container_cpu_system_seconds_total{id="22e3d69be889",pod_id="959a0a3530db",pod_name="pod02"} 0
-podman_container_cpu_system_seconds_total{id="390ac740fa80",pod_id="d05cda23085a",pod_name="pod03"} 0
 podman_container_cpu_system_seconds_total{id="482113b805f7",pod_id="",pod_name=""} 4.8591e-05
 podman_container_cpu_system_seconds_total{id="642490688d9c",pod_id="",pod_name=""} 0.00118734
 podman_container_cpu_system_seconds_total{id="ad36e85960a1",pod_id="3e8bae64e9af",pod_name="pod01"} 0
@@ -136,8 +125,6 @@ podman_container_cpu_system_seconds_total{id="dda983cc3ecf",pod_id="3e8bae64e9af
 # HELP podman_container_created_seconds Container creation time in unixtime.
 # TYPE podman_container_created_seconds gauge
 podman_container_created_seconds{id="19286a13dc23",pod_id="",pod_name=""} 1.655859887e+09
-podman_container_created_seconds{id="22e3d69be889",pod_id="959a0a3530db",pod_name="pod02"} 1.655484892e+09
-podman_container_created_seconds{id="390ac740fa80",pod_id="d05cda23085a",pod_name="pod03"} 1.655489348e+09
 podman_container_created_seconds{id="482113b805f7",pod_id="",pod_name=""} 1.655859728e+09
 podman_container_created_seconds{id="642490688d9c",pod_id="",pod_name=""} 1.655859511e+09
 podman_container_created_seconds{id="ad36e85960a1",pod_id="3e8bae64e9af",pod_name="pod01"} 1.655859858e+09
@@ -146,8 +133,6 @@ podman_container_created_seconds{id="dda983cc3ecf",pod_id="3e8bae64e9af",pod_nam
 # HELP podman_container_started_seconds Container started time in unixtime.
 # TYPE podman_container_started_seconds gauge
 podman_container_started_seconds{id="19286a13dc23",pod_id="",pod_name=""} 1.659253804e+09
-podman_container_started_seconds{id="22e3d69be889",pod_id="959a0a3530db",pod_name="pod02"} -6.21355968e+10
-podman_container_started_seconds{id="390ac740fa80",pod_id="d05cda23085a",pod_name="pod03"} 1.66064284e+09
 podman_container_started_seconds{id="482113b805f7",pod_id="",pod_name=""} 1.659253804e+09
 podman_container_started_seconds{id="642490688d9c",pod_id="",pod_name=""} 1.660642996e+09
 podman_container_started_seconds{id="ad36e85960a1",pod_id="3e8bae64e9af",pod_name="pod01"} 1.66064284e+09
@@ -156,8 +141,6 @@ podman_container_started_seconds{id="dda983cc3ecf",pod_id="3e8bae64e9af",pod_nam
 # HELP podman_container_exit_code Container exit code, if the container has not exited or restarted then the exit code will be 0.
 # TYPE podman_container_exit_code gauge
 podman_container_exit_code{id="19286a13dc23",pod_id="",pod_name=""} 0
-podman_container_exit_code{id="22e3d69be889",pod_id="959a0a3530db",pod_name="pod02"} 0
-podman_container_exit_code{id="390ac740fa80",pod_id="d05cda23085a",pod_name="pod03"} 1
 podman_container_exit_code{id="482113b805f7",pod_id="",pod_name=""} 0
 podman_container_exit_code{id="642490688d9c",pod_id="",pod_name=""} 0
 podman_container_exit_code{id="ad36e85960a1",pod_id="3e8bae64e9af",pod_name="pod01"} 130
@@ -166,8 +149,6 @@ podman_container_exit_code{id="dda983cc3ecf",pod_id="3e8bae64e9af",pod_name="pod
 # HELP podman_container_exited_seconds Container exited time in unixtime.
 # TYPE podman_container_exited_seconds gauge
 podman_container_exited_seconds{id="19286a13dc23",pod_id="",pod_name=""} 1.659253805e+09
-podman_container_exited_seconds{id="22e3d69be889",pod_id="959a0a3530db",pod_name="pod02"} -6.21355968e+10
-podman_container_exited_seconds{id="390ac740fa80",pod_id="d05cda23085a",pod_name="pod03"} 1.660643511e+09
 podman_container_exited_seconds{id="482113b805f7",pod_id="",pod_name=""} 1.659253805e+09
 podman_container_exited_seconds{id="642490688d9c",pod_id="",pod_name=""} 1.659253804e+09
 podman_container_exited_seconds{id="ad36e85960a1",pod_id="3e8bae64e9af",pod_name="pod01"} 1.660643511e+09
@@ -176,8 +157,6 @@ podman_container_exited_seconds{id="dda983cc3ecf",pod_id="3e8bae64e9af",pod_name
 # HELP podman_container_mem_limit_bytes Container memory limit.
 # TYPE podman_container_mem_limit_bytes gauge
 podman_container_mem_limit_bytes{id="19286a13dc23",pod_id="",pod_name=""} 9.713655808e+09
-podman_container_mem_limit_bytes{id="22e3d69be889",pod_id="959a0a3530db",pod_name="pod02"} 0
-podman_container_mem_limit_bytes{id="390ac740fa80",pod_id="d05cda23085a",pod_name="pod03"} 0
 podman_container_mem_limit_bytes{id="482113b805f7",pod_id="",pod_name=""} 9.713655808e+09
 podman_container_mem_limit_bytes{id="642490688d9c",pod_id="",pod_name=""} 9.713655808e+09
 podman_container_mem_limit_bytes{id="ad36e85960a1",pod_id="3e8bae64e9af",pod_name="pod01"} 0
@@ -186,8 +165,6 @@ podman_container_mem_limit_bytes{id="dda983cc3ecf",pod_id="3e8bae64e9af",pod_nam
 # HELP podman_container_mem_usage_bytes Container memory usage.
 # TYPE podman_container_mem_usage_bytes gauge
 podman_container_mem_usage_bytes{id="19286a13dc23",pod_id="",pod_name=""} 1.029062656e+09
-podman_container_mem_usage_bytes{id="22e3d69be889",pod_id="959a0a3530db",pod_name="pod02"} 0
-podman_container_mem_usage_bytes{id="390ac740fa80",pod_id="d05cda23085a",pod_name="pod03"} 0
 podman_container_mem_usage_bytes{id="482113b805f7",pod_id="",pod_name=""} 2.748416e+06
 podman_container_mem_usage_bytes{id="642490688d9c",pod_id="",pod_name=""} 3.67616e+07
 podman_container_mem_usage_bytes{id="ad36e85960a1",pod_id="3e8bae64e9af",pod_name="pod01"} 0
@@ -196,8 +173,6 @@ podman_container_mem_usage_bytes{id="dda983cc3ecf",pod_id="3e8bae64e9af",pod_nam
 # HELP podman_container_net_input_total Container network input.
 # TYPE podman_container_net_input_total counter
 podman_container_net_input_total{id="19286a13dc23",pod_id="",pod_name=""} 430
-podman_container_net_input_total{id="22e3d69be889",pod_id="959a0a3530db",pod_name="pod02"} 0
-podman_container_net_input_total{id="390ac740fa80",pod_id="d05cda23085a",pod_name="pod03"} 0
 podman_container_net_input_total{id="482113b805f7",pod_id="",pod_name=""} 430
 podman_container_net_input_total{id="642490688d9c",pod_id="",pod_name=""} 4323
 podman_container_net_input_total{id="ad36e85960a1",pod_id="3e8bae64e9af",pod_name="pod01"} 0
@@ -206,8 +181,6 @@ podman_container_net_input_total{id="dda983cc3ecf",pod_id="3e8bae64e9af",pod_nam
 # HELP podman_container_net_output_total Container network output.
 # TYPE podman_container_net_output_total counter
 podman_container_net_output_total{id="19286a13dc23",pod_id="",pod_name=""} 110
-podman_container_net_output_total{id="22e3d69be889",pod_id="959a0a3530db",pod_name="pod02"} 0
-podman_container_net_output_total{id="390ac740fa80",pod_id="d05cda23085a",pod_name="pod03"} 0
 podman_container_net_output_total{id="482113b805f7",pod_id="",pod_name=""} 110
 podman_container_net_output_total{id="642490688d9c",pod_id="",pod_name=""} 12071
 podman_container_net_output_total{id="ad36e85960a1",pod_id="3e8bae64e9af",pod_name="pod01"} 0
@@ -216,12 +189,26 @@ podman_container_net_output_total{id="dda983cc3ecf",pod_id="3e8bae64e9af",pod_na
 # HELP podman_container_pids Container pid number.
 # TYPE podman_container_pids gauge
 podman_container_pids{id="19286a13dc23",pod_id="",pod_name=""} 94
-podman_container_pids{id="22e3d69be889",pod_id="959a0a3530db",pod_name="pod02"} 0
-podman_container_pids{id="390ac740fa80",pod_id="d05cda23085a",pod_name="pod03"} 0
 podman_container_pids{id="482113b805f7",pod_id="",pod_name=""} 82
 podman_container_pids{id="642490688d9c",pod_id="",pod_name=""} 14
 podman_container_pids{id="ad36e85960a1",pod_id="3e8bae64e9af",pod_name="pod01"} 0
 podman_container_pids{id="dda983cc3ecf",pod_id="3e8bae64e9af",pod_name="pod01"} 1
+
+# HELP podman_container_rootfs_size_bytes Container root filesystem size in bytes.
+# TYPE podman_container_rootfs_size_bytes gauge
+podman_container_rootfs_size_bytes{id="19286a13dc23",pod_id="",pod_name=""} 1.452382e+06
+podman_container_rootfs_size_bytes{id="482113b805f7",pod_id="",pod_name=""} 1.135744e+06
+podman_container_rootfs_size_bytes{id="642490688d9c",pod_id="",pod_name=""} 1.72771905e+08
+podman_container_rootfs_size_bytes{id="ad36e85960a1",pod_id="3e8bae64e9af",pod_name="pod01"} 1.135744e+06
+podman_container_rootfs_size_bytes{id="dda983cc3ecf",pod_id="3e8bae64e9af",pod_name="pod01"} 1.035744e+06
+
+# HELP podman_container_rw_size_bytes Container top read-write layer size in bytes.
+# TYPE podman_container_rw_size_bytes gauge
+podman_container_rw_size_bytes{id="19286a13dc23",pod_id="",pod_name=""} 0
+podman_container_rw_size_bytes{id="482113b805f7",pod_id="",pod_name=""} 0
+podman_container_rw_size_bytes{id="642490688d9c",pod_id="",pod_name=""} 26261
+podman_container_rw_size_bytes{id="ad36e85960a1",pod_id="3e8bae64e9af",pod_name="pod01"} 3551
+podman_container_rw_size_bytes{id="dda983cc3ecf",pod_id="3e8bae64e9af",pod_name="pod01"} 0
 ```
 
 #### `pod`

diff --git a/cmd/root.go b/cmd/root.go
@@ -11,7 +11,8 @@ import (
 )
 
 const (
-	maxRequest int = 40
+	maxRequest    int   = 40
+	cacheDuration int64 = 3600
 )
 
 var (
@@ -50,13 +51,15 @@ func preRun(cmd *cobra.Command, _ []string) error {
 
 func run(cmd *cobra.Command, args []string) {
 	if err := exporter.Start(cmd, args); err != nil {
-		log.Panic(err.Error())
+		log.Print(err.Error())
+		os.Exit(1)
 	}
 }
 
 // Execute adds all child commands to the root command and sets flags appropriately.
 func Execute() {
 	if err := rootCmd.Execute(); err != nil {
+		log.Print(err.Error())
 		os.Exit(1)
 	}
 }
@@ -94,4 +97,6 @@ func init() {
 		"Comma separated list of pod/container/image labels to be converted\n"+
 			"to labels on prometheus metrics for each pod/container/image.\n"+
 			"collector.store_labels must be set to false for this to take effect.")
+	rootCmd.Flags().Int64P("collector.cache_duration", "t", cacheDuration,
+		"Duration (seconds) to retrieve container, size and refresh the cache")
 }
diff --git a/collector/container.go b/collector/container.go
@@ -23,6 +23,8 @@ type containerCollector struct {
 	netOutput   typedDesc
 	blockInput  typedDesc
 	blockOutput typedDesc
+	rwSize      typedDesc
+	rootFsSize  typedDesc
 	logger      log.Logger
 }
 
@@ -142,6 +144,20 @@ func NewContainerStatsCollector(logger log.Logger) (Collector, error) {
 				[]string{"id", "pod_id", "pod_name"}, nil,
 			), prometheus.CounterValue,
 		},
+		rwSize: typedDesc{
+			prometheus.NewDesc(
+				prometheus.BuildFQName(namespace, "container", "rw_size_bytes"),
+				"Container top read-write layer size in bytes.",
+				[]string{"id", "pod_id", "pod_name"}, nil,
+			), prometheus.GaugeValue,
+		},
+		rootFsSize: typedDesc{
+			prometheus.NewDesc(
+				prometheus.BuildFQName(namespace, "container", "rootfs_size_bytes"),
+				"Container root filesystem size in bytes.",
+				[]string{"id", "pod_id", "pod_name"}, nil,
+			), prometheus.GaugeValue,
+		},
 		logger: logger,
 	}, nil
 }
@@ -170,6 +186,8 @@ func (c *containerCollector) Update(ch chan<- prometheus.Metric) error {
 		ch <- c.started.mustNewConstMetric(float64(rep.Started), rep.ID, rep.PodID, rep.PodName)
 		ch <- c.exited.mustNewConstMetric(float64(rep.Exited), rep.ID, rep.PodID, rep.PodName)
 		ch <- c.exitCode.mustNewConstMetric(float64(rep.ExitCode), rep.ID, rep.PodID, rep.PodName)
+		ch <- c.rwSize.mustNewConstMetric(float64(rep.RwSize), rep.ID, rep.PodID, rep.PodName)
+		ch <- c.rootFsSize.mustNewConstMetric(float64(rep.RootFsSize), rep.ID, rep.PodID, rep.PodName)
 
 		if cntStat != nil {
 			ch <- c.pids.mustNewConstMetric(float64(cntStat.PIDs), rep.ID, rep.PodID, rep.PodName)