Fix nil pointer dereference with stopped container in EC2

As the updated task stats fixture for EC2 shows, stopped containers in EC2 have the empty JSON object for their stats, instead of the `null` that Fargate has, which triggered a nil pointer dereference that is now fixed. Signed-off-by: Ian Kerins <[email protected]>
prometheus-community · Feb 27, 2025 · 26990aa · 26990aa
1 parent a90443a
commit 26990aa
Show file tree

Hide file tree

Showing 7 changed files with 451 additions and 336 deletions.
diff --git a/ecscollector/collector.go b/ecscollector/collector.go
@@ -196,7 +196,7 @@ func (c *collector) Collect(ch chan<- prometheus.Metric) {
 		c.logger.Debug("Failed to retrieve metadata", "error", err)
 		return
 	}
-	c.logger.Debug("Got ECS task metadata response", "stats", metadata)
+	c.logger.Debug("Got ECS task metadata response", "metadata", metadata)
 
 	ch <- prometheus.MustNewConstMetric(
 		taskMetadataDesc,
@@ -269,10 +269,10 @@ func (c *collector) Collect(ch chan<- prometheus.Metric) {
 	networks := make(map[string]*container.NetworkStats)
 	for _, container := range metadata.Containers {
 		s := stats[container.ID]
-		if s == nil {
+		if s == nil || s.StatsJSON == nil {
 			// This can happen if the container is stopped; if it's
 			// nonessential, the task goes on.
-			c.logger.Debug("Couldn't find container with ID in stats", "id", container.ID)
+			c.logger.Debug("Couldn't find stats for container", "id", container.ID)
 			continue
 		}
 

diff --git a/ecscollector/testdata/fixtures/ec2_task_metadata.json b/ecscollector/testdata/fixtures/ec2_task_metadata.json
@@ -1,88 +1,85 @@
 {
   "Cluster": "prom-ecs-exporter-sandbox",
-  "TaskARN": "arn:aws:ecs:us-east-1:829490980523:task/prom-ecs-exporter-sandbox/2a466d5fae2e47b7a1bd718effb44d93",
+  "TaskARN": "arn:aws:ecs:us-east-1:829490980523:task/prom-ecs-exporter-sandbox/506f22fab0414cde856201584703fed9",
   "Family": "prom-ecs-exporter-sandbox-main-ec2",
-  "Revision": "12",
+  "Revision": "13",
   "DesiredStatus": "RUNNING",
   "KnownStatus": "RUNNING",
-  "PullStartedAt": "2025-02-03T02:37:20.456792378Z",
-  "PullStoppedAt": "2025-02-03T02:37:20.95024713Z",
+  "PullStartedAt": "2025-02-27T05:09:52.332595252Z",
+  "PullStoppedAt": "2025-02-27T05:10:01.206072368Z",
   "AvailabilityZone": "us-east-1a",
   "LaunchType": "EC2",
   "Containers": [
     {
-      "DockerId": "6158335d67e586c915e89b99b211962e438eb38cf4a941942a25d0ee508904f9",
-      "Name": "prometheus",
-      "DockerName": "ecs-prom-ecs-exporter-sandbox-main-ec2-12-prometheus-eab8afe7aed084f2fe01",
-      "Image": "prom/prometheus:v3.1.0",
-      "ImageID": "sha256:f3d60e89ba2d4a402d1c62dccdab300f81579355e0744670c55b9ba282f3b56d",
+      "DockerId": "213e1203f4bb72af185724d937e698d2724acf35b57ec2dd5f3c963adbd2d38c",
+      "Name": "nonessential",
+      "DockerName": "ecs-prom-ecs-exporter-sandbox-main-ec2-13-nonessential-9c9ab8aeb0e0dbdca601",
+      "Image": "alpine",
+      "ImageID": "sha256:8d591b0b7dea080ea3be9e12ae563eebf9869168ffced1cb25b2470a3d9fe15e",
       "Labels": {
         "com.amazonaws.ecs.cluster": "prom-ecs-exporter-sandbox",
-        "com.amazonaws.ecs.container-name": "prometheus",
-        "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-east-1:829490980523:task/prom-ecs-exporter-sandbox/2a466d5fae2e47b7a1bd718effb44d93",
+        "com.amazonaws.ecs.container-name": "nonessential",
+        "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-east-1:829490980523:task/prom-ecs-exporter-sandbox/506f22fab0414cde856201584703fed9",
         "com.amazonaws.ecs.task-definition-family": "prom-ecs-exporter-sandbox-main-ec2",
-        "com.amazonaws.ecs.task-definition-version": "12"
+        "com.amazonaws.ecs.task-definition-version": "13"
       },
       "DesiredStatus": "RUNNING",
-      "KnownStatus": "RUNNING",
+      "KnownStatus": "STOPPED",
+      "ExitCode": 0,
       "Limits": {
         "CPU": 128,
         "Memory": 256
       },
-      "CreatedAt": "2025-02-03T02:37:20.862634081Z",
-      "StartedAt": "2025-02-03T02:37:21.926336895Z",
+      "CreatedAt": "2025-02-27T05:09:54.959587312Z",
+      "StartedAt": "2025-02-27T05:09:56.392336771Z",
+      "FinishedAt": "2025-02-27T05:09:56.409399983Z",
       "Type": "NORMAL",
       "Volumes": [
-        {
-          "Source": "/var/lib/ecs/deps/execute-command/bin/3.3.859.0/amazon-ssm-agent",
-          "Destination": "/ecs-execute-command-5fc546da-654a-4f21-9314-3331f56c56ca/amazon-ssm-agent"
-        },
-        {
-          "Source": "/var/lib/ecs/deps/execute-command/bin/3.3.859.0/ssm-agent-worker",
-          "Destination": "/ecs-execute-command-5fc546da-654a-4f21-9314-3331f56c56ca/ssm-agent-worker"
-        },
-        {
-          "Source": "/var/lib/ecs/deps/execute-command/bin/3.3.859.0/ssm-session-worker",
-          "Destination": "/ecs-execute-command-5fc546da-654a-4f21-9314-3331f56c56ca/ssm-session-worker"
-        },
         {
           "Source": "/var/lib/ecs/deps/execute-command/config/amazon-ssm-agent-Orvj12YkCf4DKDu1cHTOVj7smDviWx1T4Kg3Q_IdNYA=.json",
-          "Destination": "/ecs-execute-command-5fc546da-654a-4f21-9314-3331f56c56ca/configuration/amazon-ssm-agent.json"
+          "Destination": "/ecs-execute-command-636764d0-0d77-44c1-96b2-207c74034dff/configuration/amazon-ssm-agent.json"
         },
         {
           "Source": "/var/lib/ecs/deps/execute-command/config/seelog-gEZ-TIvHAyOLfMC5wiWRofgDMlDzaCZ6zcswnAoop84=.xml",
-          "Destination": "/ecs-execute-command-5fc546da-654a-4f21-9314-3331f56c56ca/configuration/seelog.xml"
+          "Destination": "/ecs-execute-command-636764d0-0d77-44c1-96b2-207c74034dff/configuration/seelog.xml"
         },
         {
           "Source": "/var/lib/ecs/deps/execute-command/certs/tls-ca-bundle.pem",
-          "Destination": "/ecs-execute-command-5fc546da-654a-4f21-9314-3331f56c56ca/certs/amazon-ssm-agent.crt"
+          "Destination": "/ecs-execute-command-636764d0-0d77-44c1-96b2-207c74034dff/certs/amazon-ssm-agent.crt"
         },
         {
-          "Source": "/var/log/ecs/exec/2a466d5fae2e47b7a1bd718effb44d93/prometheus",
+          "Source": "/var/log/ecs/exec/506f22fab0414cde856201584703fed9/nonessential",
           "Destination": "/var/log/amazon/ssm"
         },
         {
-          "DockerName": "a3a147f66706aa5f6e35046cfe417e41c0e52ab1d37158d6a96199918a20f20d",
-          "Source": "/var/lib/docker/volumes/a3a147f66706aa5f6e35046cfe417e41c0e52ab1d37158d6a96199918a20f20d/_data",
-          "Destination": "/prometheus"
+          "Source": "/var/lib/ecs/deps/execute-command/bin/3.3.1802.0/amazon-ssm-agent",
+          "Destination": "/ecs-execute-command-636764d0-0d77-44c1-96b2-207c74034dff/amazon-ssm-agent"
+        },
+        {
+          "Source": "/var/lib/ecs/deps/execute-command/bin/3.3.1802.0/ssm-agent-worker",
+          "Destination": "/ecs-execute-command-636764d0-0d77-44c1-96b2-207c74034dff/ssm-agent-worker"
+        },
+        {
+          "Source": "/var/lib/ecs/deps/execute-command/bin/3.3.1802.0/ssm-session-worker",
+          "Destination": "/ecs-execute-command-636764d0-0d77-44c1-96b2-207c74034dff/ssm-session-worker"
         }
       ],
-      "ContainerARN": "arn:aws:ecs:us-east-1:829490980523:container/prom-ecs-exporter-sandbox/2a466d5fae2e47b7a1bd718effb44d93/e1b8a817-72f6-4e36-b736-856201bb01af",
+      "ContainerARN": "arn:aws:ecs:us-east-1:829490980523:container/prom-ecs-exporter-sandbox/506f22fab0414cde856201584703fed9/80b5fc27-0113-4b4f-83a4-f3d4b4b2b016",
       "Networks": [
         {
           "NetworkMode": "bridge",
           "IPv4Addresses": [
-            "172.17.0.2"
+            ""
           ]
         }
       ]
     },
     {
-      "DockerId": "08462152b05bdaafb3143848d2854eab243a426105958cbcee272ced42b206c4",
+      "DockerId": "01cf1f3208005cda71d5ac936ded65d2ecc0a8cc8ff8a82d2e00410bf4fbbd6d",
       "Name": "ecs-exporter",
-      "DockerName": "ecs-prom-ecs-exporter-sandbox-main-ec2-12-ecs-exporter-d4f1cbecc196af9cd301",
+      "DockerName": "ecs-prom-ecs-exporter-sandbox-main-ec2-13-ecs-exporter-e2aeb1e6be8998c72300",
       "Image": "quay.io/prometheuscommunity/ecs-exporter:main",
-      "ImageID": "sha256:6dda80f6ccad7aa04210a159d5d386895b0a28b960d7b2624ec01082f5fa2bc6",
+      "ImageID": "sha256:1585460bf5becf755c9f45fa931283546ca62e2d51bb638010c8958158d144bc",
       "Ports": [
         {
           "ContainerPort": 9779,
@@ -100,56 +97,123 @@
       "Labels": {
         "com.amazonaws.ecs.cluster": "prom-ecs-exporter-sandbox",
         "com.amazonaws.ecs.container-name": "ecs-exporter",
-        "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-east-1:829490980523:task/prom-ecs-exporter-sandbox/2a466d5fae2e47b7a1bd718effb44d93",
+        "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-east-1:829490980523:task/prom-ecs-exporter-sandbox/506f22fab0414cde856201584703fed9",
         "com.amazonaws.ecs.task-definition-family": "prom-ecs-exporter-sandbox-main-ec2",
-        "com.amazonaws.ecs.task-definition-version": "12"
+        "com.amazonaws.ecs.task-definition-version": "13"
       },
       "DesiredStatus": "RUNNING",
       "KnownStatus": "RUNNING",
       "Limits": {
         "CPU": 128,
         "Memory": 256
       },
-      "CreatedAt": "2025-02-03T02:37:20.966311618Z",
-      "StartedAt": "2025-02-03T02:37:21.912366566Z",
+      "CreatedAt": "2025-02-27T05:10:00.313953836Z",
+      "StartedAt": "2025-02-27T05:10:02.731563327Z",
       "Type": "NORMAL",
       "Volumes": [
         {
           "Source": "/var/lib/ecs/deps/execute-command/config/seelog-gEZ-TIvHAyOLfMC5wiWRofgDMlDzaCZ6zcswnAoop84=.xml",
-          "Destination": "/ecs-execute-command-548a3c6d-5ec7-47c7-a1e6-e2df8e80588d/configuration/seelog.xml"
+          "Destination": "/ecs-execute-command-36dfb910-8e80-47b9-8b3a-12c7308123b2/configuration/seelog.xml"
         },
         {
           "Source": "/var/lib/ecs/deps/execute-command/certs/tls-ca-bundle.pem",
-          "Destination": "/ecs-execute-command-548a3c6d-5ec7-47c7-a1e6-e2df8e80588d/certs/amazon-ssm-agent.crt"
+          "Destination": "/ecs-execute-command-36dfb910-8e80-47b9-8b3a-12c7308123b2/certs/amazon-ssm-agent.crt"
         },
         {
-          "Source": "/var/log/ecs/exec/2a466d5fae2e47b7a1bd718effb44d93/ecs-exporter",
+          "Source": "/var/log/ecs/exec/506f22fab0414cde856201584703fed9/ecs-exporter",
           "Destination": "/var/log/amazon/ssm"
         },
         {
-          "Source": "/var/lib/ecs/deps/execute-command/bin/3.3.859.0/amazon-ssm-agent",
-          "Destination": "/ecs-execute-command-548a3c6d-5ec7-47c7-a1e6-e2df8e80588d/amazon-ssm-agent"
+          "Source": "/var/lib/ecs/deps/execute-command/bin/3.3.1802.0/amazon-ssm-agent",
+          "Destination": "/ecs-execute-command-36dfb910-8e80-47b9-8b3a-12c7308123b2/amazon-ssm-agent"
         },
         {
-          "Source": "/var/lib/ecs/deps/execute-command/bin/3.3.859.0/ssm-agent-worker",
-          "Destination": "/ecs-execute-command-548a3c6d-5ec7-47c7-a1e6-e2df8e80588d/ssm-agent-worker"
+          "Source": "/var/lib/ecs/deps/execute-command/bin/3.3.1802.0/ssm-agent-worker",
+          "Destination": "/ecs-execute-command-36dfb910-8e80-47b9-8b3a-12c7308123b2/ssm-agent-worker"
         },
         {
-          "Source": "/var/lib/ecs/deps/execute-command/bin/3.3.859.0/ssm-session-worker",
-          "Destination": "/ecs-execute-command-548a3c6d-5ec7-47c7-a1e6-e2df8e80588d/ssm-session-worker"
+          "Source": "/var/lib/ecs/deps/execute-command/bin/3.3.1802.0/ssm-session-worker",
+          "Destination": "/ecs-execute-command-36dfb910-8e80-47b9-8b3a-12c7308123b2/ssm-session-worker"
         },
         {
           "Source": "/var/lib/ecs/deps/execute-command/config/amazon-ssm-agent-Orvj12YkCf4DKDu1cHTOVj7smDviWx1T4Kg3Q_IdNYA=.json",
-          "Destination": "/ecs-execute-command-548a3c6d-5ec7-47c7-a1e6-e2df8e80588d/configuration/amazon-ssm-agent.json"
+          "Destination": "/ecs-execute-command-36dfb910-8e80-47b9-8b3a-12c7308123b2/configuration/amazon-ssm-agent.json"
         }
       ],
       "LogDriver": "awslogs",
       "LogOptions": {
-        "awslogs-group": "EcsExporterCdkStack-promecsexportersandboxmainec2taskdefinitionpromecsexportersandboxmainec2ecsexporterLogGroup874A22EF-kCHiL9WfBtnb",
+        "awslogs-group": "EcsExporterCdkStack-promecsexportersandboxmainec2taskdefinitionpromecsexportersandboxmainec2ecsexporterLogGroup874A22EF-y3iGqSSTf3sz",
         "awslogs-region": "us-east-1",
-        "awslogs-stream": "ecs-exporter/ecs-exporter/2a466d5fae2e47b7a1bd718effb44d93"
+        "awslogs-stream": "ecs-exporter/ecs-exporter/506f22fab0414cde856201584703fed9"
       },
-      "ContainerARN": "arn:aws:ecs:us-east-1:829490980523:container/prom-ecs-exporter-sandbox/2a466d5fae2e47b7a1bd718effb44d93/6ec245d3-e949-4993-9e0a-6ef6929a2f85",
+      "ContainerARN": "arn:aws:ecs:us-east-1:829490980523:container/prom-ecs-exporter-sandbox/506f22fab0414cde856201584703fed9/5fba1957-462a-48b2-9295-8602b69e00be",
+      "Networks": [
+        {
+          "NetworkMode": "bridge",
+          "IPv4Addresses": [
+            "172.17.0.2"
+          ]
+        }
+      ]
+    },
+    {
+      "DockerId": "6b80adab0733f579594eccae31e5b0056b9544b805450ad6e278fed7f5e1c5ba",
+      "Name": "prometheus",
+      "DockerName": "ecs-prom-ecs-exporter-sandbox-main-ec2-13-prometheus-86f1e9bab7a8e9a65400",
+      "Image": "prom/prometheus:v3.1.0",
+      "ImageID": "sha256:f3d60e89ba2d4a402d1c62dccdab300f81579355e0744670c55b9ba282f3b56d",
+      "Labels": {
+        "com.amazonaws.ecs.cluster": "prom-ecs-exporter-sandbox",
+        "com.amazonaws.ecs.container-name": "prometheus",
+        "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-east-1:829490980523:task/prom-ecs-exporter-sandbox/506f22fab0414cde856201584703fed9",
+        "com.amazonaws.ecs.task-definition-family": "prom-ecs-exporter-sandbox-main-ec2",
+        "com.amazonaws.ecs.task-definition-version": "13"
+      },
+      "DesiredStatus": "RUNNING",
+      "KnownStatus": "RUNNING",
+      "Limits": {
+        "CPU": 128,
+        "Memory": 256
+      },
+      "CreatedAt": "2025-02-27T05:10:01.22383376Z",
+      "StartedAt": "2025-02-27T05:10:02.730952683Z",
+      "Type": "NORMAL",
+      "Volumes": [
+        {
+          "DockerName": "b4c23c0b1e1cea0ddfeab13122e911c7f52eb67720d3ffb43adba63b817e6a1e",
+          "Source": "/var/lib/docker/volumes/b4c23c0b1e1cea0ddfeab13122e911c7f52eb67720d3ffb43adba63b817e6a1e/_data",
+          "Destination": "/prometheus"
+        },
+        {
+          "Source": "/var/lib/ecs/deps/execute-command/bin/3.3.1802.0/amazon-ssm-agent",
+          "Destination": "/ecs-execute-command-12f856a9-3af4-4de7-ab82-671147b2a114/amazon-ssm-agent"
+        },
+        {
+          "Source": "/var/lib/ecs/deps/execute-command/bin/3.3.1802.0/ssm-agent-worker",
+          "Destination": "/ecs-execute-command-12f856a9-3af4-4de7-ab82-671147b2a114/ssm-agent-worker"
+        },
+        {
+          "Source": "/var/lib/ecs/deps/execute-command/bin/3.3.1802.0/ssm-session-worker",
+          "Destination": "/ecs-execute-command-12f856a9-3af4-4de7-ab82-671147b2a114/ssm-session-worker"
+        },
+        {
+          "Source": "/var/lib/ecs/deps/execute-command/config/amazon-ssm-agent-Orvj12YkCf4DKDu1cHTOVj7smDviWx1T4Kg3Q_IdNYA=.json",
+          "Destination": "/ecs-execute-command-12f856a9-3af4-4de7-ab82-671147b2a114/configuration/amazon-ssm-agent.json"
+        },
+        {
+          "Source": "/var/lib/ecs/deps/execute-command/config/seelog-gEZ-TIvHAyOLfMC5wiWRofgDMlDzaCZ6zcswnAoop84=.xml",
+          "Destination": "/ecs-execute-command-12f856a9-3af4-4de7-ab82-671147b2a114/configuration/seelog.xml"
+        },
+        {
+          "Source": "/var/lib/ecs/deps/execute-command/certs/tls-ca-bundle.pem",
+          "Destination": "/ecs-execute-command-12f856a9-3af4-4de7-ab82-671147b2a114/certs/amazon-ssm-agent.crt"
+        },
+        {
+          "Source": "/var/log/ecs/exec/506f22fab0414cde856201584703fed9/prometheus",
+          "Destination": "/var/log/amazon/ssm"
+        }
+      ],
+      "ContainerARN": "arn:aws:ecs:us-east-1:829490980523:container/prom-ecs-exporter-sandbox/506f22fab0414cde856201584703fed9/7ae35d49-867b-468f-afef-916925db8dca",
       "Networks": [
         {
           "NetworkMode": "bridge",
@@ -160,6 +224,6 @@
       ]
     }
   ],
-  "VPCID": "vpc-048a8b68a885ea10e",
+  "VPCID": "vpc-0839c743edb0c009e",
   "ServiceName": "prom-ecs-exporter-sandbox-main-ec2"
 }