feat: added pod monitor for metrics

bootstrap/rpc/config.tf

@@ -8,7 +8,9 @@ resource "kubernetes_config_map_v1" "fabric_rpc_config" {
     "rpc.toml" = "${templatefile(
       "${path.module}/rpc.toml.tftpl",
       {
-        port = local.port,
+        port            = local.port,
+        prometheus_port = local.prometheus_port,
+
         // If we change the consumer, we must rebuild the cache.
         db_path                     = "/var/cache/${var.consumer_name}.db",
         broker_urls                 = var.broker_urls
@@ -27,7 +29,6 @@ resource "kubernetes_config_map_v1" "fabric_rpc_config" {
         email_ses_secret_access_key = var.email_ses_secret_access_key
         email_ses_region            = var.email_ses_region
         email_ses_verified_email    = var.email_ses_verified_email
-        prometheus_addr             = var.prometheus_addr
       }
     )}"
   }

bootstrap/rpc/main.tf

@@ -2,6 +2,7 @@ locals {
   configmap_name      = "fabric-rpc-config"
   crds_configmap_name = "fabric-rpc-crds"
   port                = 5050
+  prometheus_port     = 9946
 }
 
 variable "namespace" {
@@ -92,12 +93,6 @@ variable "dns_zone" {
   default = "demeter.run"
 }
 
-variable "prometheus_addr" {
-  type    = string
-  default = "0.0.0.0:9946"
-}
-
-
 variable "replicas" {
   type    = number
   default = 1

bootstrap/rpc/monitor.tf

@@ -0,0 +1,27 @@
+resource "kubernetes_manifest" "rpc_monitor" {
+  manifest = {
+    apiVersion = "monitoring.coreos.com/v1"
+    kind       = "PodMonitor"
+    metadata = {
+      labels = {
+        "app.kubernetes.io/component" = "o11y"
+        "app.kubernetes.io/part-of"   = "demeter"
+      }
+      name      = "fabric-rpc"
+      namespace = var.namespace
+    }
+    spec = {
+      selector = {
+        matchLabels = {
+          role = "fabric-rpc"
+        }
+      }
+      podMetricsEndpoints = [
+        {
+          port = "metrics",
+          path = "/metrics"
+        }
+      ]
+    }
+  }
+}

bootstrap/rpc/rpc.toml.tftpl

@@ -44,5 +44,5 @@ ses_verified_email = "${email_ses_verified_email}"
 "sasl.password" = "${kafka_password}"
 
 [prometheus]
-addr = "${prometheus_addr}"
+addr = "0.0.0.0:${prometheus_port}"
 

bootstrap/rpc/sts.tf

@@ -60,6 +60,11 @@ resource "kubernetes_stateful_set_v1" "rpc" {
             container_port = local.port
           }
 
+          port {
+            name           = "metrics"
+            container_port = local.prometheus_port
+          }
+
           volume_mount {
             name       = "cache"
             mount_path = "/var/cache"