Add an otel-collector module (#32)

This produces the image and grants the service account as which it will
run permission to publish metrics.

Signed-off-by: Matt Moore <[email protected]>

.github/workflows/documentation.yaml

@@ -14,6 +14,7 @@ jobs:
         - cloudevent-broker
         - cloudevent-trigger
         - cloudevent-recorder
+        - otel-collector
         - networking
         - dashboard/service
         - dashboard/job

cloudevent-broker/README.md

@@ -93,6 +93,7 @@ No requirements.
 | <a name="module_http"></a> [http](#module\_http) | ../dashboard/sections/http | n/a |
 | <a name="module_layout"></a> [layout](#module\_layout) | ../dashboard/sections/layout | n/a |
 | <a name="module_logs"></a> [logs](#module\_logs) | ../dashboard/sections/logs | n/a |
+| <a name="module_otel-collector"></a> [otel-collector](#module\_otel-collector) | ../otel-collector | n/a |
 | <a name="module_resources"></a> [resources](#module\_resources) | ../dashboard/sections/resources | n/a |
 | <a name="module_topic"></a> [topic](#module\_topic) | ../dashboard/sections/topic | n/a |
 | <a name="module_width"></a> [width](#module\_width) | ../dashboard/sections/width | n/a |

cloudevent-broker/ingress.tf

@@ -36,6 +36,13 @@ resource "cosign_sign" "this" {
   conflict = "REPLACE"
 }
 
+module "otel-collector" {
+  source = "../otel-collector"
+
+  project_id      = var.project_id
+  service_account = google_service_account.this.email
+}
+
 resource "google_cloud_run_v2_service" "this" {
   for_each = var.regions
 
@@ -69,11 +76,14 @@ resource "google_cloud_run_v2_service" "this" {
     containers {
       image = cosign_sign.this.signed_ref
 
+      ports { container_port = 8080 }
+
       env {
         name  = "PUBSUB_TOPIC"
         value = google_pubsub_topic.this[each.key].name
       }
     }
+    containers { image = module.otel-collector.image }
   }
 }
 

cloudevent-recorder/README.md

@@ -62,6 +62,7 @@ No requirements.
 
 | Name | Source | Version |
 |------|--------|---------|
+| <a name="module_otel-collector"></a> [otel-collector](#module\_otel-collector) | ../otel-collector | n/a |
 | <a name="module_recorder-dashboard"></a> [recorder-dashboard](#module\_recorder-dashboard) | ../dashboard/cloudevent-receiver | n/a |
 | <a name="module_triggers"></a> [triggers](#module\_triggers) | ../cloudevent-trigger | n/a |
 

cloudevent-recorder/recorder.tf

@@ -24,6 +24,9 @@ resource "ko_build" "recorder-image" {
 
 resource "cosign_sign" "recorder-image" {
   image = ko_build.recorder-image.image_ref
+
+  # Only keep the latest signature.
+  conflict = "REPLACE"
 }
 
 resource "ko_build" "logrotate-image" {
@@ -34,6 +37,16 @@ resource "ko_build" "logrotate-image" {
 
 resource "cosign_sign" "logrotate-image" {
   image = ko_build.logrotate-image.image_ref
+
+  # Only keep the latest signature.
+  conflict = "REPLACE"
+}
+
+module "otel-collector" {
+  source = "../otel-collector"
+
+  project_id      = var.project_id
+  service_account = google_service_account.recorder.email
 }
 
 resource "google_cloud_run_v2_service" "recorder-service" {
@@ -91,6 +104,7 @@ resource "google_cloud_run_v2_service" "recorder-service" {
         mount_path = "/logs"
       }
     }
+    containers { image = module.otel-collector.image }
     volumes {
       name = "logs"
       empty_dir {}

otel-collector/README.md

@@ -0,0 +1,68 @@
+# `otel-collector`
+
+This module encapsulates producing a sidecar image for publishing otel collected
+metrics, and granting the service account as which the sidecar runs permission
+to write those metrics (so it's impossible to forget):
+
+```
+module "otel-collector" {
+  source = "chainguard-dev/cloudrun/glue//otel-collector"
+
+  project_id      = var.project_id
+  service_account = google_service_account.this.email
+}
+
+resource "google_cloud_run_v2_service" "this" {
+  template {
+    service_account = google_service_account.this.email
+    containers {
+      image = "..."
+
+      // Specifying port is necessary when there are multiple containers.
+      ports { container_port = 8080 }
+    }
+    // Install the sidecar!
+    containers { image = module.otel-collector.image }
+  }
+}
+```
+
+<!-- BEGIN_TF_DOCS -->
+## Requirements
+
+No requirements.
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_cosign"></a> [cosign](#provider\_cosign) | n/a |
+| <a name="provider_google"></a> [google](#provider\_google) | n/a |
+| <a name="provider_ko"></a> [ko](#provider\_ko) | n/a |
+
+## Modules
+
+No modules.
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [cosign_sign.otel-image](https://registry.terraform.io/providers/chainguard-dev/cosign/latest/docs/resources/sign) | resource |
+| [google_project_iam_member.metrics-writer](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource |
+| [ko_build.otel-image](https://registry.terraform.io/providers/ko-build/ko/latest/docs/resources/build) | resource |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_otel_collector_image"></a> [otel\_collector\_image](#input\_otel\_collector\_image) | The otel collector image to use as a base. | `string` | `"cgr.dev/chainguard/opentelemetry-collector-contrib:latest"` | no |
+| <a name="input_project_id"></a> [project\_id](#input\_project\_id) | n/a | `string` | n/a | yes |
+| <a name="input_service_account"></a> [service\_account](#input\_service\_account) | The service account as which the collector will run. | `string` | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_image"></a> [image](#output\_image) | n/a |
+<!-- END_TF_DOCS -->

otel-collector/cmd/otel-collector/kodata/config.yaml

@@ -0,0 +1,68 @@
+receivers:
+  prometheus:
+    config:
+      scrape_configs:
+      - job_name: "localhost"
+        scrape_interval: 10s
+        static_configs:
+        # TODO: make this configurable
+        - targets: ["localhost:2112"]
+        metric_relabel_configs:
+          - source_labels: [ __name__ ]
+            regex: '^prometheus_.*'
+            action: drop
+          - source_labels: [ __name__ ]
+            regex: '^process_.*'
+            action: drop
+          - source_labels: [ __name__ ]
+            regex: '^go_.*'
+            action: drop
+
+processors:
+  batch:
+    # batch metrics before sending to reduce API usage
+    send_batch_max_size: 200
+    send_batch_size: 200
+    timeout: 5s
+
+  memory_limiter:
+    # drop metrics if memory usage gets too high
+    check_interval: 1s
+    limit_percentage: 65
+    spike_limit_percentage: 20
+
+  # automatically detect Cloud Run resource metadata
+  resourcedetection:
+    detectors: [env, gcp]
+    timeout: 2s
+    override: false
+
+  resource:
+    attributes:
+      # add instance_id as a resource attribute
+    - key: service.instance.id
+      from_attribute: faas.id
+      action: upsert
+      # parse service name from K_SERVICE Cloud Run variable
+    - key: service.name
+      value: ${env:K_SERVICE}
+      action: insert
+
+exporters:
+  googlemanagedprometheus:
+
+extensions:
+  health_check:
+
+service:
+  telemetry:
+    logs:
+      # We don't want to see scraper startup logging every
+      # cold start.
+      level: "error"
+  extensions: [health_check]
+  pipelines:
+    metrics:
+      receivers: [prometheus]
+      processors: [batch, memory_limiter, resourcedetection, resource]
+      exporters: [googlemanagedprometheus]

otel-collector/cmd/otel-collector/main.go

@@ -0,0 +1,28 @@
+/*
+Copyright 2023 Chainguard, Inc.
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package main
+
+import (
+	"log"
+	"os"
+	"os/exec"
+)
+
+func main() {
+	configPath := os.Getenv("KO_DATA_PATH") + "/config.yaml"
+
+	if _, err := os.Stat(configPath); err != nil {
+		log.Fatalf("error checking config.yaml: %v", err)
+	}
+
+	// run /otel-collector binary with --config config.yaml
+	cmd := exec.Command("otelcol-contrib", "--config", configPath)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		log.Fatal(err)
+	}
+}

otel-collector/main.tf

@@ -0,0 +1,25 @@
+terraform {
+  required_providers {
+    ko     = { source = "ko-build/ko" }
+    cosign = { source = "chainguard-dev/cosign" }
+  }
+}
+
+resource "google_project_iam_member" "metrics-writer" {
+  project = var.project_id
+  role    = "roles/monitoring.metricWriter"
+  member  = "serviceAccount:${var.service_account}"
+}
+
+resource "ko_build" "otel-image" {
+  base_image  = var.otel_collector_image
+  importpath  = "./cmd/otel-collector"
+  working_dir = path.module
+}
+
+resource "cosign_sign" "otel-image" {
+  image = ko_build.otel-image.image_ref
+
+  # Only keep the latest signature.
+  conflict = "REPLACE"
+}

otel-collector/outputs.tf

@@ -0,0 +1,3 @@
+output "image" {
+  value = cosign_sign.otel-image.signed_ref
+}

otel-collector/variables.tf

@@ -0,0 +1,14 @@
+variable "project_id" {
+  type = string
+}
+
+variable "service_account" {
+  type        = string
+  description = "The service account as which the collector will run."
+}
+
+variable "otel_collector_image" {
+  type        = string
+  default     = "cgr.dev/chainguard/opentelemetry-collector-contrib:latest"
+  description = "The otel collector image to use as a base."
+}