From d6e73ebd90a1d6242b4673e2266caa380590ee86 Mon Sep 17 00:00:00 2001
From: schneefux <schneefux+commit@schneefux.xyz>
Date: Tue, 26 Mar 2024 15:45:05 +0100
Subject: [PATCH] infra: Fix random 502s

---
 deployment/jobs/conf/traefik.toml.tpl         |  3 +-
 deployment/jobs/invalid-service-cleaner.nomad | 36 +++++++++++++++++++
 2 files changed, 38 insertions(+), 1 deletion(-)
 create mode 100644 deployment/jobs/invalid-service-cleaner.nomad

diff --git a/deployment/jobs/conf/traefik.toml.tpl b/deployment/jobs/conf/traefik.toml.tpl
index 591d433b..76e40410 100644
--- a/deployment/jobs/conf/traefik.toml.tpl
+++ b/deployment/jobs/conf/traefik.toml.tpl
@@ -7,7 +7,8 @@
     trustedIPs = ["127.0.0.1/32"]
 
 [log]
-  level = "INFO"
+  #level = "INFO"
+  level = "DEBUG"
 
 [api]
   dashboard = true
diff --git a/deployment/jobs/invalid-service-cleaner.nomad b/deployment/jobs/invalid-service-cleaner.nomad
new file mode 100644
index 00000000..a7689a6b
--- /dev/null
+++ b/deployment/jobs/invalid-service-cleaner.nomad
@@ -0,0 +1,36 @@
+# workaround for zombie service instances
+# https://github.com/hashicorp/nomad/issues/16616#issuecomment-1748006168
+
+job "nomad-invalid-services-cleaner" {
+  type = "batch"
+
+  periodic {
+    prohibit_overlap = true
+    cron = "0/10 * * * * *"
+    time_zone = "Europe/Berlin"
+  }
+
+  group "services_cleaner" {
+    task "cleaner" {
+      driver = "docker"
+
+      config {
+        image = "ghcr.io/icyleaf/nomad-invalid-services-cleaner:0.1"
+      }
+
+      template {
+        destination = "secrets/.env"
+        env = true
+        data = <<-EOF
+        ONESHOT = true
+        NOMAD_ENDPOINT = http://{{ env "attr.unique.network.ip-address" }}:4646
+        EOF
+      }
+
+      resources {
+        cpu = 50
+        memory  = 50
+      }
+    }
+  }
+}