From c2355507107a961d9f0b5637c089daac94728e0e Mon Sep 17 00:00:00 2001
From: VURU <sueleyman.vurucu@amova.eu>
Date: Tue, 22 Oct 2024 09:17:13 +0200
Subject: [PATCH] Delete consul connect from deployment. See
 https://github.com/suikast42/nomadder/issues/164

---
 .../setup/stack_core/templates/security.j2    |  87 +---
 .../stack_core/templates/security_with_cc.j2  | 480 ++++++++++++++++++
 2 files changed, 499 insertions(+), 68 deletions(-)
 create mode 100644 ansible/setup/stack_core/templates/security_with_cc.j2

diff --git a/ansible/setup/stack_core/templates/security.j2 b/ansible/setup/stack_core/templates/security.j2
index 468e7ff..098152a 100644
--- a/ansible/setup/stack_core/templates/security.j2
+++ b/ansible/setup/stack_core/templates/security.j2
@@ -55,31 +55,13 @@ job "security" {
       mode = "bridge"
       port "db" {
         to = 5432
-        host_network = "local"
       }
     }
 
     service {
       name = "keycloak-postgres"
-      port = "5432"
-      address_mode = "alloc"
-      connect {
-        sidecar_service {}
-        sidecar_task{
-          config{
- {% if set_cpu_hard_limit %}
-            cpu_hard_limit = "true"
- {% endif %}
-            labels = {
-              "com.github.logunifier.application.name" = "envoy-proxy-keycloak-postgres"
-              "com.github.logunifier.application.version" = "{{version_envoy}}"
-              "com.github.logunifier.application.org" = "${var.org}"
-              "com.github.logunifier.application.env" = "${var.env}"
-              "com.github.logunifier.application.pattern.key" = "envoy"
-            }
-          }
-        }
-      }
+      port = "db"
+
      check {
         name     = "keycloak_postgres_ping"
         type     = "script"
@@ -87,7 +69,6 @@ job "security" {
         task     = "keycloak_postgres"
         interval = "10s"
         timeout  = "2s"
-        address_mode = "alloc"
         check_restart {
           limit = 3
           grace = "30s"
@@ -176,13 +157,11 @@ job "security" {
       mode = "bridge"
       port "auth" {
         to = 4181
-        host_network = "local"
       }
     }
     service {
       name = "forwardauth"
       port = "auth"
-      address_mode = "alloc"
       tags = [
         "traefik.enable=true",
         "traefik.http.routers.forwardauth.entrypoints=https",
@@ -296,10 +275,8 @@ job "security" {
       mode = "bridge"
       port "ui" {
         to = 8080
-        host_network = "local"
       }
       port "health_check" {
-        host_network = "local"
         to = 9000
       }
     }
@@ -307,7 +284,6 @@ job "security" {
     service {
         name = "keycloak-health"
         port         = "health_check"
-        address_mode = "alloc"
         check {
             name  = "health"
             type  = "http"
@@ -315,7 +291,6 @@ job "security" {
             path="/health"
             interval = "10s"
             timeout  = "2s"
-            address_mode = "alloc"
             check_restart {
             limit = 3
             grace = "60s"
@@ -329,7 +304,6 @@ job "security" {
             path="/health/live"
             interval = "10s"
             timeout  = "2s"
-            address_mode = "alloc"
         }
         check {
             name  = "started"
@@ -338,7 +312,6 @@ job "security" {
             path="/health/started"
             interval = "10s"
             timeout  = "2s"
-            address_mode = "alloc"
         }
         check {
             name  = "ready"
@@ -347,7 +320,6 @@ job "security" {
             path="/health/ready"
             interval = "10s"
             timeout  = "2s"
-            address_mode = "alloc"
             check_restart {
                 limit = 3
                 grace = "60s"
@@ -357,38 +329,11 @@ job "security" {
     }
     service {
       name = "keycloak"
-#      port = "ui"
-      port = "8080"
-      address_mode = "alloc"
-      connect {
-        sidecar_service {
-          proxy {
-            upstreams {
-              destination_name = "keycloak-postgres"
-              local_bind_port  = 5432
-            }
-          }
-        }
-        sidecar_task{
-
+      port = "ui"
+    #  port = "8080"
 
-          config{
- {% if set_cpu_hard_limit %}
-            cpu_hard_limit = "true"
- {% endif %}
-            labels = {
-              "com.github.logunifier.application.name" = "envoy-proxy-keycloak"
-              "com.github.logunifier.application.pattern.key" = "envoy"
-              "com.github.logunifier.application.version" = "{{version_envoy}}"
-              "com.github.logunifier.application.org" = "${var.org}"
-              "com.github.logunifier.application.env" = "${var.env}"
-            }
-          }
-        }
-      }
       tags = [
         "traefik.enable=true",
-        "traefik.consulcatalog.connect=true",
         "traefik.http.routers.keycloak.tls=true",
         "traefik.http.routers.keycloak.rule=Host(`{{security_dns}}`)",
       ]
@@ -437,8 +382,6 @@ job "security" {
         KC_DB                     = "postgres"
         KC_DB_SCHEMA              = "keycloak"
         KC_DB_USERNAME            = "keycloak"
-        KC_DB_URL_HOST            = "${NOMAD_UPSTREAM_IP_keycloak_postgres}"
-        KC_DB_URL_PORT            = "${NOMAD_UPSTREAM_PORT_keycloak_postgres}"
       }
       config {
         image = "{{registry_dns}}/{{stack_name}}/keycloak:{{version_keycloak_nomadder}}"
@@ -466,15 +409,23 @@ job "security" {
             destination = "${NOMAD_SECRETS_DIR}/env.vars"
             env         = true
             change_mode = "restart"
+            right_delimiter = "++"
+            left_delimiter = "++"
             data        = <<EOF
-    {{ '{{' }}- with nomadVar "{{nomad_security_job_path}}" -{{ '}}' }}
-      KEYCLOAK_ADMIN_PASSWORD      = {{ '{{' }}.keycloak_password{{ '}}' }}
-      KC_DB_PASSWORD               = {{ '{{' }}.keycloak_db_password{{ '}}' }}
-      KC_NOMADDER_CLIENT_SECRET    = {{ '{{' }}.keycloak_ingress_secret{{ '}}' }}
-      KC_NOMADDER_CLIENT_SECRET_GRAFANA    = {{ '{{' }}.keycloak_secret_observability_grafana{{ '}}' }}
-    {{ '{{' }}- end -{{ '}}' }}
+        ++- with nomadVar "{{nomad_security_job_path}}" -++
+      KEYCLOAK_ADMIN_PASSWORD      = ++.keycloak_password++
+      KC_DB_PASSWORD               = ++.keycloak_db_password++
+      KC_NOMADDER_CLIENT_SECRET    = ++.keycloak_ingress_secret++
+      KC_NOMADDER_CLIENT_SECRET_GRAFANA    = ++.keycloak_secret_observability_grafana++
+      ++- end -++
+++- range  $index, $service := service "keycloak-postgres" -++
+++- if eq $index 0  ++
+KC_DB_URL_HOST            = ++$service.Address++
+KC_DB_URL_PORT            = ++$service.Port++
+++- end ++
+++- end ++
     EOF
          }
     }
   }
-}
\ No newline at end of file
+}
diff --git a/ansible/setup/stack_core/templates/security_with_cc.j2 b/ansible/setup/stack_core/templates/security_with_cc.j2
new file mode 100644
index 0000000..468e7ff
--- /dev/null
+++ b/ansible/setup/stack_core/templates/security_with_cc.j2
@@ -0,0 +1,480 @@
+variable "org" {
+  type = string
+  description = "Default organisation"
+  default = "{{tls_san}}"
+}
+
+variable "env" {
+  type = string
+  description = "environment like prod dev testing"
+  default = "{{env}}"
+}
+
+job "security" {
+  type = "service"
+  priority    = 90
+ {% if is_env_development %}
+  meta {
+    run_uuid = "${uuidv4()}"
+  }
+  {% endif %}
+  reschedule {
+    delay          = "10s"
+    delay_function = "constant"
+    unlimited      = true
+  }
+  update {
+      health_check      = "checks"
+      # Alloc is marked as unhealthy after this time
+      healthy_deadline  = "5m"
+      auto_revert  = true
+      # Mark the task as healthy after 10s positive check
+      min_healthy_time  = "10s"
+      # Task is dead after failed checks in 1h
+      progress_deadline = "1h"
+  }
+
+  datacenters = ["{{data_center}}"]
+
+  group "keycloak-postgres" {
+
+    restart {
+      attempts = 1
+      interval = "1h"
+      delay = "5s"
+      mode = "fail"
+    }
+    volume "keycloak_postgres_volume" {
+      type      = "host"
+      source    = "stack_core_keycloak_postgres_volume"
+      read_only = false
+    }
+
+    count = 1
+    network {
+      mode = "bridge"
+      port "db" {
+        to = 5432
+        host_network = "local"
+      }
+    }
+
+    service {
+      name = "keycloak-postgres"
+      port = "5432"
+      address_mode = "alloc"
+      connect {
+        sidecar_service {}
+        sidecar_task{
+          config{
+ {% if set_cpu_hard_limit %}
+            cpu_hard_limit = "true"
+ {% endif %}
+            labels = {
+              "com.github.logunifier.application.name" = "envoy-proxy-keycloak-postgres"
+              "com.github.logunifier.application.version" = "{{version_envoy}}"
+              "com.github.logunifier.application.org" = "${var.org}"
+              "com.github.logunifier.application.env" = "${var.env}"
+              "com.github.logunifier.application.pattern.key" = "envoy"
+            }
+          }
+        }
+      }
+     check {
+        name     = "keycloak_postgres_ping"
+        type     = "script"
+        command  = "pg_isready"
+        task     = "keycloak_postgres"
+        interval = "10s"
+        timeout  = "2s"
+        address_mode = "alloc"
+        check_restart {
+          limit = 3
+          grace = "30s"
+          ignore_warnings = false
+        }
+      }
+    }
+
+    task "keycloak_postgres" {
+      volume_mount {
+        volume      = "keycloak_postgres_volume"
+        destination = "/var/lib/postgresql/data/pgdata"
+      }
+      driver = "docker"
+      env {
+        POSTGRES_USER        = "keycloak"
+        # tell pg_isready to use this user
+        # otherwise a error FATAL:  role "root" does not exist will logged
+        # on every check
+        PGUSER         = "keycloak"
+        POSTGRES_DB          = "keycloak"
+        PGDATA               = "/var/lib/postgresql/data/pgdata"
+        POSTGRES_INITDB_ARGS = "--encoding=UTF8"
+      }
+      config {
+        image = "{{registry_dns}}/postgres:{{version_postgres_keycloak}}"
+ {% if set_cpu_hard_limit %}
+        cpu_hard_limit = "true"
+ {% endif %}
+        volumes = [
+          "local/initddb.sql:/docker-entrypoint-initdb.d/initddb.sql"
+        ]
+        ports = ["db"]
+        labels = {
+           "com.github.logunifier.application.name" = "keycloak_postgres"
+           "com.github.logunifier.application.version" = "{{version_postgres_keycloak}}"
+           "com.github.logunifier.application.org" = "${var.org}"
+           "com.github.logunifier.application.env" = "${var.env}"
+        }
+      }
+      resources {
+        cpu    = 500
+        memory = 512
+        memory_max = 32768
+      }
+      template {
+        data = <<EOF
+           CREATE SCHEMA IF NOT EXISTS keycloak;
+         EOF
+        destination = "local/initddb.sql"
+      }
+      template {
+              destination = "${NOMAD_SECRETS_DIR}/env.vars"
+              env         = true
+              change_mode = "restart"
+              data        = <<EOF
+      {{ '{{' }}- with nomadVar "{{nomad_security_job_path}}" -{{ '}}' }}
+        POSTGRES_PASSWORD    = {{ '{{' }}.keycloak_db_password{{ '}}' }}
+      {{ '{{' }}- end -{{ '}}' }}
+      EOF
+           }
+    }
+  }
+
+  group "keycloak-ingress" {
+
+    update {
+      # keycloak-ingress service has no health checks
+      # rely on task_states
+      health_check      = "task_states"
+      max_parallel      = 1
+    }
+    restart {
+      attempts = 1
+      interval = "1h"
+      delay = "5s"
+      mode = "fail"
+    }
+    volume "ca_cert" {
+      type      = "host"
+      source    = "ca_cert"
+      read_only = true
+    }
+    count = 1
+    network {
+      mode = "bridge"
+      port "auth" {
+        to = 4181
+        host_network = "local"
+      }
+    }
+    service {
+      name = "forwardauth"
+      port = "auth"
+      address_mode = "alloc"
+      tags = [
+        "traefik.enable=true",
+        "traefik.http.routers.forwardauth.entrypoints=https",
+        "traefik.http.routers.forwardauth.rule= Path(`/_oauth`)",
+        "traefik.http.routers.forwardauth.middlewares=traefik-forward-auth",
+        "traefik.http.routers.traefik-forward-auth.tls=true",
+        "traefik.http.middlewares.traefik-forward-auth.forwardauth.address=http://forwardauth.service.consul:${NOMAD_HOST_PORT_auth}",
+        "traefik.http.middlewares.traefik-forward-auth.forwardauth.authResponseHeaders= X-Forwarded-User",
+        "traefik.http.middlewares.traefik-forward-auth.forwardauth.authResponseHeadersRegex= ^X-",
+        "traefik.http.middlewares.traefik-forward-auth.forwardauth.trustForwardHeader=true",
+      #  "traefik.http.middlewares.test-auth.forwardauth.tls.insecureSkipVerify=true"
+      ]
+
+
+    }
+      task "await-for-keycloak" {
+        driver = "docker"
+
+        config {
+          image        = "{{registry_dns}}/busybox:{{version_busy_box_init_container}}"
+          labels = {
+            "com.github.logunifier.application.name" = "await-for-keycloak"
+             "com.github.logunifier.application.version" = "{{version_busy_box_init_container}}"
+             "com.github.logunifier.application.org" = "${var.org}"
+             "com.github.logunifier.application.env" = "${var.env}"
+          }
+ {% if set_cpu_hard_limit %}
+          cpu_hard_limit = "true"
+ {% endif %}
+          command      = "sh"
+          args         = ["-c", "echo -n 'Waiting for service keycloak'; until nslookup keycloak.service.consul 2>&1 >/dev/null; do echo '.'; sleep 2; done"]
+          #network_mode = "host"
+        }
+
+        resources {
+          cpu    = 200
+          memory = 128
+          memory_max = 1024
+        }
+
+        lifecycle {
+          hook    = "prestart"
+          sidecar = false
+        }
+      }
+    task "forwardauth" {
+      driver = "docker"
+      env {
+        #        https://brianturchyn.net/traefik-forwardauth-support-with-keycloak/
+        #        https://github.com/mesosphere/traefik-forward-auth/issues/36
+        #        INSECURE_COOKIE = "1"
+        ENCRYPTION_KEY = "45659373957778734945638459467936" #32 character encryption key
+        #        SCOPE = "profile email openid" # scope openid is necessary for keycloak...
+        SECRET        = "9e7d7b0776f032e3a1996272c2fe22d2"
+        PROVIDER_URI  = "https://security.{{tls_san}}/realms/nomadder"
+        #        OIDC_ISSUER   = "https://security.{{tls_san}}/realms/nomadder"
+        CLIENT_ID     = "ingress"
+        LOG_LEVEL     = "debug"
+        # Lifetime of cookie 60s
+        LIFETIME = "60"
+
+      }
+      volume_mount {
+        volume      = "ca_cert"
+        destination = "/etc/ssl/certs/"
+      }
+      config {
+        image = "{{registry_dns}}/mesosphere/traefik-forward-auth:{{version_mesosphere}}"
+ {% if set_cpu_hard_limit %}
+        cpu_hard_limit = "true"
+ {% endif %}
+        labels = {
+          "com.github.logunifier.application.name" = "mesosphere"
+          "com.github.logunifier.application.version" = "{{version_mesosphere}}"
+          "com.github.logunifier.application.pattern.key" = "logfmt"
+          "com.github.logunifier.application.org" = "${var.org}"
+          "com.github.logunifier.application.env" = "${var.env}"
+        }
+        ports = ["auth"]
+      }
+      resources {
+        cpu    = 500
+        memory = 256
+        memory_max = 32768
+      }
+      template {
+              destination = "${NOMAD_SECRETS_DIR}/env.vars"
+              env         = true
+              change_mode = "restart"
+              data        = <<EOF
+      {{ '{{' }}- with nomadVar "{{nomad_security_job_path}}" -{{ '}}' }}
+        CLIENT_SECRET      = {{ '{{' }}.keycloak_ingress_secret{{ '}}' }}
+      {{ '{{' }}- end -{{ '}}' }}
+      EOF
+           }
+      }
+  }
+
+  group "keycloak" {
+    restart {
+      attempts = 1
+      interval = "1h"
+      delay = "5s"
+      mode = "fail"
+    }
+     update {
+       max_parallel      = 1
+     }
+    count = 1
+    network {
+      mode = "bridge"
+      port "ui" {
+        to = 8080
+        host_network = "local"
+      }
+      port "health_check" {
+        host_network = "local"
+        to = 9000
+      }
+    }
+
+    service {
+        name = "keycloak-health"
+        port         = "health_check"
+        address_mode = "alloc"
+        check {
+            name  = "health"
+            type  = "http"
+            port ="health_check"
+            path="/health"
+            interval = "10s"
+            timeout  = "2s"
+            address_mode = "alloc"
+            check_restart {
+            limit = 3
+            grace = "60s"
+            ignore_warnings = false
+            }
+        }
+        check {
+            name  = "live"
+            type  = "http"
+            port ="health_check"
+            path="/health/live"
+            interval = "10s"
+            timeout  = "2s"
+            address_mode = "alloc"
+        }
+        check {
+            name  = "started"
+            type  = "http"
+            port ="health_check"
+            path="/health/started"
+            interval = "10s"
+            timeout  = "2s"
+            address_mode = "alloc"
+        }
+        check {
+            name  = "ready"
+            type  = "http"
+            port ="health_check"
+            path="/health/ready"
+            interval = "10s"
+            timeout  = "2s"
+            address_mode = "alloc"
+            check_restart {
+                limit = 3
+                grace = "60s"
+                ignore_warnings = false
+            }
+        }
+    }
+    service {
+      name = "keycloak"
+#      port = "ui"
+      port = "8080"
+      address_mode = "alloc"
+      connect {
+        sidecar_service {
+          proxy {
+            upstreams {
+              destination_name = "keycloak-postgres"
+              local_bind_port  = 5432
+            }
+          }
+        }
+        sidecar_task{
+
+
+          config{
+ {% if set_cpu_hard_limit %}
+            cpu_hard_limit = "true"
+ {% endif %}
+            labels = {
+              "com.github.logunifier.application.name" = "envoy-proxy-keycloak"
+              "com.github.logunifier.application.pattern.key" = "envoy"
+              "com.github.logunifier.application.version" = "{{version_envoy}}"
+              "com.github.logunifier.application.org" = "${var.org}"
+              "com.github.logunifier.application.env" = "${var.env}"
+            }
+          }
+        }
+      }
+      tags = [
+        "traefik.enable=true",
+        "traefik.consulcatalog.connect=true",
+        "traefik.http.routers.keycloak.tls=true",
+        "traefik.http.routers.keycloak.rule=Host(`{{security_dns}}`)",
+      ]
+
+
+
+    }
+      task "await-for-keycloak-postgres" {
+        driver = "docker"
+
+        config {
+          image        = "{{registry_dns}}/busybox:{{version_busy_box_init_container}}"
+           labels = {
+               "com.github.logunifier.application.name" = "await-for-keycloak-postgres"
+               "com.github.logunifier.application.version" = "{{version_busy_box_init_container}}"
+               "com.github.logunifier.application.org" = "${var.org}"
+               "com.github.logunifier.application.env" = "${var.env}"
+           }
+ {% if set_cpu_hard_limit %}
+          cpu_hard_limit = "true"
+ {% endif %}
+          command      = "sh"
+          args         = ["-c", "echo -n 'Waiting for service keycloak-postgres'; until nslookup keycloak-postgres.service.consul 2>&1 >/dev/null; do echo '.'; sleep 2; done"]
+          #network_mode = "host"
+        }
+
+        resources {
+          cpu    = 200
+          memory = 128
+          memory_max = 1024
+        }
+
+        lifecycle {
+          hook    = "prestart"
+          sidecar = false
+        }
+      }
+    task "keycloak" {
+      driver = "docker"
+      env {
+        KEYCLOAK_ADMIN  = "admin"
+        KC_HTTP_ENABLED= "true"
+        KC_HOSTNAME="https://{{security_dns}}"
+        KC_HEALTH_ENABLED= "true"
+        KC_METRICS_ENABLED= "true"
+        KC_DB                     = "postgres"
+        KC_DB_SCHEMA              = "keycloak"
+        KC_DB_USERNAME            = "keycloak"
+        KC_DB_URL_HOST            = "${NOMAD_UPSTREAM_IP_keycloak_postgres}"
+        KC_DB_URL_PORT            = "${NOMAD_UPSTREAM_PORT_keycloak_postgres}"
+      }
+      config {
+        image = "{{registry_dns}}/{{stack_name}}/keycloak:{{version_keycloak_nomadder}}"
+ {% if set_cpu_hard_limit %}
+        cpu_hard_limit = "true"
+ {% endif %}
+        labels = {
+          "com.github.logunifier.application.name" = "keycloak"
+          "com.github.logunifier.application.version" = "{{version_keycloak_nomadder}}"
+          "com.github.logunifier.application.pattern.key" = "tslevelmsg"
+          "com.github.logunifier.application.org" = "${var.org}"
+          "com.github.logunifier.application.env" = "${var.env}"
+        }
+        ports = ["ui"]
+        args = [
+          "start", "--import-realm" , "--optimized"
+        ]
+      }
+      resources {
+        cpu    = 500
+        memory = 2048
+        memory_max = 32768
+      }
+    template {
+            destination = "${NOMAD_SECRETS_DIR}/env.vars"
+            env         = true
+            change_mode = "restart"
+            data        = <<EOF
+    {{ '{{' }}- with nomadVar "{{nomad_security_job_path}}" -{{ '}}' }}
+      KEYCLOAK_ADMIN_PASSWORD      = {{ '{{' }}.keycloak_password{{ '}}' }}
+      KC_DB_PASSWORD               = {{ '{{' }}.keycloak_db_password{{ '}}' }}
+      KC_NOMADDER_CLIENT_SECRET    = {{ '{{' }}.keycloak_ingress_secret{{ '}}' }}
+      KC_NOMADDER_CLIENT_SECRET_GRAFANA    = {{ '{{' }}.keycloak_secret_observability_grafana{{ '}}' }}
+    {{ '{{' }}- end -{{ '}}' }}
+    EOF
+         }
+    }
+  }
+}
\ No newline at end of file