chore: Add reloader (#51)

* chore: Add PgBouncer reloader on DbSync * Fix data
demeter-run · Sep 11, 2024 · e97b8b0 · e97b8b0
1 parent 2cd2f00
commit e97b8b0
Show file tree

Hide file tree

Showing 9 changed files with 148 additions and 49 deletions.
diff --git a/bootstrap/cell/main.tf b/bootstrap/cell/main.tf
@@ -34,12 +34,12 @@ module "dbsync_pgbouncer" {
   namespace                     = var.namespace
   pg_bouncer_replicas           = var.pgbouncer_replicas
   certs_configmap_name          = var.certs_configmap_name
-  pg_bouncer_user_settings      = var.pgbouncer_user_settings
   pg_bouncer_auth_user_password = var.pgbouncer_auth_user_password
   instance_role                 = "pgbouncer"
   postgres_secret_name          = var.postgres_secret_name
   instance_name                 = "postgres-dbsync-v3-${var.salt}"
   postgres_instance_name        = local.postgres_host
+  pgbouncer_reloader_image_tag  = var.pgbouncer_reloader_image_tag
 }
 
 module "dbsync_instances" {

diff --git a/bootstrap/cell/variables.tf b/bootstrap/cell/variables.tf
@@ -71,16 +71,11 @@ variable "pgbouncer_replicas" {
   default = 1
 }
 
-variable "pgbouncer_user_settings" {
-  default = []
-  type = list(object({
-    name            = string
-    password        = string
-    max_connections = number
-  }))
+variable "pgbouncer_auth_user_password" {
+  type = string
 }
 
-variable "pgbouncer_auth_user_password" {
+variable "pgbouncer_reloader_image_tag" {
   type = string
 }
 

diff --git a/bootstrap/main.tf b/bootstrap/main.tf
@@ -53,8 +53,8 @@ module "dbsync_cells" {
   // PGBouncer
   pgbouncer_image_tag          = var.pgbouncer_image_tag
   pgbouncer_replicas           = each.value.pgbouncer.replicas
-  pgbouncer_user_settings      = var.pgbouncer_user_settings
   pgbouncer_auth_user_password = var.pgbouncer_auth_user_password
+  pgbouncer_reloader_image_tag = var.pgbouncer_reloader_image_tag
 
   // Instances
   instances = each.value.instances

diff --git a/bootstrap/pgbouncer/main.tf b/bootstrap/pgbouncer/main.tf
@@ -31,13 +31,8 @@ variable "certs_configmap_name" {
   default = "pgbouncer-certs"
 }
 
-variable "pg_bouncer_user_settings" {
-  default = []
-  type = list(object({
-    name            = string
-    password        = string
-    max_connections = number
-  }))
+variable "pgbouncer_reloader_image_tag" {
+  type = string
 }
 
 variable "pg_bouncer_auth_user_password" {

diff --git a/bootstrap/pgbouncer/pg-bouncer.tf b/bootstrap/pgbouncer/pg-bouncer.tf
@@ -1,3 +1,8 @@
+locals {
+  users_volume         = "/etc/pgbouncer"
+  tiers_configmap_name = "${var.instance_name}-tiers"
+}
+
 resource "kubernetes_deployment_v1" "pgbouncer" {
   wait_for_rollout = false
   metadata {
@@ -85,14 +90,9 @@ resource "kubernetes_deployment_v1" "pgbouncer" {
             value = "5432"
           }
 
-          env {
-            name  = "PGBOUNCER_USERLIST_FILE"
-            value = "/etc/pgbouncer/users.txt"
-          }
-
           volume_mount {
             name       = "pgbouncer-users"
-            mount_path = "/etc/pgbouncer"
+            mount_path = local.users_volume
           }
 
           volume_mount {
@@ -107,6 +107,91 @@ resource "kubernetes_deployment_v1" "pgbouncer" {
 
         }
 
+        container {
+          name  = "pgbouncer-reloader"
+          image = "ghcr.io/demeter-run/pgbouncer-reloader:${var.pgbouncer_reloader_image_tag}"
+
+          resources {
+            limits = {
+              memory = "250Mi"
+            }
+            requests = {
+              cpu    = "100m"
+              memory = "250Mi"
+            }
+          }
+
+          env {
+            name  = "TIERS_PATH"
+            value = "/etc/tiers/tiers.toml"
+          }
+
+          env {
+            name  = "API_RESOURCE_GROUP"
+            value = "demeter.run"
+          }
+
+          env {
+            name  = "API_RESOURCE_VERSION"
+            value = "v1alpha1"
+          }
+
+          env {
+            name  = "API_RESOURCE_API_VERSION"
+            value = "demeter.run/v1alpha1"
+          }
+
+          env {
+            name  = "API_RESOURCE_KIND"
+            value = "DbSyncPort"
+          }
+
+          env {
+            name  = "API_RESOURCE_PLURAL"
+            value = "dbsyncports"
+          }
+
+          env {
+            name = "POSTGRES_PASSWORD"
+            value_from {
+              secret_key_ref {
+                name = var.postgres_secret_name
+                key  = "password"
+              }
+            }
+          }
+
+          env {
+            name  = "CONNECTION_OPTIONS"
+            value = "host=localhost user=pgbouncer password=${var.pg_bouncer_auth_user_password} dbname=pgbouncer port=6432"
+          }
+
+          env {
+            name  = "PGBOUNCER_PASSWORD"
+            value = var.pg_bouncer_auth_user_password
+          }
+
+          env {
+            name  = "USERS_INI_FILEPATH"
+            value = "${local.users_volume}/users.ini"
+          }
+
+          env {
+            name  = "USERLIST_FILEPATH"
+            value = "${local.users_volume}/userlist.txt"
+          }
+
+          volume_mount {
+            name       = "pgbouncer-users"
+            mount_path = local.users_volume
+          }
+
+          volume_mount {
+            name       = "tiers"
+            mount_path = "/etc/tiers"
+          }
+        }
+
         container {
           name  = "readiness"
           image = "ghcr.io/demeter-run/cardano-dbsync-probe:${var.dbsync_probe_image_tag}"
@@ -156,11 +241,23 @@ resource "kubernetes_deployment_v1" "pgbouncer" {
 
         }
 
+        init_container {
+          name  = "init-user-files"
+          image = "busybox:1.28"
+          command = [
+            "sh", "-c",
+            "touch ${local.users_volume}/users.ini ${local.users_volume}/userlist.txt; echo '\"pgbouncer\" \"${var.pg_bouncer_auth_user_password}\"' > ${local.users_volume}/userlist.txt"
+          ]
+
+          volume_mount {
+            name       = "pgbouncer-users"
+            mount_path = local.users_volume
+          }
+        }
+
         volume {
           name = "pgbouncer-users"
-          config_map {
-            name = "${var.instance_name}-pgbouncer-users"
-          }
+          empty_dir {}
         }
 
         volume {
@@ -177,6 +274,13 @@ resource "kubernetes_deployment_v1" "pgbouncer" {
           }
         }
 
+        volume {
+          name = "tiers"
+          config_map {
+            name = local.tiers_configmap_name
+          }
+        }
+
         toleration {
           effect   = "NoSchedule"
           key      = "demeter.run/compute-profile"
@@ -202,14 +306,14 @@ resource "kubernetes_deployment_v1" "pgbouncer" {
 }
 
 
-resource "kubernetes_config_map" "dbsync_pgbouncer_users" {
+resource "kubernetes_config_map" "dbsync_pgbouncer_tiers" {
   metadata {
     namespace = var.namespace
-    name      = "${var.instance_name}-pgbouncer-users"
+    name      = local.tiers_configmap_name
   }
 
   data = {
-    "users.txt" = "${templatefile("${path.module}/users.txt.tftpl", { auth_user_password = "${var.pg_bouncer_auth_user_password}", users = var.pg_bouncer_user_settings })}"
+    "tiers.toml" = file("${path.module}/tiers.toml")
   }
 }
 
@@ -221,6 +325,11 @@ resource "kubernetes_config_map" "dbsync_pgbouncer_ini_config" {
   }
 
   data = {
-    "pgbouncer.ini" = "${templatefile("${path.module}/pgbouncer.ini.tftpl", { db_host = "${var.postgres_instance_name}", users = var.pg_bouncer_user_settings })}"
+    "pgbouncer.ini" = "${templatefile("${path.module}/pgbouncer.ini.tftpl", {
+      db_host      = "${var.postgres_instance_name}",
+      users_volume = local.users_volume
+    })}"
+    # Empty file to bypass bitnami userlist bootstrapping, which we do ourselves.
+    "userlist.txt" = ""
   }
 }
diff --git a/bootstrap/pgbouncer/pgbouncer.ini.tftpl b/bootstrap/pgbouncer/pgbouncer.ini.tftpl
@@ -10,7 +10,7 @@ listen_port=6432
 listen_addr=0.0.0.0
 unix_socket_dir=/tmp/
 unix_socket_mode=0777
-auth_file=/opt/bitnami/pgbouncer/conf/userlist.txt
+auth_file=${users_volume}/userlist.txt
 auth_type=scram-sha-256
 auth_query=SELECT usename, passwd FROM user_search($1)
 pidfile=/opt/bitnami/pgbouncer/tmp/pgbouncer.pid
@@ -29,7 +29,4 @@ tcp_keepintvl=75
 admin_users=pgbouncer
 max_db_connections=80
 [users]
-%{ for user in users ~}
-${user.name} = max_user_connections=${user.max_connections}
-%{ endfor ~}
-
+%include ${users_volume}/users.ini
diff --git a/bootstrap/pgbouncer/tiers.toml b/bootstrap/pgbouncer/tiers.toml
@@ -0,0 +1,12 @@
+[[tiers]]
+name = "0"
+max_connections = 1
+[[tiers]]
+name = "1"
+max_connections = 3
+[[tiers]]
+name = "2"
+max_connections = 10
+[[tiers]]
+name = "3"
+max_connections = 40
diff --git a/bootstrap/pgbouncer/users.txt.tftpl b/bootstrap/pgbouncer/users.txt.tftpl
diff --git a/bootstrap/variables.tf b/bootstrap/variables.tf
@@ -39,6 +39,10 @@ variable "pgbouncer_server_key" {
   type = string
 }
 
+variable "pgbouncer_reloader_image_tag" {
+  type = string
+}
+
 variable "postgres_hosts" {
   type    = list(string)
   default = null
@@ -73,15 +77,6 @@ variable "pgbouncer_image_tag" {
   default = "1.21.0"
 }
 
-variable "pgbouncer_user_settings" {
-  default = []
-  type = list(object({
-    name            = string
-    password        = string
-    max_connections = number
-  }))
-}
-
 variable "pgbouncer_auth_user_password" {
   type = string
 }
@@ -125,7 +120,7 @@ variable "cells" {
       empty_args            = optional(bool, false)
       custom_config         = optional(bool, true)
       network_env_var       = optional(string, false)
-      topology_zone = optional(string)
+      topology_zone         = optional(string)
       dbsync_resources = optional(object({
         requests = map(string)
         limits   = map(string)