From e97b8b0585854764fb353f235743581ca363fd81 Mon Sep 17 00:00:00 2001 From: Felipe Gonzalez Date: Wed, 11 Sep 2024 15:29:24 -0300 Subject: [PATCH] chore: Add reloader (#51) * chore: Add PgBouncer reloader on DbSync * Fix data --- bootstrap/cell/main.tf | 2 +- bootstrap/cell/variables.tf | 11 +- bootstrap/main.tf | 2 +- bootstrap/pgbouncer/main.tf | 9 +- bootstrap/pgbouncer/pg-bouncer.tf | 135 +++++++++++++++++++++--- bootstrap/pgbouncer/pgbouncer.ini.tftpl | 7 +- bootstrap/pgbouncer/tiers.toml | 12 +++ bootstrap/pgbouncer/users.txt.tftpl | 4 - bootstrap/variables.tf | 15 +-- 9 files changed, 148 insertions(+), 49 deletions(-) create mode 100644 bootstrap/pgbouncer/tiers.toml delete mode 100644 bootstrap/pgbouncer/users.txt.tftpl diff --git a/bootstrap/cell/main.tf b/bootstrap/cell/main.tf index 19bf2ba..cd782f3 100644 --- a/bootstrap/cell/main.tf +++ b/bootstrap/cell/main.tf @@ -34,12 +34,12 @@ module "dbsync_pgbouncer" { namespace = var.namespace pg_bouncer_replicas = var.pgbouncer_replicas certs_configmap_name = var.certs_configmap_name - pg_bouncer_user_settings = var.pgbouncer_user_settings pg_bouncer_auth_user_password = var.pgbouncer_auth_user_password instance_role = "pgbouncer" postgres_secret_name = var.postgres_secret_name instance_name = "postgres-dbsync-v3-${var.salt}" postgres_instance_name = local.postgres_host + pgbouncer_reloader_image_tag = var.pgbouncer_reloader_image_tag } module "dbsync_instances" { diff --git a/bootstrap/cell/variables.tf b/bootstrap/cell/variables.tf index 2598085..de9cdb8 100644 --- a/bootstrap/cell/variables.tf +++ b/bootstrap/cell/variables.tf @@ -71,16 +71,11 @@ variable "pgbouncer_replicas" { default = 1 } -variable "pgbouncer_user_settings" { - default = [] - type = list(object({ - name = string - password = string - max_connections = number - })) +variable "pgbouncer_auth_user_password" { + type = string } -variable "pgbouncer_auth_user_password" { +variable "pgbouncer_reloader_image_tag" { type = string } diff --git a/bootstrap/main.tf b/bootstrap/main.tf index 040f764..e00c83d 100644 --- a/bootstrap/main.tf +++ b/bootstrap/main.tf @@ -53,8 +53,8 @@ module "dbsync_cells" { // PGBouncer pgbouncer_image_tag = var.pgbouncer_image_tag pgbouncer_replicas = each.value.pgbouncer.replicas - pgbouncer_user_settings = var.pgbouncer_user_settings pgbouncer_auth_user_password = var.pgbouncer_auth_user_password + pgbouncer_reloader_image_tag = var.pgbouncer_reloader_image_tag // Instances instances = each.value.instances diff --git a/bootstrap/pgbouncer/main.tf b/bootstrap/pgbouncer/main.tf index b985a95..0cebc73 100644 --- a/bootstrap/pgbouncer/main.tf +++ b/bootstrap/pgbouncer/main.tf @@ -31,13 +31,8 @@ variable "certs_configmap_name" { default = "pgbouncer-certs" } -variable "pg_bouncer_user_settings" { - default = [] - type = list(object({ - name = string - password = string - max_connections = number - })) +variable "pgbouncer_reloader_image_tag" { + type = string } variable "pg_bouncer_auth_user_password" { diff --git a/bootstrap/pgbouncer/pg-bouncer.tf b/bootstrap/pgbouncer/pg-bouncer.tf index 8aa0839..9f42ada 100644 --- a/bootstrap/pgbouncer/pg-bouncer.tf +++ b/bootstrap/pgbouncer/pg-bouncer.tf @@ -1,3 +1,8 @@ +locals { + users_volume = "/etc/pgbouncer" + tiers_configmap_name = "${var.instance_name}-tiers" +} + resource "kubernetes_deployment_v1" "pgbouncer" { wait_for_rollout = false metadata { @@ -85,14 +90,9 @@ resource "kubernetes_deployment_v1" "pgbouncer" { value = "5432" } - env { - name = "PGBOUNCER_USERLIST_FILE" - value = "/etc/pgbouncer/users.txt" - } - volume_mount { name = "pgbouncer-users" - mount_path = "/etc/pgbouncer" + mount_path = local.users_volume } volume_mount { @@ -107,6 +107,91 @@ resource "kubernetes_deployment_v1" "pgbouncer" { } + container { + name = "pgbouncer-reloader" + image = "ghcr.io/demeter-run/pgbouncer-reloader:${var.pgbouncer_reloader_image_tag}" + + resources { + limits = { + memory = "250Mi" + } + requests = { + cpu = "100m" + memory = "250Mi" + } + } + + env { + name = "TIERS_PATH" + value = "/etc/tiers/tiers.toml" + } + + env { + name = "API_RESOURCE_GROUP" + value = "demeter.run" + } + + env { + name = "API_RESOURCE_VERSION" + value = "v1alpha1" + } + + env { + name = "API_RESOURCE_API_VERSION" + value = "demeter.run/v1alpha1" + } + + env { + name = "API_RESOURCE_KIND" + value = "DbSyncPort" + } + + env { + name = "API_RESOURCE_PLURAL" + value = "dbsyncports" + } + + env { + name = "POSTGRES_PASSWORD" + value_from { + secret_key_ref { + name = var.postgres_secret_name + key = "password" + } + } + } + + env { + name = "CONNECTION_OPTIONS" + value = "host=localhost user=pgbouncer password=${var.pg_bouncer_auth_user_password} dbname=pgbouncer port=6432" + } + + env { + name = "PGBOUNCER_PASSWORD" + value = var.pg_bouncer_auth_user_password + } + + env { + name = "USERS_INI_FILEPATH" + value = "${local.users_volume}/users.ini" + } + + env { + name = "USERLIST_FILEPATH" + value = "${local.users_volume}/userlist.txt" + } + + volume_mount { + name = "pgbouncer-users" + mount_path = local.users_volume + } + + volume_mount { + name = "tiers" + mount_path = "/etc/tiers" + } + } + container { name = "readiness" image = "ghcr.io/demeter-run/cardano-dbsync-probe:${var.dbsync_probe_image_tag}" @@ -156,11 +241,23 @@ resource "kubernetes_deployment_v1" "pgbouncer" { } + init_container { + name = "init-user-files" + image = "busybox:1.28" + command = [ + "sh", "-c", + "touch ${local.users_volume}/users.ini ${local.users_volume}/userlist.txt; echo '\"pgbouncer\" \"${var.pg_bouncer_auth_user_password}\"' > ${local.users_volume}/userlist.txt" + ] + + volume_mount { + name = "pgbouncer-users" + mount_path = local.users_volume + } + } + volume { name = "pgbouncer-users" - config_map { - name = "${var.instance_name}-pgbouncer-users" - } + empty_dir {} } volume { @@ -177,6 +274,13 @@ resource "kubernetes_deployment_v1" "pgbouncer" { } } + volume { + name = "tiers" + config_map { + name = local.tiers_configmap_name + } + } + toleration { effect = "NoSchedule" key = "demeter.run/compute-profile" @@ -202,14 +306,14 @@ resource "kubernetes_deployment_v1" "pgbouncer" { } -resource "kubernetes_config_map" "dbsync_pgbouncer_users" { +resource "kubernetes_config_map" "dbsync_pgbouncer_tiers" { metadata { namespace = var.namespace - name = "${var.instance_name}-pgbouncer-users" + name = local.tiers_configmap_name } data = { - "users.txt" = "${templatefile("${path.module}/users.txt.tftpl", { auth_user_password = "${var.pg_bouncer_auth_user_password}", users = var.pg_bouncer_user_settings })}" + "tiers.toml" = file("${path.module}/tiers.toml") } } @@ -221,6 +325,11 @@ resource "kubernetes_config_map" "dbsync_pgbouncer_ini_config" { } data = { - "pgbouncer.ini" = "${templatefile("${path.module}/pgbouncer.ini.tftpl", { db_host = "${var.postgres_instance_name}", users = var.pg_bouncer_user_settings })}" + "pgbouncer.ini" = "${templatefile("${path.module}/pgbouncer.ini.tftpl", { + db_host = "${var.postgres_instance_name}", + users_volume = local.users_volume + })}" + # Empty file to bypass bitnami userlist bootstrapping, which we do ourselves. + "userlist.txt" = "" } } diff --git a/bootstrap/pgbouncer/pgbouncer.ini.tftpl b/bootstrap/pgbouncer/pgbouncer.ini.tftpl index 5387dd0..1fb9512 100644 --- a/bootstrap/pgbouncer/pgbouncer.ini.tftpl +++ b/bootstrap/pgbouncer/pgbouncer.ini.tftpl @@ -10,7 +10,7 @@ listen_port=6432 listen_addr=0.0.0.0 unix_socket_dir=/tmp/ unix_socket_mode=0777 -auth_file=/opt/bitnami/pgbouncer/conf/userlist.txt +auth_file=${users_volume}/userlist.txt auth_type=scram-sha-256 auth_query=SELECT usename, passwd FROM user_search($1) pidfile=/opt/bitnami/pgbouncer/tmp/pgbouncer.pid @@ -29,7 +29,4 @@ tcp_keepintvl=75 admin_users=pgbouncer max_db_connections=80 [users] -%{ for user in users ~} -${user.name} = max_user_connections=${user.max_connections} -%{ endfor ~} - +%include ${users_volume}/users.ini diff --git a/bootstrap/pgbouncer/tiers.toml b/bootstrap/pgbouncer/tiers.toml new file mode 100644 index 0000000..c7ac8e5 --- /dev/null +++ b/bootstrap/pgbouncer/tiers.toml @@ -0,0 +1,12 @@ +[[tiers]] +name = "0" +max_connections = 1 +[[tiers]] +name = "1" +max_connections = 3 +[[tiers]] +name = "2" +max_connections = 10 +[[tiers]] +name = "3" +max_connections = 40 diff --git a/bootstrap/pgbouncer/users.txt.tftpl b/bootstrap/pgbouncer/users.txt.tftpl deleted file mode 100644 index 0b4d582..0000000 --- a/bootstrap/pgbouncer/users.txt.tftpl +++ /dev/null @@ -1,4 +0,0 @@ -"pgbouncer" "${auth_user_password}" -%{ for user in users ~} -"${user.name}" "${user.password}" -%{ endfor ~} diff --git a/bootstrap/variables.tf b/bootstrap/variables.tf index 820abc1..4222086 100644 --- a/bootstrap/variables.tf +++ b/bootstrap/variables.tf @@ -39,6 +39,10 @@ variable "pgbouncer_server_key" { type = string } +variable "pgbouncer_reloader_image_tag" { + type = string +} + variable "postgres_hosts" { type = list(string) default = null @@ -73,15 +77,6 @@ variable "pgbouncer_image_tag" { default = "1.21.0" } -variable "pgbouncer_user_settings" { - default = [] - type = list(object({ - name = string - password = string - max_connections = number - })) -} - variable "pgbouncer_auth_user_password" { type = string } @@ -125,7 +120,7 @@ variable "cells" { empty_args = optional(bool, false) custom_config = optional(bool, true) network_env_var = optional(string, false) - topology_zone = optional(string) + topology_zone = optional(string) dbsync_resources = optional(object({ requests = map(string) limits = map(string)