ISSUE-106 Utilize pgbouncer to pool connections into PostGIS

docker-compose.yml

@@ -2,40 +2,43 @@ version: '3.9'
 services:
   dbbackups:
     depends_on:
-      postgis:
+      pgbouncer:
         condition: service_healthy
     environment:
+      - DBLIST=tagbase
       - DUMPPREFIX=PG_db
       - POSTGRES_USER=tagbase
       - POSTGRES_PASS=${POSTGRES_PASSWORD}
-      - POSTGRES_PORT=${POSTGRES_PORT}
-      - POSTGRES_HOST=postgis
+      - POSTGRES_PORT=${PGBOUNCER_PORT}
+      - POSTGRES_HOST=pgbouncer
     hostname: pg-backups
     image: kartoza/pg-backup:14-3.3
     labels:
       "docker_compose_diagram.cluster": "Internal Network"
       "docker_compose_diagram.description": "Docker PostGIS backup"
       "docker_compose_diagram.icon": "docker"
     links:
-      - postgis
+      - pgbouncer
+    networks:
+      - internal-network
+    restart: unless-stopped
     volumes:
       - ./dbbackups:/backups
-    restart: on-failure
   docker-cron:
     build:
       context: ./services/docker-cron
     depends_on:
-      postgis:
+      pgbouncer:
         condition: service_healthy
     environment:
+      - PGBOUNCER_PORT=${PGBOUNCER_PORT}
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
-      - POSTGRES_PORT=${POSTGRES_PORT}
     labels:
       "docker_compose_diagram.cluster": "Internal Network"
       "docker_compose_diagram.description": "'metadata_types' and\n'observation_types' synchronization"
       "docker_compose_diagram.icon": "python"
     links:
-      - postgis
+      - pgbouncer
     networks:
       - internal-network
     restart: unless-stopped
@@ -71,14 +74,15 @@ services:
     ports:
       - 81:81
       - 443:443
+    restart: unless-stopped
     volumes:
       - ./services/nginx/config/nginx.conf:/etc/nginx/nginx.conf
       - ./services/nginx/proxy/:/usr/share/nginx/html/:ro
       - ./services/nginx/ssl/cert.pem:/etc/nginx/certs/cert.pem
       - ./services/nginx/ssl/key.pem:/etc/nginx/certs/key.pem
   pgadmin4:
     depends_on:
-      postgis:
+      pgbouncer:
         condition: service_healthy
     environment:
       - PGADMIN_DEFAULT_EMAIL=${PGADMIN_DEFAULT_EMAIL}
@@ -92,11 +96,47 @@ services:
       "docker_compose_diagram.cluster": "Internal Network"
       "docker_compose_diagram.description": "PostgreSQL administration"
       "docker_compose_diagram.icon": "docker"
+    links:
+      - pgbouncer
+    networks:
+      - internal-network
+    restart: unless-stopped
+  pgbouncer:
+    depends_on:
+      postgis:
+        condition: service_healthy
+    environment:
+      - PGBOUNCER_AUTH_TYPE=trust
+      - PGBOUNCER_DATABASE=tagbase
+      # - PGBOUNCER_CLIENT_TLS_SSLMODE=require
+      # - PGBOUNCER_CLIENT_TLS_CERT_FILE=/opt/bitnami/pgbouncer/certs/cert.pem
+      # - PGBOUNCER_CLIENT_TLS_KEY_FILE=/opt/bitnami/pgbouncer/certs/key.pem
+      - POSTGRESQL_DATABASE=tagbase
+      - POSTGRESQL_HOST=postgis
+      - POSTGRESQL_PASSWORD=${POSTGRES_PASSWORD}
+      - POSTGRESQL_PORT=${POSTGRES_PORT}
+      - POSTGRESQL_USERNAME=tagbase
+    expose:
+      - ${PGBOUNCER_PORT}
+    healthcheck:
+      test: "pgbouncer --version"
+      interval: 15s
+      timeout: 5s
+      retries: 5
+      start_period: 15s
+    hostname: pgbouncer
+    image: bitnami/pgbouncer:latest
+    labels:
+      "docker_compose_diagram.cluster": "Internal Network"
+      "docker_compose_diagram.description": "PostgreSQL connection pooling"
+      "docker_compose_diagram.icon": "postgres"
     links:
       - postgis
     networks:
       - internal-network
     restart: unless-stopped
+    # volumes:
+    #   - ./services/nginx/ssl:/opt/bitnami/pgbouncer/certs
   # postgres:
   #   build:
   #     context: ./services/postgres
@@ -147,7 +187,7 @@ services:
     networks:
       - internal-network
     ports:
-      - "5432:5432"
+      - 5432:5432
     restart: unless-stopped
     volumes:
       - ./dbbackups:/backups
@@ -167,12 +207,12 @@ services:
     build:
       context: ./tagbase_server
     depends_on:
-      postgis:
+      pgbouncer:
         condition: service_healthy
     environment:
       - PGADMIN_DEFAULT_EMAIL=${PGADMIN_DEFAULT_EMAIL}
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
-      - POSTGRES_PORT=${POSTGRES_PORT}
+      - PGBOUNCER_PORT=${PGBOUNCER_PORT}
       - SLACK_BOT_CHANNEL=${SLACK_BOT_CHANNEL}
       - SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN}
     expose:
@@ -183,7 +223,7 @@ services:
       "docker_compose_diagram.description": "tagbase-server tag \ningestion and administration"
       "docker_compose_diagram.icon": "flask"
     links:
-      - postgis
+      - pgbouncer
     networks:
       - internal-network
     restart: unless-stopped

services/docker-cron/Dockerfile

@@ -3,7 +3,7 @@ FROM python:bullseye
 ENV PYTHONDONTWRITEBYTECODE 1
 ENV PYTHONUNBUFFERED 1
 ARG POSTGRES_PASSWORD
-ARG POSTGRES_PORT
+ARG PGBOUNCER_PORT
 
 WORKDIR /usr/src/app
 
@@ -28,7 +28,7 @@ RUN python3 -m pip install pip --upgrade pip && \
 
 RUN touch .env \
     && echo "export POSTGRES_PASSWORD=${POSTGRES_PASSWORD}" >> /usr/src/app/.env \
-    && echo "export POSTGRES_PORT=${POSTGRES_PORT}" >> /usr/src/app/.env
+    && echo "export PGBOUNCER_PORT=${PGBOUNCER_PORT}" >> /usr/src/app/.env
 
 RUN chmod 777 -R /usr/src/app
 

services/docker-cron/poll_metadata_and_obs_types.py

@@ -83,8 +83,8 @@ def connect():
         conn = psycopg2.connect(
             dbname="tagbase",
             user="tagbase",
-            host="postgis",
-            port=os.getenv("POSTGRES_PORT"),
+            host="pgbouncer",
+            port=os.getenv("PGBOUNCER_PORT"),
             password=os.getenv("POSTGRES_PASSWORD"),
         )
     except psycopg2.OperationalError as poe:

tagbase_server/tagbase_server/utils/db_utils.py

@@ -18,8 +18,8 @@ def connect():
         conn = psycopg2.connect(
             dbname="tagbase",
             user="tagbase",
-            host="postgis",
-            port=os.getenv("POSTGRES_PORT"),
+            host="pgbouncer",
+            port=os.getenv("PGBOUNCER_PORT"),
             password=os.getenv("POSTGRES_PASSWORD"),
         )
     except psycopg2.OperationalError as poe: