fix: add DB keepalives

Resolves #317

.env-example

@@ -0,0 +1,31 @@
+# Used for local development, when running in docker or on command line
+
+DB_USER=
+DB_PASS=
+DB_HOST=
+DB_NAME=
+DB_PORT=
+DB_SSL_MODE=
+DB_KEEPALIVE_IDLE=
+DB_KEEPALIVE_INTERVAL=
+DB_KEEPALIVE_COUNT=
+
+AZURE_STORAGE_CONNECTION_STRING=
+AZURE_STORAGE_CONTAINER_SOURCE=source
+AZURE_STORAGE_CONTAINER_CLEAN=clean
+ACTIVITIES_LAKE_CONTAINER_NAME=lake
+
+SCHEMA_VALIDATION_API_URL=
+SCHEMA_VALIDATION_KEY_NAME=x-functions-key
+SCHEMA_VALIDATION_KEY_VALUE=
+
+VALIDATOR_API_URL=
+VALIDATOR_API_KEY_NAME=x-functions-key
+VALIDATOR_API_KEY_VALUE=
+
+SOLR_API_URL=
+SOLR_USER=
+SOLR_PASSWORD=
+LOG_LEVEL=
+SOLR_PARALLEL_PROCESSES=
+SOLR_500_SLEEP=

.github/workflows/develop.yml

@@ -34,6 +34,9 @@ env:
   DB_HOST: ${{ secrets.DB_HOST }}
   DB_NAME: ${{ secrets.DB_NAME }}
   DB_PORT: ${{ secrets.DB_PORT }}
+  DB_KEEPALIVE_IDLE: ${{ secrets.DEV_DB_KEEPALIVE_IDLE }}
+  DB_KEEPALIVE_INTERVAL: ${{ secrets.DEV_DB_KEEPALIVE_INTERVAL }}
+  DB_KEEPALIVE_COUNT: ${{ secrets.DEV_DB_KEEPALIVE_COUNT }}
   LOG_WORKSPACE_ID: ${{ secrets.DEV_LOG_WORKSPACE_ID }}
   LOG_WORKSPACE_KEY: ${{ secrets.DEV_LOG_WORKSPACE_KEY }}
   COMMSHUB_URL: ${{ secrets.DEV_COMMSHUB_URL }}
@@ -115,6 +118,9 @@ jobs:
           sed -i ''s^#DB_USER#^$DB_USER^g'' ./deployment/deployment.yml
           sed -i ''s^#DB_PASS#^$DB_PASS^g'' ./deployment/deployment.yml
           sed -i ''s^#DB_NAME#^$DB_NAME^g'' ./deployment/deployment.yml
+          sed -i ''s^#DB_KEEPALIVE_IDLE#^$DB_KEEPALIVE_IDLE^g'' ./deployment/deployment.yml
+          sed -i ''s^#DB_KEEPALIVE_INTERVAL#^$DB_KEEPALIVE_INTERVAL^g'' ./deployment/deployment.yml
+          sed -i ''s^#DB_KEEPALIVE_COUNT#^$DB_KEEPALIVE_COUNT^g'' ./deployment/deployment.yml
 
           sed -i ''s^#SOLR_API_URL#^$SOLR_API_URL^g'' ./deployment/deployment.yml
           sed -i ''s^#SOLR_USER#^$SOLR_USER^g'' ./deployment/deployment.yml

.github/workflows/main.yml

@@ -29,6 +29,9 @@ env:
   DB_HOST: ${{ secrets.PROD_DB_HOST }}
   DB_NAME: ${{ secrets.PROD_DB_NAME }}
   DB_PORT: ${{ secrets.PROD_DB_PORT }}
+  DB_KEEPALIVE_IDLE: ${{ secrets.PROD_DB_KEEPALIVE_IDLE }}
+  DB_KEEPALIVE_INTERVAL: ${{ secrets.PROD_DB_KEEPALIVE_INTERVAL }}
+  DB_KEEPALIVE_COUNT: ${{ secrets.PROD_DB_KEEPALIVE_COUNT }}
   LOG_WORKSPACE_ID: ${{ secrets.PROD_LOG_WORKSPACE_ID }}
   LOG_WORKSPACE_KEY: ${{ secrets.PROD_LOG_WORKSPACE_KEY }}
   COMMSHUB_URL: ${{ secrets.PROD_COMMSHUB_URL }}
@@ -109,6 +112,9 @@ jobs:
           sed -i ''s^#DB_USER#^$DB_USER^g'' ./deployment/deployment.yml
           sed -i ''s^#DB_PASS#^$DB_PASS^g'' ./deployment/deployment.yml
           sed -i ''s^#DB_NAME#^$DB_NAME^g'' ./deployment/deployment.yml
+          sed -i ''s^#DB_KEEPALIVE_IDLE#^$DB_KEEPALIVE_IDLE^g'' ./deployment/deployment.yml
+          sed -i ''s^#DB_KEEPALIVE_INTERVAL#^$DB_KEEPALIVE_INTERVAL^g'' ./deployment/deployment.yml
+          sed -i ''s^#DB_KEEPALIVE_COUNT#^$DB_KEEPALIVE_COUNT^g'' ./deployment/deployment.yml
 
           sed -i ''s^#SOLR_API_URL#^$SOLR_API_URL^g'' ./deployment/deployment.yml
           sed -i ''s^#SOLR_USER#^$SOLR_USER^g'' ./deployment/deployment.yml

src/constants/config.py

@@ -12,11 +12,14 @@
     DB_PORT=os.getenv("DB_PORT"),
     DB_NAME=os.getenv("DB_NAME"),
     DB_SSL_MODE=os.getenv("DB_SSL_MODE") or "require",
-    # Database retry/timeout constants
+    # Database retry/timeout/keepalive
     DB_CONN_RETRY_LIMIT=8,
     DB_CONN_SLEEP_START=5,
     DB_CONN_SLEEP_MAX=60,
     DB_CONN_TIMEOUT=5,
+    DB_KEEPALIVE_IDLE=os.getenv("DB_KEEPALIVE_IDLE"),
+    DB_KEEPALIVE_INTERVAL=os.getenv("DB_KEEPALIVE_INTERVAL"),
+    DB_KEEPALIVE_COUNT=os.getenv("DB_KEEPALIVE_COUNT"),
     # Azure Storage Account Connection String
     # This can be found in the Azure Portal > Storage Account > Access Keys
     STORAGE_CONNECTION_STR=os.getenv("AZURE_STORAGE_CONNECTION_STRING"),

src/library/db.py

@@ -22,6 +22,10 @@ def getDirectConnection(retry_counter=0):
             port=config["DB_PORT"],
             sslmode=config["DB_SSL_MODE"],
             connect_timeout=config["DB_CONN_TIMEOUT"],
+            keepalives=1,
+            keepalives_idle=config["DB_KEEPALIVE_IDLE"],
+            keepalives_interval=config["DB_KEEPALIVE_INTERVAL"],
+            keepalives_count=config["DB_KEEPALIVE_COUNT"],
         )
         retry_counter = 0
         return connection