From 915890cc808bd77dc1597e79928a5fb0cc23750b Mon Sep 17 00:00:00 2001 From: Florian Aucomte Date: Sat, 2 Nov 2024 02:36:26 +0000 Subject: [PATCH] Switch to PSQL DB setup (#334) * PSQL env vars * Install psycopg2 * Proxy on new DB instance * Update ping to DB * Do it properly * Correct port * Typo * Try other stuff * Print info about pg service * Start PG service * Sudo everything * whatever * Fix psycopg2 version and add to requirements * Remove anymail * Make PSQL settings default * Remove duplicate field * Final settings * Remove unnecessary input --- .github/actions/deploy_gcloud/action.yml | 19 ++++++++++++++++--- .github/workflows/deploy_default.yml | 5 +++-- .github/workflows/deploy_dev.yml | 5 +++-- .github/workflows/deploy_staging.yml | 5 +++-- app.yaml.tmpl | 1 + django_site/pipeline_settings.py | 1 - django_site/settings.py | 5 +++-- generate_requirements.py | 1 + 8 files changed, 30 insertions(+), 12 deletions(-) diff --git a/.github/actions/deploy_gcloud/action.yml b/.github/actions/deploy_gcloud/action.yml index 8458016..6b86fd2 100644 --- a/.github/actions/deploy_gcloud/action.yml +++ b/.github/actions/deploy_gcloud/action.yml @@ -22,6 +22,12 @@ inputs: database-host: description: Database host required: true + database-name: + description: Database name + required: true + database-password: + description: Database password + required: true django-secret: description: Django secret required: true @@ -112,8 +118,8 @@ runs: pip install -t lib --upgrade "git+https://github.com/ocadotechnology/rapid-router@${RAPID_ROUTER_BRANCH}#egg=rapid-router" fi - pip install -t lib django-anymail[amazon_ses] pip install -t lib google-auth==1.* + pip install -t lib psycopg2==2.9.10 # Install custom common package after everything else if deploying a specific portal branch to ensure the # changes from common come through @@ -139,6 +145,12 @@ runs: sudo mkdir /cloudsql sudo chown runner:runner /cloudsql + + # Start PSQL service + sudo systemctl start postgresql + + # Check PSQL status + sudo systemctl status postgresql # Start Cloud SQL Proxy and migrate /tmp/cloud_sql_proxy -dir=/cloudsql & @@ -147,7 +159,7 @@ runs: is_sql_proxy_ready=0 for i in {1..30}; do echo "Waiting for cloudsql-proxy to be ready $i/30" - if mysqladmin -S /cloudsql/decent-digit-629:europe-west1:db ping; then + if pg_isready --host=/cloudsql/decent-digit-629:europe-west1:db-new --port=5432; then echo "cloudsql-proxy is ready" is_sql_proxy_ready=1 break @@ -179,8 +191,9 @@ runs: VERSION: ${{ inputs.version }} CLOUDSDK_CORE_DISABLE_PROMPTS: "1" CLOUDSDK_PYTHON_SITEPACKAGES: "1" - DATABASE_NAME: cfl_${{env.DATABASE_POSTFIX}} + DATABASE_NAME: ${{ inputs.database-name }} DATABASE_HOST: ${{ inputs.database-host }} + DATABASE_PASSWORD: ${{ inputs.database-password }} CACHE_PREFIX: ${{ env.MODULE_NAME }}- AWS_ACCESS_KEY_ID: ${{ inputs.aws-access-key-id }} AWS_SECRET_ACCESS_KEY: ${{ inputs.aws-secret-access-key }} diff --git a/.github/workflows/deploy_default.yml b/.github/workflows/deploy_default.yml index 0ceb315..9d60470 100644 --- a/.github/workflows/deploy_default.yml +++ b/.github/workflows/deploy_default.yml @@ -10,11 +10,10 @@ jobs: env: APP_ID: decent-digit-629 NODE_ENV: production - DB_PORT: 3306 + DB_PORT: 5432 GKE_ZONE: europe-west1-b MODULE_NAME: default VERSION: ${{ github.run_number }} - DATABASE_POSTFIX: ${{ secrets.DATABASE_POSTFIX }} permissions: id-token: 'write' steps: @@ -48,6 +47,8 @@ jobs: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} database-host: ${{ secrets.DATABASE_HOST }} + database-name: ${{ secrets.DATABASE_NAME }} + database-password: ${{ secrets.DATABASE_PASSWORD }} django-secret: ${{ secrets.DJANGO_SECRET }} django-portal-contact-form-email: ${{ secrets.DJANGO_PORTAL_CONTACT_FORM_EMAIL }} dotmailer-create-contact-url: ${{ secrets.DOTMAILER_CREATE_CONTACT_URL }} diff --git a/.github/workflows/deploy_dev.yml b/.github/workflows/deploy_dev.yml index 4979276..a72885a 100644 --- a/.github/workflows/deploy_dev.yml +++ b/.github/workflows/deploy_dev.yml @@ -20,11 +20,10 @@ jobs: env: APP_ID: decent-digit-629 NODE_ENV: production - DB_PORT: 3306 + DB_PORT: 5432 GKE_ZONE: europe-west1-b MODULE_NAME: dev VERSION: ${{ github.run_number }} - DATABASE_POSTFIX: ${{ secrets.DATABASE_POSTFIX }} permissions: id-token: 'write' steps: @@ -58,6 +57,8 @@ jobs: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} database-host: ${{ secrets.DATABASE_HOST }} + database-name: ${{ secrets.DATABASE_NAME }} + database-password: ${{ secrets.DATABASE_PASSWORD }} django-secret: ${{ secrets.DJANGO_SECRET }} django-portal-contact-form-email: ${{ secrets.DJANGO_PORTAL_CONTACT_FORM_EMAIL }} dotmailer-create-contact-url: ${{ secrets.DOTMAILER_CREATE_CONTACT_URL }} diff --git a/.github/workflows/deploy_staging.yml b/.github/workflows/deploy_staging.yml index a8f4599..fa25bbc 100644 --- a/.github/workflows/deploy_staging.yml +++ b/.github/workflows/deploy_staging.yml @@ -10,11 +10,10 @@ jobs: env: APP_ID: decent-digit-629 NODE_ENV: production - DB_PORT: 3306 + DB_PORT: 5432 GKE_ZONE: europe-west1-b MODULE_NAME: staging VERSION: ${{ github.run_number }} - DATABASE_POSTFIX: ${{ secrets.DATABASE_POSTFIX }} permissions: id-token: 'write' steps: @@ -48,6 +47,8 @@ jobs: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} database-host: ${{ secrets.DATABASE_HOST }} + database-name: ${{ secrets.DATABASE_NAME }} + database-password: ${{ secrets.DATABASE_PASSWORD }} django-secret: ${{ secrets.DJANGO_SECRET }} django-portal-contact-form-email: ${{ secrets.DJANGO_PORTAL_CONTACT_FORM_EMAIL }} dotmailer-create-contact-url: ${{ secrets.DOTMAILER_CREATE_CONTACT_URL }} diff --git a/app.yaml.tmpl b/app.yaml.tmpl index 117b3ba..3165a18 100644 --- a/app.yaml.tmpl +++ b/app.yaml.tmpl @@ -27,6 +27,7 @@ env_variables: CACHE_PREFIX: '${CACHE_PREFIX}' DATABASE_NAME: '${DATABASE_NAME}' DATABASE_HOST: '${DATABASE_HOST}' + DATABASE_PASSWORD: '${DATABASE_PASSWORD}' RECAPTCHA_PRIVATE_KEY: '${RECAPTCHA_PRIVATE_KEY}' RECAPTCHA_PUBLIC_KEY: '${RECAPTCHA_PUBLIC_KEY}' DJANGO_PORTAL_CONTACT_FORM_EMAIL: '${DJANGO_PORTAL_CONTACT_FORM_EMAIL}' diff --git a/django_site/pipeline_settings.py b/django_site/pipeline_settings.py index 35d84b4..ec50ab4 100644 --- a/django_site/pipeline_settings.py +++ b/django_site/pipeline_settings.py @@ -20,7 +20,6 @@ # Application definition INSTALLED_APPS = ( - "anymail", "deploy", "game", "pipeline", diff --git a/django_site/settings.py b/django_site/settings.py index 2af5460..e47f29c 100644 --- a/django_site/settings.py +++ b/django_site/settings.py @@ -168,10 +168,11 @@ DATABASES = { "default": { - "ENGINE": "django.db.backends.mysql", + "ENGINE": "django.db.backends.postgresql", "HOST": os.getenv("DATABASE_HOST"), "NAME": os.getenv("DATABASE_NAME"), - "USER": "root", + "USER": "postgres", + "PASSWORD": os.getenv("DATABASE_PASSWORD"), "ATOMIC_REQUESTS": True, } } diff --git a/generate_requirements.py b/generate_requirements.py index 9f97588..a7c1e0b 100644 --- a/generate_requirements.py +++ b/generate_requirements.py @@ -33,6 +33,7 @@ "django-redis==4.11.*", "google-cloud-logging==1.*", "google-auth==1.*", + "psycopg2==2.9.10" ] )