feat(ci): Throw error on sync script failure, wrap in single transact…

…ion (#2177)

.github/workflows/sync-staging-db.yml

@@ -33,6 +33,23 @@ jobs:
         with:
           files: ".env, .env.staging, api.planx.uk/.env.test, hasura.planx.uk/.env.test"
           fail: true
-      - run: docker run --rm -v "$(pwd)/scripts/seed-database:/app" --workdir="/app" postgis/postgis:12-3.0-alpine "./container.sh" "${STAGING_PG_URL}" "${PRODUCTION_PG_URL_FOR_USER_GITHUB_ACTIONS}" "reset_flows" "include_published_flows"
+      - name: Run sync script
+        run: | 
+          {
+            docker run --rm -v "$(pwd)/scripts/seed-database:/app" --workdir="/app" postgis/postgis:12-3.0-alpine "./container.sh" "${STAGING_PG_URL}" "${PRODUCTION_PG_URL_FOR_USER_GITHUB_ACTIONS}" "reset_flows"
+          } || {
+            echo "Database sync failed!"
+            exit 1
+          }
         env:
           PRODUCTION_PG_URL_FOR_USER_GITHUB_ACTIONS: ${{ secrets.PRODUCTION_PG_URL_FOR_USER_GITHUB_ACTIONS }}
+      - name: Slack Notification
+        uses: rtCamp/action-slack-notify@v2
+        if: failure()
+        env:
+          SLACK_CHANNEL: planx-notifications-internal
+          SLACK_USERNAME: GitHub Actions
+          SLACK_COLOR: "#FF5733"
+          SLACK_TITLE: Staging sync failure
+          SLACK_MESSAGE: Failed to sync the staging database with production
+          SLACK_WEBHOOK: ${{ secrets.SLACK_DEPLOYMENT_WEBHOOK }}

docker-compose.seed.yml

@@ -7,7 +7,7 @@ services:
       - "./scripts/seed-database:/app"
     working_dir: "/app"
     entrypoint: "./container.sh postgres://${PG_USERNAME}:${PG_PASSWORD}@postgres/${PG_DATABASE} ${PRODUCTION_PG_URL_FOR_USER_GITHUB_ACTIONS}"
-    command: "no_reset include_published_flows"
+    command: "no_reset"
     restart: "no"
     depends_on:
       hasura-proxy:

scripts/seed-database/container.sh

@@ -10,9 +10,6 @@ REMOTE_PG=${2}
 # RESET set to "reset_all" will truncate all synced tables
 RESET=${3}
 
-# INCLUDE_PUBLISHED_FLOWS set to "include_published_flows" will sync published flows
-INCLUDE_PUBLISHED_FLOWS=${4}
-
 echo downloading data from production
 
 # set-up tmp dir for remote data
@@ -26,27 +23,27 @@ for table in "${tables[@]}"; do
   read_cmd="\\copy (SELECT * FROM ${table}) TO ${target} WITH (FORMAT csv, DELIMITER ';');"
   psql --quiet ${REMOTE_PG} --command="${read_cmd}"
   echo ${table} downloaded
+
   if [[ ${RESET} == "reset_all" ]]; then
     reset_cmd="TRUNCATE TABLE ${table} CASCADE;"
-    psql ${LOCAL_PG} --command="${reset_cmd}"
+
+    # Start building single sync.sql file which will be executed in a single transaction
+    cat $reset_cmd > sync.sql
   fi
 done
 
-if [[ ${RESET} == "reset_flows" ]]; then
-  psql ${LOCAL_PG} --command="TRUNCATE TABLE flows CASCADE;"
-fi
+psql --quiet ${REMOTE_PG} --command="\\copy (SELECT DISTINCT ON (flow_id) id, data, flow_id, summary, publisher_id, created_at FROM published_flows ORDER BY flow_id, created_at DESC) TO '/tmp/published_flows.csv' (FORMAT csv, DELIMITER ';');"
+echo published_flows downloaded
 
-if [[ ${INCLUDE_PUBLISHED_FLOWS} == "include_published_flows" ]]; then
-  psql --quiet ${REMOTE_PG} --command="\\copy (SELECT DISTINCT ON (flow_id) id, data, flow_id, summary, publisher_id, created_at FROM published_flows ORDER BY flow_id, created_at DESC) TO '/tmp/published_flows.csv' (FORMAT csv, DELIMITER ';');"
-  echo published_flows downloaded
+if [[ ${RESET} == "reset_flows" ]]; then
+  cat write/truncate_flows.sql > sync.sql
 fi
 
-echo beginning write transaction
-psql --quiet ${LOCAL_PG} -f write/main.sql
+# Add main operations
+cat write/main.sql > sync.sql
 
-if [[ ${INCLUDE_PUBLISHED_FLOWS} == "include_published_flows" ]]; then
-  psql --quiet ${LOCAL_PG} -f write/published_flows.sql
-fi
+echo "Beginning write transaction..."
+psql --quiet ${LOCAL_PG} -f sync.sql --single-transaction -v ON_ERROR_STOP=on
 
 # clean-up tmp dir
 rm -rf /tmp

scripts/seed-database/write/main.sql

@@ -1,6 +1,5 @@
-BEGIN;
 \include write/users.sql
 \include write/teams.sql
 \include write/flows.sql
 \include write/flow_document_templates.sql
-COMMIT;
+\include write/published_flows.sql

scripts/seed-database/write/truncate_flows.sql

@@ -0,0 +1 @@
+TRUNCATE TABLE flows CASCADE;