chore: remove zerotier dependency on db connection (#305)

MobilityData · Feb 21, 2024 · 541504c · 541504c
1 parent b6f7508
commit 541504c
Show file tree

Hide file tree

Showing 6 changed files with 220 additions and 233 deletions.
diff --git a/.github/workflows/db-update-dev.yml b/.github/workflows/db-update-dev.yml
@@ -11,81 +11,19 @@ on:
     types: [ catalog-sources-updated ]
   workflow_dispatch:
 jobs:
-  get_database_ip:
-    name: 'Retrieve DB IP'
-    permissions: write-all
-    runs-on: ubuntu-latest
-    outputs:
-      db_ip: ${{ steps.get_ip.outputs.DB_IP }}
-    steps:
-    - name: Checkout code
-      uses: actions/checkout@v4
-
-    - name: Authenticate to Google Cloud
-      uses: google-github-actions/auth@v2
-      with:
-        credentials_json: ${{ secrets.QA_GCP_MOBILITY_FEEDS_SA_KEY }}
-
-    - name: Google Cloud Setup
-      uses: google-github-actions/setup-gcloud@v2
-
-    - name: Get Database Instance IP
-      id: get_ip
-      run: |
-        gcloud config set project ${{ vars.QA_MOBILITY_FEEDS_PROJECT_ID }}
-        DB_IP=$(gcloud sql instances describe ${{ secrets.DB_INSTANCE_NAME }} --format=json | jq -r '.ipAddresses[] | select(.type=="PRIVATE") | .ipAddress')
-        echo "DB_IP=$DB_IP" >> "$GITHUB_OUTPUT"
-
-  gw_status:
-    name: 'Check GCP GW Machine Status'
-    permissions: write-all
-    runs-on: ubuntu-latest
-    steps:
-    - name: Authenticate to Google Cloud
-      uses: google-github-actions/auth@v2
-      with:
-        credentials_json: ${{ secrets.QA_GCP_MOBILITY_FEEDS_SA_KEY }}
-
-    - name: Google Cloud Setup
-      uses: google-github-actions/setup-gcloud@v2
-
-    - name: Check GCP Machine Status
-      id: check_gw_status
-      run: |        
-        gcloud config set component_manager/disable_update_check true
-        STATUS=$(gcloud compute instances describe $INSTANCE_NAME --zone=$ZONE --format='get(status)')
-        echo "Instance status: $STATUS"
-        echo "status=$STATUS" >> $GITHUB_OUTPUT
-      env:
-        ZONE: ${{ vars.MOBILITY_FEEDS_REGION }}-a
-        INSTANCE_NAME: 'zerotier-instance-qa'
-        GCP_PROJECT_ID: ${{ vars.QA_MOBILITY_FEEDS_PROJECT_ID }}
-
-    - name: Start GCP Machine if Not Running
-      if: steps.check_gw_status.outputs.status != 'RUNNING'
-      run: |
-        echo "Instance is not running. Starting instance..."
-        gcloud compute instances start $INSTANCE_NAME --zone=$ZONE
-        echo "Instance is starting. Please wait..."
-        sleep 60
-      env:
-        ZONE: ${{ vars.MOBILITY_FEEDS_REGION }}-a
-        INSTANCE_NAME: 'zerotier-instance-qa'
-        GCP_PROJECT_ID: ${{ vars.QA_MOBILITY_FEEDS_PROJECT_ID }}
-
   update:
-    needs: [get_database_ip, gw_status]
     uses: ./.github/workflows/db-update.yml
     with:
       PROJECT_ID: ${{ vars.DEV_MOBILITY_FEEDS_PROJECT_ID }}
       REGION: ${{ vars.MOBILITY_FEEDS_REGION }}
       DB_NAME: ${{ vars.DEV_POSTGRE_SQL_DB_NAME }}
       ENVIRONMENT: ${{ vars.DEV_MOBILITY_FEEDS_ENVIRONMENT }}
-      DB_IP: ${{needs.get_database_ip.outputs.db_ip}}
-      ZEROTIER_NETWORK_GW_KEY: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/GCP_FEED_QA_ZEROTIER_NETWORK_GW/credential"
+      DB_ENVIRONMENT: ${{ vars.QA_MOBILITY_FEEDS_ENVIRONMENT }}
     secrets:
       DB_USER_PASSWORD: ${{ secrets.DEV_POSTGRE_USER_PASSWORD }}
       DB_USER_NAME: ${{ secrets.DEV_POSTGRE_USER_NAME }}
       DB_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }}
       GCP_MOBILITY_FEEDS_SA_KEY: ${{ secrets.DEV_GCP_MOBILITY_FEEDS_SA_KEY }}
-      OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
+      DB_GCP_MOBILITY_FEEDS_SA_KEY: ${{ secrets.QA_GCP_MOBILITY_FEEDS_SA_KEY }}
+      OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
+      POSTGRE_SQL_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }}
diff --git a/.github/workflows/db-update-prod.yml b/.github/workflows/db-update-prod.yml
@@ -8,81 +8,19 @@ on:
   repository_dispatch: # Update on mobility-database-catalog repo dispatch
     types: [ catalog-sources-updated ]
 jobs:
-  get_database_ip:
-    name: 'Retrieve DB IP'
-    permissions: write-all
-    runs-on: ubuntu-latest
-    outputs:
-      db_ip: ${{ steps.get_ip.outputs.DB_IP }}
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Authenticate to Google Cloud
-        uses: google-github-actions/auth@v2
-        with:
-          credentials_json: ${{ secrets.PROD_GCP_MOBILITY_FEEDS_SA_KEY }}
-
-      - name: Google Cloud Setup
-        uses: google-github-actions/setup-gcloud@v2
-
-      - name: Get Database Instance IP
-        id: get_ip
-        run: |
-          gcloud config set project ${{ vars.PROD_MOBILITY_FEEDS_PROJECT_ID }}
-          DB_IP=$(gcloud sql instances describe ${{ secrets.DB_INSTANCE_NAME }} --format=json | jq -r '.ipAddresses[] | select(.type=="PRIVATE") | .ipAddress')
-          echo "DB_IP=$DB_IP" >> "$GITHUB_OUTPUT"
-
-  gw_status:
-    name: 'Check GCP GW Machine Status'
-    permissions: write-all
-    runs-on: ubuntu-latest
-    steps:
-    - name: Authenticate to Google Cloud
-      uses: google-github-actions/auth@v2
-      with:
-        credentials_json: ${{ secrets.PROD_GCP_MOBILITY_FEEDS_SA_KEY }}
-
-    - name: Google Cloud Setup
-      uses: google-github-actions/setup-gcloud@v2
-
-    - name: Check GCP Machine Status
-      id: check_gw_status
-      run: |        
-        gcloud config set component_manager/disable_update_check true
-        STATUS=$(gcloud compute instances describe $INSTANCE_NAME --zone=$ZONE --format='get(status)')
-        echo "Instance status: $STATUS"
-        echo "status=$STATUS" >> $GITHUB_OUTPUT
-      env:
-        ZONE: ${{ vars.MOBILITY_FEEDS_REGION }}-a
-        INSTANCE_NAME: 'zerotier-instance-prod'
-        GCP_PROJECT_ID: ${{ vars.PROD_MOBILITY_FEEDS_PROJECT_ID }}
-
-    - name: Start GCP Machine if Not Running
-      if: steps.check_gw_status.outputs.status != 'RUNNING'
-      run: |
-        echo "Instance is not running. Starting instance..."
-        gcloud compute instances start $INSTANCE_NAME --zone=$ZONE
-        echo "Instance is starting. Please wait..."
-        sleep 60
-      env:
-        ZONE: ${{ vars.MOBILITY_FEEDS_REGION }}-a
-        INSTANCE_NAME: 'zerotier-instance-prod'
-        GCP_PROJECT_ID: ${{ vars.PROD_MOBILITY_FEEDS_PROJECT_ID }}
-
   update:
     uses: ./.github/workflows/db-update.yml
-    needs: [get_database_ip, gw_status]
     with:
       PROJECT_ID: ${{ vars.PROD_MOBILITY_FEEDS_PROJECT_ID }}
       REGION: ${{ vars.MOBILITY_FEEDS_REGION }}
       DB_NAME: ${{ vars.PROD_POSTGRE_SQL_DB_NAME }}
       ENVIRONMENT: ${{ vars.PROD_MOBILITY_FEEDS_ENVIRONMENT }}
-      DB_IP: ${{needs.get_database_ip.outputs.db_ip}}
-      ZEROTIER_NETWORK_GW_KEY: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/GCP_FEED_PROD_ZEROTIER_NETWORK_GW/credential"
+      DB_ENVIRONMENT: ${{ vars.PROD_MOBILITY_FEEDS_ENVIRONMENT }}
     secrets:
       DB_USER_PASSWORD: ${{ secrets.PROD_POSTGRE_USER_PASSWORD }}
       DB_USER_NAME: ${{ secrets.PROD_POSTGRE_USER_NAME }}
       DB_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }}
       GCP_MOBILITY_FEEDS_SA_KEY: ${{ secrets.PROD_GCP_MOBILITY_FEEDS_SA_KEY }}
-      OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
+      DB_GCP_MOBILITY_FEEDS_SA_KEY: ${{ secrets.PROD_GCP_MOBILITY_FEEDS_SA_KEY }}
+      OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
+      POSTGRE_SQL_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }}
diff --git a/.github/workflows/db-update-qa.yml b/.github/workflows/db-update-qa.yml
@@ -10,80 +10,19 @@ on:
   repository_dispatch: # Update on mobility-database-catalog repo dispatch
     types: [ catalog-sources-updated ]
 jobs:
-  get_database_ip:
-    name: 'Retrieve DB IP'
-    permissions: write-all
-    runs-on: ubuntu-latest
-    outputs:
-      db_ip: ${{ steps.get_ip.outputs.DB_IP }}
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Authenticate to Google Cloud
-        uses: google-github-actions/auth@v2
-        with:
-          credentials_json: ${{ secrets.QA_GCP_MOBILITY_FEEDS_SA_KEY }}
-
-      - name: Google Cloud Setup
-        uses: google-github-actions/setup-gcloud@v2
-
-      - name: Get Database Instance IP
-        id: get_ip
-        run: |
-          gcloud config set project ${{ vars.QA_MOBILITY_FEEDS_PROJECT_ID }}
-          DB_IP=$(gcloud sql instances describe ${{ secrets.DB_INSTANCE_NAME }} --format=json | jq -r '.ipAddresses[] | select(.type=="PRIVATE") | .ipAddress')
-          echo "DB_IP=$DB_IP" >> "$GITHUB_OUTPUT"
-  gw_status:
-    name: 'Check GCP GW Machine Status'
-    permissions: write-all
-    runs-on: ubuntu-latest
-    steps:
-    - name: Authenticate to Google Cloud
-      uses: google-github-actions/auth@v2
-      with:
-        credentials_json: ${{ secrets.QA_GCP_MOBILITY_FEEDS_SA_KEY }}
-
-    - name: Google Cloud Setup
-      uses: google-github-actions/setup-gcloud@v2
-
-    - name: Check GCP Machine Status
-      id: check_gw_status
-      run: |        
-        gcloud config set component_manager/disable_update_check true
-        STATUS=$(gcloud compute instances describe $INSTANCE_NAME --zone=$ZONE --format='get(status)')
-        echo "Instance status: $STATUS"
-        echo "status=$STATUS" >> $GITHUB_OUTPUT
-      env:
-        ZONE: ${{ vars.MOBILITY_FEEDS_REGION }}-a
-        INSTANCE_NAME: 'zerotier-instance-qa'
-        GCP_PROJECT_ID: ${{ vars.QA_MOBILITY_FEEDS_PROJECT_ID }}
-
-    - name: Start GCP Machine if Not Running
-      if: steps.check_gw_status.outputs.status != 'RUNNING'
-      run: |
-        echo "Instance is not running. Starting instance..."
-        gcloud compute instances start $INSTANCE_NAME --zone=$ZONE
-        echo "Instance is starting. Please wait..."
-        sleep 60
-      env:
-        ZONE: ${{ vars.MOBILITY_FEEDS_REGION }}-a
-        INSTANCE_NAME: 'zerotier-instance-qa'
-        GCP_PROJECT_ID: ${{ vars.QA_MOBILITY_FEEDS_PROJECT_ID }}
-
   update:
     uses: ./.github/workflows/db-update.yml
-    needs: [get_database_ip, gw_status]
     with:
       PROJECT_ID: ${{ vars.QA_MOBILITY_FEEDS_PROJECT_ID }}
       REGION: ${{ vars.MOBILITY_FEEDS_REGION }}
       DB_NAME: ${{ vars.QA_POSTGRE_SQL_DB_NAME }}
       ENVIRONMENT: ${{ vars.QA_MOBILITY_FEEDS_ENVIRONMENT }}
-      DB_IP: ${{needs.get_database_ip.outputs.db_ip}}
-      ZEROTIER_NETWORK_GW_KEY: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/GCP_FEED_QA_ZEROTIER_NETWORK_GW/credential"
+      DB_ENVIRONMENT: ${{ vars.QA_MOBILITY_FEEDS_ENVIRONMENT }}
     secrets:
       DB_USER_PASSWORD: ${{ secrets.QA_POSTGRE_USER_PASSWORD }}
       DB_USER_NAME: ${{ secrets.QA_POSTGRE_USER_NAME }}
       DB_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }}
       GCP_MOBILITY_FEEDS_SA_KEY: ${{ secrets.QA_GCP_MOBILITY_FEEDS_SA_KEY }}
-      OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
+      DB_GCP_MOBILITY_FEEDS_SA_KEY: ${{ secrets.QA_GCP_MOBILITY_FEEDS_SA_KEY }}
+      OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
+      POSTGRE_SQL_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }}
diff --git a/.github/workflows/db-update.yml b/.github/workflows/db-update.yml
@@ -25,6 +25,9 @@ on:
       GCP_MOBILITY_FEEDS_SA_KEY:
         description: Service account key
         required: true
+      DB_GCP_MOBILITY_FEEDS_SA_KEY:
+        description: Service account key where the DB in deployed
+        required: true
       DB_USER_NAME:
         description: PostgreSQL User Name
         required: true
@@ -37,6 +40,9 @@ on:
       OP_SERVICE_ACCOUNT_TOKEN:
         description: OnePassword Service Account Token
         required: true
+      POSTGRE_SQL_INSTANCE_NAME:
+        description: PostgreSQL Instance Name
+        required: true        
     inputs:
       PROJECT_ID:
         description: GCP Project ID
@@ -50,14 +56,10 @@ on:
         description: GCP ENVIRONMENT
         required: true
         type: string
-      DB_IP:
-        description: Database IP Address
-        required: true
-        type: string
-      ZEROTIER_NETWORK_GW_KEY:
-        description: ZeroTier Network Gateway
-        type: string
+      DB_ENVIRONMENT:
+        description: GCP ENVIRONMENT where DB is deployed.
         required: true
+        type: string        
       REGION:
         description: GCP region
         required: true
@@ -71,29 +73,30 @@ jobs:
     - name: Checkout code
       uses: actions/checkout@v4
 
+    - name: Authenticate to Google Cloud QA/PROD
+      uses: google-github-actions/auth@v2
+      with:
+        credentials_json: ${{ secrets.DB_GCP_MOBILITY_FEEDS_SA_KEY }}
+
+    - name: Google Cloud Setup
+      uses: google-github-actions/setup-gcloud@v2
+
     - name: Load secrets from 1Password
       uses: 1password/[email protected]
       with:
         export-env: true # Export loaded secrets as environment variables
       env:
         OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
-        GCP_FEED_ZEROTIER_NETWORK_ID: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/4c7ut35delweauflq7g2kphl5u/credential"
-        GCP_FEED_ZEROTIER_TOKEN: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/4lsfgxbtegqicjj2fz6skkj2pm/credential"
-        GCP_FEED_ZEROTIER_SSH_KEY: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/h4zgaa7wyk3ml3mrggedezyxkm/private key"
-        GCP_FEED_ZEROTIER_NETWORK_GW: ${{ inputs.ZEROTIER_NETWORK_GW_KEY }}
-
-    - name: ZeroTier
-      uses: zerotier/github-action@v1
-      with:
-        network_id: ${{ env.GCP_FEED_ZEROTIER_NETWORK_ID }}
-        auth_token: ${{ env.GCP_FEED_ZEROTIER_TOKEN }}
-
+        GCP_FEED_SSH_USER: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/GCP_FEED_SSH_USER/username"
+        GCP_FEED_BASTION_NAME: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/GCP_FEED_BASTION_NAME/username"
+        GCP_FEED_BASTION_SSH_KEY: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/GCP_FEED_BASTION_SSH_KEY/private key"
+
     - name: Tunnel
       run: |
         mkdir -p ~/.ssh
-        echo "${{ env.GCP_FEED_ZEROTIER_SSH_KEY }}" > ~/.ssh/id_rsa
-        chmod 600 ~/.ssh/id_rsa        
-        ssh -o StrictHostKeyChecking=no -L 5432:${{ inputs.DB_IP }}:5432 it@${{ env.GCP_FEED_ZEROTIER_NETWORK_GW }} -N &
+        echo "${{ env.GCP_FEED_BASTION_SSH_KEY }}" > ~/.ssh/id_rsa
+        chmod 600 ~/.ssh/id_rsa             
+        ./scripts/tunnel-create.sh -project_id ${{ inputs.PROJECT_ID }} -zone ${{ inputs.REGION }}-a -instance ${{ env.GCP_FEED_BASTION_NAME }}-${{ inputs.DB_ENVIRONMENT}} -target_account ${{ env.GCP_FEED_SSH_USER }} -db_instance ${{ secrets.POSTGRE_SQL_INSTANCE_NAME }}
         sleep 10 # Wait for the tunnel to establish
 
 #    Uncomment the following block to test the database connection through the tunnel
@@ -135,6 +138,14 @@ jobs:
       with:
         python-version: '3.10'
 
+    - name: Authenticate to Google Cloud QA/PROD
+      uses: google-github-actions/auth@v2
+      with:
+        credentials_json: ${{ secrets.DB_GCP_MOBILITY_FEEDS_SA_KEY }}
+
+    - name: Google Cloud Setup
+      uses: google-github-actions/setup-gcloud@v2
+
     - name: Update .env file
       run: |
         echo "PGUSER=${{ secrets.DB_USER_NAME }}" > config/.env.local
@@ -153,24 +164,17 @@ jobs:
         export-env: true # Export loaded secrets as environment variables
       env:
         OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
-        GCP_FEED_ZEROTIER_NETWORK_ID: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/4c7ut35delweauflq7g2kphl5u/credential"
-        GCP_FEED_ZEROTIER_TOKEN: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/4lsfgxbtegqicjj2fz6skkj2pm/credential"
-        GCP_FEED_ZEROTIER_SSH_KEY: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/h4zgaa7wyk3ml3mrggedezyxkm/private key"
-        GCP_FEED_ZEROTIER_NETWORK_GW: ${{ inputs.ZEROTIER_NETWORK_GW_KEY }}
-
-    - name: ZeroTier
-      uses: zerotier/github-action@v1
-      with:
-        network_id: ${{ env.GCP_FEED_ZEROTIER_NETWORK_ID }}
-        auth_token: ${{ env.GCP_FEED_ZEROTIER_TOKEN }}
-
+        GCP_FEED_SSH_USER: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/GCP_FEED_SSH_USER/username"
+        GCP_FEED_BASTION_NAME: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/GCP_FEED_BASTION_NAME/username"
+        GCP_FEED_BASTION_SSH_KEY: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/GCP_FEED_BASTION_SSH_KEY/private key"
+
     - name: Tunnel
       run: |
         mkdir -p ~/.ssh
-        echo "${{ env.GCP_FEED_ZEROTIER_SSH_KEY }}" > ~/.ssh/id_rsa
-        chmod 600 ~/.ssh/id_rsa        
-        ssh -o StrictHostKeyChecking=no -L 5432:${{ inputs.DB_IP }}:5432 it@${{ env.GCP_FEED_ZEROTIER_NETWORK_GW }} -N &
-        sleep 10 # Wait for the tunnel to establish      
+        echo "${{ env.GCP_FEED_BASTION_SSH_KEY }}" > ~/.ssh/id_rsa
+        chmod 600 ~/.ssh/id_rsa             
+        ./scripts/tunnel-create.sh -project_id ${{ inputs.PROJECT_ID }} -zone ${{ inputs.REGION }}-a -instance ${{ env.GCP_FEED_BASTION_NAME }}-${{ inputs.DB_ENVIRONMENT}} -target_account ${{ env.GCP_FEED_SSH_USER }} -db_instance ${{ secrets.POSTGRE_SQL_INSTANCE_NAME }}
+        sleep 10 # Wait for the tunnel to establish
 
     - name: Install requirements and generate db model
       run: scripts/db-gen.sh