persistent volume and option to deploy to the same VM (#2211)

ten-protocol · Dec 18, 2024 · 0867530 · 0867530
1 parent 6328e9a
commit 0867530
Showing 1 changed file with 93 additions and 49 deletions.
diff --git a/.github/workflows/manual-deploy-obscuro-gateway.yml b/.github/workflows/manual-deploy-obscuro-gateway.yml
@@ -35,6 +35,14 @@ on:
         options:
           - "primary"
           - "DEXYNTH"
+      recreate_vm:
+        description: "Delete and recreate VM"
+        required: false
+        default: "false"
+        type: choice
+        options:
+          - "false"
+          - "true"
 
 jobs:
   validate-inputs:
@@ -59,19 +67,18 @@ jobs:
           INSTANCE_PREFIX=""
 
           if [[ "${{ github.event.inputs.instance_type }}" != "primary" ]]; then
-            INSTANCE_SUFFIX="_${{ github.event.inputs.instance_type }}"
-            INSTANCE_SUFFIX2="-${{ github.event.inputs.instance_type }}"
+            INSTANCE_SUFFIX="-${{ github.event.inputs.instance_type }}"
             INSTANCE_PREFIX="${{ github.event.inputs.instance_type }}_"
           fi
 
           echo "INSTANCE_SUFFIX=$INSTANCE_SUFFIX" >> $GITHUB_ENV
           echo "INSTANCE_PREFIX=$INSTANCE_PREFIX" >> $GITHUB_ENV
 
           # Set infrastructure variables
-          PUBLIC_IP="${{ github.event.inputs.testnet_type }}-OG-static${INSTANCE_SUFFIX2,,}"
-          DNS_NAME="obscurogateway-${{ github.event.inputs.testnet_type }}${INSTANCE_SUFFIX2,,}"
-          VM_NAME="${{ github.event.inputs.testnet_type }}-OG-${{ github.run_number }}${INSTANCE_SUFFIX}"
-          DEPLOY_GROUP="ObscuroGateway-${{ github.event.inputs.testnet_type }}-${{ github.run_number }}${INSTANCE_SUFFIX}"
+          PUBLIC_IP="${{ github.event.inputs.testnet_type }}-OG-static${INSTANCE_SUFFIX,,}"
+          DNS_NAME="obscurogateway-${{ github.event.inputs.testnet_type }}${INSTANCE_SUFFIX,,}"
+          VM_NAME="${{ github.event.inputs.testnet_type }}-OG${INSTANCE_SUFFIX}"
+          DEPLOY_GROUP="ObscuroGateway-${{ github.event.inputs.testnet_type }}${INSTANCE_SUFFIX}"
           VNET_NAME="ObscuroGateway-${{ github.event.inputs.testnet_type }}-01VNET${INSTANCE_SUFFIX}"
           SUBNET_NAME="ObscuroGateway-${{ github.event.inputs.testnet_type }}-01Subnet${INSTANCE_SUFFIX}"
 
@@ -105,7 +112,6 @@ jobs:
           done
 
       - name: "Print environment variables"
-        # This is a useful record of what the environment variables were at the time the job ran, for debugging and reference
         run: |
           echo "INSTANCE_SUFFIX: $INSTANCE_SUFFIX"
           echo "INSTANCE_PREFIX: $INSTANCE_PREFIX"
@@ -125,7 +131,6 @@ jobs:
           echo "GATEWAY_TLS_DOMAIN: $GATEWAY_TLS_DOMAIN"
 
       - name: "Print GitHub variables"
-        # This is a useful record of what the environment variables were at the time the job ran, for debugging and reference
         run: |
           echo "GitHub Variables = ${{ toJSON(vars) }}"
 
@@ -157,57 +162,79 @@ jobs:
           DOCKER_BUILDKIT=1 docker build --build-arg TESTNET_TYPE=${{ github.event.inputs.testnet_type }} -t ${{ env.DOCKER_BUILD_TAG_GATEWAY }} -f ./tools/walletextension/enclave.Dockerfile .
           docker push ${{ env.DOCKER_BUILD_TAG_GATEWAY }}
 
-      # This will fail some deletions due to resource dependencies ( ie. you must first delete the vm before deleting the disk)
+      # If recreate_vm = true, delete VMs and their dependencies
       - name: "Delete deployed VMs"
+        if: ${{ github.event.inputs.recreate_vm == 'true' }}
         uses: azure/CLI@v1
         with:
           inlineScript: |
             $(az resource list --tag ${{ env.AZURE_DEPLOY_GROUP_GATEWAY }}=true --query '[]."id"' -o tsv | xargs -n1 az resource delete --verbose -g Testnet --ids) || true
 
-      # This will clean up any lingering dependencies - might fail if there are no resources to cleanup
       - name: "Delete VMs dependencies"
+        if: ${{ github.event.inputs.recreate_vm == 'true' }}
         uses: azure/CLI@v1
         with:
           inlineScript: |
             $(az resource list --tag ${{ env.AZURE_DEPLOY_GROUP_GATEWAY }}=true --query '[]."id"' -o tsv | xargs -n1 az resource delete --verbose -g Testnet --ids) || true
 
-      - name: "Ensure VM Static Public IP Exists"
-        uses: azure/CLI@v1
-        with:
-          inlineScript: |
-            az network public-ip show -g Testnet -n "${{ env.PUBLIC_IP }}" || az network public-ip create -g Testnet -n "${{ env.PUBLIC_IP }}" --allocation-method Static --sku Standard
+      # If recreate_vm = false, check if VM exists
+      - name: "Check if VM exists"
+        if: ${{ github.event.inputs.recreate_vm == 'false' }}
+        id: check_vm
+        shell: bash
+        run: |
+          if ! az vm show -g Testnet -n "${{ env.VM_NAME }}" &> /dev/null; then
+            echo "vm_exists=false" >> $GITHUB_ENV
+          else
+            echo "vm_exists=true" >> $GITHUB_ENV
+          fi
 
-      - name: "Assign/Update DNS Name for Public IP"
+      - name: "Ensure VM Static Public IP and DNS if needed"
+        if: ${{ github.event.inputs.recreate_vm == 'true' || env.vm_exists == 'false' }}
         uses: azure/CLI@v1
         with:
           inlineScript: |
+            az network public-ip show -g Testnet -n "${{ env.PUBLIC_IP }}" || az network public-ip create -g Testnet -n "${{ env.PUBLIC_IP }}" --allocation-method Static --sku Standard
             existing_dns_name=$(az network public-ip show -g Testnet -n "${{ env.PUBLIC_IP }}" --query dnsSettings.domainNameLabel -o tsv)
             if [ -z "$existing_dns_name" ]; then
               az network public-ip update -g Testnet -n "${{ env.PUBLIC_IP }}" --dns-name "${{ env.DNS_NAME }}"
             fi
 
-      - name: "Create VM for Gateway node on Azure"
+      - name: "Create VM if it doesn't exist (recreate_vm=false)"
+        if: ${{ github.event.inputs.recreate_vm == 'false' && env.vm_exists == 'false' }}
         uses: azure/CLI@v1
         with:
           inlineScript: |
             az vm create -g Testnet -n "${{ env.VM_NAME }}" \
-            --admin-username obscurouser --admin-password "${{ secrets.OBSCURO_NODE_VM_PWD }}" \
-            --public-ip-address "${{ env.PUBLIC_IP }}" \
-            --tags deploygroup="${{ env.DEPLOY_GROUP }}"  ${{ env.AZURE_DEPLOY_GROUP_GATEWAY }}=true \
-            --vnet-name "${{ env.VNET_NAME }}" --subnet "${{ env.SUBNET_NAME }}" \
-            --size Standard_DC2s_v3 --storage-sku StandardSSD_LRS --image ObscuroConfUbuntu \
-            --authentication-type password
-
-      - name: "Open TEN node-${{ matrix.host_id }} ports on Azure"
+              --admin-username obscurouser --admin-password "${{ secrets.OBSCURO_NODE_VM_PWD }}" \
+              --public-ip-address "${{ env.PUBLIC_IP }}" \
+              --tags deploygroup="${{ env.DEPLOY_GROUP }}"  ${{ env.AZURE_DEPLOY_GROUP_GATEWAY }}=true \
+              --vnet-name "${{ env.VNET_NAME }}" --subnet "${{ env.SUBNET_NAME }}" \
+              --size Standard_DC2s_v3 --storage-sku StandardSSD_LRS --image ObscuroConfUbuntu \
+              --authentication-type password
+              
+            az vm open-port -g Testnet -n "${{ env.VM_NAME }}" --port 80,81,443
+            
+            # Allow time for VM initialization
+            sleep 30
+
+      - name: "Create VM if recreate_vm = true"
+        if: ${{ github.event.inputs.recreate_vm == 'true' }}
         uses: azure/CLI@v1
         with:
           inlineScript: |
-            az vm open-port -g Testnet -n "${{ env.VM_NAME }}"  --port 80,81,443
-
-        # To overcome issues with critical VM resources being unavailable, we need to wait for the VM to be ready
-      - name: "Allow time for VM initialization"
-        shell: bash
-        run: sleep 30
+            az vm create -g Testnet -n "${{ env.VM_NAME }}" \
+              --admin-username obscurouser --admin-password "${{ secrets.OBSCURO_NODE_VM_PWD }}" \
+              --public-ip-address "${{ env.PUBLIC_IP }}" \
+              --tags deploygroup="${{ env.DEPLOY_GROUP }}"  ${{ env.AZURE_DEPLOY_GROUP_GATEWAY }}=true \
+              --vnet-name "${{ env.VNET_NAME }}" --subnet "${{ env.SUBNET_NAME }}" \
+              --size Standard_DC2s_v3 --storage-sku StandardSSD_LRS --image ObscuroConfUbuntu \
+              --authentication-type password
+              
+            az vm open-port -g Testnet -n "${{ env.VM_NAME }}" --port 80,81,443
+            
+            # Allow time for VM initialization
+            sleep 30
 
       - name: "Start TEN Gateway on Azure"
         uses: azure/CLI@v1
@@ -238,8 +265,11 @@ jobs:
             done
 
             curl -fsSL https://get.docker.com -o get-docker.sh && sh ./get-docker.sh
+            rm -rf /home/obscuro/go-obscuro
             git clone --depth 1 -b "${{ env.BRANCH_NAME }}" https://github.com/ten-protocol/go-ten.git /home/obscuro/go-obscuro
-            docker network create --driver bridge node_network || true
+            if ! docker network inspect node_network >/dev/null 2>&1; then
+              docker network create --driver bridge node_network
+            fi
             cd /home/obscuro/go-obscuro/
 
             # Promtail Integration Start
@@ -280,6 +310,9 @@ jobs:
                 - replacement: "${{ env.VM_NAME }}"
                   target_label: "node_name"
             EOF
+
+            docker stop promtail || true
+            docker rm promtail || true
             
             docker run -d --name promtail \
             --network node_network \
@@ -304,29 +337,33 @@ jobs:
                   password: "${{ secrets.LOKI_PASSWORD }}"
             scrape_configs:
               # Node metrics
-              - job_name:  node-${{ env.VM_NAME }}
-                scrape_interval: 5s  # Frequent scrapes for node metrics
+              - job_name: node-${{ env.VM_NAME }}
+                scrape_interval: 5s
                 static_configs:
                   - targets:
-                      - node_exporter:9100  # Node Exporter instance
+                      - node_exporter:9100
                 relabel_configs:
                   - source_labels: [job]
-                    target_label: 'node'
-                    replacement:  node-${{ env.VM_NAME }}
+                    target_label: "node"
+                    replacement: node-${{ env.VM_NAME }}
 
               # Container metrics
-              - job_name:  container-${{ env.VM_NAME }}
+              - job_name: container-${{ env.VM_NAME }}
                 scrape_interval: 5s
                 static_configs:
                   - targets:
-                      - cadvisor:8080  # cAdvisor instance for container metrics
+                      - cadvisor:8080
                 relabel_configs:
                   - source_labels: [job]
-                    target_label: 'node'
-                    replacement:  container-${{ env.VM_NAME }}
+                    target_label: "node"
+                    replacement: container-${{ env.VM_NAME }}
             EOF
 
-            docker volume create prometheus-data
+
+            docker stop prometheus || true
+            docker rm prometheus || true
+
+            docker volume create prometheus-data || true
             docker run -d --name prometheus \
             --network node_network \
             -p 9090:9090 \
@@ -335,6 +372,10 @@ jobs:
             prom/prometheus:latest \
             --config.file=/etc/prometheus/prometheus.yml
 
+
+            docker stop node_exporter || true
+            docker rm node_exporter || true
+
             docker run -d --name node_exporter \
             --network node_network \
             -p 9100:9100 \
@@ -343,6 +384,10 @@ jobs:
             quay.io/prometheus/node-exporter:latest \
             --path.rootfs=/host
 
+
+            docker stop cadvisor || true
+            docker rm cadvisor || true
+
             docker run -d --name cadvisor \
             --network node_network \
             -p 8080:8080 \
@@ -355,13 +400,16 @@ jobs:
             gcr.io/cadvisor/cadvisor:latest
             # Promtail Integration End
             
-            # Create a named volume for persistence
-            docker volume create "${{ env.VM_NAME }}-data"
+            docker volume create "TENGateway-${{ github.event.inputs.testnet_type }}-data" || true
+
+            # Stop and remove existing container if it exists
+            docker stop "${{ env.VM_NAME }}" || true
+            docker rm "${{ env.VM_NAME }}" || true
             
             # Start Ten Gateway Container
             docker run -d -p 80:80 -p 81:81 -p 443:443 --name "${{ env.VM_NAME }}" \
             --device /dev/sgx_enclave --device /dev/sgx_provision \
-            -v "${{ env.VM_NAME }}-data:/data" \
+            -v "TENGateway-${{ github.event.inputs.testnet_type }}-data:/data" \
             -e OBSCURO_GATEWAY_VERSION="${{ github.run_number }}-${{ github.sha }}" \
             -e OE_SIMULATION=0 \
             "${{ env.DOCKER_BUILD_TAG_GATEWAY }}" \
@@ -376,8 +424,6 @@ jobs:
             -enableTLS=true \
             -tlsDomain="${{ env.GATEWAY_TLS_DOMAIN }}"
             
-
-            # After starting the container, verify the volume mount
             docker exec "${{ env.VM_NAME }}" sh -c "
                 echo \"Checking volume mount...\";
                 df -h | grep /data;
@@ -391,5 +437,3 @@ jobs:
                 ps aux;
             "
             '
-            
-