manual-deploy-testnet-l2.yml

# Deploys an Obscuro network on Azure for Testnet and Dev Testnet
#
# The Obscuro network is composed of 2 obscuro nodes running on individual vms with SGX. At the moment the workflow
# can only be triggered manually as a workflow dispatch.
#
# This script uses GitHub Environments for variables (vars) and secrets - these are configured on GitHub and
#  the environments match the input.testnet_type options

name: '[M] Deploy Testnet L2'
run-name: '[M] Deploy Testnet L2 ( ${{ github.event.inputs.testnet_type }} )'
on:
  workflow_dispatch:
    inputs:
      testnet_type:
        description: 'Testnet Type'
        required: true
        default: 'dev-testnet'
        type: choice
        options:
          - 'dev-testnet'
          - 'uat-testnet'
          - 'sepolia-testnet'
      log_level:
        description: 'Log Level 1-Error 5-Trace'
        required: true
        default: 3
        type: number

jobs:
  build:
    runs-on: ubuntu-latest
    environment:
      name: ${{ github.event.inputs.testnet_type }}

    # Map a step output to a job output
    outputs:
      MGMT_CONTRACT_ADDR: ${{ steps.deployContracts.outputs.MGMT_CONTRACT_ADDR }}
      MSG_BUS_CONTRACT_ADDR: ${{ steps.deployContracts.outputs.MSG_BUS_CONTRACT_ADDR }}
      L1_START_HASH: ${{ steps.deployContracts.outputs.L1_START_HASH }}
      HOC_ERC20_ADDR: ${{ steps.deployContracts.outputs.HOC_ERC20_ADDR }}
      POC_ERC20_ADDR: ${{ steps.deployContracts.outputs.POC_ERC20_ADDR }}


    steps:
      - name: 'Print GitHub variables'
        # This is a useful record of what the environment variables were at the time the job ran, for debugging and reference
        run: |
          echo "GitHub Variables = ${{ toJSON(vars) }}"

      - uses: actions/checkout@v4

      - uses: actions/setup-go@v5
        with:
          go-version: 1.21.8

      - name: 'Login via Azure CLI'
        uses: azure/login@v1
        with:
          creds: ${{ secrets.AZURE_CREDENTIALS }}

      - name: 'Login to Azure docker registry'
        uses: azure/docker-login@v1
        with:
          login-server: testnetobscuronet.azurecr.io
          username: testnetobscuronet
          password: ${{ secrets.REGISTRY_PASSWORD }}

      - name: 'Build and push obscuro node images'
        run: |
          DOCKER_BUILDKIT=1 docker build -t ${{ vars.DOCKER_BUILD_TAG_ENCLAVE }} -f dockerfiles/enclave.Dockerfile  .
          docker push ${{ vars.DOCKER_BUILD_TAG_ENCLAVE }}
          DOCKER_BUILDKIT=1 docker build -t ${{ vars.DOCKER_BUILD_TAG_HOST }} -f dockerfiles/host.Dockerfile .
          docker push ${{ vars.DOCKER_BUILD_TAG_HOST }}
          DOCKER_BUILDKIT=1 docker build -t ${{ vars.DOCKER_BUILD_TAG_L2_HARDHAT_DEPLOYER }} -f tools/hardhatdeployer/Dockerfile .
          docker push ${{ vars.DOCKER_BUILD_TAG_L2_HARDHAT_DEPLOYER }}

      - name: 'Deploy Contracts'
        id: deployContracts
        shell: bash
        run: |
          go run ./testnet/launcher/l1contractdeployer/cmd \
          -l1_http_url=${{ secrets.L1_HTTP_URL }} \
          -private_key=${{ secrets.ACCOUNT_PK_WORKER }} \
          -docker_image=${{ vars.DOCKER_BUILD_TAG_L2_HARDHAT_DEPLOYER }} \
          -contracts_env_file=./testnet/.env
          source ./testnet/.env
          echo "Contracts deployed to $MGMTCONTRACTADDR"
          echo "MGMT_CONTRACT_ADDR=$MGMTCONTRACTADDR" >> $GITHUB_ENV
          echo "MGMT_CONTRACT_ADDR=$MGMTCONTRACTADDR" >> $GITHUB_OUTPUT
          echo "MSG_BUS_CONTRACT_ADDR=$MSGBUSCONTRACTADDR" >> $GITHUB_ENV
          echo "MSG_BUS_CONTRACT_ADDR=$MSGBUSCONTRACTADDR" >> $GITHUB_OUTPUT
          echo "L1_START_HASH=$L1START" >> $GITHUB_ENV
          echo "L1_START_HASH=$L1START" >> $GITHUB_OUTPUT

      - name: 'Save L1 deployer container logs'
        # Wait to make sure the logs are available in the container
        run: |
          sleep 60
          docker logs `docker ps -aqf "name=hh-l1-deployer"` > deploy-l1-contracts.out 2>&1

      # This will fail some deletions due to resource dependencies ( ie. you must first delete the vm before deleting the disk)
      - name: 'Delete deployed VMs'
        uses: azure/CLI@v1
        with:
          inlineScript: |
            $(az resource list --tag ${{ vars.AZURE_DEPLOY_GROUP_L2 }}=true --query '[]."id"' -o tsv | xargs -n1 az resource delete --verbose -g Testnet --ids) || true

      # This will clean up any lingering dependencies - might fail if there are no resources to cleanup
      - name: 'Delete VMs dependencies'
        uses: azure/CLI@v1
        with:
          inlineScript: |
            $(az resource list --tag ${{ vars.AZURE_DEPLOY_GROUP_L2 }}=true --query '[]."id"' -o tsv | xargs -n1 az resource delete --verbose -g Testnet --ids) || true

      # Delete old database tables from previous deployment
      - name: 'Delete host databases'
        uses: azure/CLI@v1
        with:
          inlineScript: |
            databases=$(az postgres flexible-server db list --resource-group Testnet --server-name postgres-ten-${{ github.event.inputs.testnet_type }} --query "[?starts_with(name, 'host_')].[name]" -o tsv)

            for db in $databases; do
              az postgres flexible-server db delete --database-name "$db" --resource-group Testnet --server-name postgres-ten-${{ github.event.inputs.testnet_type }} --yes
            done

      - name: 'Upload L1 deployer container logs'
        uses: actions/upload-artifact@v4
        with:
          name: deploy-l1-artifacts
          path: |
            deploy-l1-contracts.out
          retention-days: 7

  deploy:
    needs: build
    runs-on: ubuntu-latest
    environment:
      name: ${{ github.event.inputs.testnet_type }}

    strategy:
      matrix:
        host_id: [ 0,1,2 ]
        include:
          # Ensure there is a single genesis node
          - is_genesis: true
            host_id: 0
          - is_genesis: false
            host_id: 1
          - is_genesis: false
            host_id: 2
          # Ensure there is a single sequencer
          - node_type: sequencer
            host_id: 0
          - node_type: validator
            host_id: 1
          - node_type: validator
            host_id: 2
          # Hardcoded lookup keys because GH actions doesn't let you build them inline with the host_id
          - node_pk_lookup: ACCOUNT_PK_NODE_0
            host_id: 0
          - node_pk_lookup: ACCOUNT_PK_NODE_1
            host_id: 1
          - node_pk_lookup: ACCOUNT_PK_NODE_2
            host_id: 2
          - node_addr_lookup: ACCOUNT_ADDR_NODE_0
            host_id: 0
          - node_addr_lookup: ACCOUNT_ADDR_NODE_1
            host_id: 1
          - node_addr_lookup: ACCOUNT_ADDR_NODE_2
            host_id: 2
          - node_l1_ws_lookup: L1_WS_URL_0
            host_id: 0
          - node_l1_ws_lookup: L1_WS_URL_1
            host_id: 1
          - node_l1_ws_lookup: L1_WS_URL_2
            host_id: 2
          # sequencer has an HA setup with 2 enclaves
          - num_enclaves: 2
            host_id: 0
          - num_enclaves: 1
            host_id: 1
          - num_enclaves: 1
            host_id: 2

    steps:
      - name: 'Extract branch name'
        shell: bash
        run: |
          echo "Branch Name: ${GITHUB_REF_NAME}"
          echo "BRANCH_NAME=${GITHUB_REF_NAME}" >> $GITHUB_ENV

        # The Azure API will sometimes supersede PUT requests that come in close together. This sleep will stagger the VM requests.
        # It expects host_id to be an int and then multiplies it by 60s (i.e. host 0: sleep 0, host 1: sleep 60,...)
      - name: 'Stagger VM creation'
        shell: bash
        run: sleep `expr ${{matrix.host_id}} \* 60`

      - name: 'Login via Azure CLI'
        uses: azure/login@v1
        with:
          creds: ${{ secrets.AZURE_CREDENTIALS }}

      - name: 'Create VM for Obscuro node-${{ matrix.host_id }} on Azure'
        uses: azure/CLI@v1
        with:
          inlineScript: |
            az vm create -g Testnet -n "${{ vars.AZURE_RESOURCE_PREFIX }}-${{ matrix.host_id }}-${{ GITHUB.RUN_NUMBER }}" \
            --admin-username obscurouser --admin-password "${{ secrets.OBSCURO_NODE_VM_PWD }}" \
            --public-ip-address-dns-name "obscuronode-${{ matrix.host_id }}-${{ github.event.inputs.testnet_type }}" \
            --tags deploygroup=ObscuroNode-${{ github.event.inputs.testnet_type }}-${{ GITHUB.RUN_NUMBER }}  ${{ vars.AZURE_DEPLOY_GROUP_L2 }}=true \
            --vnet-name ${{ github.event.inputs.testnet_type }}-virtual-network --subnet ${{ github.event.inputs.testnet_type }}-sub-network \
            --size Standard_DC8_v2 --storage-sku StandardSSD_LRS --image ObscuroConfUbuntu \
            --public-ip-sku Standard --authentication-type password

      - name: 'Open Obscuro node-${{ matrix.host_id }} ports on Azure'
        uses: azure/CLI@v1
        with:
          inlineScript: |
            az vm open-port -g Testnet -n "${{ vars.AZURE_RESOURCE_PREFIX }}-${{ matrix.host_id }}-${{ GITHUB.RUN_NUMBER }}"  --port 80,81,6060,6061,10000

        # To overcome issues with critical VM resources being unavailable, we need to wait for the VM to be ready
      - name: 'Allow time for VM initialization'
        shell: bash
        run: sleep 60

      - name: 'Start Obscuro node-${{ matrix.host_id }} on Azure'
        uses: azure/CLI@v1
        with:
          inlineScript: |
            az vm run-command invoke -g Testnet -n "${{ vars.AZURE_RESOURCE_PREFIX }}-${{ matrix.host_id }}-${{ GITHUB.RUN_NUMBER }}"  \
            --command-id RunShellScript \
            --scripts 'mkdir -p /home/obscuro \
            && git clone --depth 1 -b ${{ env.BRANCH_NAME }} https://github.com/ten-protocol/go-ten.git /home/obscuro/go-obscuro \
            && cp /home/obscuro/go-obscuro/tools/edbconnect/edb-connect.sh /home/obscurouser/edb-connect.sh \
            && chown obscurouser:obscurouser /home/obscurouser/edb-connect.sh \
            && chmod u+x /home/obscurouser/edb-connect.sh \
            && docker network create --driver bridge node_network || true \
            && mkdir -p /home/obscuro/metrics \
            && echo "
            server:
              http_listen_port: 9080
              grpc_listen_port: 0

            positions:
              filename: /tmp/positions.yaml

            clients:
              - url: ${{ vars.LOKI_URI }}
                batchwait: 3s
                batchsize: 1048576
                tls_config:
                  insecure_skip_verify: true
                basic_auth:
                  username: ${{ secrets.LOKI_USER }}
                  password: ${{ secrets.LOKI_PASSWORD }}

            scrape_configs:
            - job_name: flog_scrape
              docker_sd_configs:
                - host: unix:///var/run/docker.sock
                  refresh_interval: 5s
              relabel_configs:
                - source_labels: [\"__meta_docker_container_name\"]
                  regex: \"/(.*)\"
                  target_label: \"container\"
                - source_labels: [\"__meta_docker_container_log_stream\"]
                  target_label: \"logstream\"
                - source_labels: [\"__meta_docker_container_label_logging_jobname\"]
                  target_label: \"job\"
                - replacement: ${{ matrix.host_id }}-${{ github.event.inputs.testnet_type }}-${{ GITHUB.RUN_NUMBER }}
                  target_label: "node_name"
            " > /home/obscuro/metrics/promtail-config.yaml \
            && echo "
            global:
              scrape_interval: 15s
              evaluation_interval: 15s
            remote_write:
              - url: ${{ vars.PROMETHEUS_URI }}
                tls_config:
                  insecure_skip_verify: true
                basic_auth:
                  username: ${{ secrets.LOKI_USER }}
                  password: ${{ secrets.LOKI_PASSWORD }}
            scrape_configs:
              # Node metrics
              - job_name:  node-${{ matrix.host_id }}-${{ github.event.inputs.testnet_type }}-${{ GITHUB.RUN_NUMBER }}
                scrape_interval: 5s  # Frequent scrapes for node metrics
                static_configs:
                  - targets:
                      - node_exporter:9100  # Node Exporter instance
                relabel_configs:
                  - source_labels: [job]
                    target_label: 'node'
                    replacement:  ${{ matrix.host_id }}-${{ github.event.inputs.testnet_type }}-${{ GITHUB.RUN_NUMBER }}

              # Container metrics
              - job_name:  container-${{ matrix.host_id }}-${{ github.event.inputs.testnet_type }}-${{ GITHUB.RUN_NUMBER }}
                scrape_interval: 5s
                static_configs:
                  - targets:
                      - cadvisor:8080  # cAdvisor instance for container metrics
                relabel_configs:
                  - source_labels: [job]
                    target_label: 'node'
                    replacement:  ${{ matrix.host_id }}-${{ github.event.inputs.testnet_type }}-${{ GITHUB.RUN_NUMBER }}

            " > /home/obscuro/metrics/prometheus.yaml \
            && docker run -d --name promtail \
              --network node_network \
              -e HOSTNAME=${{ matrix.host_id }}-${{ github.event.inputs.testnet_type }}-${{ GITHUB.RUN_NUMBER }} \
              -v /var/log:/var/log \
              -v /home/obscuro/metrics:/etc/promtail \
              -v /var/lib/docker/containers:/var/lib/docker/containers:ro \
              -v /var/run/docker.sock:/var/run/docker.sock \
              grafana/promtail:latest \
              -config.file=/etc/promtail/promtail-config.yaml -config.expand-env=true \
            && docker volume create prometheus-data \
            && docker run -d --name prometheus \
              --network node_network \
              -p 9090:9090 \
              -v /home/obscuro/metrics/prometheus.yaml:/etc/prometheus/prometheus.yml \
              -v prometheus-data:/prometheus \
              prom/prometheus:latest \
              --config.file=/etc/prometheus/prometheus.yml \
            && docker run -d --name node_exporter \
              --network node_network \
              -p 9100:9100 \
              --pid="host" \
              -v /:/host:ro \
              quay.io/prometheus/node-exporter:latest \
              --path.rootfs=/host \
            && docker run -d --name cadvisor \
              --network node_network \
              -p 8080:8080 \
              --privileged \
              -v /:/rootfs:ro \
              -v /var/run:/var/run:ro \
              -v /sys:/sys:ro \
              -v /var/lib/docker/:/var/lib/docker:ro \
              -v /dev/disk/:/dev/disk:ro \
              gcr.io/cadvisor/cadvisor:latest \
            && cd /home/obscuro/go-obscuro/ \
            && sudo go run /home/obscuro/go-obscuro/go/node/cmd  \
               -is_genesis=${{ matrix.is_genesis }} \
               -node_type=${{ matrix.node_type }} \
               -num_enclaves=${{ matrix.num_enclaves }} \
               -is_sgx_enabled=true \
               -host_id=${{ vars[matrix.node_addr_lookup] }} \
               -l1_ws_url=${{ secrets[matrix.node_l1_ws_lookup] }} \
               -management_contract_addr=${{needs.build.outputs.MGMT_CONTRACT_ADDR}} \
               -message_bus_contract_addr=${{needs.build.outputs.MSG_BUS_CONTRACT_ADDR}} \
               -l1_start=${{needs.build.outputs.L1_START_HASH}} \
               -private_key=${{ secrets[matrix.node_pk_lookup] }} \
               -sequencer_addr=obscuronode-0-${{ github.event.inputs.testnet_type }}.uksouth.cloudapp.azure.com:10000 \
               -host_public_p2p_addr=obscuronode-${{ matrix.host_id }}-${{ github.event.inputs.testnet_type }}.uksouth.cloudapp.azure.com:10000 \
               -host_p2p_port=10000 \
               -enclave_docker_image=${{ vars.L2_ENCLAVE_DOCKER_BUILD_TAG }} \
               -host_docker_image=${{ vars.L2_HOST_DOCKER_BUILD_TAG }} \
               -is_debug_namespace_enabled=true \
               -log_level=${{ github.event.inputs.log_level }} \
               -batch_interval=${{ vars.L2_BATCH_INTERVAL }} \
               -max_batch_interval=${{ vars.L2_MAX_BATCH_INTERVAL }} \
               -rollup_interval=${{ vars.L2_ROLLUP_INTERVAL }} \
               -l1_chain_id=${{ vars.L1_CHAIN_ID }} \
               -l1_beacon_url=${{ vars.L1_BEACON_URL }} \
               -l1_blob_archive_url=${{ vars.L1_BLOB_ARCHIVE_URL }} \
               -system_contracts_upgrader=${{ vars.NETWORK_SEQUENCER_SYSTEMCONTRACTSUPGRADER }} \
               -postgres_db_host=postgres://tenuser:${{ secrets.TEN_POSTGRES_USER_PWD }}@postgres-ten-${{  github.event.inputs.testnet_type }}.postgres.database.azure.com:5432/ \
               start'

  check-obscuro-is-healthy:
    needs:
      - build
      - deploy
    runs-on: ubuntu-latest
    environment:
      name: ${{ github.event.inputs.testnet_type }}
    steps:
      - uses: actions/checkout@v4

      - name: "Wait until obscuro node is healthy"
        shell: bash
        run: |
          ./.github/workflows/runner-scripts/wait-node-healthy.sh --host=obscuronode-0-${{ github.event.inputs.testnet_type }}.uksouth.cloudapp.azure.com
          ./.github/workflows/runner-scripts/wait-node-healthy.sh --host=obscuronode-1-${{ github.event.inputs.testnet_type }}.uksouth.cloudapp.azure.com
          ./.github/workflows/runner-scripts/wait-node-healthy.sh --host=obscuronode-2-${{ github.event.inputs.testnet_type }}.uksouth.cloudapp.azure.com
          

  grant-sequencer-enclaves:
    needs:
      - build
      - check-obscuro-is-healthy
    runs-on: ubuntu-latest
    environment:
      name: ${{ github.event.inputs.testnet_type }}
    steps:
      -  uses: actions/checkout@v4

      -  name: 'Grant permission to sequencer enclave(s)'
         id: grantSequencerPermission
         shell: bash
         run: |
           go run ./testnet/launcher/l1grantsequencers/cmd \
           -l1_http_url=${{ secrets.L1_HTTP_URL }} \
           -private_key=${{ secrets.ACCOUNT_PK_WORKER }} \
           -management_contract_addr=${{ needs.build.outputs.MGMT_CONTRACT_ADDR }} \
           -docker_image=${{ vars.L2_HARDHATDEPLOYER_DOCKER_BUILD_TAG }} \
           -sequencer_url=http://obscuronode-0-${{ github.event.inputs.testnet_type }}.uksouth.cloudapp.azure.com:80

      -  name: 'Save sequencer permissioning container logs'
         run: |
           docker logs `docker ps -aqf "name=grant-sequencers"` > grant-sequencers.out 2>&1

      -  name: 'Upload sequencer permissioning container logs'
         uses: actions/upload-artifact@v4
         with:
           name: grant-sequencers
           path: |
             grant-sequencers.out
           retention-days: 7

  deploy-l2-contracts:
    needs:
      - build
      - grant-sequencer-enclaves
    runs-on: ubuntu-latest
    environment:
      name: ${{ github.event.inputs.testnet_type }}
    steps:
      - uses: actions/checkout@v4

      - name: 'Deploy L2 contracts'
        id: deployL2Contracts
        shell: bash
        run: |
          go run ./testnet/launcher/l2contractdeployer/cmd \
          -l2_host=obscuronode-1-${{ github.event.inputs.testnet_type }}.uksouth.cloudapp.azure.com \
          -l1_http_url=${{ secrets.L1_HTTP_URL }} \
          -l2_ws_port=81 \
          -private_key=${{ secrets.ACCOUNT_PK_WORKER }} \
          -l2_private_key=${{ secrets.L2_DEPLOYER_KEY }} \
          -l2_hoc_private_key=${{ secrets.L2_HOC_KEY }} \
          -l2_poc_private_key=${{ secrets.L2_POC_KEY}} \
          -management_contract_addr=${{ needs.build.outputs.MGMT_CONTRACT_ADDR }} \
          -message_bus_contract_addr=${{ needs.build.outputs.MSG_BUS_CONTRACT_ADDR }} \
          -docker_image=${{ vars.L2_HARDHATDEPLOYER_DOCKER_BUILD_TAG }} \
          -faucet_funds=${{ vars.FAUCET_INITIAL_FUNDS }}

      - name: 'Save L2 deployer container logs'
        run: |
          docker logs `docker ps -aqf "name=hh-l2-deployer"` > deploy-l2-contracts.out 2>&1

      - name: 'Upload L2 deployer container logs'
        uses: actions/upload-artifact@v4
        with:
          name: deploy-l2-artifacts
          path: |
            deploy-l2-contracts.out
          retention-days: 7

  update-loadbalancer:
    needs:
      - grant-sequencer-enclaves
    runs-on: ubuntu-latest
    environment:
      name: ${{ github.event.inputs.testnet_type }}
    steps:
      - uses: actions/checkout@v4

      - name: 'Login via Azure CLI'
        uses: azure/login@v1
        with:
          creds: ${{ secrets.AZURE_CREDENTIALS }}

      - name: 'Remove existing backend nodes from the load balancer'
        run: ./.github/workflows/runner-scripts/testnet-clear-loadbalancer.sh ${{ github.event.inputs.testnet_type }}

      - name: 'Add load balancer address pool to the IP configuration'
        uses: azure/CLI@v1
        with:
          inlineScript: |
            az network nic ip-config address-pool add \
              --address-pool ${{ github.event.inputs.testnet_type }}-backend-pool \
              --ip-config-name ipconfig${{ vars.AZURE_RESOURCE_PREFIX }}-1-${{ GITHUB.RUN_NUMBER }} \
              --nic-name ${{ vars.AZURE_RESOURCE_PREFIX }}-1-${{ GITHUB.RUN_NUMBER }}VMNic \
              --resource-group Testnet \
              --lb-name ${{ github.event.inputs.testnet_type }}-loadbalancer

  deploy-faucet:
    name: 'Trigger Faucet deployment for dev- / testnet on a new deployment'
    uses: ./.github/workflows/manual-deploy-testnet-faucet.yml
    with:
      testnet_type: ${{ github.event.inputs.testnet_type }}
    secrets: inherit
    needs:
      - grant-sequencer-enclaves

  obscuro-test-repository-dispatch:
    runs-on: ubuntu-latest
    environment:
      name: ${{ github.event.inputs.testnet_type }}
    needs:
      - deploy-faucet
    steps:
      - name: 'Send a repository dispatch to obscuro-test on deployment of dev-testnet'
        if: ${{ (github.event.inputs.testnet_type == 'dev-testnet') }}
        run: |
          curl -XPOST -H "Authorization: Bearer ${{ secrets.GH_TOKEN }}" -H "Accept:application/vnd.github" -H "Content-Type:application/json" https://api.github.com/repos/ten-protocol/ten-test/dispatches --data '{ "event_type": "dev_testnet_deployed", "client_payload": { "ref": "${{ github.ref_name }}" }'

      - name: 'Send a repository dispatch to obscuro-test on deployment of testnet'
        if: ${{ (github.event.inputs.testnet_type == 'uat-testnet') }}
        run: |
          curl -XPOST -H "Authorization: Bearer ${{ secrets.GH_TOKEN }}" -H "Accept:application/vnd.github" -H "Content-Type:application/json" https://api.github.com/repos/ten-protocol/ten-test/dispatches --data '{ "event_type": "uat_testnet_deployed", "client_payload": { "ref": "${{ github.ref_name }}" }'