Skip to content

Commit

Permalink
Merged origin/main.
Browse files Browse the repository at this point in the history
  • Loading branch information
StefanIliev545 committed Sep 17, 2023
2 parents 51b79a0 + c638627 commit 3e98fe7
Show file tree
Hide file tree
Showing 88 changed files with 1,293 additions and 903 deletions.
120 changes: 120 additions & 0 deletions .github/workflows/manual-deploy-obscuro-gateway-database.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
# Deploys Obscuro Gateway's Database on Azure for Testnet
# Starts the Obscuro Gateway's database on Azure VM

name: '[M] Deploy Obscuro Gateway Database'
run-name: '[M] Deploy Obscuro Gateway Database ( ${{ github.event.inputs.testnet_type }} )'
on:
workflow_dispatch:
inputs:
testnet_type:
description: 'Testnet Type'
required: true
default: 'dev-testnet'
type: choice
options:
- 'dev-testnet'
- 'testnet'

jobs:
build-and-deploy:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3

- name: 'Extract branch name'
shell: bash
run: |
echo "Branch Name: ${GITHUB_REF_NAME}"
echo "BRANCH_NAME=${GITHUB_REF_NAME}" >> $GITHUB_ENV
- name: 'Set Obscuro Gateway variables for testnet'
if: ${{ github.event.inputs.testnet_type == 'testnet' }}
run: |
echo "OBSCURO_GATEWAY_DB_BUILD_TAG=testnetobscuronet.azurecr.io/obscuronet/obscuro_gateway_mariadb_testnet:latest" >> $GITHUB_ENV
echo "RESOURCE_STARTING_NAME=testnet" >> $GITHUB_ENV
echo "RESOURCE_TAG_NAME=obscurogatewaydbtestnetlatest" >> $GITHUB_ENV
- name: 'Set Obscuro Gateway variables for dev-testnet'
if: ${{ github.event.inputs.testnet_type == 'dev-testnet' }}
run: |
echo "OBSCURO_GATEWAY_DB_BUILD_TAG=testnetobscuronet.azurecr.io/obscuronet/dev_obscuro_gateway_mariadb_testnet:latest" >> $GITHUB_ENV
echo "RESOURCE_STARTING_NAME=dev-testnet" >> $GITHUB_ENV
echo "RESOURCE_TAG_NAME=obscurogatewaydbdevtestnetlatest" >> $GITHUB_ENV
- name: 'Login via Azure CLI'
uses: azure/login@v1
with:
creds: ${{ secrets.AZURE_CREDENTIALS }}

# This will fail some deletions due to resource dependencies ( ie. you must first delete the vm before deleting the disk)
- name: 'Delete deployed VMs'
uses: azure/CLI@v1
with:
inlineScript: |
$(az resource list --tag ${{env.RESOURCE_TAG_NAME}}=true --query '[]."id"' -o tsv | xargs -n1 az resource delete --verbose -g Testnet --ids) || true
# This will clean up any lingering dependencies - might fail if there are no resources to cleanup
- name: 'Delete VMs dependencies'
uses: azure/CLI@v1
with:
inlineScript: |
$(az resource list --tag ${{env.RESOURCE_TAG_NAME}}=true --query '[]."id"' -o tsv | xargs -n1 az resource delete --verbose -g Testnet --ids) || true
- name: 'Create VM for Gateway database node on Azure'
uses: azure/CLI@v1
with:
inlineScript: |
az vm create -g Testnet -n "${{env.RESOURCE_STARTING_NAME}}-OG-MariaDB-${{ GITHUB.RUN_NUMBER }}" \
--admin-username obscurouser --admin-password "${{ secrets.OBSCURO_NODE_VM_PWD }}" \
--public-ip-address-dns-name "obscurogateway-mariadb-${{env.RESOURCE_STARTING_NAME}}" \
--tags deploygroup=ObscuroGateway-mariaDB-${{env.RESOURCE_STARTING_NAME}}-${{ GITHUB.RUN_NUMBER }} ${{env.RESOURCE_TAG_NAME}}=true \
--size Standard_D4_v5 --image Canonical:0001-com-ubuntu-server-focal:20_04-lts-gen2:latest \
--public-ip-sku Basic --authentication-type password
- name: 'Open Obscuro Gateway db ports on Azure'
uses: azure/CLI@v1
with:
inlineScript: |
az vm open-port -g Testnet -n "${{env.RESOURCE_STARTING_NAME}}-OG-MariaDB-${{ GITHUB.RUN_NUMBER }}" --port 3306
# To overcome issues with critical VM resources being unavailable, we need to wait for the VM to be ready
- name: 'Allow time for VM initialization'
shell: bash
run: sleep 30

- name: 'Start Obscuro gateway Database on Azure'
uses: azure/CLI@v1
with:
inlineScript: |
az vm run-command invoke -g Testnet -n "${{env.RESOURCE_STARTING_NAME}}-OG-MariaDB-${{ GITHUB.RUN_NUMBER }}" \
--command-id RunShellScript \
--scripts 'mkdir -p /home/obscuro \
&& sudo apt-get update \
&& sudo apt-get install -y gcc \
&& sudo snap refresh && sudo snap install --channel=1.18 go --classic \
&& curl -fsSL https://get.docker.com -o get-docker.sh && sh ./get-docker.sh \
&& git clone --depth 1 -b ${{ env.BRANCH_NAME }} https://github.com/obscuronet/go-obscuro.git /home/obscuro/go-obscuro \
&& docker network create --driver bridge node_network || true \
&& docker run -d --name datadog-agent \
--network node_network \
-e DD_API_KEY=${{ secrets.DD_API_KEY }} \
-e DD_LOGS_ENABLED=true \
-e DD_LOGS_CONFIG_CONTAINER_COLLECT_ALL=true \
-e DD_LOGS_CONFIG_AUTO_MULTI_LINE_DETECTION=true \
-e DD_CONTAINER_EXCLUDE_LOGS="name:datadog-agent" \
-e DD_SITE="datadoghq.eu" \
-v /var/run/docker.sock:/var/run/docker.sock:ro \
-v /proc/:/host/proc/:ro \
-v /opt/datadog-agent/run:/opt/datadog-agent/run:rw \
-v /sys/fs/cgroup/:/host/sys/fs/cgroup:ro \
datadog/agent:latest \
&& cd /home/obscuro/go-obscuro/ \
&& docker run -d --name ${{env.RESOURCE_STARTING_NAME}}-OG-MariaDB-${{ GITHUB.RUN_NUMBER }} \
-p 3306:3306 \
-e MARIADB_ROOT_PASSWORD=${{ secrets.OBSCURO_GATEWAY_MARIADB_ROOT_PWD }} \
-e MARIADB_USER=obscurouser \
-e MARIADB_PASSWORD=${{ secrets.OBSCURO_GATEWAY_MARIADB_USER_PWD }} \
-v /home/obscuro/go-obscuro/tools/walletextension/storage/database/001_init.sql:/docker-entrypoint-initdb.d/schema.sql \
mariadb:11.1.2-jammy'
98 changes: 82 additions & 16 deletions .github/workflows/manual-deploy-obscuro-gateway.yml
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
# Deploys Obscuro Gateway on Azure for Testnet
# Builds the Obscuro Gateway image, pushes the image to dockerhub and starts the Obscuro Gateway on Azure
# Builds the Obscuro Gateway image, pushes the image to dockerhub and starts the Obscuro Gateway on Azure VM

name: '[M] Deploy Obscuro Gateway Testnet'
run-name: '[M] Deploy Obscuro Gateway Testnet ( ${{ github.event.inputs.testnet_type }} )'
name: '[M] Deploy Obscuro Gateway'
run-name: '[M] Deploy Obscuro Gateway ( ${{ github.event.inputs.testnet_type }} )'
on:
workflow_dispatch:
inputs:
Expand All @@ -21,6 +21,12 @@ jobs:
steps:
- uses: actions/checkout@v3

- name: 'Extract branch name'
shell: bash
run: |
echo "Branch Name: ${GITHUB_REF_NAME}"
echo "BRANCH_NAME=${GITHUB_REF_NAME}" >> $GITHUB_ENV
- name: 'Set up Docker'
uses: docker/setup-buildx-action@v1

Expand All @@ -29,12 +35,17 @@ jobs:
run: |
echo "OBSCURO_GATEWAY_BUILD_TAG=testnetobscuronet.azurecr.io/obscuronet/obscuro_gateway_testnet:latest" >> $GITHUB_ENV
echo "OBSCURO_GATEWAY_NODE_HOST=testnet.obscu.ro" >> $GITHUB_ENV
echo "RESOURCE_STARTING_NAME=testnet" >> $GITHUB_ENV
echo "RESOURCE_TAG_NAME=obscurogatewaytestnetlatest" >> $GITHUB_ENV
- name: 'Set Obscuro Gateway variables for dev-testnet'
if: ${{ github.event.inputs.testnet_type == 'dev-testnet' }}
run: |
echo "OBSCURO_GATEWAY_BUILD_TAG=testnetobscuronet.azurecr.io/obscuronet/dev_obscuro_gateway_testnet:latest" >> $GITHUB_ENV
echo "OBSCURO_GATEWAY_NODE_HOST=dev-testnet.obscu.ro" >> $GITHUB_ENV
echo "RESOURCE_STARTING_NAME=dev-testnet" >> $GITHUB_ENV
echo "RESOURCE_TAG_NAME=obscurogatewaydevtestnetlatest" >> $GITHUB_ENV
- name: 'Login to Azure docker registry'
uses: azure/docker-login@v1
Expand All @@ -53,17 +64,72 @@ jobs:
DOCKER_BUILDKIT=1 docker build -t ${{ env.OBSCURO_GATEWAY_BUILD_TAG }} -f ./tools/walletextension/Dockerfile .
docker push ${{ env.OBSCURO_GATEWAY_BUILD_TAG }}
- name: 'Deploy to Azure Container Instances'
uses: 'azure/aci-deploy@v1'
# This will fail some deletions due to resource dependencies ( ie. you must first delete the vm before deleting the disk)
- name: 'Delete deployed VMs'
uses: azure/CLI@v1
with:
inlineScript: |
$(az resource list --tag ${{env.RESOURCE_TAG_NAME}}=true --query '[]."id"' -o tsv | xargs -n1 az resource delete --verbose -g Testnet --ids) || true
# This will clean up any lingering dependencies - might fail if there are no resources to cleanup
- name: 'Delete VMs dependencies'
uses: azure/CLI@v1
with:
inlineScript: |
$(az resource list --tag ${{env.RESOURCE_TAG_NAME}}=true --query '[]."id"' -o tsv | xargs -n1 az resource delete --verbose -g Testnet --ids) || true
- name: 'Create VM for Gateway node on Azure'
uses: azure/CLI@v1
with:
inlineScript: |
az vm create -g Testnet -n "${{env.RESOURCE_STARTING_NAME}}-OG-${{ GITHUB.RUN_NUMBER }}" \
--admin-username obscurouser --admin-password "${{ secrets.OBSCURO_NODE_VM_PWD }}" \
--public-ip-address-dns-name "obscurogateway-${{env.RESOURCE_STARTING_NAME}}" \
--tags deploygroup=ObscuroNode-${{env.RESOURCE_STARTING_NAME}}-${{ GITHUB.RUN_NUMBER }} ${{env.RESOURCE_TAG_NAME}}=true \
--vnet-name ObscuroGateway-${{env.RESOURCE_STARTING_NAME}}-01VNET --subnet ObscuroGateway-${{env.RESOURCE_STARTING_NAME}}-01Subnet \
--size Standard_D4_v5 --image Canonical:0001-com-ubuntu-server-focal:20_04-lts-gen2:latest \
--public-ip-sku Basic --authentication-type password
- name: 'Open Obscuro node-${{ matrix.host_id }} ports on Azure'
uses: azure/CLI@v1
with:
inlineScript: |
az vm open-port -g Testnet -n "${{env.RESOURCE_STARTING_NAME}}-OG-${{ GITHUB.RUN_NUMBER }}" --port 80,81
# To overcome issues with critical VM resources being unavailable, we need to wait for the VM to be ready
- name: 'Allow time for VM initialization'
shell: bash
run: sleep 30

- name: 'Start Obscuro gateway on Azure'
uses: azure/CLI@v1
with:
resource-group: ${{ secrets.RESOURCE_GROUP }}
dns-name-label: ${{ github.event.inputs.testnet_type }}-obscuro-gateway
image: ${{ env.OBSCURO_GATEWAY_BUILD_TAG }}
name: ${{ github.event.inputs.testnet_type }}-obscuro-gateway
location: 'uksouth'
restart-policy: 'Never'
environment-variables: PORT=80
command-line: ./wallet_extension_linux -host=0.0.0.0 -port=80 -portWS=81 -nodeHost=${{ env.OBSCURO_GATEWAY_NODE_HOST }}
ports: '80,81'
cpu: 2
memory: 2
inlineScript: |
az vm run-command invoke -g Testnet -n "${{env.RESOURCE_STARTING_NAME}}-OG-${{ GITHUB.RUN_NUMBER }}" \
--command-id RunShellScript \
--scripts 'mkdir -p /home/obscuro \
&& sudo apt-get update \
&& sudo apt-get install -y gcc \
&& sudo snap refresh && sudo snap install --channel=1.18 go --classic \
&& curl -fsSL https://get.docker.com -o get-docker.sh && sh ./get-docker.sh \
&& git clone --depth 1 -b ${{ env.BRANCH_NAME }} https://github.com/obscuronet/go-obscuro.git /home/obscuro/go-obscuro \
&& docker network create --driver bridge node_network || true \
&& docker run -d --name datadog-agent \
--network node_network \
-e DD_API_KEY=${{ secrets.DD_API_KEY }} \
-e DD_LOGS_ENABLED=true \
-e DD_LOGS_CONFIG_CONTAINER_COLLECT_ALL=true \
-e DD_LOGS_CONFIG_AUTO_MULTI_LINE_DETECTION=true \
-e DD_CONTAINER_EXCLUDE_LOGS="name:datadog-agent" \
-e DD_SITE="datadoghq.eu" \
-v /var/run/docker.sock:/var/run/docker.sock:ro \
-v /proc/:/host/proc/:ro \
-v /opt/datadog-agent/run:/opt/datadog-agent/run:rw \
-v /sys/fs/cgroup/:/host/sys/fs/cgroup:ro \
datadog/agent:latest \
&& cd /home/obscuro/go-obscuro/ \
&& docker run -d -p 80:80 -p 81:81 --name ${{env.RESOURCE_STARTING_NAME}}-OG-${{ GITHUB.RUN_NUMBER }} \
${{ env.OBSCURO_GATEWAY_BUILD_TAG }} \
./wallet_extension_linux -host=0.0.0.0 -port=80 -portWS=81 -nodeHost=${{ env.OBSCURO_GATEWAY_NODE_HOST }} \
-logPath=sys_out -dbType=mariaDB -dbConnectionURL="obscurouser:${{ secrets.OBSCURO_GATEWAY_MARIADB_USER_PWD }}@tcp(obscurogateway-mariadb-${{ github.event.inputs.testnet_type }}.uksouth.cloudapp.azure.com:3306)/ogdb"'
18 changes: 11 additions & 7 deletions .github/workflows/manual-deploy-testnet-l2.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,8 @@ jobs:
RESOURCE_TAG_NAME: ${{ steps.outputVars.outputs.RESOURCE_TAG_NAME }}
RESOURCE_STARTING_NAME: ${{ steps.outputVars.outputs.RESOURCE_STARTING_NAME }}
RESOURCE_TESTNET_NAME: ${{ steps.outputVars.outputs.RESOURCE_TESTNET_NAME }}
L1_HOST: ${{ steps.outputVars.outputs.L1_HOST }}
L1_WS_URL: ${{ steps.outputVars.outputs.L1_WS_URL }}
L1_HTTP_URL: ${{ steps.outputVars.outputs.L1_HTTP_URL }}


steps:
Expand All @@ -62,7 +63,8 @@ jobs:
echo "RESOURCE_TAG_NAME=testnetlatest" >> $GITHUB_ENV
echo "RESOURCE_STARTING_NAME=T" >> $GITHUB_ENV
echo "RESOURCE_TESTNET_NAME=testnet" >> $GITHUB_ENV
echo "L1_HOST=testnet-eth2network.uksouth.cloudapp.azure.com" >> $GITHUB_ENV
echo "L1_WS_URL=ws://testnet-eth2network.uksouth.cloudapp.azure.com:9000" >> $GITHUB_ENV
echo "L1_HTTP_URL=http://testnet-eth2network.uksouth.cloudapp.azure.com:8025" >> $GITHUB_ENV
- name: 'Sets env vars for dev-testnet'
if: ${{ (github.event.inputs.testnet_type == 'dev-testnet') }}
Expand All @@ -73,7 +75,8 @@ jobs:
echo "RESOURCE_TAG_NAME=devtestnetlatest" >> $GITHUB_ENV
echo "RESOURCE_STARTING_NAME=D" >> $GITHUB_ENV
echo "RESOURCE_TESTNET_NAME=devtestnet" >> $GITHUB_ENV
echo "L1_HOST=dev-testnet-eth2network.uksouth.cloudapp.azure.com" >> $GITHUB_ENV
echo "L1_WS_URL=ws://dev-testnet-eth2network.uksouth.cloudapp.azure.com:9000" >> $GITHUB_ENV
echo "L1_HTTP_URL=http://dev-testnet-eth2network.uksouth.cloudapp.azure.com:8025" >> $GITHUB_ENV
- name: 'Output env vars'
id: outputVars
Expand All @@ -84,7 +87,8 @@ jobs:
echo "RESOURCE_TAG_NAME=${{env.RESOURCE_TAG_NAME}}" >> $GITHUB_OUTPUT
echo "RESOURCE_STARTING_NAME=${{env.RESOURCE_STARTING_NAME}}" >> $GITHUB_OUTPUT
echo "RESOURCE_TESTNET_NAME=${{env.RESOURCE_TESTNET_NAME}}" >> $GITHUB_OUTPUT
echo "L1_HOST=${{env.L1_HOST}}" >> $GITHUB_OUTPUT
echo "L1_WS_URL=${{env.L1_WS_URL}}" >> $GITHUB_OUTPUT
echo "L1_HTTP_URL=${{env.L1_HTTP_URL}}" >> $GITHUB_OUTPUT
- name: 'Login to Azure docker registry'
uses: azure/docker-login@v1
Expand All @@ -107,7 +111,7 @@ jobs:
shell: bash
run: |
go run ./testnet/launcher/l1contractdeployer/cmd \
-l1_host=${{ env.L1_HOST }} \
-l1_http_url=${{ env.L1_HTTP_URL }} \
-private_key=${{ secrets.GETHNETWORK_PREFUNDED_PKSTR_WORKER }} \
-docker_image=${{env.L2_HARDHATDEPLOYER_DOCKER_BUILD_TAG}} \
-contracts_env_file=./testnet/.env
Expand Down Expand Up @@ -252,7 +256,7 @@ jobs:
-node_type=${{ matrix.node_type }} \
-is_sgx_enabled=true \
-host_id=${{ secrets[matrix.node_pk_addr] }} \
-l1_host=${{needs.build.outputs.L1_HOST}} \
-l1_ws_url=${{needs.build.outputs.L1_WS_URL}} \
-management_contract_addr=${{needs.build.outputs.MGMT_CONTRACT_ADDR}} \
-message_bus_contract_addr=${{needs.build.outputs.MSG_BUS_CONTRACT_ADDR}} \
-l1_start=${{needs.build.outputs.L1_START_HASH}} \
Expand Down Expand Up @@ -322,7 +326,7 @@ jobs:
run: |
go run ./testnet/launcher/l2contractdeployer/cmd \
-l2_host=obscuronode-0-${{needs.build.outputs.RESOURCE_TESTNET_NAME}}-${{ GITHUB.RUN_NUMBER }}.uksouth.cloudapp.azure.com \
-l1_host=${{ needs.build.outputs.L1_HOST }} \
-l1_http_url=${{ needs.build.outputs.L1_HTTP_URL }} \
-l2_ws_port=81 \
-private_key=${{ secrets.GETHNETWORK_PREFUNDED_PKSTR_WORKER }} \
-l2_private_key=8dfb8083da6275ae3e4f41e3e8a8c19d028d32c9247e24530933782f2a05035b \
Expand Down
16 changes: 8 additions & 8 deletions .github/workflows/manual-upgrade-testnet-l2.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@
name: '[M] Upgrade Testnet L2'

on:
schedule:
- cron: '05 03 * * *'
# schedule:
# - cron: '05 03 * * *'
workflow_dispatch:
inputs:
testnet_type:
Expand All @@ -39,7 +39,7 @@ jobs:
RESOURCE_TAG_NAME: ${{ steps.outputVars.outputs.RESOURCE_TAG_NAME }}
RESOURCE_STARTING_NAME: ${{ steps.outputVars.outputs.RESOURCE_STARTING_NAME }}
RESOURCE_TESTNET_NAME: ${{ steps.outputVars.outputs.RESOURCE_TESTNET_NAME }}
L1_HOST: ${{ steps.outputVars.outputs.L1_HOST }}
L1_WS_URL: ${{ steps.outputVars.outputs.L1_WS_URL }}
VM_BUILD_NUMBER: ${{ steps.outputVars.outputs.VM_BUILD_NUMBER }}

steps:
Expand All @@ -62,8 +62,8 @@ jobs:
echo "RESOURCE_TAG_NAME=testnetlatest" >> $GITHUB_ENV
echo "RESOURCE_STARTING_NAME=T" >> $GITHUB_ENV
echo "RESOURCE_TESTNET_NAME=testnet" >> $GITHUB_ENV
echo "L1_HOST=testnet-eth2network.uksouth.cloudapp.azure.com" >> $GITHUB_ENV
echo "L1_WS_URL=ws://testnet-eth2network.uksouth.cloudapp.azure.com:9000" >> $GITHUB_ENV
- name: 'Sets env vars for dev-testnet'
if: ${{ (github.event.inputs.testnet_type == 'dev-testnet') || (github.event_name == 'schedule') }}
run: |
Expand All @@ -72,7 +72,7 @@ jobs:
echo "RESOURCE_TAG_NAME=devtestnetlatest" >> $GITHUB_ENV
echo "RESOURCE_STARTING_NAME=D" >> $GITHUB_ENV
echo "RESOURCE_TESTNET_NAME=devtestnet" >> $GITHUB_ENV
echo "L1_HOST=dev-testnet-eth2network.uksouth.cloudapp.azure.com" >> $GITHUB_ENV
echo "L1_WS_URL=ws://dev-testnet-eth2network.uksouth.cloudapp.azure.com:9000" >> $GITHUB_ENV
- name: 'Fetch latest VM hostnames by env tag and extract build number'
id: fetch_hostnames
Expand All @@ -95,7 +95,7 @@ jobs:
echo "RESOURCE_TAG_NAME=${{env.RESOURCE_TAG_NAME}}" >> $GITHUB_OUTPUT
echo "RESOURCE_STARTING_NAME=${{env.RESOURCE_STARTING_NAME}}" >> $GITHUB_OUTPUT
echo "RESOURCE_TESTNET_NAME=${{env.RESOURCE_TESTNET_NAME}}" >> $GITHUB_OUTPUT
echo "L1_HOST=${{env.L1_HOST}}" >> $GITHUB_OUTPUT
echo "L1_WS_URL=${{env.L1_WS_URL}}" >> $GITHUB_OUTPUT
echo "VM_BUILD_NUMBER=${{env.VM_BUILD_NUMBER}}" >> $GITHUB_OUTPUT
- name: 'Login to Azure docker registry'
Expand Down Expand Up @@ -174,7 +174,7 @@ jobs:
-node_type=${{ matrix.node_type }} \
-is_sgx_enabled=true \
-host_id=${{ secrets[matrix.node_pk_addr] }} \
-l1_host=${{needs.build.outputs.L1_HOST}} \
-l1_ws_url=${{needs.build.outputs.L1_WS_URL}} \
-private_key=${{ secrets[matrix.node_pk_str] }} \
-sequencer_id=${{ secrets.GETHNETWORK_PREFUNDED_ADDR_0 }} \
-host_public_p2p_addr=obscuronode-${{ matrix.host_id }}-${{needs.build.outputs.RESOURCE_TESTNET_NAME}}-${{needs.build.outputs.VM_BUILD_NUMBER}}.uksouth.cloudapp.azure.com:10000 \
Expand Down
Loading

0 comments on commit 3e98fe7

Please sign in to comment.