From 60222106d29dca3c3a980ff1a1bc7eaba556b94b Mon Sep 17 00:00:00 2001 From: hatemosphere Date: Thu, 2 Nov 2023 12:55:21 +0100 Subject: [PATCH 01/27] ci: Fix load testing workflow --- .github/workflows/load_testing.yaml | 32 ++++++++++++++++------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/.github/workflows/load_testing.yaml b/.github/workflows/load_testing.yaml index 412f48e7..adebc266 100644 --- a/.github/workflows/load_testing.yaml +++ b/.github/workflows/load_testing.yaml @@ -18,11 +18,14 @@ on: required: false type: number default: 8081 + pull_request: env: + NUM_INSTANCES: ${{ github.event.inputs.num_instances || '100' }} + NODE_PORT: ${{ github.event.inputs.node_port || '8080' }} + METRICS_PORT: ${{ github.event.inputs.metrics_port || '8081' }} VM_AUTH_USERNAME: ${{ secrets.VM_AUTH_USERNAME }} VM_AUTH_PASSWORD: ${{ secrets.VM_AUTH_PASSWORD }} - METRICS_PORT: ${{ inputs.metrics_port }} jobs: run_load_tests: @@ -30,14 +33,14 @@ jobs: steps: - name: Fail-fast on incorrect inputs run: | - if [[ "${{ github.event.inputs.num_instances }}" -ge 1 && "${{ github.event.inputs.num_instances }}" -le 200 ]]; then + if [[ "${{ env.NUM_INSTANCES }}" -ge 1 && "${{ env.NUM_INSTANCES }}" -le 200 ]]; then echo "Number of instances is within range." else echo "Error: Number of instances is not within range 1-200." exit 1 fi - if [[ "${{ github.event.inputs.node_port }}" == "${{ github.event.inputs.metrics_port }}" ]]; then + if [[ "${{ env.NODE_PORT }}" == "${{ env.METRICS_PORT }}" ]]; then echo "Error: node_port and metrics_port should not be equal." exit 1 fi @@ -62,6 +65,7 @@ jobs: # to-do pin versions and move to requirements.yaml python -m pip install --upgrade pip pip install ansible ansible-core google-auth + echo "/home/runner/.local/bin" >> $GITHUB_PATH ansible-galaxy install -r infrastructure/loadtests/ansible/requirements.yml - name: Terraform Init And Apply @@ -69,10 +73,10 @@ jobs: run: | terraform init terraform apply -auto-approve \ - -parallelism=${{ inputs.num_instances }} \ - -var="num_instances=${{ inputs.num_instances }}" \ - -var="node_port=${{ inputs.node_port }}" \ - -var="metrics_port=${{ inputs.metrics_port }}" \ + -parallelism=${{ env.NUM_INSTANCES }} \ + -var="num_instances=${{ env.NUM_INSTANCES }}" \ + -var="node_port=${{ env.NODE_PORT }}" \ + -var="metrics_port=${{ env.METRICS_PORT }}" \ -var="test_id=${{ env.TEST_ID }}" - name: Generate list of host:port for config generator @@ -83,7 +87,7 @@ jobs: echo "${tmp_inventory}" | envsubst > ../infrastructure/loadtests/ansible/gcp.yml ansible-inventory -i ../infrastructure/loadtests/ansible/gcp.yml --list | jq -r '.gcp_loadtest.hosts[]' | \ - awk -v port=${{ inputs.node_port }} -F\" '{print $1 ":" port}' > ips_prts.txt + awk -v port=${{ env.NODE_PORT }} -F\" '{print $1 ":" port}' > ips_prts.txt - name: Install node build dependencies run: sudo apt update && sudo apt install -y clang @@ -107,7 +111,7 @@ jobs: cargo run -p tools \ --bin localnet_config -- \ --input-addrs ips_prts.txt \ - --metrics-server-port ${{ inputs.metrics_port }} \ + --metrics-server-port ${{ env.METRICS_PORT }} \ --output-dir artifacts/node_configs - name: Build executor binary @@ -126,15 +130,15 @@ jobs: ansible-playbook -i gcp.yml \ --user sa_${sa_name} \ --private-key .ssh/google_compute_engine playbook.yml \ - --forks ${{ inputs.num_instances }} + --forks ${{ env.NUM_INSTANCES }} - name: Terraform Destroy working-directory: infrastructure/loadtests if: always() run: | terraform destroy -auto-approve \ - -parallelism=${{ inputs.num_instances }} \ - -var="num_instances=${{ inputs.num_instances }}" \ - -var="node_port=${{ inputs.node_port }}" \ - -var="metrics_port=${{ inputs.metrics_port }}" \ + -parallelism=${{ env.NUM_INSTANCES }} \ + -var="num_instances=${{ env.NUM_INSTANCES }}" \ + -var="node_port=${{ env.NODE_PORT }}" \ + -var="metrics_port=${{ env.METRICS_PORT }}" \ -var="test_id=${{ env.TEST_ID }}" From 5ab938f393a58b766c580bc914cacdb8fa084148 Mon Sep 17 00:00:00 2001 From: hatemosphere Date: Thu, 2 Nov 2023 12:58:55 +0100 Subject: [PATCH 02/27] Pin Ansible deps in requirements.txt --- .github/workflows/load_testing.yaml | 3 +-- infrastructure/loadtests/ansible/extract.py | 11 +++++++++++ infrastructure/loadtests/ansible/requirements.txt | 3 +++ 3 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 infrastructure/loadtests/ansible/extract.py create mode 100644 infrastructure/loadtests/ansible/requirements.txt diff --git a/.github/workflows/load_testing.yaml b/.github/workflows/load_testing.yaml index adebc266..c5597ded 100644 --- a/.github/workflows/load_testing.yaml +++ b/.github/workflows/load_testing.yaml @@ -62,9 +62,8 @@ jobs: - name: Install Ansible run: | - # to-do pin versions and move to requirements.yaml python -m pip install --upgrade pip - pip install ansible ansible-core google-auth + pip install -r infrastructure/loadtests/ansible/requirements.txt echo "/home/runner/.local/bin" >> $GITHUB_PATH ansible-galaxy install -r infrastructure/loadtests/ansible/requirements.yml diff --git a/infrastructure/loadtests/ansible/extract.py b/infrastructure/loadtests/ansible/extract.py new file mode 100644 index 00000000..636a3cbe --- /dev/null +++ b/infrastructure/loadtests/ansible/extract.py @@ -0,0 +1,11 @@ +# packages_to_extract.py +packages = {"ansible", "ansible-core", "google-auth"} + +with open("requirements_all.txt", "r") as file: + lines = file.readlines() + +with open("requirements.txt", "w") as file: + for line in lines: + pkg = line.split("==")[0] + if pkg in packages: + file.write(line) diff --git a/infrastructure/loadtests/ansible/requirements.txt b/infrastructure/loadtests/ansible/requirements.txt new file mode 100644 index 00000000..eda88637 --- /dev/null +++ b/infrastructure/loadtests/ansible/requirements.txt @@ -0,0 +1,3 @@ +ansible==8.3.0 +ansible-core==2.15.3 +google-auth==2.19.1 From cba5955c67d17e2b18d95a3a6ff906c31bae75b5 Mon Sep 17 00:00:00 2001 From: hatemosphere Date: Thu, 2 Nov 2023 13:06:10 +0100 Subject: [PATCH 03/27] Latest pypi packages --- infrastructure/loadtests/ansible/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/infrastructure/loadtests/ansible/requirements.txt b/infrastructure/loadtests/ansible/requirements.txt index eda88637..5aa8f89d 100644 --- a/infrastructure/loadtests/ansible/requirements.txt +++ b/infrastructure/loadtests/ansible/requirements.txt @@ -1,3 +1,3 @@ -ansible==8.3.0 -ansible-core==2.15.3 -google-auth==2.19.1 +ansible==8.5.0 +ansible-core==2.15.5 +google-auth==2.23.4 From 72f24f80403dea65b4802735a391c9aada6ed3bc Mon Sep 17 00:00:00 2001 From: hatemosphere Date: Thu, 2 Nov 2023 13:09:05 +0100 Subject: [PATCH 04/27] Use python3 --- .github/workflows/load_testing.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/load_testing.yaml b/.github/workflows/load_testing.yaml index c5597ded..8c3368a9 100644 --- a/.github/workflows/load_testing.yaml +++ b/.github/workflows/load_testing.yaml @@ -62,8 +62,8 @@ jobs: - name: Install Ansible run: | - python -m pip install --upgrade pip - pip install -r infrastructure/loadtests/ansible/requirements.txt + python3 -m pip install --upgrade pip + pip3 install -r infrastructure/loadtests/ansible/requirements.txt echo "/home/runner/.local/bin" >> $GITHUB_PATH ansible-galaxy install -r infrastructure/loadtests/ansible/requirements.yml From ba77b0c7cb4fd2df9aea4dd36759623e8d0af037 Mon Sep 17 00:00:00 2001 From: hatemosphere Date: Thu, 2 Nov 2023 13:34:11 +0100 Subject: [PATCH 05/27] debug --- .github/workflows/load_testing.yaml | 160 ++++++++++++++-------------- 1 file changed, 81 insertions(+), 79 deletions(-) diff --git a/.github/workflows/load_testing.yaml b/.github/workflows/load_testing.yaml index 8c3368a9..96d074b5 100644 --- a/.github/workflows/load_testing.yaml +++ b/.github/workflows/load_testing.yaml @@ -62,82 +62,84 @@ jobs: - name: Install Ansible run: | - python3 -m pip install --upgrade pip - pip3 install -r infrastructure/loadtests/ansible/requirements.txt - echo "/home/runner/.local/bin" >> $GITHUB_PATH - ansible-galaxy install -r infrastructure/loadtests/ansible/requirements.yml - - - name: Terraform Init And Apply - working-directory: infrastructure/loadtests - run: | - terraform init - terraform apply -auto-approve \ - -parallelism=${{ env.NUM_INSTANCES }} \ - -var="num_instances=${{ env.NUM_INSTANCES }}" \ - -var="node_port=${{ env.NODE_PORT }}" \ - -var="metrics_port=${{ env.METRICS_PORT }}" \ - -var="test_id=${{ env.TEST_ID }}" - - - name: Generate list of host:port for config generator - working-directory: node - run: | - sudo apt update && sudo apt install -y gettext-base - tmp_inventory=$(cat ../infrastructure/loadtests/ansible/gcp.yml) - echo "${tmp_inventory}" | envsubst > ../infrastructure/loadtests/ansible/gcp.yml - - ansible-inventory -i ../infrastructure/loadtests/ansible/gcp.yml --list | jq -r '.gcp_loadtest.hosts[]' | \ - awk -v port=${{ env.NODE_PORT }} -F\" '{print $1 ":" port}' > ips_prts.txt - - - name: Install node build dependencies - run: sudo apt update && sudo apt install -y clang - - - uses: actions-rust-lang/setup-rust-toolchain@v1 - id: setup-rust - - - name: Print used Rust versions - run: | - echo "Rustc version: ${{ steps.setup-rust.outputs.rustc-version }}" - echo "Cargo version: ${{ steps.setup-rust.outputs.cargo-version }}" - echo "Rustup version: ${{ steps.setup-rust.outputs.rustup-version }}" - - - name: Pre-create dirs for node artifacts - working-directory: node - run: mkdir -p artifacts/{node_configs,binaries} - - - name: Generate node configs - working-directory: node - run: | - cargo run -p tools \ - --bin localnet_config -- \ - --input-addrs ips_prts.txt \ - --metrics-server-port ${{ env.METRICS_PORT }} \ - --output-dir artifacts/node_configs - - - name: Build executor binary - working-directory: node - run: | - build_output=$(cargo build --release -p tools --bin executor --message-format=json) || exit 1 - echo "$build_output" | jq -r 'select(.executable != null) | .executable' \ - | while read binary; do - cp "$binary" artifacts/binaries/ - done - - - name: Run ansible - working-directory: infrastructure/loadtests/ansible - run: | - sa_name=$(gcloud iam service-accounts describe deployer-sandbox@matterlabs-infra.iam.gserviceaccount.com --format='value(uniqueId)') - ansible-playbook -i gcp.yml \ - --user sa_${sa_name} \ - --private-key .ssh/google_compute_engine playbook.yml \ - --forks ${{ env.NUM_INSTANCES }} - - - name: Terraform Destroy - working-directory: infrastructure/loadtests - if: always() - run: | - terraform destroy -auto-approve \ - -parallelism=${{ env.NUM_INSTANCES }} \ - -var="num_instances=${{ env.NUM_INSTANCES }}" \ - -var="node_port=${{ env.NODE_PORT }}" \ - -var="metrics_port=${{ env.METRICS_PORT }}" \ - -var="test_id=${{ env.TEST_ID }}" + python3 --version + python --version + # echo "/home/runner/.local/bin" >> $GITHUB_PATH + # python3 -m pip install --upgrade pip + # pip3 install -r infrastructure/loadtests/ansible/requirements.txt + # ansible-galaxy install -r infrastructure/loadtests/ansible/requirements.yml + + # - name: Terraform Init And Apply + # working-directory: infrastructure/loadtests + # run: | + # terraform init + # terraform apply -auto-approve \ + # -parallelism=${{ env.NUM_INSTANCES }} \ + # -var="num_instances=${{ env.NUM_INSTANCES }}" \ + # -var="node_port=${{ env.NODE_PORT }}" \ + # -var="metrics_port=${{ env.METRICS_PORT }}" \ + # -var="test_id=${{ env.TEST_ID }}" + + # - name: Generate list of host:port for config generator + # working-directory: node + # run: | + # sudo apt update && sudo apt install -y gettext-base + # tmp_inventory=$(cat ../infrastructure/loadtests/ansible/gcp.yml) + # echo "${tmp_inventory}" | envsubst > ../infrastructure/loadtests/ansible/gcp.yml + + # ansible-inventory -i ../infrastructure/loadtests/ansible/gcp.yml --list | jq -r '.gcp_loadtest.hosts[]' | \ + # awk -v port=${{ env.NODE_PORT }} -F\" '{print $1 ":" port}' > ips_prts.txt + + # - name: Install node build dependencies + # run: sudo apt update && sudo apt install -y clang + + # - uses: actions-rust-lang/setup-rust-toolchain@v1 + # id: setup-rust + + # - name: Print used Rust versions + # run: | + # echo "Rustc version: ${{ steps.setup-rust.outputs.rustc-version }}" + # echo "Cargo version: ${{ steps.setup-rust.outputs.cargo-version }}" + # echo "Rustup version: ${{ steps.setup-rust.outputs.rustup-version }}" + + # - name: Pre-create dirs for node artifacts + # working-directory: node + # run: mkdir -p artifacts/{node_configs,binaries} + + # - name: Generate node configs + # working-directory: node + # run: | + # cargo run -p tools \ + # --bin localnet_config -- \ + # --input-addrs ips_prts.txt \ + # --metrics-server-port ${{ env.METRICS_PORT }} \ + # --output-dir artifacts/node_configs + + # - name: Build executor binary + # working-directory: node + # run: | + # build_output=$(cargo build --release -p tools --bin executor --message-format=json) || exit 1 + # echo "$build_output" | jq -r 'select(.executable != null) | .executable' \ + # | while read binary; do + # cp "$binary" artifacts/binaries/ + # done + + # - name: Run ansible + # working-directory: infrastructure/loadtests/ansible + # run: | + # sa_name=$(gcloud iam service-accounts describe deployer-sandbox@matterlabs-infra.iam.gserviceaccount.com --format='value(uniqueId)') + # ansible-playbook -i gcp.yml \ + # --user sa_${sa_name} \ + # --private-key .ssh/google_compute_engine playbook.yml \ + # --forks ${{ env.NUM_INSTANCES }} + + # - name: Terraform Destroy + # working-directory: infrastructure/loadtests + # if: always() + # run: | + # terraform destroy -auto-approve \ + # -parallelism=${{ env.NUM_INSTANCES }} \ + # -var="num_instances=${{ env.NUM_INSTANCES }}" \ + # -var="node_port=${{ env.NODE_PORT }}" \ + # -var="metrics_port=${{ env.METRICS_PORT }}" \ + # -var="test_id=${{ env.TEST_ID }}" From 5f99c0a5fc4a4fd0cb1bc1d2f3df3a6f83a9a9ce Mon Sep 17 00:00:00 2001 From: hatemosphere Date: Thu, 2 Nov 2023 13:53:05 +0100 Subject: [PATCH 06/27] force python 3.12 usage --- .github/workflows/load_testing.yaml | 164 ++++++++++++++-------------- 1 file changed, 83 insertions(+), 81 deletions(-) diff --git a/.github/workflows/load_testing.yaml b/.github/workflows/load_testing.yaml index 96d074b5..8d6b45ee 100644 --- a/.github/workflows/load_testing.yaml +++ b/.github/workflows/load_testing.yaml @@ -55,6 +55,10 @@ jobs: with: node-version: 16 + - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4 + with: + python-version: '3.12' + - name: Setup Terraform uses: hashicorp/setup-terraform@v2 with: @@ -62,84 +66,82 @@ jobs: - name: Install Ansible run: | - python3 --version - python --version - # echo "/home/runner/.local/bin" >> $GITHUB_PATH - # python3 -m pip install --upgrade pip - # pip3 install -r infrastructure/loadtests/ansible/requirements.txt - # ansible-galaxy install -r infrastructure/loadtests/ansible/requirements.yml - - # - name: Terraform Init And Apply - # working-directory: infrastructure/loadtests - # run: | - # terraform init - # terraform apply -auto-approve \ - # -parallelism=${{ env.NUM_INSTANCES }} \ - # -var="num_instances=${{ env.NUM_INSTANCES }}" \ - # -var="node_port=${{ env.NODE_PORT }}" \ - # -var="metrics_port=${{ env.METRICS_PORT }}" \ - # -var="test_id=${{ env.TEST_ID }}" - - # - name: Generate list of host:port for config generator - # working-directory: node - # run: | - # sudo apt update && sudo apt install -y gettext-base - # tmp_inventory=$(cat ../infrastructure/loadtests/ansible/gcp.yml) - # echo "${tmp_inventory}" | envsubst > ../infrastructure/loadtests/ansible/gcp.yml - - # ansible-inventory -i ../infrastructure/loadtests/ansible/gcp.yml --list | jq -r '.gcp_loadtest.hosts[]' | \ - # awk -v port=${{ env.NODE_PORT }} -F\" '{print $1 ":" port}' > ips_prts.txt - - # - name: Install node build dependencies - # run: sudo apt update && sudo apt install -y clang - - # - uses: actions-rust-lang/setup-rust-toolchain@v1 - # id: setup-rust - - # - name: Print used Rust versions - # run: | - # echo "Rustc version: ${{ steps.setup-rust.outputs.rustc-version }}" - # echo "Cargo version: ${{ steps.setup-rust.outputs.cargo-version }}" - # echo "Rustup version: ${{ steps.setup-rust.outputs.rustup-version }}" - - # - name: Pre-create dirs for node artifacts - # working-directory: node - # run: mkdir -p artifacts/{node_configs,binaries} - - # - name: Generate node configs - # working-directory: node - # run: | - # cargo run -p tools \ - # --bin localnet_config -- \ - # --input-addrs ips_prts.txt \ - # --metrics-server-port ${{ env.METRICS_PORT }} \ - # --output-dir artifacts/node_configs - - # - name: Build executor binary - # working-directory: node - # run: | - # build_output=$(cargo build --release -p tools --bin executor --message-format=json) || exit 1 - # echo "$build_output" | jq -r 'select(.executable != null) | .executable' \ - # | while read binary; do - # cp "$binary" artifacts/binaries/ - # done - - # - name: Run ansible - # working-directory: infrastructure/loadtests/ansible - # run: | - # sa_name=$(gcloud iam service-accounts describe deployer-sandbox@matterlabs-infra.iam.gserviceaccount.com --format='value(uniqueId)') - # ansible-playbook -i gcp.yml \ - # --user sa_${sa_name} \ - # --private-key .ssh/google_compute_engine playbook.yml \ - # --forks ${{ env.NUM_INSTANCES }} - - # - name: Terraform Destroy - # working-directory: infrastructure/loadtests - # if: always() - # run: | - # terraform destroy -auto-approve \ - # -parallelism=${{ env.NUM_INSTANCES }} \ - # -var="num_instances=${{ env.NUM_INSTANCES }}" \ - # -var="node_port=${{ env.NODE_PORT }}" \ - # -var="metrics_port=${{ env.METRICS_PORT }}" \ - # -var="test_id=${{ env.TEST_ID }}" + echo "/home/runner/.local/bin" >> $GITHUB_PATH + python -m pip install --upgrade pip + pip install -r infrastructure/loadtests/ansible/requirements.txt + ansible-galaxy install -r infrastructure/loadtests/ansible/requirements.yml + + - name: Terraform Init And Apply + working-directory: infrastructure/loadtests + run: | + terraform init + terraform apply -auto-approve \ + -parallelism=${{ env.NUM_INSTANCES }} \ + -var="num_instances=${{ env.NUM_INSTANCES }}" \ + -var="node_port=${{ env.NODE_PORT }}" \ + -var="metrics_port=${{ env.METRICS_PORT }}" \ + -var="test_id=${{ env.TEST_ID }}" + + - name: Generate list of host:port for config generator + working-directory: node + run: | + sudo apt update && sudo apt install -y gettext-base + tmp_inventory=$(cat ../infrastructure/loadtests/ansible/gcp.yml) + echo "${tmp_inventory}" | envsubst > ../infrastructure/loadtests/ansible/gcp.yml + + ansible-inventory -i ../infrastructure/loadtests/ansible/gcp.yml --list | jq -r '.gcp_loadtest.hosts[]' | \ + awk -v port=${{ env.NODE_PORT }} -F\" '{print $1 ":" port}' > ips_prts.txt + + - name: Install node build dependencies + run: sudo apt update && sudo apt install -y clang + + - uses: actions-rust-lang/setup-rust-toolchain@v1 + id: setup-rust + + - name: Print used Rust versions + run: | + echo "Rustc version: ${{ steps.setup-rust.outputs.rustc-version }}" + echo "Cargo version: ${{ steps.setup-rust.outputs.cargo-version }}" + echo "Rustup version: ${{ steps.setup-rust.outputs.rustup-version }}" + + - name: Pre-create dirs for node artifacts + working-directory: node + run: mkdir -p artifacts/{node_configs,binaries} + + - name: Generate node configs + working-directory: node + run: | + cargo run -p tools \ + --bin localnet_config -- \ + --input-addrs ips_prts.txt \ + --metrics-server-port ${{ env.METRICS_PORT }} \ + --output-dir artifacts/node_configs + + - name: Build executor binary + working-directory: node + run: | + build_output=$(cargo build --release -p tools --bin executor --message-format=json) || exit 1 + echo "$build_output" | jq -r 'select(.executable != null) | .executable' \ + | while read binary; do + cp "$binary" artifacts/binaries/ + done + + - name: Run ansible + working-directory: infrastructure/loadtests/ansible + run: | + sa_name=$(gcloud iam service-accounts describe deployer-sandbox@matterlabs-infra.iam.gserviceaccount.com --format='value(uniqueId)') + ansible-playbook -i gcp.yml \ + --user sa_${sa_name} \ + --private-key .ssh/google_compute_engine playbook.yml \ + --forks ${{ env.NUM_INSTANCES }} + + - name: Terraform Destroy + working-directory: infrastructure/loadtests + if: always() + run: | + terraform destroy -auto-approve \ + -parallelism=${{ env.NUM_INSTANCES }} \ + -var="num_instances=${{ env.NUM_INSTANCES }}" \ + -var="node_port=${{ env.NODE_PORT }}" \ + -var="metrics_port=${{ env.METRICS_PORT }}" \ + -var="test_id=${{ env.TEST_ID }}" From 1f90aecf6068a57d40d5e8666d57a7300672d936 Mon Sep 17 00:00:00 2001 From: hatemosphere Date: Thu, 2 Nov 2023 15:08:14 +0100 Subject: [PATCH 07/27] error on empty inventory --- .github/workflows/load_testing.yaml | 8 +++++++- infrastructure/loadtests/ansible/extract.py | 11 ----------- 2 files changed, 7 insertions(+), 12 deletions(-) delete mode 100644 infrastructure/loadtests/ansible/extract.py diff --git a/.github/workflows/load_testing.yaml b/.github/workflows/load_testing.yaml index 8d6b45ee..e7953685 100644 --- a/.github/workflows/load_testing.yaml +++ b/.github/workflows/load_testing.yaml @@ -85,11 +85,17 @@ jobs: - name: Generate list of host:port for config generator working-directory: node run: | + set -o pipefail + sudo apt update && sudo apt install -y gettext-base tmp_inventory=$(cat ../infrastructure/loadtests/ansible/gcp.yml) echo "${tmp_inventory}" | envsubst > ../infrastructure/loadtests/ansible/gcp.yml - ansible-inventory -i ../infrastructure/loadtests/ansible/gcp.yml --list | jq -r '.gcp_loadtest.hosts[]' | \ + ansible-inventory -i ../infrastructure/loadtests/ansible/gcp.yml --list | \ + tee /tmp/ansible_output.txt | \ + grep -q "No inventory was parsed" && { echo "Error: No inventory was parsed."; exit 1; } + + jq -r '.gcp_loadtest.hosts[]' < /tmp/ansible_output.txt | \ awk -v port=${{ env.NODE_PORT }} -F\" '{print $1 ":" port}' > ips_prts.txt - name: Install node build dependencies diff --git a/infrastructure/loadtests/ansible/extract.py b/infrastructure/loadtests/ansible/extract.py deleted file mode 100644 index 636a3cbe..00000000 --- a/infrastructure/loadtests/ansible/extract.py +++ /dev/null @@ -1,11 +0,0 @@ -# packages_to_extract.py -packages = {"ansible", "ansible-core", "google-auth"} - -with open("requirements_all.txt", "r") as file: - lines = file.readlines() - -with open("requirements.txt", "w") as file: - for line in lines: - pkg = line.split("==")[0] - if pkg in packages: - file.write(line) From 758ff1d44a50731d81c07d8002086461c7bc038a Mon Sep 17 00:00:00 2001 From: hatemosphere Date: Thu, 2 Nov 2023 15:21:36 +0100 Subject: [PATCH 08/27] debug --- .github/workflows/load_testing.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/load_testing.yaml b/.github/workflows/load_testing.yaml index e7953685..19b80931 100644 --- a/.github/workflows/load_testing.yaml +++ b/.github/workflows/load_testing.yaml @@ -67,8 +67,10 @@ jobs: - name: Install Ansible run: | echo "/home/runner/.local/bin" >> $GITHUB_PATH + python --version python -m pip install --upgrade pip - pip install -r infrastructure/loadtests/ansible/requirements.txt + pip install ansible ansible-core google-auth + # pip install -r infrastructure/loadtests/ansible/requirements.txt ansible-galaxy install -r infrastructure/loadtests/ansible/requirements.yml - name: Terraform Init And Apply From 2f3f313ca92c3a2c358d4f5ac9506f99ccf089b3 Mon Sep 17 00:00:00 2001 From: hatemosphere Date: Thu, 2 Nov 2023 15:34:46 +0100 Subject: [PATCH 09/27] debug --- .github/workflows/load_testing.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/load_testing.yaml b/.github/workflows/load_testing.yaml index 19b80931..8246c626 100644 --- a/.github/workflows/load_testing.yaml +++ b/.github/workflows/load_testing.yaml @@ -93,6 +93,8 @@ jobs: tmp_inventory=$(cat ../infrastructure/loadtests/ansible/gcp.yml) echo "${tmp_inventory}" | envsubst > ../infrastructure/loadtests/ansible/gcp.yml + cat ../infrastructure/loadtests/ansible/gcp.yml + ansible-inventory -i ../infrastructure/loadtests/ansible/gcp.yml --list | \ tee /tmp/ansible_output.txt | \ grep -q "No inventory was parsed" && { echo "Error: No inventory was parsed."; exit 1; } From f2b95139d2ebacdfb0dabf78dd04b8d9251544ac Mon Sep 17 00:00:00 2001 From: hatemosphere Date: Thu, 2 Nov 2023 15:36:57 +0100 Subject: [PATCH 10/27] remove extra path --- .github/workflows/load_testing.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/load_testing.yaml b/.github/workflows/load_testing.yaml index 8246c626..5c81df0f 100644 --- a/.github/workflows/load_testing.yaml +++ b/.github/workflows/load_testing.yaml @@ -66,7 +66,6 @@ jobs: - name: Install Ansible run: | - echo "/home/runner/.local/bin" >> $GITHUB_PATH python --version python -m pip install --upgrade pip pip install ansible ansible-core google-auth From a35ac8475d85968226de108b2e9dd0e654ce0070 Mon Sep 17 00:00:00 2001 From: hatemosphere Date: Thu, 2 Nov 2023 15:58:06 +0100 Subject: [PATCH 11/27] force ansible to use installed python --- .github/workflows/load_testing.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/load_testing.yaml b/.github/workflows/load_testing.yaml index 5c81df0f..87c8b4eb 100644 --- a/.github/workflows/load_testing.yaml +++ b/.github/workflows/load_testing.yaml @@ -65,11 +65,12 @@ jobs: terraform_version: 1.5.6 - name: Install Ansible + env: + ANSIBLE_PYTHON_INTERPRETER: /home/runner/_work/_tool/Python/3.12.0/x64/bin/python run: | - python --version python -m pip install --upgrade pip - pip install ansible ansible-core google-auth - # pip install -r infrastructure/loadtests/ansible/requirements.txt + # pip install ansible ansible-core google-auth + pip install -r infrastructure/loadtests/ansible/requirements.txt ansible-galaxy install -r infrastructure/loadtests/ansible/requirements.yml - name: Terraform Init And Apply From 7c8857b676c5d4ae01077b3165eb6898f4408ddc Mon Sep 17 00:00:00 2001 From: hatemosphere Date: Thu, 2 Nov 2023 16:44:12 +0100 Subject: [PATCH 12/27] force ansible to use installed python --- .github/workflows/load_testing.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/load_testing.yaml b/.github/workflows/load_testing.yaml index 87c8b4eb..9363069d 100644 --- a/.github/workflows/load_testing.yaml +++ b/.github/workflows/load_testing.yaml @@ -68,9 +68,9 @@ jobs: env: ANSIBLE_PYTHON_INTERPRETER: /home/runner/_work/_tool/Python/3.12.0/x64/bin/python run: | - python -m pip install --upgrade pip + /home/runner/_work/_tool/Python/3.12.0/x64/bin/python -m pip install --upgrade pip # pip install ansible ansible-core google-auth - pip install -r infrastructure/loadtests/ansible/requirements.txt + /home/runner/_work/_tool/Python/3.12.0/x64/bin/python pip install -r infrastructure/loadtests/ansible/requirements.txt ansible-galaxy install -r infrastructure/loadtests/ansible/requirements.yml - name: Terraform Init And Apply From 7e4131485a7dcfaaccf3df6cbaef494f14812f1c Mon Sep 17 00:00:00 2001 From: hatemosphere Date: Thu, 2 Nov 2023 16:58:53 +0100 Subject: [PATCH 13/27] fix pip --- .github/workflows/load_testing.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/load_testing.yaml b/.github/workflows/load_testing.yaml index 9363069d..45fe0474 100644 --- a/.github/workflows/load_testing.yaml +++ b/.github/workflows/load_testing.yaml @@ -70,7 +70,7 @@ jobs: run: | /home/runner/_work/_tool/Python/3.12.0/x64/bin/python -m pip install --upgrade pip # pip install ansible ansible-core google-auth - /home/runner/_work/_tool/Python/3.12.0/x64/bin/python pip install -r infrastructure/loadtests/ansible/requirements.txt + /home/runner/_work/_tool/Python/3.12.0/x64/bin/pip install -r infrastructure/loadtests/ansible/requirements.txt ansible-galaxy install -r infrastructure/loadtests/ansible/requirements.yml - name: Terraform Init And Apply From b8d649f5b218747e1587dfff019420fa657fada3 Mon Sep 17 00:00:00 2001 From: hatemosphere Date: Fri, 3 Nov 2023 10:02:04 +0100 Subject: [PATCH 14/27] try something else --- .github/workflows/load_testing.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/load_testing.yaml b/.github/workflows/load_testing.yaml index 45fe0474..895f8c0e 100644 --- a/.github/workflows/load_testing.yaml +++ b/.github/workflows/load_testing.yaml @@ -57,7 +57,7 @@ jobs: - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4 with: - python-version: '3.12' + python-version: '3.11' - name: Setup Terraform uses: hashicorp/setup-terraform@v2 @@ -65,12 +65,12 @@ jobs: terraform_version: 1.5.6 - name: Install Ansible - env: - ANSIBLE_PYTHON_INTERPRETER: /home/runner/_work/_tool/Python/3.12.0/x64/bin/python + # env: + # ANSIBLE_PYTHON_INTERPRETER: /home/runner/_work/_tool/Python/3.12.0/x64/bin/python run: | - /home/runner/_work/_tool/Python/3.12.0/x64/bin/python -m pip install --upgrade pip - # pip install ansible ansible-core google-auth - /home/runner/_work/_tool/Python/3.12.0/x64/bin/pip install -r infrastructure/loadtests/ansible/requirements.txt + python -m pip install --upgrade pip + pip install ansible ansible-core google-auth + # /home/runner/_work/_tool/Python/3.12.0/x64/bin/pip install -r infrastructure/loadtests/ansible/requirements.txt ansible-galaxy install -r infrastructure/loadtests/ansible/requirements.yml - name: Terraform Init And Apply From a541ea1decb603d4a764d28bcb1e68df11da715d Mon Sep 17 00:00:00 2001 From: hatemosphere Date: Fri, 3 Nov 2023 10:03:16 +0100 Subject: [PATCH 15/27] and more --- .github/workflows/load_testing.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/load_testing.yaml b/.github/workflows/load_testing.yaml index 895f8c0e..ff793ed3 100644 --- a/.github/workflows/load_testing.yaml +++ b/.github/workflows/load_testing.yaml @@ -68,6 +68,8 @@ jobs: # env: # ANSIBLE_PYTHON_INTERPRETER: /home/runner/_work/_tool/Python/3.12.0/x64/bin/python run: | + which python + which pip python -m pip install --upgrade pip pip install ansible ansible-core google-auth # /home/runner/_work/_tool/Python/3.12.0/x64/bin/pip install -r infrastructure/loadtests/ansible/requirements.txt From d68657d9701ae393633fb7b64c322c2186eef790 Mon Sep 17 00:00:00 2001 From: hatemosphere Date: Fri, 3 Nov 2023 10:04:57 +0100 Subject: [PATCH 16/27] and more --- .github/workflows/load_testing.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/load_testing.yaml b/.github/workflows/load_testing.yaml index ff793ed3..ff5cac8f 100644 --- a/.github/workflows/load_testing.yaml +++ b/.github/workflows/load_testing.yaml @@ -57,7 +57,7 @@ jobs: - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4 with: - python-version: '3.11' + python-version: '3.12' - name: Setup Terraform uses: hashicorp/setup-terraform@v2 From 9699d18196b748c17a9dfaf4c5cfdca60ea4e1b4 Mon Sep 17 00:00:00 2001 From: hatemosphere Date: Mon, 6 Nov 2023 12:47:57 +0100 Subject: [PATCH 17/27] test something else --- .github/workflows/load_testing.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/load_testing.yaml b/.github/workflows/load_testing.yaml index ff5cac8f..3f386f0d 100644 --- a/.github/workflows/load_testing.yaml +++ b/.github/workflows/load_testing.yaml @@ -88,6 +88,8 @@ jobs: - name: Generate list of host:port for config generator working-directory: node + env: + ANSIBLE_PYTHON_INTERPRETER: /home/runner/_work/_tool/Python/3.12.0/x64/bin/python run: | set -o pipefail From 713914db5b99e36f65b8f896ee4ff54b11ad0b47 Mon Sep 17 00:00:00 2001 From: hatemosphere Date: Mon, 6 Nov 2023 12:49:22 +0100 Subject: [PATCH 18/27] and more --- .github/workflows/load_testing.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/load_testing.yaml b/.github/workflows/load_testing.yaml index 3f386f0d..d1341ee0 100644 --- a/.github/workflows/load_testing.yaml +++ b/.github/workflows/load_testing.yaml @@ -71,7 +71,7 @@ jobs: which python which pip python -m pip install --upgrade pip - pip install ansible ansible-core google-auth + pip install requests ansible ansible-core google-auth # /home/runner/_work/_tool/Python/3.12.0/x64/bin/pip install -r infrastructure/loadtests/ansible/requirements.txt ansible-galaxy install -r infrastructure/loadtests/ansible/requirements.yml @@ -88,8 +88,8 @@ jobs: - name: Generate list of host:port for config generator working-directory: node - env: - ANSIBLE_PYTHON_INTERPRETER: /home/runner/_work/_tool/Python/3.12.0/x64/bin/python + # env: + # ANSIBLE_PYTHON_INTERPRETER: /home/runner/_work/_tool/Python/3.12.0/x64/bin/python run: | set -o pipefail From e9f27a1bbd789b8a3a0091c67874f8e07d6207ce Mon Sep 17 00:00:00 2001 From: hatemosphere Date: Mon, 6 Nov 2023 13:09:46 +0100 Subject: [PATCH 19/27] install openssh --- .github/workflows/load_testing.yaml | 10 ++-------- infrastructure/loadtests/ansible/requirements.txt | 1 + 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/load_testing.yaml b/.github/workflows/load_testing.yaml index d1341ee0..81eeb87b 100644 --- a/.github/workflows/load_testing.yaml +++ b/.github/workflows/load_testing.yaml @@ -65,14 +65,10 @@ jobs: terraform_version: 1.5.6 - name: Install Ansible - # env: - # ANSIBLE_PYTHON_INTERPRETER: /home/runner/_work/_tool/Python/3.12.0/x64/bin/python run: | - which python - which pip + sudo apt update && apt install -y openssh-client python -m pip install --upgrade pip - pip install requests ansible ansible-core google-auth - # /home/runner/_work/_tool/Python/3.12.0/x64/bin/pip install -r infrastructure/loadtests/ansible/requirements.txt + pip install -r infrastructure/loadtests/ansible/requirements.txt ansible-galaxy install -r infrastructure/loadtests/ansible/requirements.yml - name: Terraform Init And Apply @@ -88,8 +84,6 @@ jobs: - name: Generate list of host:port for config generator working-directory: node - # env: - # ANSIBLE_PYTHON_INTERPRETER: /home/runner/_work/_tool/Python/3.12.0/x64/bin/python run: | set -o pipefail diff --git a/infrastructure/loadtests/ansible/requirements.txt b/infrastructure/loadtests/ansible/requirements.txt index 5aa8f89d..e8a3d89d 100644 --- a/infrastructure/loadtests/ansible/requirements.txt +++ b/infrastructure/loadtests/ansible/requirements.txt @@ -1,3 +1,4 @@ ansible==8.5.0 ansible-core==2.15.5 google-auth==2.23.4 +requests=2.31.0 From b71335662de712f220ce2a9e029f2990384dff8c Mon Sep 17 00:00:00 2001 From: hatemosphere Date: Mon, 6 Nov 2023 13:17:22 +0100 Subject: [PATCH 20/27] install openssh --- .github/workflows/load_testing.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/load_testing.yaml b/.github/workflows/load_testing.yaml index 81eeb87b..235f25f9 100644 --- a/.github/workflows/load_testing.yaml +++ b/.github/workflows/load_testing.yaml @@ -66,7 +66,7 @@ jobs: - name: Install Ansible run: | - sudo apt update && apt install -y openssh-client + sudo apt update && sudo apt install -y openssh-client python -m pip install --upgrade pip pip install -r infrastructure/loadtests/ansible/requirements.txt ansible-galaxy install -r infrastructure/loadtests/ansible/requirements.yml From 8b88d8dd34ab3ec00d82f4fd88205e25f9a66e28 Mon Sep 17 00:00:00 2001 From: hatemosphere Date: Mon, 6 Nov 2023 13:22:18 +0100 Subject: [PATCH 21/27] typo fix --- infrastructure/loadtests/ansible/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/loadtests/ansible/requirements.txt b/infrastructure/loadtests/ansible/requirements.txt index e8a3d89d..bb820249 100644 --- a/infrastructure/loadtests/ansible/requirements.txt +++ b/infrastructure/loadtests/ansible/requirements.txt @@ -1,4 +1,4 @@ ansible==8.5.0 ansible-core==2.15.5 google-auth==2.23.4 -requests=2.31.0 +requests==2.31.0 From ef4428d0ed1ecc23405db35cd7fa160482e653cc Mon Sep 17 00:00:00 2001 From: Maksym Kryva Date: Mon, 6 Nov 2023 17:57:41 +0200 Subject: [PATCH 22/27] add sleep block --- .github/workflows/load_testing.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/load_testing.yaml b/.github/workflows/load_testing.yaml index 235f25f9..92bea6e8 100644 --- a/.github/workflows/load_testing.yaml +++ b/.github/workflows/load_testing.yaml @@ -137,6 +137,7 @@ jobs: - name: Run ansible working-directory: infrastructure/loadtests/ansible run: | + sleep 3600 sa_name=$(gcloud iam service-accounts describe deployer-sandbox@matterlabs-infra.iam.gserviceaccount.com --format='value(uniqueId)') ansible-playbook -i gcp.yml \ --user sa_${sa_name} \ From 8fa58ec067a6487c993b61852eb2a6bc33062d1e Mon Sep 17 00:00:00 2001 From: hatemosphere Date: Tue, 7 Nov 2023 10:20:55 +0100 Subject: [PATCH 23/27] force enable oslogin --- .github/workflows/load_testing.yaml | 2 +- infrastructure/loadtests/compute.tf | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/load_testing.yaml b/.github/workflows/load_testing.yaml index 92bea6e8..a3b4c094 100644 --- a/.github/workflows/load_testing.yaml +++ b/.github/workflows/load_testing.yaml @@ -64,7 +64,7 @@ jobs: with: terraform_version: 1.5.6 - - name: Install Ansible + - name: Install Ansible and deps run: | sudo apt update && sudo apt install -y openssh-client python -m pip install --upgrade pip diff --git a/infrastructure/loadtests/compute.tf b/infrastructure/loadtests/compute.tf index 9279a4e4..62700867 100644 --- a/infrastructure/loadtests/compute.tf +++ b/infrastructure/loadtests/compute.tf @@ -30,6 +30,10 @@ resource "google_compute_instance" "zksync_bft_node" { machine_type = "e2-highcpu-8" zone = local.instances_distribution[count.index].zone + metadata = { + enable-oslogin : "TRUE" + } + tags = ["allow-zksync-bft-node-port", "allow-zksync-bft-metrics-port"] labels = { From 5ef01603651361cd441b7a9e13aa3b2eab50d450 Mon Sep 17 00:00:00 2001 From: hatemosphere Date: Tue, 7 Nov 2023 10:21:14 +0100 Subject: [PATCH 24/27] remove debug sleep --- .github/workflows/load_testing.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/load_testing.yaml b/.github/workflows/load_testing.yaml index a3b4c094..59ff8322 100644 --- a/.github/workflows/load_testing.yaml +++ b/.github/workflows/load_testing.yaml @@ -137,7 +137,6 @@ jobs: - name: Run ansible working-directory: infrastructure/loadtests/ansible run: | - sleep 3600 sa_name=$(gcloud iam service-accounts describe deployer-sandbox@matterlabs-infra.iam.gserviceaccount.com --format='value(uniqueId)') ansible-playbook -i gcp.yml \ --user sa_${sa_name} \ From fc6c6e183edcfba8461891b2a0fbd7e7d926516c Mon Sep 17 00:00:00 2001 From: hatemosphere Date: Tue, 7 Nov 2023 16:59:35 +0100 Subject: [PATCH 25/27] force enable oslogin 2 --- infrastructure/loadtests/compute.tf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/infrastructure/loadtests/compute.tf b/infrastructure/loadtests/compute.tf index 62700867..3e671421 100644 --- a/infrastructure/loadtests/compute.tf +++ b/infrastructure/loadtests/compute.tf @@ -66,6 +66,10 @@ resource "google_compute_instance" "vmagent" { machine_type = "e2-highcpu-4" zone = "us-central1-a" + metadata = { + enable-oslogin : "TRUE" + } + labels = { repo = "zksync-bft" purpose = "monitoring" From 156672eca388ff979935d022d59c08d8f90cf7e3 Mon Sep 17 00:00:00 2001 From: hatemosphere Date: Wed, 8 Nov 2023 09:50:17 +0100 Subject: [PATCH 26/27] upd tools crate name --- .github/workflows/load_testing.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/load_testing.yaml b/.github/workflows/load_testing.yaml index 59ff8322..c9cc02a3 100644 --- a/.github/workflows/load_testing.yaml +++ b/.github/workflows/load_testing.yaml @@ -119,7 +119,7 @@ jobs: - name: Generate node configs working-directory: node run: | - cargo run -p tools \ + cargo run -p zksync_consensus_tools \ --bin localnet_config -- \ --input-addrs ips_prts.txt \ --metrics-server-port ${{ env.METRICS_PORT }} \ @@ -128,7 +128,7 @@ jobs: - name: Build executor binary working-directory: node run: | - build_output=$(cargo build --release -p tools --bin executor --message-format=json) || exit 1 + build_output=$(cargo build --release -p zksync_consensus_tools --bin executor --message-format=json) || exit 1 echo "$build_output" | jq -r 'select(.executable != null) | .executable' \ | while read binary; do cp "$binary" artifacts/binaries/ From 1df300a9f6dc69c04306da9b30afb2a74aea2d75 Mon Sep 17 00:00:00 2001 From: hatemosphere Date: Thu, 9 Nov 2023 10:58:50 +0100 Subject: [PATCH 27/27] final touch --- .github/workflows/load_testing.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/load_testing.yaml b/.github/workflows/load_testing.yaml index c9cc02a3..b19b6ff8 100644 --- a/.github/workflows/load_testing.yaml +++ b/.github/workflows/load_testing.yaml @@ -18,7 +18,6 @@ on: required: false type: number default: 8081 - pull_request: env: NUM_INSTANCES: ${{ github.event.inputs.num_instances || '100' }} @@ -134,7 +133,7 @@ jobs: cp "$binary" artifacts/binaries/ done - - name: Run ansible + - name: Run Ansible working-directory: infrastructure/loadtests/ansible run: | sa_name=$(gcloud iam service-accounts describe deployer-sandbox@matterlabs-infra.iam.gserviceaccount.com --format='value(uniqueId)')