Merge branch 'main' into switch_bn254

.github/workflows/load_testing.yaml

@@ -20,24 +20,26 @@ on:
         default: 8081
 
 env:
+  NUM_INSTANCES: ${{ github.event.inputs.num_instances || '100' }}
+  NODE_PORT: ${{ github.event.inputs.node_port || '8080' }}
+  METRICS_PORT: ${{ github.event.inputs.metrics_port || '8081' }}
   VM_AUTH_USERNAME: ${{ secrets.VM_AUTH_USERNAME }}
   VM_AUTH_PASSWORD: ${{ secrets.VM_AUTH_PASSWORD }}
-  METRICS_PORT: ${{ inputs.metrics_port }}
 
 jobs:
   run_load_tests:
     runs-on: ["matterlabs-deployer-sandbox-infra"]
     steps:
       - name: Fail-fast on incorrect inputs
         run: |
-          if [[ "${{ github.event.inputs.num_instances }}" -ge 1 && "${{ github.event.inputs.num_instances }}" -le 200 ]]; then
+          if [[ "${{ env.NUM_INSTANCES }}" -ge 1 && "${{ env.NUM_INSTANCES }}" -le 200 ]]; then
             echo "Number of instances is within range."
           else
             echo "Error: Number of instances is not within range 1-200."
             exit 1
           fi
 
-          if [[ "${{ github.event.inputs.node_port }}" == "${{ github.event.inputs.metrics_port }}" ]]; then
+          if [[ "${{ env.NODE_PORT }}" == "${{ env.METRICS_PORT }}" ]]; then
             echo "Error: node_port and metrics_port should not be equal."
             exit 1
           fi
@@ -52,38 +54,50 @@ jobs:
         with:
           node-version: 16
 
+      - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4
+        with:
+          python-version: '3.12'
+
       - name: Setup Terraform
         uses: hashicorp/setup-terraform@v2
         with:
           terraform_version: 1.5.6
 
-      - name: Install Ansible
+      - name: Install Ansible and deps
         run: |
-          # to-do pin versions and move to requirements.yaml
+          sudo apt update && sudo apt install -y openssh-client
           python -m pip install --upgrade pip
-          pip install ansible ansible-core google-auth
+          pip install -r infrastructure/loadtests/ansible/requirements.txt
           ansible-galaxy install -r infrastructure/loadtests/ansible/requirements.yml
 
       - name: Terraform Init And Apply
         working-directory: infrastructure/loadtests
         run: |
           terraform init
           terraform apply -auto-approve \
-            -parallelism=${{ inputs.num_instances }} \
-            -var="num_instances=${{ inputs.num_instances }}" \
-            -var="node_port=${{ inputs.node_port }}" \
-            -var="metrics_port=${{ inputs.metrics_port }}" \
+            -parallelism=${{ env.NUM_INSTANCES }} \
+            -var="num_instances=${{ env.NUM_INSTANCES }}" \
+            -var="node_port=${{ env.NODE_PORT }}" \
+            -var="metrics_port=${{ env.METRICS_PORT }}" \
             -var="test_id=${{ env.TEST_ID }}"
 
       - name: Generate list of host:port for config generator
         working-directory: node
         run: |
+          set -o pipefail
+
           sudo apt update && sudo apt install -y gettext-base
           tmp_inventory=$(cat ../infrastructure/loadtests/ansible/gcp.yml)
           echo "${tmp_inventory}" | envsubst > ../infrastructure/loadtests/ansible/gcp.yml
 
-          ansible-inventory -i ../infrastructure/loadtests/ansible/gcp.yml --list  | jq -r '.gcp_loadtest.hosts[]' | \
-          awk -v port=${{ inputs.node_port }} -F\" '{print $1 ":" port}' > ips_prts.txt
+          cat ../infrastructure/loadtests/ansible/gcp.yml
+
+          ansible-inventory -i ../infrastructure/loadtests/ansible/gcp.yml --list | \
+          tee /tmp/ansible_output.txt | \
+          grep -q "No inventory was parsed" && { echo "Error: No inventory was parsed."; exit 1; }
+
+          jq -r '.gcp_loadtest.hosts[]' < /tmp/ansible_output.txt | \
+          awk -v port=${{ env.NODE_PORT }} -F\" '{print $1 ":" port}' > ips_prts.txt
 
       - name: Install node build dependencies
         run: sudo apt update && sudo apt install -y clang
@@ -104,37 +118,37 @@ jobs:
       - name: Generate node configs
         working-directory: node
         run: |
-          cargo run -p tools \
+          cargo run -p zksync_consensus_tools \
           --bin localnet_config -- \
           --input-addrs ips_prts.txt \
-          --metrics-server-port ${{ inputs.metrics_port }} \
+          --metrics-server-port ${{ env.METRICS_PORT }} \
           --output-dir artifacts/node_configs
 
       - name: Build executor binary
         working-directory: node
         run: |
-          build_output=$(cargo build --release -p tools --bin executor --message-format=json) || exit 1
+          build_output=$(cargo build --release -p zksync_consensus_tools --bin executor --message-format=json) || exit 1
           echo "$build_output" | jq -r 'select(.executable != null) | .executable' \
           | while read binary; do
             cp "$binary" artifacts/binaries/
           done
 
-      - name: Run ansible
+      - name: Run Ansible
         working-directory: infrastructure/loadtests/ansible
         run: |
           sa_name=$(gcloud iam service-accounts describe [email protected] --format='value(uniqueId)')
           ansible-playbook -i gcp.yml \
             --user sa_${sa_name} \
             --private-key .ssh/google_compute_engine playbook.yml \
-            --forks ${{ inputs.num_instances }}
+            --forks ${{ env.NUM_INSTANCES }}
 
       - name: Terraform Destroy
         working-directory: infrastructure/loadtests
         if: always()
         run: |
           terraform destroy -auto-approve \
-            -parallelism=${{ inputs.num_instances }} \
-            -var="num_instances=${{ inputs.num_instances }}" \
-            -var="node_port=${{ inputs.node_port }}" \
-            -var="metrics_port=${{ inputs.metrics_port }}" \
+            -parallelism=${{ env.NUM_INSTANCES }} \
+            -var="num_instances=${{ env.NUM_INSTANCES }}" \
+            -var="node_port=${{ env.NODE_PORT }}" \
+            -var="metrics_port=${{ env.METRICS_PORT }}" \
             -var="test_id=${{ env.TEST_ID }}"

infrastructure/loadtests/ansible/requirements.txt

@@ -0,0 +1,4 @@
+ansible==8.5.0
+ansible-core==2.15.5
+google-auth==2.23.4
+requests==2.31.0

infrastructure/loadtests/compute.tf

@@ -30,6 +30,10 @@ resource "google_compute_instance" "zksync_bft_node" {
   machine_type = "e2-highcpu-8"
   zone         = local.instances_distribution[count.index].zone
 
+  metadata = {
+    enable-oslogin : "TRUE"
+  }
+
   tags = ["allow-zksync-bft-node-port", "allow-zksync-bft-metrics-port"]
 
   labels = {
@@ -62,6 +66,10 @@ resource "google_compute_instance" "vmagent" {
   machine_type = "e2-highcpu-4"
   zone         = "us-central1-a"
 
+  metadata = {
+    enable-oslogin : "TRUE"
+  }
+
   labels = {
     repo    = "zksync-bft"
     purpose = "monitoring"