Merge branch 'main' into aov-bft-370-fix-extra-getblock-requests-for-…

…validators

.github/workflows/load_testing.yaml

@@ -20,24 +20,26 @@ on:
         default: 8081
 
 env:
+  NUM_INSTANCES: ${{ github.event.inputs.num_instances || '100' }}
+  NODE_PORT: ${{ github.event.inputs.node_port || '8080' }}
+  METRICS_PORT: ${{ github.event.inputs.metrics_port || '8081' }}
   VM_AUTH_USERNAME: ${{ secrets.VM_AUTH_USERNAME }}
   VM_AUTH_PASSWORD: ${{ secrets.VM_AUTH_PASSWORD }}
-  METRICS_PORT: ${{ inputs.metrics_port }}
 
 jobs:
   run_load_tests:
     runs-on: ["matterlabs-deployer-sandbox-infra"]
     steps:
       - name: Fail-fast on incorrect inputs
         run: |
-          if [[ "${{ github.event.inputs.num_instances }}" -ge 1 && "${{ github.event.inputs.num_instances }}" -le 200 ]]; then
+          if [[ "${{ env.NUM_INSTANCES }}" -ge 1 && "${{ env.NUM_INSTANCES }}" -le 200 ]]; then
             echo "Number of instances is within range."
           else
             echo "Error: Number of instances is not within range 1-200."
             exit 1
           fi
 
-          if [[ "${{ github.event.inputs.node_port }}" == "${{ github.event.inputs.metrics_port }}" ]]; then
+          if [[ "${{ env.NODE_PORT }}" == "${{ env.METRICS_PORT }}" ]]; then
             echo "Error: node_port and metrics_port should not be equal."
             exit 1
           fi
@@ -52,38 +54,50 @@ jobs:
         with:
           node-version: 16
 
+      - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4
+        with:
+          python-version: '3.12'
+
       - name: Setup Terraform
         uses: hashicorp/setup-terraform@v2
         with:
           terraform_version: 1.5.6
 
-      - name: Install Ansible
+      - name: Install Ansible and deps
         run: |
-          # to-do pin versions and move to requirements.yaml
+          sudo apt update && sudo apt install -y openssh-client
           python -m pip install --upgrade pip
-          pip install ansible ansible-core google-auth
+          pip install -r infrastructure/loadtests/ansible/requirements.txt
           ansible-galaxy install -r infrastructure/loadtests/ansible/requirements.yml
 
       - name: Terraform Init And Apply
         working-directory: infrastructure/loadtests
         run: |
           terraform init
           terraform apply -auto-approve \
-            -parallelism=${{ inputs.num_instances }} \
-            -var="num_instances=${{ inputs.num_instances }}" \
-            -var="node_port=${{ inputs.node_port }}" \
-            -var="metrics_port=${{ inputs.metrics_port }}" \
+            -parallelism=${{ env.NUM_INSTANCES }} \
+            -var="num_instances=${{ env.NUM_INSTANCES }}" \
+            -var="node_port=${{ env.NODE_PORT }}" \
+            -var="metrics_port=${{ env.METRICS_PORT }}" \
             -var="test_id=${{ env.TEST_ID }}"
 
       - name: Generate list of host:port for config generator
         working-directory: node
         run: |
+          set -o pipefail
+
           sudo apt update && sudo apt install -y gettext-base
           tmp_inventory=$(cat ../infrastructure/loadtests/ansible/gcp.yml)
           echo "${tmp_inventory}" | envsubst > ../infrastructure/loadtests/ansible/gcp.yml
 
-          ansible-inventory -i ../infrastructure/loadtests/ansible/gcp.yml --list  | jq -r '.gcp_loadtest.hosts[]' | \
-          awk -v port=${{ inputs.node_port }} -F\" '{print $1 ":" port}' > ips_prts.txt
+          cat ../infrastructure/loadtests/ansible/gcp.yml
+
+          ansible-inventory -i ../infrastructure/loadtests/ansible/gcp.yml --list | \
+          tee /tmp/ansible_output.txt | \
+          grep -q "No inventory was parsed" && { echo "Error: No inventory was parsed."; exit 1; }
+
+          jq -r '.gcp_loadtest.hosts[]' < /tmp/ansible_output.txt | \
+          awk -v port=${{ env.NODE_PORT }} -F\" '{print $1 ":" port}' > ips_prts.txt
 
       - name: Install node build dependencies
         run: sudo apt update && sudo apt install -y clang
@@ -104,37 +118,37 @@ jobs:
       - name: Generate node configs
         working-directory: node
         run: |
-          cargo run -p tools \
+          cargo run -p zksync_consensus_tools \
           --bin localnet_config -- \
           --input-addrs ips_prts.txt \
-          --metrics-server-port ${{ inputs.metrics_port }} \
+          --metrics-server-port ${{ env.METRICS_PORT }} \
           --output-dir artifacts/node_configs
 
       - name: Build executor binary
         working-directory: node
         run: |
-          build_output=$(cargo build --release -p tools --bin executor --message-format=json) || exit 1
+          build_output=$(cargo build --release -p zksync_consensus_tools --bin executor --message-format=json) || exit 1
           echo "$build_output" | jq -r 'select(.executable != null) | .executable' \
           | while read binary; do
             cp "$binary" artifacts/binaries/
           done
 
-      - name: Run ansible
+      - name: Run Ansible
         working-directory: infrastructure/loadtests/ansible
         run: |
           sa_name=$(gcloud iam service-accounts describe [email protected] --format='value(uniqueId)')
           ansible-playbook -i gcp.yml \
             --user sa_${sa_name} \
             --private-key .ssh/google_compute_engine playbook.yml \
-            --forks ${{ inputs.num_instances }}
+            --forks ${{ env.NUM_INSTANCES }}
 
       - name: Terraform Destroy
         working-directory: infrastructure/loadtests
         if: always()
         run: |
           terraform destroy -auto-approve \
-            -parallelism=${{ inputs.num_instances }} \
-            -var="num_instances=${{ inputs.num_instances }}" \
-            -var="node_port=${{ inputs.node_port }}" \
-            -var="metrics_port=${{ inputs.metrics_port }}" \
+            -parallelism=${{ env.NUM_INSTANCES }} \
+            -var="num_instances=${{ env.NUM_INSTANCES }}" \
+            -var="node_port=${{ env.NODE_PORT }}" \
+            -var="metrics_port=${{ env.METRICS_PORT }}" \
             -var="test_id=${{ env.TEST_ID }}"

.github/workflows/protobuf.yaml

@@ -14,24 +14,46 @@ on:
     branches: ["main"]
 
 env:
-  PROTOS_DIR: "node/libs/schema/proto"
+  CARGO_TERM_COLOR: always
+  CARGO_INCREMENTAL: "0"
+  RUSTFLAGS: "-Dwarnings -C linker=clang -C link-arg=-fuse-ld=lld -C link-arg=-Wl,-z,nostart-stop-gc"
+  RUSTC_WRAPPER: "sccache"
+  SCCACHE_GHA_ENABLED: "true"
+  RUST_BACKTRACE: "1"
 
 jobs:
   compatibility:
-    runs-on: ubuntu-latest
+    runs-on: [ubuntu-22.04-github-hosted-16core]
     steps:
       # github.base_ref -> github.head_ref for pull_request
       # github.event.before -> github.event.after for push
-      - uses: actions/checkout@v3
+      - uses: mozilla-actions/[email protected]
+      - uses: actions/checkout@v4
         with:
           ref: ${{ github.base_ref || github.event.before }}
           path: before
-      - uses: actions/checkout@v3
+      - name: compile before
+        run: cargo build --all-targets
+        working-directory: ./before/node
+      - name: build before.binpb
+        run: >
+          perl -ne 'print "$1\n" if /PROTOBUF_DESCRIPTOR="(.*)"/'
+          `find ./before/node/target/debug/build/*/output`
+          | xargs cat > ./before.binpb
+      - uses: actions/checkout@v4
         with:
           ref: ${{ github.head_ref || github.event.after }}
           path: after
+      - name: compile after
+        run: cargo build --all-targets
+        working-directory: ./after/node
+      - name: build after.binpb
+        run: >
+          perl -ne 'print "$1\n" if /PROTOBUF_DESCRIPTOR="(.*)"/'
+          `find ./after/node/target/debug/build/*/output`
+          | xargs cat > ./after.binpb
       - uses: bufbuild/buf-setup-action@v1
       - uses: bufbuild/buf-breaking-action@v1
         with:
-          input: "after/$PROTOS_DIR"
-          against: "before/$PROTOS_DIR"
+          input: "./after.binpb"
+          against: "./before.binpb"

.github/workflows/protobuf_conformance.yaml

@@ -28,7 +28,7 @@ jobs:
           path: "protobuf"
       - uses: mozilla-actions/[email protected]
       - name: build test
-        run: cargo build -p zksync_consensus_schema --bin conformance_test
+        run: cargo build -p zksync_protobuf --bin conformance_test
         working-directory: "this/node"
       - name: Cache Bazel
         uses: actions/cache@v3
@@ -41,6 +41,6 @@ jobs:
       - name: run test
         run: >
           bazel run //conformance:conformance_test_runner --
-          --failure_list "${{ github.workspace }}/this/node/libs/schema/src/bin/conformance_test_failure_list.txt"
+          --failure_list "${{ github.workspace }}/this/node/libs/protobuf/src/bin/conformance_test/failure_list.txt"
           "${{ github.workspace }}/this/node/target/debug/conformance_test"
         working-directory: "protobuf"

infrastructure/loadtests/ansible/requirements.txt

@@ -0,0 +1,4 @@
+ansible==8.5.0
+ansible-core==2.15.5
+google-auth==2.23.4
+requests==2.31.0

infrastructure/loadtests/compute.tf

@@ -30,6 +30,10 @@ resource "google_compute_instance" "zksync_bft_node" {
   machine_type = "e2-highcpu-8"
   zone         = local.instances_distribution[count.index].zone
 
+  metadata = {
+    enable-oslogin : "TRUE"
+  }
+
   tags = ["allow-zksync-bft-node-port", "allow-zksync-bft-metrics-port"]
 
   labels = {
@@ -62,6 +66,10 @@ resource "google_compute_instance" "vmagent" {
   machine_type = "e2-highcpu-4"
   zone         = "us-central1-a"
 
+  metadata = {
+    enable-oslogin : "TRUE"
+  }
+
   labels = {
     repo    = "zksync-bft"
     purpose = "monitoring"