Merge branch 'protocol-defense-calldata' into v25-chain-registration

.github/release-please/manifest.json

@@ -1,5 +1,5 @@
 {
-  "core": "24.29.0",
-  "prover": "16.5.0",
+  "core": "25.1.0",
+  "prover": "17.0.0",
   "zkstack_cli": "0.1.2"
 }

.github/workflows/build-contract-verifier-template.yml

@@ -101,7 +101,6 @@ jobs:
       - name: start-services
         run: |
           echo "IMAGE_TAG_SUFFIX=${{ env.IMAGE_TAG_SUFFIX }}" >> .env
-          mkdir -p ./volumes/postgres
           run_retried docker compose pull zk postgres
           docker compose up -d zk postgres
           ci_run pre_download_compilers.sh

.github/workflows/build-core-template.yml

@@ -114,7 +114,6 @@ jobs:
       - name: start-services
         run: |
           echo "IMAGE_TAG_SUFFIX=${{ env.IMAGE_TAG_SUFFIX }}" >> .env
-          mkdir -p ./volumes/postgres
           run_retried docker compose pull zk postgres
           docker compose up -d zk postgres
           ci_run pre_download_compilers.sh

.github/workflows/build-local-node-docker.yml

@@ -50,7 +50,6 @@ jobs:
 
       - name: start-services
         run: |
-          mkdir -p ./volumes/postgres
           run_retried docker compose pull zk postgres
           docker compose up -d zk postgres
         

.github/workflows/build-prover-fri-gpu-gar-and-circuit-prover-gpu-gar.yml

@@ -28,7 +28,6 @@ jobs:
       - name: Download Setup data
         run: |
           gsutil -m rsync -r gs://matterlabs-setup-data-us/${{ inputs.setup_keys_id }} docker/prover-gpu-fri-gar
-          cp -v docker/prover-gpu-fri-gar/*.bin docker/circuit-prover-gpu-gar/
 
       - name: Login to us-central1 GAR
         run: |
@@ -70,6 +69,10 @@ jobs:
             --tag europe-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }} \
             us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }}
 
+      - name: Move Setup data from prover-gpu-fri-gar to circuit-prover-gpu-gar
+        run: |
+          mv -v docker/prover-gpu-fri-gar/*.bin docker/circuit-prover-gpu-gar/
+
       - name: Build and push circuit-prover-gpu-gar
         uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v6.7.0
         with:

.github/workflows/build-prover-template.yml

@@ -45,7 +45,7 @@ jobs:
       RUNNER_COMPOSE_FILE: "docker-compose-runner-nightly.yml"
       ERA_BELLMAN_CUDA_RELEASE: ${{ inputs.ERA_BELLMAN_CUDA_RELEASE }}
       CUDA_ARCH: ${{ inputs.CUDA_ARCH }}
-    runs-on: [ matterlabs-ci-runner-high-performance ]
+    runs-on: [matterlabs-ci-runner-high-performance]
     strategy:
       matrix:
         component:
@@ -56,6 +56,7 @@ jobs:
           - prover-fri-gateway
           - prover-job-monitor
           - proof-fri-gpu-compressor
+          - prover-autoscaler
     outputs:
       protocol_version: ${{ steps.protocolversion.outputs.protocol_version }}
     steps:
@@ -74,7 +75,6 @@ jobs:
       - name: start-services
         run: |
           echo "IMAGE_TAG_SUFFIX=${{ env.IMAGE_TAG_SUFFIX }}" >> .env
-          mkdir -p ./volumes/postgres
           run_retried docker compose pull zk postgres
           docker compose up -d zk postgres
           ci_run sccache --start-server
@@ -91,7 +91,6 @@ jobs:
         run: |
           ci_run run_retried curl -LO https://storage.googleapis.com/matterlabs-setup-keys-us/setup-keys/setup_2\^24.key
 
-
       - name: login to Docker registries
         if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
         run: |

.github/workflows/build-tee-prover-template.yml

@@ -76,4 +76,3 @@ jobs:
               docker push "${repo}/${tag}"
             done
           done
-

.github/workflows/build-witness-generator-template.yml

@@ -75,7 +75,6 @@ jobs:
       - name: start-services
         run: |
           echo "IMAGE_TAG_SUFFIX=${{ env.IMAGE_TAG_SUFFIX }}" >> .env
-          mkdir -p ./volumes/postgres
           run_retried docker compose pull zk postgres
           docker compose up -d zk postgres
           ci_run sccache --start-server

.github/workflows/ci-common-reusable.yml

@@ -27,9 +27,8 @@ jobs:
       - name: Start services
         run: |
           run_retried docker-compose -f ${RUNNER_COMPOSE_FILE} pull
-          mkdir -p ./volumes/postgres
           docker-compose -f ${RUNNER_COMPOSE_FILE} up --build -d zk postgres
-      
+
       - name: Install zkstack
         run: |
           ci_run ./zkstack_cli/zkstackup/install -g --path ./zkstack_cli/zkstackup/zkstackup
@@ -39,4 +38,3 @@ jobs:
       # `zk lint prover` = cargo clippy, which does cargo check behind the scenes, which is a lightweight version of cargo build
       - name: Lints
         run: ci_run zkstack dev lint -t rs --check
-

.github/workflows/ci-core-lint-reusable.yml

@@ -49,6 +49,8 @@ jobs:
           ci_run zkstack dev lint -t js --check
           ci_run zkstack dev lint -t ts --check
           ci_run zkstack dev lint -t rs --check
+          ci_run zkstack dev lint -t autocompletion --check
+          ci_run zkstack dev lint -t rust-toolchain
 
       - name: Check Database
         run: |

.github/workflows/ci-core-reusable.yml

@@ -8,6 +8,10 @@ on:
         required: false
         default: '[{ "zksolc": ["1.3.14", "1.3.16", "1.3.17", "1.3.1", "1.3.7", "1.3.18", "1.3.19", "1.3.21"] } , { "zkvyper": ["1.3.13"] }]'
 
+env:
+  RUST_BACKTRACE: 1
+  PASSED_ENV_VARS: RUST_BACKTRACE
+
 jobs:
   lint:
     name: lint
@@ -30,6 +34,7 @@ jobs:
           echo "SCCACHE_GCS_SERVICE_ACCOUNT=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com" >> .env
           echo "SCCACHE_GCS_RW_MODE=READ_WRITE" >> .env
           echo "RUSTC_WRAPPER=sccache" >> .env
+          echo RUN_CONTRACT_VERIFICATION_TEST=true >> .env
 
       # TODO: Remove when we after upgrade of hardhat-plugins
       - name: pre-download compilers
@@ -56,7 +61,7 @@ jobs:
       - name: Init
         run: |
           ci_run run_retried rustup show
-      
+
       - name: Install zkstack
         run: |
           ci_run ./zkstack_cli/zkstackup/install -g --path ./zkstack_cli/zkstackup/zkstackup
@@ -69,6 +74,9 @@ jobs:
       - name: Contracts unit tests
         run: ci_run yarn l1-contracts test
 
+      - name: Download compilers for contract verifier tests
+        run: ci_run zkstack contract-verifier init --zksolc-version=v1.5.3 --zkvyper-version=v1.5.4 --solc-version=0.8.26 --vyper-version=v0.3.10 --era-vm-solc-version=0.8.26-1.0.1 --only --chain era
+
       - name: Rust unit tests
         run: |
           ci_run zkstack dev test rust
@@ -101,7 +109,7 @@ jobs:
 
       - name: Loadtest configuration
         run: |
-          echo EXPECTED_TX_COUNT=${{ matrix.vm_mode == 'NEW' && 21000 || 16000 }} >> .env
+          echo EXPECTED_TX_COUNT=${{ matrix.vm_mode == 'NEW' && 30000 || 16000 }} >> .env
           echo ACCOUNTS_AMOUNT="100" >> .env
           echo MAX_INFLIGHT_TXS="10" >> .env
           echo SYNC_API_REQUESTS_LIMIT="15" >> .env
@@ -139,10 +147,10 @@ jobs:
             --base-token-price-denominator 1 \
             --set-as-default false \
             --ignore-prerequisites \
-            --legacy-bridge
+            --legacy-bridge \
+            --evm-emulator false
 
           ci_run zkstack ecosystem init --dev --verbose
-          ci_run zkstack dev contracts --test-contracts
 
       # `sleep 60` because we need to wait until server added all the tokens
       - name: Run server
@@ -193,7 +201,7 @@ jobs:
         run: |
           ci_run ./zkstack_cli/zkstackup/install -g --path ./zkstack_cli/zkstackup/zkstackup || true
           ci_run zkstackup -g --local
-  
+
       - name: Create log directories
         run: |
           SERVER_LOGS_DIR=logs/server
@@ -254,7 +262,8 @@ jobs:
           --base-token-price-nominator 1 \
           --base-token-price-denominator 1 \
           --set-as-default false \
-          --ignore-prerequisites
+          --ignore-prerequisites \
+          --evm-emulator false
 
           ci_run zkstack chain init \
           --deploy-paymaster \
@@ -272,10 +281,11 @@ jobs:
           --wallet-creation localhost \
           --l1-batch-commit-data-generator-mode rollup \
           --base-token-address ${{ env.CUSTOM_TOKEN_ADDRESS }} \
-          --base-token-price-nominator 3 \
-          --base-token-price-denominator 2 \
+          --base-token-price-nominator 314 \
+          --base-token-price-denominator 1000 \
           --set-as-default false \
-          --ignore-prerequisites
+          --ignore-prerequisites \
+          --evm-emulator false
 
           ci_run zkstack chain init \
           --deploy-paymaster \
@@ -296,7 +306,8 @@ jobs:
           --base-token-price-nominator 1 \
           --base-token-price-denominator 1 \
           --set-as-default false \
-          --ignore-prerequisites
+          --ignore-prerequisites \
+          --evm-emulator false
 
           ci_run zkstack chain build-transactions --chain offline_chain --l1-rpc-url http://127.0.0.1:8545
 
@@ -328,10 +339,11 @@ jobs:
           --wallet-creation localhost \
           --l1-batch-commit-data-generator-mode validium \
           --base-token-address ${{ env.CUSTOM_TOKEN_ADDRESS }} \
-          --base-token-price-nominator 3 \
-          --base-token-price-denominator 2 \
+          --base-token-price-nominator 314 \
+          --base-token-price-denominator 1000 \
           --set-as-default false \
-          --ignore-prerequisites
+          --ignore-prerequisites \
+          --evm-emulator false
 
           ci_run zkstack chain init \
           --deploy-paymaster \
@@ -349,24 +361,39 @@ jobs:
         run: |
           ci_run zkstack dev test build
 
+      - name: Build tested binaries
+        run: |
+          ci_run zkstack server build
+          ci_run zkstack external-node build
+          ci_run zkstack contract-verifier build
+
       - name: Initialize Contract verifier
         run: |
           ci_run zkstack contract-verifier init --zksolc-version=v1.5.3 --zkvyper-version=v1.5.4 --solc-version=0.8.26 --vyper-version=v0.3.10 --era-vm-solc-version=0.8.26-1.0.1 --only --chain era
           ci_run zkstack contract-verifier run --chain era &> ${{ env.SERVER_LOGS_DIR }}/contract-verifier-rollup.log &
+          ci_run zkstack contract-verifier wait --chain era --verbose
 
       - name: Run servers
         run: |
+          # Override config for part of chains to test the default config as well
+          ci_run zkstack dev config-writer --path etc/env/file_based/overrides/tests/integration.yaml --chain era
+          ci_run zkstack dev config-writer --path etc/env/file_based/overrides/tests/integration.yaml --chain validium
+          
           ci_run zkstack server --ignore-prerequisites --chain era &> ${{ env.SERVER_LOGS_DIR }}/rollup.log &
           ci_run zkstack server --ignore-prerequisites --chain validium &> ${{ env.SERVER_LOGS_DIR }}/validium.log &
           ci_run zkstack server --ignore-prerequisites --chain custom_token &> ${{ env.SERVER_LOGS_DIR }}/custom_token.log &
           ci_run zkstack server --ignore-prerequisites --chain consensus \
-          --components=api,tree,eth,state_keeper,housekeeper,commitment_generator,vm_runner_protective_reads,vm_runner_bwip,vm_playground,da_dispatcher,consensus \
-          &> ${{ env.SERVER_LOGS_DIR }}/consensus.log &
+            --components=api,tree,eth,state_keeper,housekeeper,commitment_generator,vm_runner_protective_reads,vm_runner_bwip,vm_playground,da_dispatcher,consensus \
+            &> ${{ env.SERVER_LOGS_DIR }}/consensus.log &
 
-          ci_run sleep 5
+          ci_run zkstack server wait --ignore-prerequisites --verbose --chain era
+          ci_run zkstack server wait --ignore-prerequisites --verbose --chain validium
+          ci_run zkstack server wait --ignore-prerequisites --verbose --chain custom_token
+          ci_run zkstack server wait --ignore-prerequisites --verbose --chain consensus
 
-      - name: Setup attester committee for the consensus chain
+      - name: Set up attester committee for the consensus chain
         run: |
+          ci_run zkstack consensus wait-for-registry --ignore-prerequisites --verbose --chain consensus
           ci_run zkstack consensus set-attester-committee --chain consensus --from-genesis &> ${{ env.INTEGRATION_TESTS_LOGS_DIR }}/consensus.log 
 
       - name: Run integration tests
@@ -399,12 +426,17 @@ jobs:
         run: |
           ci_run ./bin/run_on_all_chains.sh "zkstack dev test recovery --no-deps --no-kill --ignore-prerequisites --verbose" ${{ env.CHAINS }} ${{ env.INTEGRATION_TESTS_LOGS_DIR }}
 
-      - name: Run external node server
+      - name: Run external nodes
         run: |
           ci_run zkstack external-node run --ignore-prerequisites --chain era &> ${{ env.EXTERNAL_NODE_LOGS_DIR }}/rollup.log &
           ci_run zkstack external-node run --ignore-prerequisites --chain validium &> ${{ env.EXTERNAL_NODE_LOGS_DIR }}/validium.log &
           ci_run zkstack external-node run --ignore-prerequisites --chain custom_token &> ${{ env.EXTERNAL_NODE_LOGS_DIR }}/custom_token.log &
           ci_run zkstack external-node run --ignore-prerequisites --chain consensus --enable-consensus &> ${{ env.EXTERNAL_NODE_LOGS_DIR }}/consensus.log &
+          
+          ci_run zkstack external-node wait --ignore-prerequisites --verbose --chain era
+          ci_run zkstack external-node wait --ignore-prerequisites --verbose --chain validium
+          ci_run zkstack external-node wait --ignore-prerequisites --verbose --chain custom_token
+          ci_run zkstack external-node wait --ignore-prerequisites --verbose --chain consensus
 
       - name: Run integration tests en
         run: |

.github/workflows/ci-prover-e2e.yml

@@ -29,7 +29,6 @@ jobs:
       - name: Start services
         run: |
           run_retried docker-compose -f ${RUNNER_COMPOSE_FILE} pull
-          mkdir -p ./volumes/postgres ./volumes/reth/data
           docker-compose -f ${RUNNER_COMPOSE_FILE} --profile runner up -d --wait
           ci_run sccache --start-server
 
@@ -87,7 +86,7 @@ jobs:
           ci_run zkstack prover run --component=witness-generator --round=all-rounds --docker=false &>prover_logs/witness-generator.log &
       - name: Run Circuit Prover
         run: |
-          ci_run zkstack prover run --component=circuit-prover --witness-vector-generator-count=10 --docker=false &>prover_logs/circuit_prover.log &
+          ci_run zkstack prover run --component=circuit-prover -l=23 -h=3 --docker=false &>prover_logs/circuit_prover.log &
       - name: Wait for prover jobs to finish
         env:
           DATABASE_URL: postgres://postgres:notsecurepassword@localhost:5432/zksync_prover_localhost_proving_chain
@@ -101,7 +100,7 @@ jobs:
       - name: Kill prover & start compressor
         run: |
           sudo ./bin/prover_checkers/kill_prover
-          
+
           ci_run zkstack prover run --component=compressor --docker=false &>prover_logs/compressor.log &
       - name: Wait for batch to be executed on L1
         env:

.github/workflows/ci-prover-reusable.yml

@@ -27,7 +27,6 @@ jobs:
       - name: Start services
         run: |
           run_retried docker-compose -f ${RUNNER_COMPOSE_FILE} pull
-          mkdir -p ./volumes/postgres
           docker-compose -f ${RUNNER_COMPOSE_FILE} up --build -d zk postgres
 
       - name: Install zkstack
@@ -40,7 +39,9 @@ jobs:
           ci_run zkstack dev db setup --prover-url=${{ env.prover_url }} --core-url=${{ env.core_url }}
 
       - name: Formatting
-        run: ci_run bash -c "cd prover && cargo fmt --check"
+        run: |
+          ci_run git config --global --add safe.directory /usr/src/zksync
+          ci_run zkstack dev fmt --check rustfmt
 
   unit-tests:
     runs-on: [ matterlabs-ci-runner-highmem-long ]
@@ -66,7 +67,6 @@ jobs:
       - name: Start services
         run: |
           run_retried docker-compose -f ${RUNNER_COMPOSE_FILE} pull
-          mkdir -p ./volumes/postgres
           docker-compose -f ${RUNNER_COMPOSE_FILE} up --build -d zk postgres
 
       - name: Install zkstack

.github/workflows/ci.yml

@@ -20,7 +20,6 @@ jobs:
     outputs:
       core: ${{ steps.changed-files.outputs.core_any_changed }}
       prover: ${{ steps.changed-files.outputs.prover_any_changed }}
-      zkstack_cli: ${{ steps.changed-files.outputs.zkstack_cli_any_changed }}
       docs: ${{ steps.changed-files.outputs.docs_any_changed }}
       all: ${{ steps.changed-files.outputs.all_any_changed }}
     steps:
@@ -178,6 +177,7 @@ jobs:
       image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }}-avx512
       action: "build"
       WITNESS_GENERATOR_RUST_FLAGS: "-Ctarget_feature=+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512vl"
+      ERA_BELLMAN_CUDA_RELEASE: ${{ vars.ERA_BELLMAN_CUDA_RELEASE }}
     secrets:
       DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
       DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}