Skip to content

Commit

Permalink
Merge branch 'main' into bf-consensus-default
Browse files Browse the repository at this point in the history
  • Loading branch information
brunoffranca committed Dec 11, 2024
2 parents e0f38ef + 51958f6 commit aa64fe9
Show file tree
Hide file tree
Showing 462 changed files with 13,792 additions and 9,331 deletions.
4 changes: 2 additions & 2 deletions .github/release-please/manifest.json
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"core": "25.1.0",
"prover": "17.0.0",
"core": "25.2.0",
"prover": "17.1.1",
"zkstack_cli": "0.1.2"
}
28 changes: 15 additions & 13 deletions .github/workflows/build-docker-from-tag.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ concurrency: docker-build
jobs:
setup:
name: Setup
runs-on: [ ubuntu-latest ]
runs-on: [ubuntu-latest]
outputs:
image_tag_suffix: ${{ steps.set.outputs.image_tag_suffix }}
prover_fri_gpu_key_id: ${{ steps.extract-prover-fri-setup-key-ids.outputs.gpu_short_commit_sha }}
Expand All @@ -48,19 +48,20 @@ jobs:
build-push-core-images:
name: Build and push image
needs: [ setup ]
uses: ./.github/workflows/build-core-template.yml
needs: [setup]
uses: ./.github/workflows/new-build-core-template.yml
if: contains(github.ref_name, 'core')
secrets:
DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
with:
image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }}
en_alpha_release: true
action: "push"

build-push-tee-prover-images:
name: Build and push images
needs: [ setup ]
needs: [setup]
uses: ./.github/workflows/build-tee-prover-template.yml
if: contains(github.ref_name, 'core')
secrets:
Expand All @@ -72,45 +73,46 @@ jobs:

build-push-contract-verifier:
name: Build and push image
needs: [ setup ]
uses: ./.github/workflows/build-contract-verifier-template.yml
needs: [setup]
uses: ./.github/workflows/new-build-contract-verifier-template.yml
if: contains(github.ref_name, 'contract_verifier')
secrets:
DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
with:
image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }}
action: "push"

build-push-prover-images:
name: Build and push image
needs: [ setup ]
uses: ./.github/workflows/build-prover-template.yml
needs: [setup]
uses: ./.github/workflows/new-build-prover-template.yml
if: contains(github.ref_name, 'prover')
with:
image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }}
ERA_BELLMAN_CUDA_RELEASE: ${{ vars.ERA_BELLMAN_CUDA_RELEASE }}
CUDA_ARCH: "60;70;75;80;89"
action: "push"
secrets:
DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}

build-push-witness-generator-image-avx512:
name: Build and push image
needs: [ setup ]
uses: ./.github/workflows/build-witness-generator-template.yml
needs: [setup]
uses: ./.github/workflows/new-build-witness-generator-template.yml
if: contains(github.ref_name, 'prover')
with:
image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }}-avx512
ERA_BELLMAN_CUDA_RELEASE: ${{ vars.ERA_BELLMAN_CUDA_RELEASE }}
CUDA_ARCH: "60;70;75;80;89"
WITNESS_GENERATOR_RUST_FLAGS: "-Ctarget_feature=+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512vl"
action: "push"
secrets:
DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}

build-gar-prover-fri-gpu-and-circuit-prover-gpu-gar:
name: Build GAR prover FRI GPU
needs: [ setup, build-push-prover-images ]
needs: [setup, build-push-prover-images]
uses: ./.github/workflows/build-prover-fri-gpu-gar-and-circuit-prover-gpu-gar.yml
if: contains(github.ref_name, 'prover')
with:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/build-prover-template.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ on:
CUDA_ARCH:
description: "CUDA Arch to build"
type: string
default: "89"
default: "75;80;89"
required: false
outputs:
protocol_version:
Expand Down
13 changes: 3 additions & 10 deletions .github/workflows/build-witness-generator-template.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,6 @@ on:
type: boolean
default: false
required: false
CUDA_ARCH:
description: "CUDA Arch to build"
type: string
default: "89"
required: false
WITNESS_GENERATOR_RUST_FLAGS:
description: "Rust flags for witness_generator compilation"
type: string
Expand All @@ -49,10 +44,9 @@ jobs:
IMAGE_TAG_SUFFIX: ${{ inputs.image_tag_suffix }}
RUNNER_COMPOSE_FILE: "docker-compose-runner-nightly.yml"
ERA_BELLMAN_CUDA_RELEASE: ${{ inputs.ERA_BELLMAN_CUDA_RELEASE }}
CUDA_ARCH: ${{ inputs.CUDA_ARCH }}
WITNESS_GENERATOR_RUST_FLAGS: ${{ inputs.WITNESS_GENERATOR_RUST_FLAGS }}
ZKSYNC_USE_CUDA_STUBS: true
runs-on: [ matterlabs-ci-runner-c3d ]
runs-on: [matterlabs-ci-runner-c3d]
strategy:
matrix:
component:
Expand Down Expand Up @@ -91,7 +85,6 @@ jobs:
run: |
ci_run run_retried curl -LO https://storage.googleapis.com/matterlabs-setup-keys-us/setup-keys/setup_2\^24.key
- name: login to Docker registries
if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
run: |
Expand Down Expand Up @@ -162,11 +155,11 @@ jobs:
DOCKER_ACTION: ${{ inputs.action }}
COMPONENT: ${{ matrix.component }}
run: |
PASSED_ENV_VARS="ERA_BELLMAN_CUDA_RELEASE,CUDA_ARCH,PROTOCOL_VERSION,RUST_FLAGS" \
PASSED_ENV_VARS="ERA_BELLMAN_CUDA_RELEASE,PROTOCOL_VERSION,RUST_FLAGS" \
ci_run zk docker $DOCKER_ACTION $COMPONENT
- name: Show sccache stats
if: always()
run: |
ci_run sccache --show-stats || true
ci_run cat /tmp/sccache_log.txt || true
ci_run cat /tmp/sccache_log.txt || true
7 changes: 3 additions & 4 deletions .github/workflows/ci-core-reusable.yml
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ jobs:
- name: Init
run: |
ci_run run_retried rustup show
- name: Install zkstack
run: |
ci_run ./zkstack_cli/zkstackup/install -g --path ./zkstack_cli/zkstackup/zkstackup
Expand Down Expand Up @@ -151,7 +151,6 @@ jobs:
--evm-emulator false
ci_run zkstack ecosystem init --dev --verbose
ci_run zkstack dev contracts --test-contracts
# `sleep 60` because we need to wait until server added all the tokens
- name: Run server
Expand Down Expand Up @@ -202,7 +201,7 @@ jobs:
run: |
ci_run ./zkstack_cli/zkstackup/install -g --path ./zkstack_cli/zkstackup/zkstackup || true
ci_run zkstackup -g --local
- name: Create log directories
run: |
SERVER_LOGS_DIR=logs/server
Expand Down Expand Up @@ -427,7 +426,7 @@ jobs:
run: |
ci_run ./bin/run_on_all_chains.sh "zkstack dev test recovery --no-deps --no-kill --ignore-prerequisites --verbose" ${{ env.CHAINS }} ${{ env.INTEGRATION_TESTS_LOGS_DIR }}
- name: Run external node server
- name: Run external nodes
run: |
ci_run zkstack external-node run --ignore-prerequisites --chain era &> ${{ env.EXTERNAL_NODE_LOGS_DIR }}/rollup.log &
ci_run zkstack external-node run --ignore-prerequisites --chain validium &> ${{ env.EXTERNAL_NODE_LOGS_DIR }}/validium.log &
Expand Down
3 changes: 2 additions & 1 deletion .github/workflows/ci-prover-e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,8 @@ jobs:
--base-token-price-nominator 1 \
--base-token-price-denominator 1 \
--set-as-default true \
--ignore-prerequisites
--ignore-prerequisites \
--evm-emulator false
ci_run zkstack ecosystem init --dev --verbose
ci_run zkstack prover init --dev --verbose
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -186,7 +186,7 @@ jobs:
name: Github Status Check
runs-on: ubuntu-latest
if: always() && !cancelled()
needs: [ ci-for-core-lint, ci-for-common, ci-for-core, ci-for-prover, ci-for-docs, build-core-images, build-contract-verifier, build-prover-images ]
needs: [ ci-for-core-lint, ci-for-common, ci-for-core, ci-for-prover, ci-for-docs, build-core-images, build-contract-verifier, build-prover-images, e2e-for-prover ]
steps:
- name: Status
run: |
Expand Down
4 changes: 1 addition & 3 deletions .github/workflows/new-build-core-template.yml
Original file line number Diff line number Diff line change
Expand Up @@ -197,10 +197,8 @@ jobs:
shell: bash
run: |
echo PLATFORM=$(echo ${{ matrix.platforms }} | tr '/' '-') >> $GITHUB_ENV
echo IMAGE_TAG_SHA=$(git rev-parse --short HEAD) >> $GITHUB_ENV
# Support for custom tag suffix
if [ -n "${{ inputs.image_tag_suffix }}" ]; then
echo IMAGE_TAG_SHA_TS="${{ inputs.image_tag_suffix }}" >> $GITHUB_ENV
echo IMAGE_TAG_SHA_TS="${{ env.IMAGE_TAG_SUFFIX }}" >> $GITHUB_ENV
else
echo IMAGE_TAG_SHA_TS=$(git rev-parse --short HEAD)-$(date +%s) >> $GITHUB_ENV
fi
Expand Down
11 changes: 9 additions & 2 deletions .github/workflows/new-build-prover-template.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,12 @@ on:
CUDA_ARCH:
description: "CUDA Arch to build"
type: string
default: "89"
default: "75;80;89"
required: false
# Details: https://arnon.dk/matching-sm-architectures-arch-and-gencode-for-various-nvidia-cards/
# L4: 89
# T4: 75
# A100: 80
outputs:
protocol_version:
description: "Protocol version of the binary"
Expand Down Expand Up @@ -168,6 +172,8 @@ jobs:
matterlabs/${{ matrix.components }}:2.0-${{ env.PROTOCOL_VERSION }}-${{ env.IMAGE_TAG_SHA_TS }}
us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.components }}:2.0-${{ env.IMAGE_TAG_SHA_TS }}
matterlabs/${{ matrix.components }}:2.0-${{ env.IMAGE_TAG_SHA_TS }}
us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.components }}:latest
matterlabs/${{ matrix.components }}:latest
- name: Push docker image
if: ${{ inputs.action == 'push' }}
Expand All @@ -176,6 +182,8 @@ jobs:
docker push matterlabs/${{ matrix.components }}:2.0-${{ env.PROTOCOL_VERSION }}-${{ env.IMAGE_TAG_SHA_TS }}
docker push us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.components }}:2.0-${{ env.IMAGE_TAG_SHA_TS }}
docker push matterlabs/${{ matrix.components }}:2.0-${{ env.IMAGE_TAG_SHA_TS }}
docker push us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.components }}:latest
docker push matterlabs/${{ matrix.components }}:latest
copy-images:
name: Copy images between docker registries
Expand Down Expand Up @@ -206,7 +214,6 @@ jobs:
--tag asia-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component }}:2.0-${{ inputs.image_tag_suffix }} \
us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component }}:2.0-${{ inputs.image_tag_suffix }}
- name: Login and push to Europe GAR
run: |
gcloud auth print-access-token --lifetime=7200 --impersonate-service-account=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com | docker login -u oauth2accesstoken --password-stdin https://europe-docker.pkg.dev
Expand Down
12 changes: 4 additions & 8 deletions .github/workflows/new-build-witness-generator-template.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,6 @@ on:
type: string
default: non-push
required: false
CUDA_ARCH:
description: "CUDA Arch to build"
type: string
default: "89"
required: false
WITNESS_GENERATOR_RUST_FLAGS:
description: "Rust flags for witness_generator compilation"
type: string
Expand All @@ -39,7 +34,7 @@ on:
jobs:
get-protocol-version:
name: Get protocol version
runs-on: [ matterlabs-ci-runner-high-performance ]
runs-on: [matterlabs-ci-runner-high-performance]
outputs:
protocol_version: ${{ steps.protocolversion.outputs.protocol_version }}
steps:
Expand Down Expand Up @@ -85,7 +80,7 @@ jobs:
needs: get-protocol-version
env:
PROTOCOL_VERSION: ${{ needs.get-protocol-version.outputs.protocol_version }}
runs-on: [ matterlabs-ci-runner-c3d ]
runs-on: [matterlabs-ci-runner-c3d]
strategy:
matrix:
components:
Expand Down Expand Up @@ -126,7 +121,6 @@ jobs:
context: .
push: ${{ inputs.action == 'push' }}
build-args: |
CUDA_ARCH=${{ inputs.CUDA_ARCH }}
SCCACHE_GCS_BUCKET=matterlabs-infra-sccache-storage
SCCACHE_GCS_SERVICE_ACCOUNT=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com
SCCACHE_GCS_RW_MODE=READ_WRITE
Expand All @@ -140,3 +134,5 @@ jobs:
matterlabs/${{ matrix.components }}:2.0-${{ env.PROTOCOL_VERSION }}-${{ env.IMAGE_TAG_SHA_TS }}
us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.components }}:2.0-${{ env.IMAGE_TAG_SHA_TS }}
matterlabs/${{ matrix.components }}:2.0-${{ env.IMAGE_TAG_SHA_TS }}
us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.components }}:latest
matterlabs/${{ matrix.components }}:latest
6 changes: 4 additions & 2 deletions .github/workflows/protobuf.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ env:
RUSTC_WRAPPER: "sccache"
SCCACHE_GHA_ENABLED: "true"
RUST_BACKTRACE: "1"
SQLX_OFFLINE: true,
SQLX_OFFLINE: true
# github.base_ref -> github.head_ref for pull_request
BASE: ${{ github.event.pull_request.base.sha || github.event.before }}
# github.event.before -> github.event.after for push
Expand All @@ -41,9 +41,10 @@ jobs:
ref: ${{ env.BASE }}
path: before
fetch-depth: 0 # fetches all branches and tags, which is needed to compute the LCA.
submodules: "recursive"
- name: checkout LCA
run:
git checkout $(git merge-base $BASE $HEAD)
git checkout $(git merge-base $BASE $HEAD) --recurse-submodules
working-directory: ./before
- name: compile before
run: cargo check --all-targets
Expand All @@ -59,6 +60,7 @@ jobs:
with:
ref: ${{ env.HEAD }}
path: after
submodules: recursive
- name: compile after
run: cargo check --all-targets
working-directory: ./after
Expand Down
2 changes: 0 additions & 2 deletions .github/workflows/release-test-stage.yml
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,6 @@ jobs:
with:
image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }}
ERA_BELLMAN_CUDA_RELEASE: ${{ vars.ERA_BELLMAN_CUDA_RELEASE }}
CUDA_ARCH: "60;70;75;80;89"
action: "push"
secrets:
DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
Expand All @@ -116,7 +115,6 @@ jobs:
with:
image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }}-avx512
ERA_BELLMAN_CUDA_RELEASE: ${{ vars.ERA_BELLMAN_CUDA_RELEASE }}
CUDA_ARCH: "60;70;75;80;89"
WITNESS_GENERATOR_RUST_FLAGS: "-Ctarget_feature=+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512vl "
action: "push"
secrets:
Expand Down
3 changes: 1 addition & 2 deletions .github/workflows/zk-environment-publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -128,8 +128,7 @@ jobs:
push: ${{ (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'workflow_dispatch') }}

zk_environment_multiarch_manifest:
# We'll update the 'latest' tag, only on environments generated from 'main'.
if: needs.changed_files.outputs.zk_environment == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/main'
if: ${{ (needs.changed_files.outputs.zk_environment == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'workflow_dispatch') }}
# Needed to push to Gihub Package Registry
permissions:
packages: write
Expand Down
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -120,3 +120,6 @@ configs/*
era-observability/
core/tests/ts-integration/deployments-zk
transactions/

# foundry-zksync
install
Loading

0 comments on commit aa64fe9

Please sign in to comment.