Skip to content

Commit

Permalink
Merge branch 'matter-labs:main' into main
Browse files Browse the repository at this point in the history
  • Loading branch information
Tleaoo authored Feb 4, 2024
2 parents 2179400 + cdca260 commit 654a05f
Show file tree
Hide file tree
Showing 1,390 changed files with 60,713 additions and 55,038 deletions.
5 changes: 2 additions & 3 deletions .dockerignore
Original file line number Diff line number Diff line change
Expand Up @@ -31,12 +31,11 @@ contracts/.git
!infrastructure/zk
!sdk/zksync-rs
!contracts/system-contracts/bootloader/build/artifacts
!contracts/system-contracts/contracts/artifacts
!contracts/system-contracts/contracts/precompiles/artifacts
!contracts/system-contracts/contracts-preprocessed/artifacts
!contracts/system-contracts/contracts-preprocessed/precompiles/artifacts
!contracts/system-contracts/artifacts-zk
!etc/multivm_bootloaders
!cargo
!bellman-cuda
!core/bin/verification_key_generator_and_server/data/
!prover/vk_setup_data_generator_server_fri/data/
!.github/release-please/manifest.json
3 changes: 2 additions & 1 deletion .github/pull_request_template.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,5 @@
- [ ] Tests for the changes have been added / updated.
- [ ] Documentation comments have been added / updated.
- [ ] Code has been formatted via `zk fmt` and `zk lint`.
- [ ] Spellcheck has been run via `cargo spellcheck --cfg=./spellcheck/era.cfg --code 1`.
- [ ] Spellcheck has been run via `zk spellcheck`.
- [ ] Linkcheck has been run via `zk linkcheck`.
4 changes: 2 additions & 2 deletions .github/release-please/manifest.json
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"sdk/zksync-rs": "0.4.0",
"core": "18.12.0",
"prover": "10.0.2"
"core": "20.5.2",
"prover": "11.0.0"
}
83 changes: 83 additions & 0 deletions .github/workflows/build-contracts.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
name: Build contracts
on:
workflow_call:
inputs:
compilers:
description: 'JSON of required compilers and their versions'
type: string
required: false
default: '[{ "zksolc": ["1.3.14", "1.3.16", "1.3.17", "1.3.1", "1.3.7", "1.3.18", "1.3.19", "1.3.21"] } , { "zkvyper": ["1.3.13"] }]'

jobs:
build-images:
name: Build and upload contracts
runs-on: [matterlabs-ci-runner]
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3
with:
submodules: "recursive"
- name: setup-env
run: |
echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
echo CI=1 >> $GITHUB_ENV
echo $(pwd)/bin >> $GITHUB_PATH
echo CI=1 >> .env
echo IN_DOCKER=1 >> .env
# TODO: Remove after when we can upgrade hardhat-plugins
- name: pre-download compiilers
run: |
# Download needed versions of vyper compiler
# Not sanitized due to unconventional path and tags
mkdir -p ./hardhat-nodejs/compilers-v2/vyper/linux
wget -nv -O ./hardhat-nodejs/compilers-v2/vyper/linux/0.3.10 https://github.com/vyperlang/vyper/releases/download/v0.3.10/vyper.0.3.10+commit.91361694.linux
wget -nv -O ./hardhat-nodejs/compilers-v2/vyper/linux/0.3.3 https://github.com/vyperlang/vyper/releases/download/v0.3.3/vyper.0.3.3+commit.48e326f0.linux
chmod +x ./hardhat-nodejs/compilers-v2/vyper/linux/0.3.10
chmod +x ./hardhat-nodejs/compilers-v2/vyper/linux/0.3.3
COMPILERS_JSON='${{ inputs.compilers }}'
echo "$COMPILERS_JSON" | jq -r '.[] | to_entries[] | .key as $compiler | .value[] | "\(.),\($compiler)"' | while IFS=, read -r version compiler; do
mkdir -p "./hardhat-nodejs/compilers-v2/$compiler"
wget -nv -O "./hardhat-nodejs/compilers-v2/$compiler/${compiler}-v${version}" "https://github.com/matter-labs/${compiler}-bin/releases/download/v${version}/${compiler}-linux-amd64-musl-v${version}"
chmod +x "./hardhat-nodejs/compilers-v2/$compiler/${compiler}-v${version}"
done
- name: start-services
run: |
echo "IMAGE_TAG_SUFFIX=${{ env.IMAGE_TAG_SUFFIX }}" >> .env
mkdir -p ./volumes/postgres
docker compose up -d zk postgres
ci_run sccache --start-server
- name: build contracts
run: |
ci_run git config --global --add safe.directory /usr/src/zksync
ci_run git config --global --add safe.directory /usr/src/zksync/sdk/binaryen
ci_run git config --global --add safe.directory /usr/src/zksync/contracts/system-contracts
ci_run git config --global --add safe.directory /usr/src/zksync/contracts
ci_run zk
ci_run zk clean --all
ci_run zk run yarn
ci_run cp etc/tokens/{test,localhost}.json
ci_run zk compiler all
ci_run zk contract build
ci_run zk f yarn run l2-contracts build
- name: upload contracts
uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392
with:
name: contracts
path: |
./contracts/system-contracts/**/artifacts/
./contracts/system-contracts/**/artifacts-zk/
./contracts/l1-contracts/**/artifacts/
./contracts/l1-contracts/**/artifacts-zk/
./contracts/l2-contracts/**/artifacts/
./contracts/l2-contracts/**/artifacts-zk/
compression-level: 0

- name: Show sccache stats
if: always()
run: |
ci_run sccache --show-stats
ci_run cat /tmp/sccache_log.txt
116 changes: 69 additions & 47 deletions .github/workflows/build-core-template.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,6 @@ on:
description: "DOCKERHUB_TOKEN"
required: true
inputs:
image_tag:
description: "Tag of a built image to deploy"
type: string
required: true
image_tag_suffix:
description: "Optional suffix to override tag name generation"
type: string
Expand All @@ -22,27 +18,25 @@ on:
type: string
default: "push"
required: false
compilers:
description: 'JSON of required compilers and their versions'
type: string
required: false
default: '[{ "zksolc": ["1.3.14", "1.3.16", "1.3.17", "1.3.1", "1.3.7", "1.3.18"] } , { "zkvyper": ["1.3.13"] }]'

jobs:
build-images:
name: Build and Push Docker Images
env:
image_tag: ${{ inputs.image_tag }}
IMAGE_TAG_SUFFIX: ${{ inputs.image_tag_suffix }}
runs-on: [matterlabs-ci-runner]
runs-on: ${{ fromJSON('["matterlabs-ci-runner", "matterlabs-ci-runner-arm"]')[contains(matrix.platforms, 'arm')] }}
strategy:
matrix:
component:
- server-v2
- external-node
- contract-verifier
- cross-external-nodes-checker
- snapshots-creator
components:
- server-v2
- external-node
- contract-verifier
- snapshots-creator
platforms:
- linux/amd64
include:
- components: external-node
platforms: linux/arm64

steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3
with:
Expand All @@ -56,23 +50,11 @@ jobs:
echo CI=1 >> .env
echo IN_DOCKER=1 >> .env
# TODO: Remove after when we can upgrade hardhat-plugins
- name: pre-download compiilers
run: |
# Download needed versions of vyper compiler
# Not sanitized due to unconventional path and tags
mkdir -p ./hardhat-nodejs/compilers-v2/vyper/linux
wget -nv -O ./hardhat-nodejs/compilers-v2/vyper/linux/0.3.10 https://github.com/vyperlang/vyper/releases/download/v0.3.10/vyper.0.3.10+commit.91361694.linux
wget -nv -O ./hardhat-nodejs/compilers-v2/vyper/linux/0.3.3 https://github.com/vyperlang/vyper/releases/download/v0.3.3/vyper.0.3.3+commit.48e326f0.linux
chmod +x ./hardhat-nodejs/compilers-v2/vyper/linux/0.3.10
chmod +x ./hardhat-nodejs/compilers-v2/vyper/linux/0.3.3
COMPILERS_JSON='${{ inputs.compilers }}'
echo "$COMPILERS_JSON" | jq -r '.[] | to_entries[] | .key as $compiler | .value[] | "\(.),\($compiler)"' | while IFS=, read -r version compiler; do
mkdir -p "./hardhat-nodejs/compilers-v2/$compiler"
wget -nv -O "./hardhat-nodejs/compilers-v2/$compiler/${compiler}-v${version}" "https://github.com/matter-labs/${compiler}-bin/releases/download/v${version}/${compiler}-linux-amd64-musl-v${version}"
chmod +x "./hardhat-nodejs/compilers-v2/$compiler/${compiler}-v${version}"
done
- name: Download contracts
uses: actions/download-artifact@v4
with:
name: contracts
path: ./contracts/

- name: start-services
run: |
Expand All @@ -87,30 +69,70 @@ jobs:
ci_run git config --global --add safe.directory /usr/src/zksync/sdk/binaryen
ci_run git config --global --add safe.directory /usr/src/zksync/contracts/system-contracts
ci_run git config --global --add safe.directory /usr/src/zksync/contracts
ci_run zk
ci_run zk clean --all
ci_run zk run yarn
ci_run cp etc/tokens/{test,localhost}.json
ci_run zk compiler all
ci_run zk contract build
ci_run zk f yarn run l2-contracts build
ci_run zk || true
ci_run yarn zk build
ci_run curl -LO https://storage.googleapis.com/matterlabs-setup-keys-us/setup-keys/setup_2\^26.key
- name: login to Docker registries
if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
- name: Login to Docker registries
if: ${{ inputs.action == 'push' }}
run: |
ci_run docker login -u ${{ secrets.DOCKERHUB_USER }} -p ${{ secrets.DOCKERHUB_TOKEN }}
ci_run gcloud auth configure-docker us-docker.pkg.dev,asia-docker.pkg.dev,europe-docker.pkg.dev -q
ci_run gcloud auth configure-docker us-docker.pkg.dev -q
- name: update-images
env:
DOCKER_ACTION: ${{ inputs.action }}
COMPONENT: ${{ matrix.component }}
COMPONENT: ${{ matrix.components }}
PLATFORM: ${{ matrix.platforms }}
run: |
ci_run rustup default nightly-2023-08-21
ci_run zk docker $DOCKER_ACTION $COMPONENT
platform=$(echo $PLATFORM | tr '/' '-')
ci_run zk docker $DOCKER_ACTION --custom-tag=${IMAGE_TAG_SUFFIX} --platform=${PLATFORM} $COMPONENT
- name: Show sccache stats
if: always()
run: |
ci_run sccache --show-stats
ci_run cat /tmp/sccache_log.txt
create_manifest:
name: Create release manifest
runs-on: matterlabs-ci-runner
needs: build-images
if: ${{ inputs.action == 'push' }}
strategy:
matrix:
component:
- name: server-v2
platform: linux/amd64
- name: external-node
platform: linux/amd64,linux/arm64
- name: contract-verifier
platform: linux/amd64
- name: snapshots-creator
platform: linux/amd64
env:
IMAGE_TAG_SUFFIX: ${{ inputs.image_tag_suffix }}
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3
with:
submodules: "recursive"
- name: login to Docker registries
run: |
docker login -u ${{ secrets.DOCKERHUB_USER }} -p ${{ secrets.DOCKERHUB_TOKEN }}
gcloud auth configure-docker us-docker.pkg.dev -q
- name: Create Docker manifest
run: |
docker_repositories=("matterlabs/${{ matrix.component.name }}" "us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component.name }}")
platforms=${{ matrix.component.platform }}
for repo in "${docker_repositories[@]}"; do
platform_tags=""
for platform in ${platforms//,/ }; do
platform=$(echo $platform | tr '/' '-')
platform_tags+=" --amend ${repo}:${IMAGE_TAG_SUFFIX}-${platform}"
done
for manifest in "${repo}:${IMAGE_TAG_SUFFIX}" "${repo}:2.0-${IMAGE_TAG_SUFFIX}" "${repo}:latest" "${repo}:latest2.0"; do
docker manifest create ${manifest} ${platform_tags}
docker manifest push ${manifest}
done
done
9 changes: 6 additions & 3 deletions .github/workflows/build-docker-from-tag.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,16 +49,20 @@ jobs:
run: |
./prover/extract-setup-data-keys.sh >> $GITHUB_OUTPUT
build-contracts:
name: Build contracts
if: contains(github.ref_name, 'core')
uses: ./.github/workflows/build-contracts.yml

build-push-core-images:
name: Build and push image
needs: [setup]
needs: [setup, build-contracts]
uses: ./.github/workflows/build-core-template.yml
if: contains(github.ref_name, 'core')
secrets:
DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
with:
image_tag: ${{ needs.setup.outputs.image_tag }}
image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }}

build-push-prover-images:
Expand All @@ -67,7 +71,6 @@ jobs:
uses: ./.github/workflows/build-prover-template.yml
if: contains(github.ref_name, 'prover')
with:
image_tag: ${{ needs.setup.outputs.image_tag }}
image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }}
ERA_BELLMAN_CUDA_RELEASE: ${{ vars.ERA_BELLMAN_CUDA_RELEASE }}
CUDA_ARCH: "60;70;75;89"
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/build-local-node-docker.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ on:
description: 'JSON of required compilers and their versions'
type: string
required: false
default: '[{ "zksolc": ["1.3.14", "1.3.16", "1.3.17", "1.3.1", "1.3.7", "1.3.18"] } , { "zkvyper": ["1.3.13"] }]'
default: '[{ "zksolc": ["1.3.14", "1.3.16", "1.3.17", "1.3.1", "1.3.7", "1.3.18", "1.3.19", "1.3.21"] } , { "zkvyper": ["1.3.13"] }]'

jobs:
build-images:
Expand Down
45 changes: 39 additions & 6 deletions .github/workflows/build-prover-template.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,6 @@ on:
description: "ERA_BELLMAN_CUDA_RELEASE"
type: string
required: true
image_tag:
description: "Tag of a built image to deploy"
type: string
required: true
image_tag_suffix:
description: "Optional suffix to override tag name generation"
type: string
Expand All @@ -41,7 +37,6 @@ jobs:
build-images:
name: Build and Push Docker Images
env:
image_tag: ${{ inputs.image_tag }}
IMAGE_TAG_SUFFIX: ${{ inputs.image_tag_suffix }}
RUNNER_COMPOSE_FILE: "docker-compose-runner-nightly.yml"
ERA_BELLMAN_CUDA_RELEASE: ${{ inputs.ERA_BELLMAN_CUDA_RELEASE }}
Expand Down Expand Up @@ -93,7 +88,7 @@ jobs:
if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
run: |
ci_run docker login -u ${{ secrets.DOCKERHUB_USER }} -p ${{ secrets.DOCKERHUB_TOKEN }}
ci_run gcloud auth configure-docker us-docker.pkg.dev,asia-docker.pkg.dev,europe-docker.pkg.dev -q
ci_run gcloud auth configure-docker us-docker.pkg.dev -q
# We need to run this only when ERA_BELLMAN_CUDA_RELEASE is not available
# In our case it happens only when PR is created from fork
Expand Down Expand Up @@ -180,3 +175,41 @@ jobs:
run: |
ci_run sccache --show-stats
ci_run cat /tmp/sccache_log.txt
copy-images:
name: Copy images between docker registries
env:
IMAGE_TAG_SUFFIX: ${{ inputs.image_tag_suffix }}
runs-on: matterlabs-ci-runner
needs: build-images
if: ${{ inputs.action == 'push' }}
strategy:
matrix:
component:
- witness-vector-generator
steps:

- name: Set up QEMU
uses: docker/setup-qemu-action@v3

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3

- name: Login to us-central1 GAR
run: |
gcloud auth print-access-token --lifetime=7200 --impersonate-service-account=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com | docker login -u oauth2accesstoken --password-stdin https://us-docker.pkg.dev
- name: Login and push to Asia GAR
run: |
gcloud auth print-access-token --lifetime=7200 --impersonate-service-account=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com | docker login -u oauth2accesstoken --password-stdin https://asia-docker.pkg.dev
docker buildx imagetools create \
--tag asia-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component }}:2.0-${{ inputs.image_tag_suffix }} \
us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component }}:2.0-${{ inputs.image_tag_suffix }}
- name: Login and push to Europe GAR
run: |
gcloud auth print-access-token --lifetime=7200 --impersonate-service-account=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com | docker login -u oauth2accesstoken --password-stdin https://europe-docker.pkg.dev
docker buildx imagetools create \
--tag europe-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component }}:2.0-${{ inputs.image_tag_suffix }} \
us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component }}:2.0-${{ inputs.image_tag_suffix }}
Loading

0 comments on commit 654a05f

Please sign in to comment.