JSON-generator added #86 #236
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and Test | |
| on: | |
| push: | |
| branches: [ master ] | |
| pull_request: | |
| concurrency: | |
| # In master we want to run for every commit, in other branches — only for the last one | |
| group: ${{ | |
| ( github.ref == 'refs/heads/master' && | |
| format('{0}/{1}/{2}', github.workflow, github.ref, github.sha) ) | |
| || | |
| format('{0}/{1}', github.workflow, github.ref) }} | |
| cancel-in-progress: true | |
| env: | |
| BOOST_VERSION: "1.83.0" | |
| CAA_ARTIFACT_NAME: circuits-and-assignments | |
| TO_ARTIFACT_NAME: transpiler-output | |
| INTEGRATION_TESTING_TARGETS: | | |
| arithmetics_cpp_example | |
| polynomial_cpp_example | |
| poseidon_cpp_example | |
| merkle_tree_poseidon_cpp_example | |
| uint_remainder_cpp | |
| uint_shift_left | |
| uint_bit_decomposition | |
| uint_bit_composition | |
| compare_eq_cpp | |
| private_input_cpp | |
| jobs: | |
| handle-syncwith: | |
| if: github.event_name == 'pull_request' | |
| name: Call Reusable SyncWith Handler | |
| uses: NilFoundation/ci-cd/.github/workflows/[email protected] | |
| with: | |
| ci-cd-ref: 'v1.1.2' | |
| secrets: inherit | |
| prepare-targets: | |
| name: Prepare targets strings | |
| runs-on: ubuntu-22.04 | |
| if: | | |
| always() && !cancelled() | |
| outputs: | |
| evm-targets: ${{ steps.get-targets.outputs.evm-targets }} | |
| prover-targets: ${{ steps.get-targets.outputs.prover-targets }} | |
| steps: | |
| - name: Set targets for integration testing | |
| id: get-targets | |
| run: | | |
| targets_str=$(echo "${{ env.INTEGRATION_TESTING_TARGETS }}" | awk 'NF {print "transpiler_output_" $1}') | |
| echo "evm-targets<<EOF" >> $GITHUB_OUTPUT | |
| echo "${targets_str}" >> $GITHUB_OUTPUT | |
| echo "EOF" >> $GITHUB_OUTPUT | |
| echo "prover-targets<<EOF" >> $GITHUB_OUTPUT | |
| echo "${{ env.INTEGRATION_TESTING_TARGETS }}" >> $GITHUB_OUTPUT | |
| echo "EOF" >> $GITHUB_OUTPUT | |
| get-zkllvm-run: | |
| name: Get zkLLVM run to use artifacts from | |
| needs: | |
| - handle-syncwith | |
| if: | | |
| always() && !cancelled() && | |
| (needs.handle-syncwith.result == 'success' || needs.handle-syncwith.result == 'skipped') | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| run-id: ${{ steps.get-run-id.outputs.run-id }} | |
| steps: | |
| - name: Get run ID of zkLLVM | |
| id: get-run-id | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| prs_refs="${{ needs.handle-syncwith.outputs.prs-refs }}" | |
| zkllvm_ref="master" | |
| zkllvm_repo="NilFoundation/zkLLVM" | |
| while read -r line; do | |
| echo "$line" | |
| if [[ $line == "${zkllvm_repo}:"* ]]; then | |
| zkllvm_ref=$(echo "$line" | cut -d ' ' -f 2) | |
| break | |
| fi | |
| done <<< "$prs_refs" | |
| if [[ $zkllvm_ref == refs/pull/* ]]; then | |
| echo "Considering reference ${zkllvm_ref} a pr ref" | |
| pr_number=${zkllvm_ref#refs/pull/} | |
| pr_number=${pr_number%/merge} | |
| sha=$(gh api repos/${zkllvm_repo}/pulls/$pr_number --jq '.head.sha') | |
| elif [[ $zkllvm_ref == refs/tags/* ]]; then | |
| echo "Considering reference ${zkllvm_ref} a tag" | |
| tag=${zkllvm_ref#refs/tags/} | |
| sha=$(gh api repos/${zkllvm_repo}/git/ref/tags/$tag --jq '.object.sha') | |
| else | |
| echo "Considering reference ${zkllvm_ref} a branch" | |
| branch=${zkllvm_ref#refs/heads/} | |
| # We can already fetch run_id here, but better fit to common approach with extra query by sha | |
| sha=$(gh api "repos/${zkllvm_repo}/actions/workflows/build_linux.yml/runs?branch=${branch}&status=completed&per_page=1" \ | |
| --jq '.workflow_runs[0].head_sha') | |
| fi | |
| echo "Using head sha: ${sha}" | |
| run_id=$(gh api "repos/${zkllvm_repo}/actions/workflows/build_linux.yml/runs?head_sha=${sha}&status=completed&per_page=1" \ | |
| --jq '.workflow_runs[0].id') | |
| if [ -z "${run_id}" ]; then | |
| echo no run ID fetched | |
| exit 1 | |
| fi | |
| echo "Run ID: ${run_id}" | |
| for artifact in "${{ env.CAA_ARTIFACT_NAME }}" "${{ env.TO_ARTIFACT_NAME }}"; do | |
| # Check if the artifact exists in the run | |
| if ! gh run view ${run_id} --repo ${zkllvm_repo} | grep "$artifact"; then | |
| echo "Artifact '$artifact' not found in run ${run_id}" | |
| exit 1 | |
| fi | |
| done | |
| echo "run-id=${run_id}" >> $GITHUB_OUTPUT | |
| # This is copied from reusable-generate-proofs-linux.yml | |
| # Disgusting duplication, we should better move all preparation and build to nix, then | |
| # make testing steps a composite action. | |
| build-and-generate-proofs: | |
| name: Build prover, generate proofs for circuits | |
| runs-on: ubuntu-22.04 | |
| container: ubuntu:jammy-20240111 | |
| needs: | |
| - handle-syncwith | |
| - get-zkllvm-run | |
| - prepare-targets | |
| if: | | |
| always() && !cancelled() && | |
| (needs.handle-syncwith.result == 'success' || needs.handle-syncwith.result == 'skipped') && | |
| (needs.get-zkllvm-run.result == 'success' || needs.get-zkllvm-run.result == 'skipped') && | |
| (needs.prepare-targets.result == 'success' || needs.prepare-targets.result == 'skipped') | |
| strategy: | |
| matrix: | |
| cpp-compiler: [ g++, clang++ ] | |
| build-type: [ Release ] | |
| fail-fast: false | |
| outputs: | |
| artifact-name: ${{ steps.artifact-name.outputs.merged }} | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| # It takes centuries for GH developers to fix bugs (https://github.com/actions/runner/issues/2058), | |
| # that's why we use `/__w/proof-producer/proof-producer` instead of ${{ github.workspace }} | |
| - name: Install Git CLI for Ubuntu container | |
| # GH runner image does not need it | |
| run: | | |
| apt-get update | |
| apt-get install git -y | |
| - name: Checkout proof-producer | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: 'recursive' | |
| fetch-depth: 0 # We have to fetch all to find nearest tag for version | |
| - name: Install dependencies | |
| uses: ./.github/actions/composite-install-dependecies | |
| - name: Print toolchain information | |
| run: | | |
| git --version | |
| cc --version | |
| cmake --version | |
| - name: Run clang-format | |
| run: | | |
| if find bin -iname '*.cpp' -o -iname '*.hpp' | xargs clang-format -n --Werror; then | |
| echo "Code formatting is correct" | |
| else | |
| echo "Code formatting differs from clang-format's expectations, run 'find bin -iname *.hpp -o -iname *.cpp | xargs clang-format -i'" | |
| exit 1 | |
| fi | |
| - name: Mark git directory as safe | |
| # https://github.com/actions/checkout/issues/766 | |
| run: | | |
| git config --global --add safe.directory $PWD | |
| - name: Checkout modules to specified refs | |
| if: ${{ needs.handle-syncwith.outputs.prs-refs }} != '' | |
| uses: NilFoundation/ci-cd/actions/[email protected] | |
| with: | |
| paths: | | |
| /__w/proof-producer/proof-producer/** | |
| !/__w/proof-producer/proof-producer/ | |
| !/__w/proof-producer/proof-producer/**/.git/** | |
| refs: ${{ needs.handle-syncwith.outputs.prs-refs }} | |
| - name: Set usefull strings | |
| id: strings | |
| run: | | |
| echo "build-dir=/__w/proof-producer/proof-producer/build" >> $GITHUB_OUTPUT | |
| echo "dependencies-dir=/__w/proof-producer/proof-producer/../dependencies" >> $GITHUB_OUTPUT | |
| echo "artifact-dir=$(realpath /__w/proof-producer/proof-producer/../artifacts)" >> $GITHUB_OUTPUT | |
| echo "platform-version=$(lsb_release -rs)" >> $GITHUB_OUTPUT | |
| - name: Download circuits and assignments artifact | |
| uses: dawidd6/action-download-artifact@v3 | |
| with: | |
| repo: NilFoundation/zkLLVM | |
| name: ${{ env.CAA_ARTIFACT_NAME }} | |
| path: ${{ steps.strings.outputs.artifact-dir }} | |
| run_id: ${{ needs.get-zkllvm-run.outputs.run-id }} | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| skip_unpack: true # It can't unpack such large files (>2Gb for some circuits) | |
| - name: Extract circuits and assignments artifact | |
| working-directory: ${{ steps.strings.outputs.artifact-dir }} | |
| run: | | |
| unzip ${{ env.CAA_ARTIFACT_NAME }}.zip | |
| - name: Install Boost | |
| uses: MarkusJx/[email protected] | |
| id: install-boost | |
| with: | |
| boost_version: ${{ env.BOOST_VERSION }} | |
| # A list of supported versions can be found here: | |
| # https://github.com/MarkusJx/prebuilt-boost/blob/main/versions-manifest.json | |
| platform_version: ${{ steps.strings.outputs.platform-version }} | |
| boost_install_dir: ${{ steps.strings.outputs.dependencies-dir }} | |
| - name: Configure CMake | |
| env: | |
| BOOST_ROOT: "${{ steps.install-boost.outputs.BOOST_ROOT }}" | |
| # The MarkusJx/install-boost action names libraries in the format lib...-x64.(a|so). CMake's FindBoost module | |
| # checks whether the CMAKE_CXX_COMPILER_ARCHITECTURE_ID variable is set and adds an architecture suffix based on it. However, | |
| # this variable is not set, so we need to manually specify the suffix using Boost_ARCHITECTURE. | |
| run: | | |
| cmake \ | |
| -G "Unix Makefiles" \ | |
| -B build \ | |
| -DCMAKE_CXX_COMPILER=${{ matrix.cpp-compiler }} \ | |
| -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} \ | |
| -DBoost_ARCHITECTURE=-x64 | |
| - name: Build proof-producer | |
| working-directory: ${{ steps.strings.outputs.build-dir }} | |
| run: | | |
| make package | |
| - name: List artifacts | |
| working-directory: ${{ steps.strings.outputs.artifact-dir }} | |
| run: find . | |
| - name: Make proofs for pairs | |
| working-directory: ${{ steps.strings.outputs.artifact-dir }} | |
| run: | | |
| extra_args="" | |
| if [[ "true" == "true" ]]; then | |
| extra_args="--multithreaded " | |
| fi | |
| targets_str=$(echo "${{ needs.prepare-targets.outputs.prover-targets }}" | awk '{$1=$1};1' | sed '/^$/d' | tr '\n' ' ' | sed 's/ $//') | |
| echo "targets from input: ${targets_str}" | |
| /__w/proof-producer/proof-producer/tests/make_proof_for_pairs.sh ${extra_args} ${targets_str} | |
| - name: Download transpiler output artifact | |
| id: download-to-artifact | |
| uses: dawidd6/action-download-artifact@v3 | |
| with: | |
| repo: NilFoundation/zkLLVM | |
| name: ${{ env.TO_ARTIFACT_NAME }} | |
| path: ${{ steps.strings.outputs.artifact-dir }} | |
| run_id: ${{ needs.get-zkllvm-run.outputs.run-id }} | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Merge proofs into transpiler output | |
| working-directory: ${{ steps.strings.outputs.artifact-dir }} | |
| run: | | |
| copy_failed=0 | |
| while read dir; do | |
| base_name=${dir#./transpiler_output_} | |
| if [[ -d "$base_name" ]]; then | |
| if ! cp "${base_name}/proof.bin" "${dir}/"; then | |
| echo "Failed to copy proof.bin to ${dir}" >&2 | |
| copy_failed=1 | |
| else | |
| echo "proof.bin added to ${dir}" | |
| fi | |
| else | |
| echo "Error: No matching directory found for ${dir}" >&2 | |
| fi | |
| done < <(find . -type d -name "transpiler_output_*") | |
| if [ $copy_failed -eq 1 ]; then | |
| echo "One or more copy operations failed." | |
| exit 1 | |
| fi | |
| - name: Set aritfact name | |
| id: artifact-name | |
| run: | | |
| echo "merged=transpiler-output-merged-proofs" >> $GITHUB_OUTPUT | |
| - name: Upload merged artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: ${{ steps.artifact-name.outputs.merged }} | |
| path: | | |
| ${{ steps.strings.outputs.artifact-dir }}/transpiler_output_* | |
| - name: Upload .deb package | |
| if: matrix.cpp-compiler == 'g++' && matrix.build-type == 'Release' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: proof-producer.deb | |
| path: | | |
| ${{ steps.strings.outputs.build-dir }}/proof-producer*.deb | |
| verify-proof-producer-proofs: | |
| name: Verify proof-producer proofs with EVM-placeholder | |
| needs: | |
| - handle-syncwith | |
| - build-and-generate-proofs | |
| - prepare-targets | |
| if: | | |
| always() && !cancelled() && | |
| (needs.handle-syncwith.result == 'success' || needs.handle-syncwith.result == 'skipped') && | |
| (needs.build-and-generate-proofs.result == 'success' || needs.build-and-generate-proofs.result == 'skipped') && | |
| (needs.prepare-targets.result == 'success' || needs.prepare-targets.result == 'skipped') | |
| uses: NilFoundation/evm-placeholder-verification/.github/workflows/reusable-verify-proofs.yml@2a0ef5fc67e97be8e3c7b59338cf9eecd030c57d | |
| with: | |
| artifact-name: ${{ needs.build-and-generate-proofs.outputs.artifact-name }} | |
| evm-placeholder-verification-ref: 2a0ef5fc67e97be8e3c7b59338cf9eecd030c57d | |
| refs: ${{ needs.handle-syncwith.outputs.prs-refs }} | |
| test-names: ${{ needs.prepare-targets.outputs.evm-targets }} |