From d2ebacf0ebbf594f3eb8e42ac22e2af3d590e14e Mon Sep 17 00:00:00 2001 From: x-mass <36629999+x-mass@users.noreply.github.com> Date: Wed, 17 Jan 2024 14:53:04 +0000 Subject: [PATCH] Fix GH Actions build --- .../composite-install-dependecies/action.yml | 11 +++ .github/workflows/pull_request.yml | 89 +++++++++++-------- .../reusable-generate-proofs-linux.yml | 8 +- bin/jsonizer/CMakeLists.txt | 21 +---- bin/proof-generator/CMakeLists.txt | 21 +---- 5 files changed, 74 insertions(+), 76 deletions(-) diff --git a/.github/actions/composite-install-dependecies/action.yml b/.github/actions/composite-install-dependecies/action.yml index 540ca3e0..a19d35bc 100644 --- a/.github/actions/composite-install-dependecies/action.yml +++ b/.github/actions/composite-install-dependecies/action.yml @@ -3,11 +3,21 @@ description: 'Installs required dependencies for the build' runs: using: "composite" steps: + - name: Ensure sudo is installed + shell: bash + run: | + if ! command -v sudo &>/dev/null; then + apt-get update + apt-get install -y sudo + fi + - name: Update and install dependencies shell: bash run: | sudo apt update sudo apt install -y \ + lsb-release \ + coreutils \ build-essential \ libssl-dev \ cmake \ @@ -17,6 +27,7 @@ runs: curl \ pkg-config \ wget \ + \ libspdlog-dev \ liblz4-dev=1.9.3-2build2 \ libgnutls28-dev=3.7.3-4ubuntu1.3 \ diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index e86cf963..21776fc1 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -44,6 +44,8 @@ jobs: prepare-targets: name: Prepare targets strings runs-on: ubuntu-22.04 + if: | + always() && !cancelled() outputs: evm-targets: ${{ steps.get-targets.outputs.evm-targets }} prover-targets: ${{ steps.get-targets.outputs.prover-targets }} @@ -65,6 +67,9 @@ jobs: name: Get zkLLVM run to use artifacts from needs: - handle-syncwith + if: | + always() && !cancelled() && + (needs.handle-syncwith.result == 'success' || needs.handle-syncwith.result == 'skipped') runs-on: ubuntu-22.04 outputs: run-id: ${{ steps.get-run-id.outputs.run-id }} @@ -127,10 +132,16 @@ jobs: build-and-generate-proofs: name: Build prover, generate proofs for circuits runs-on: ubuntu-22.04 + container: ubuntu:jammy-20240111 needs: - handle-syncwith - get-zkllvm-run - prepare-targets + if: | + always() && !cancelled() && + (needs.handle-syncwith.result == 'success' || needs.handle-syncwith.result == 'skipped') && + (needs.get-zkllvm-run.result == 'success' || needs.get-zkllvm-run.result == 'skipped') && + (needs.prepare-targets.result == 'success' || needs.prepare-targets.result == 'skipped') strategy: matrix: cpp-compiler: [ g++, clang++ ] @@ -140,6 +151,15 @@ jobs: artifact-name: ${{ steps.artifact-name.outputs.merged }} steps: + # It takes centuries for GH developers to fix bugs (https://github.com/actions/runner/issues/2058), + # that's why we use `/__w/proof-producer/proof-producer` instead of ${{ github.workspace }} + + - name: Install Git CLI for Ubuntu container + # GH runner image does not need it + run: | + apt-get update + apt-get install git -y + - name: Checkout proof-producer uses: actions/checkout@v4 with: @@ -149,56 +169,46 @@ jobs: - name: Install dependencies uses: ./.github/actions/composite-install-dependecies - - name: Install fixes to GH worker - run: | - # https://github.com/actions/runner-images/issues/8659#issuecomment-1853177960 - sudo apt-get install --allow-downgrades \ - libstdc++6=12.3.0-1ubuntu1~22.04 \ - libgcc-s1=12.3.0-1ubuntu1~22.04 - - name: Print toolchain information run: | git --version cc --version cmake --version + - name: Mark git directory as safe + # https://github.com/actions/checkout/issues/766 + run: | + git config --global --add safe.directory $PWD + - name: Checkout modules to specified refs if: ${{ needs.handle-syncwith.outputs.prs-refs }} != '' uses: NilFoundation/ci-cd/actions/recursive-checkout@v1.1.1 with: paths: | - ${{ github.workspace }}/** - !${{ github.workspace }}/**/.git/** + /__w/proof-producer/proof-producer/** + !/__w/proof-producer/proof-producer/**/.git/** refs: ${{ needs.handle-syncwith.outputs.prs-refs }} - name: Set usefull strings id: strings shell: bash run: | - echo "build-dir=${{ github.workspace }}/build" >> $GITHUB_OUTPUT - echo "dependencies-dir=${{ github.workspace }}/../dependencies" >> $GITHUB_OUTPUT - echo "artifact-dir=$(realpath ${{ github.workspace }}/../artifacts)" >> $GITHUB_OUTPUT - - # Get platform version - if [[ ${{ runner.os }} == 'Linux' ]]; then - platform_version=$(lsb_release -rs) - proc_number=$(nproc) - elif [[ ${{ runner.os }} == 'macOS' ]]; then - platform_version=$(sw_vers -productVersion) - proc_number=$(sysctl -n hw.logicalcpu) - fi - echo "platform-version=${platform_version}" >> $GITHUB_OUTPUT - - - name: Download circuits and assignments artifact - uses: dawidd6/action-download-artifact@v3 - with: - repo: NilFoundation/zkLLVM - name: ${{ env.CAA_ARTIFACT_NAME }} - path: ${{ steps.strings.outputs.artifact-dir }} - run_id: ${{ needs.get-zkllvm-run.outputs.run-id }} - github_token: ${{ secrets.GITHUB_TOKEN }} - - - name: Install boost + echo "build-dir=/__w/proof-producer/proof-producer/build" >> $GITHUB_OUTPUT + echo "dependencies-dir=/__w/proof-producer/proof-producer/../dependencies" >> $GITHUB_OUTPUT + echo "artifact-dir=$(realpath /__w/proof-producer/proof-producer/../artifacts)" >> $GITHUB_OUTPUT + + echo "platform-version=$(lsb_release -rs)" >> $GITHUB_OUTPUT + + # - name: Download circuits and assignments artifact + # uses: dawidd6/action-download-artifact@v3 + # with: + # repo: NilFoundation/zkLLVM + # name: ${{ env.CAA_ARTIFACT_NAME }} + # path: ${{ steps.strings.outputs.artifact-dir }} + # run_id: ${{ needs.get-zkllvm-run.outputs.run-id }} + # github_token: ${{ secrets.GITHUB_TOKEN }} + + - name: Download Boost uses: MarkusJx/install-boost@v2.4.4 id: install-boost with: @@ -211,12 +221,16 @@ jobs: - name: Configure CMake env: BOOST_ROOT: "${{ steps.install-boost.outputs.BOOST_ROOT }}" + # The MarkusJx/install-boost action names libraries in the format lib...-x64.(a|so). CMake's FindBoost module + # checks whether the CMAKE_CXX_COMPILER_ARCHITECTURE_ID variable is set and adds an architecture suffix based on it. However, + # this variable is not set, so we need to manually specify the suffix using Boost_ARCHITECTURE. run: | cmake \ -G "Unix Makefiles" \ -B build \ -DCMAKE_CXX_COMPILER=${{ matrix.cpp-compiler }} \ - -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} + -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} \ + -DBoost_ARCHITECTURE=-x64 - name: Build proof-producer working-directory: ${{ steps.strings.outputs.build-dir }} @@ -236,7 +250,7 @@ jobs: fi targets_str=$(echo "${{ needs.prepare-targets.outputs.prover-targets }}" | awk '{$1=$1};1' | sed '/^$/d' | tr '\n' ' ' | sed 's/ $//') echo "targets from input: ${targets_str}" - ${{ github.workspace }}/tests/make_proof_for_pairs.sh ${extra_args} ${targets_str} + /__w/proof-producer/proof-producer/tests/make_proof_for_pairs.sh ${extra_args} ${targets_str} - name: Download transpiler output artifact id: download-to-artifact @@ -291,6 +305,11 @@ jobs: - handle-syncwith - build-and-generate-proofs - prepare-targets + if: | + always() && !cancelled() && + (needs.handle-syncwith.result == 'success' || needs.handle-syncwith.result == 'skipped') && + (needs.build-and-generate-proofs.result == 'success' || needs.build-and-generate-proofs.result == 'skipped') && + (needs.prepare-targets.result == 'success' || needs.prepare-targets.result == 'skipped') uses: NilFoundation/evm-placeholder-verification/.github/workflows/reusable-verify-proofs.yml@5bb7c4b0790f56b3a339e33ba442816fac17ab88 with: artifact-name: ${{ needs.build-and-generate-proofs.outputs.artifact-name }} diff --git a/.github/workflows/reusable-generate-proofs-linux.yml b/.github/workflows/reusable-generate-proofs-linux.yml index 133940f3..29af0a7d 100644 --- a/.github/workflows/reusable-generate-proofs-linux.yml +++ b/.github/workflows/reusable-generate-proofs-linux.yml @@ -43,6 +43,8 @@ on: jobs: build-and-test: + # Testing for proof producer's PRs is run inside container. If GH worker update broke current action, + # consider using container here as well. name: Build prover, prove provided circuits runs-on: ubuntu-22.04 strategy: @@ -123,12 +125,16 @@ jobs: - name: Configure CMake env: BOOST_ROOT: "${{ steps.install-boost.outputs.BOOST_ROOT }}" + # The MarkusJx/install-boost action names libraries in the format lib...-x64.(a|so). CMake's FindBoost module + # checks whether the CMAKE_CXX_COMPILER_ARCHITECTURE_ID variable is set and adds an architecture suffix based on it. However, + # this variable is not set, so we need to manually specify the suffix using Boost_ARCHITECTURE. run: | cmake \ -G "Unix Makefiles" \ -B build \ -DCMAKE_CXX_COMPILER=${{ matrix.cpp-compiler }} \ - -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} + -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} \ + -DBoost_ARCHITECTURE=-x64 - name: Build proof-producer working-directory: ${{ steps.strings.outputs.build-dir }} diff --git a/bin/jsonizer/CMakeLists.txt b/bin/jsonizer/CMakeLists.txt index d1abf78a..a99b17df 100644 --- a/bin/jsonizer/CMakeLists.txt +++ b/bin/jsonizer/CMakeLists.txt @@ -19,25 +19,6 @@ if (CPACK_PACKAGE_VERSION) add_compile_definitions(${UPPER_CURRENT_PROJECT_NAME}_VERSION=${CPACK_PACKAGE_VERSION}) endif() -if (NOT Boost_FOUND AND NOT CMAKE_CROSSCOMPILING) - find_package(Boost COMPONENTS REQUIRED filesystem log log_setup program_options) -elseif (CMAKE_CROSSCOMPILING) - if (NOT TARGET boost) - include(ExternalProject) - set(Boost_LIBRARIES boost_random) - externalproject_add(boost - PREFIX ${CMAKE_BINARY_DIR}/libs/boost - GIT_REPOSITORY git@github.com:boostorg/boost.git - GIT_TAG boost-1.77.0 - BUILD_IN_SOURCE TRUE - CMAKE_ARGS -DCMAKE_CROSSCOMPILING_EMULATOR=${CMAKE_CROSSCOMPILING_EMULATOR} -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} - BUILD_COMMAND cmake --build . --target ${Boost_LIBRARIES} - INSTALL_COMMAND "") - else () - set(Boost_LIBRARIES boost_random) - endif () -endif () - cm_setup_version(VERSION 0.1.0) # get header files; only needed by CMake generators, @@ -107,4 +88,4 @@ elseif (CMAKE_CROSSCOMPILING AND CMAKE_SYSTEM_NAME STREQUAL "Emscripten") add_dependencies(${CURRENT_PROJECT_NAME} boost) endif () -install(TARGETS ${CURRENT_PROJECT_NAME} RUNTIME DESTINATION bin) \ No newline at end of file +install(TARGETS ${CURRENT_PROJECT_NAME} RUNTIME DESTINATION bin) diff --git a/bin/proof-generator/CMakeLists.txt b/bin/proof-generator/CMakeLists.txt index 18bd0ce8..fa64d1fe 100644 --- a/bin/proof-generator/CMakeLists.txt +++ b/bin/proof-generator/CMakeLists.txt @@ -43,25 +43,6 @@ if(ZK_PLACEHOLDER_DEBUG) add_definitions(-DZK_PLACEHOLDER_DEBUG_ENABLED) endif() -if (NOT Boost_FOUND AND NOT CMAKE_CROSSCOMPILING) - find_package(Boost COMPONENTS REQUIRED filesystem log log_setup program_options thread system) -elseif (CMAKE_CROSSCOMPILING) - if (NOT TARGET boost) - include(ExternalProject) - set(Boost_LIBRARIES boost_random) - externalproject_add(boost - PREFIX ${CMAKE_BINARY_DIR}/libs/boost - GIT_REPOSITORY git@github.com:boostorg/boost.git - GIT_TAG boost-1.77.0 - BUILD_IN_SOURCE TRUE - CMAKE_ARGS -DCMAKE_CROSSCOMPILING_EMULATOR=${CMAKE_CROSSCOMPILING_EMULATOR} -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} - BUILD_COMMAND cmake --build . --target ${Boost_LIBRARIES} - INSTALL_COMMAND "") - else () - set(Boost_LIBRARIES boost_random) - endif () -endif () - cm_setup_version(VERSION 0.1.0) # get header files; only needed by CMake generators, @@ -150,4 +131,4 @@ elseif (CMAKE_CROSSCOMPILING AND CMAKE_SYSTEM_NAME STREQUAL "Emscripten") add_dependencies(${CURRENT_PROJECT_NAME} boost) endif () -install(TARGETS ${CURRENT_PROJECT_NAME} RUNTIME DESTINATION bin) \ No newline at end of file +install(TARGETS ${CURRENT_PROJECT_NAME} RUNTIME DESTINATION bin)