Fix GH Actions build

.github/actions/composite-install-dependecies/action.yml

@@ -3,11 +3,21 @@ description: 'Installs required dependencies for the build'
 runs:
   using: "composite"
   steps:
+    - name: Ensure sudo is installed
+      shell: bash
+      run: |
+        if ! command -v sudo &>/dev/null; then
+            apt-get update
+            apt-get install -y sudo
+        fi
+
     - name: Update and install dependencies
       shell: bash
       run: |
         sudo apt update
         sudo apt install -y \
+          lsb-release \
+          coreutils \
           build-essential \
           libssl-dev \
           cmake \
@@ -17,9 +27,10 @@ runs:
           curl \
           pkg-config \
           wget \
+          \
           libspdlog-dev \
           liblz4-dev=1.9.3-2build2 \
-          libgnutls28-dev=3.7.3-4ubuntu1.3 \
+          libgnutls28-dev \
           libprotobuf-dev=3.12.4-1ubuntu7.22.04.1 \
           libyaml-cpp-dev=0.7.0+dfsg-8build1 \
           libsctp-dev \
@@ -30,4 +41,5 @@ runs:
           libhwloc-dev=2.7.0-2ubuntu1 \
           libc-dev \
           lksctp-tools \
-          libatomic1
+          libatomic1 \
+          clang

.github/workflows/pull_request.yml

@@ -44,6 +44,8 @@ jobs:
   prepare-targets:
     name: Prepare targets strings
     runs-on: ubuntu-22.04
+    if: |
+      always() && !cancelled()
     outputs:
       evm-targets: ${{ steps.get-targets.outputs.evm-targets }}
       prover-targets: ${{ steps.get-targets.outputs.prover-targets }}
@@ -65,6 +67,9 @@ jobs:
     name: Get zkLLVM run to use artifacts from
     needs:
       - handle-syncwith
+    if: |
+      always() && !cancelled() &&
+      (needs.handle-syncwith.result == 'success' || needs.handle-syncwith.result == 'skipped')
     runs-on: ubuntu-22.04
     outputs:
       run-id: ${{ steps.get-run-id.outputs.run-id }}
@@ -127,19 +132,37 @@ jobs:
   build-and-generate-proofs:
     name: Build prover, generate proofs for circuits
     runs-on: ubuntu-22.04
+    container: ubuntu:jammy-20240111
     needs:
       - handle-syncwith
       - get-zkllvm-run
       - prepare-targets
+    if: |
+      always() && !cancelled() &&
+      (needs.handle-syncwith.result == 'success' || needs.handle-syncwith.result == 'skipped') &&
+      (needs.get-zkllvm-run.result == 'success' || needs.get-zkllvm-run.result == 'skipped') &&
+      (needs.prepare-targets.result == 'success' || needs.prepare-targets.result == 'skipped')
     strategy:
       matrix:
         cpp-compiler: [ g++, clang++ ]
         build-type: [ Release ]
       fail-fast: false
     outputs:
       artifact-name: ${{ steps.artifact-name.outputs.merged }}
+    defaults:
+      run:
+        shell: bash
 
     steps:
+      # It takes centuries for GH developers to fix bugs (https://github.com/actions/runner/issues/2058),
+      #   that's why we use `/__w/proof-producer/proof-producer` instead of ${{ github.workspace }}
+
+      - name: Install Git CLI for Ubuntu container
+        # GH runner image does not need it
+        run: |
+          apt-get update
+          apt-get install git -y
+
       - name: Checkout proof-producer
         uses: actions/checkout@v4
         with:
@@ -149,45 +172,34 @@ jobs:
       - name: Install dependencies
         uses: ./.github/actions/composite-install-dependecies
 
-      - name: Install fixes to GH worker
-        run: |
-          # https://github.com/actions/runner-images/issues/8659#issuecomment-1853177960
-          sudo apt-get install --allow-downgrades \
-            libstdc++6=12.3.0-1ubuntu1~22.04 \
-            libgcc-s1=12.3.0-1ubuntu1~22.04
-
       - name: Print toolchain information
         run: |
           git --version
           cc --version
           cmake --version
 
+      - name: Mark git directory as safe
+        # https://github.com/actions/checkout/issues/766
+        run: |
+          git config --global --add safe.directory $PWD
+
       - name: Checkout modules to specified refs
         if: ${{ needs.handle-syncwith.outputs.prs-refs }} != ''
         uses: NilFoundation/ci-cd/actions/[email protected]
         with:
           paths: |
-            ${{ github.workspace }}/**
-            !${{ github.workspace }}/**/.git/**
+            /__w/proof-producer/proof-producer/**
+            !/__w/proof-producer/proof-producer/**/.git/**
           refs: ${{ needs.handle-syncwith.outputs.prs-refs }}
 
       - name: Set usefull strings
         id: strings
-        shell: bash
         run: |
-          echo "build-dir=${{ github.workspace }}/build" >> $GITHUB_OUTPUT
-          echo "dependencies-dir=${{ github.workspace }}/../dependencies" >> $GITHUB_OUTPUT
-          echo "artifact-dir=$(realpath ${{ github.workspace }}/../artifacts)" >> $GITHUB_OUTPUT
-
-          # Get platform version
-          if [[ ${{ runner.os }} == 'Linux' ]]; then
-            platform_version=$(lsb_release -rs)
-            proc_number=$(nproc)
-          elif [[ ${{ runner.os }} == 'macOS' ]]; then
-            platform_version=$(sw_vers -productVersion)
-            proc_number=$(sysctl -n hw.logicalcpu)
-          fi
-          echo "platform-version=${platform_version}" >> $GITHUB_OUTPUT
+          echo "build-dir=/__w/proof-producer/proof-producer/build" >> $GITHUB_OUTPUT
+          echo "dependencies-dir=/__w/proof-producer/proof-producer/../dependencies" >> $GITHUB_OUTPUT
+          echo "artifact-dir=$(realpath /__w/proof-producer/proof-producer/../artifacts)" >> $GITHUB_OUTPUT
+
+          echo "platform-version=$(lsb_release -rs)" >> $GITHUB_OUTPUT
 
       - name: Download circuits and assignments artifact
         uses: dawidd6/action-download-artifact@v3
@@ -198,7 +210,7 @@ jobs:
           run_id: ${{ needs.get-zkllvm-run.outputs.run-id }}
           github_token: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Install boost
+      - name: Install Boost
         uses: MarkusJx/[email protected]
         id: install-boost
         with:
@@ -211,12 +223,16 @@ jobs:
       - name: Configure CMake
         env:
           BOOST_ROOT: "${{ steps.install-boost.outputs.BOOST_ROOT }}"
+        # The MarkusJx/install-boost action names libraries in the format lib...-x64.(a|so). CMake's FindBoost module
+        #   checks whether the CMAKE_CXX_COMPILER_ARCHITECTURE_ID variable is set and adds an architecture suffix based on it. However,
+        #   this variable is not set, so we need to manually specify the suffix using Boost_ARCHITECTURE.
         run: |
           cmake \
             -G "Unix Makefiles" \
             -B build \
             -DCMAKE_CXX_COMPILER=${{ matrix.cpp-compiler }} \
-            -DCMAKE_BUILD_TYPE=${{ matrix.build-type }}
+            -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} \
+            -DBoost_ARCHITECTURE=-x64
 
       - name: Build proof-producer
         working-directory: ${{ steps.strings.outputs.build-dir }}
@@ -236,7 +252,7 @@ jobs:
           fi
           targets_str=$(echo "${{ needs.prepare-targets.outputs.prover-targets }}" | awk '{$1=$1};1' | sed '/^$/d' | tr '\n' ' ' | sed 's/ $//')
           echo "targets from input: ${targets_str}"
-          ${{ github.workspace }}/tests/make_proof_for_pairs.sh ${extra_args} ${targets_str}
+          /__w/proof-producer/proof-producer/tests/make_proof_for_pairs.sh ${extra_args} ${targets_str}
 
       - name: Download transpiler output artifact
         id: download-to-artifact
@@ -291,6 +307,11 @@ jobs:
       - handle-syncwith
       - build-and-generate-proofs
       - prepare-targets
+    if: |
+      always() && !cancelled() &&
+      (needs.handle-syncwith.result == 'success' || needs.handle-syncwith.result == 'skipped') &&
+      (needs.build-and-generate-proofs.result == 'success' || needs.build-and-generate-proofs.result == 'skipped') &&
+      (needs.prepare-targets.result == 'success' || needs.prepare-targets.result == 'skipped')
     uses: NilFoundation/evm-placeholder-verification/.github/workflows/reusable-verify-proofs.yml@5bb7c4b0790f56b3a339e33ba442816fac17ab88
     with:
       artifact-name: ${{ needs.build-and-generate-proofs.outputs.artifact-name }}

.github/workflows/reusable-generate-proofs-linux.yml

@@ -43,6 +43,8 @@ on:
 
 jobs:
   build-and-test:
+    # Testing of proof producer's PRs is run inside container. If GH worker update broke current action,
+    #   consider using container here as well.
     name: Build prover, prove provided circuits
     runs-on: ubuntu-22.04
     strategy:
@@ -123,12 +125,16 @@ jobs:
       - name: Configure CMake
         env:
           BOOST_ROOT: "${{ steps.install-boost.outputs.BOOST_ROOT }}"
+        # The MarkusJx/install-boost action names libraries in the format lib...-x64.(a|so). CMake's FindBoost module
+        #   checks whether the CMAKE_CXX_COMPILER_ARCHITECTURE_ID variable is set and adds an architecture suffix based on it. However,
+        #   this variable is not set, so we need to manually specify the suffix using Boost_ARCHITECTURE.
         run: |
           cmake \
             -G "Unix Makefiles" \
             -B build \
             -DCMAKE_CXX_COMPILER=${{ matrix.cpp-compiler }} \
-            -DCMAKE_BUILD_TYPE=${{ matrix.build-type }}
+            -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} \
+            -DBoost_ARCHITECTURE=-x64
 
       - name: Build proof-producer
         working-directory: ${{ steps.strings.outputs.build-dir }}

bin/jsonizer/CMakeLists.txt

@@ -19,23 +19,8 @@ if (CPACK_PACKAGE_VERSION)
     add_compile_definitions(${UPPER_CURRENT_PROJECT_NAME}_VERSION=${CPACK_PACKAGE_VERSION})
 endif()
 
-if (NOT Boost_FOUND AND NOT CMAKE_CROSSCOMPILING)
+if (NOT Boost_FOUND)
     find_package(Boost COMPONENTS REQUIRED filesystem log log_setup program_options)
-elseif (CMAKE_CROSSCOMPILING)
-    if (NOT TARGET boost)
-        include(ExternalProject)
-        set(Boost_LIBRARIES boost_random)
-        externalproject_add(boost
-                PREFIX ${CMAKE_BINARY_DIR}/libs/boost
-                GIT_REPOSITORY [email protected]:boostorg/boost.git
-                GIT_TAG boost-1.77.0
-                BUILD_IN_SOURCE TRUE
-                CMAKE_ARGS -DCMAKE_CROSSCOMPILING_EMULATOR=${CMAKE_CROSSCOMPILING_EMULATOR} -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}
-                BUILD_COMMAND cmake --build . --target ${Boost_LIBRARIES}
-                INSTALL_COMMAND "")
-    else ()
-        set(Boost_LIBRARIES boost_random)
-    endif ()
 endif ()
 
 cm_setup_version(VERSION 0.1.0)
@@ -107,4 +92,4 @@ elseif (CMAKE_CROSSCOMPILING AND CMAKE_SYSTEM_NAME STREQUAL "Emscripten")
     add_dependencies(${CURRENT_PROJECT_NAME} boost)
 endif ()
 
-install(TARGETS ${CURRENT_PROJECT_NAME} RUNTIME DESTINATION bin)
+install(TARGETS ${CURRENT_PROJECT_NAME} RUNTIME DESTINATION bin)

bin/proof-generator/CMakeLists.txt

@@ -43,23 +43,8 @@ if(ZK_PLACEHOLDER_DEBUG)
     add_definitions(-DZK_PLACEHOLDER_DEBUG_ENABLED)
 endif()
 
-if (NOT Boost_FOUND AND NOT CMAKE_CROSSCOMPILING)
+if (NOT Boost_FOUND)
     find_package(Boost COMPONENTS REQUIRED filesystem log log_setup program_options thread system)
-elseif (CMAKE_CROSSCOMPILING)
-    if (NOT TARGET boost)
-        include(ExternalProject)
-        set(Boost_LIBRARIES boost_random)
-        externalproject_add(boost
-                PREFIX ${CMAKE_BINARY_DIR}/libs/boost
-                GIT_REPOSITORY [email protected]:boostorg/boost.git
-                GIT_TAG boost-1.77.0
-                BUILD_IN_SOURCE TRUE
-                CMAKE_ARGS -DCMAKE_CROSSCOMPILING_EMULATOR=${CMAKE_CROSSCOMPILING_EMULATOR} -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}
-                BUILD_COMMAND cmake --build . --target ${Boost_LIBRARIES}
-                INSTALL_COMMAND "")
-    else ()
-        set(Boost_LIBRARIES boost_random)
-    endif ()
 endif ()
 
 cm_setup_version(VERSION 0.1.0)
@@ -150,4 +135,4 @@ elseif (CMAKE_CROSSCOMPILING AND CMAKE_SYSTEM_NAME STREQUAL "Emscripten")
     add_dependencies(${CURRENT_PROJECT_NAME} boost)
 endif ()
 
-install(TARGETS ${CURRENT_PROJECT_NAME} RUNTIME DESTINATION bin)
+install(TARGETS ${CURRENT_PROJECT_NAME} RUNTIME DESTINATION bin)

bin/proof-generator/src/aspects/configuration.cpp

@@ -87,6 +87,10 @@ namespace nil {
             }
 
             void configuration::write_default_config(const boost::filesystem::path &path) {
+                if (!path.parent_path().empty() && !boost::filesystem::exists(path.parent_path())) {
+                    boost::filesystem::create_directories(path.parent_path());
+                }
+
                 boost::property_tree::ptree tree;
 
                 for (const auto &option : cfgv.options()) {

tests/make_proof_for_pairs.sh

@@ -10,18 +10,17 @@ get_script_dir() {
 make_proof_for_pair() {
     local tbl_file=$1
     local crct_file="${tbl_file%assignment.tbl}circuit.crct"
-    local relative_path="$(realpath --relative-to="$base_dir" "$tbl_file")"
-    # Updated to use the specified output directory
-    local proof_dir="${output_dir}/$(dirname ${relative_path})"
+    local relative_tbl_path="$(realpath --relative-to="$base_dir" "$tbl_file")"
+    local proof_dir="${output_dir}/$(dirname ${relative_tbl_path})"
 
-    local proof_generator_binary="${script_dir}/../build/bin/proof-generator/proof-generator"
+    local proof_generator_binary="${script_dir}/../build/bin/proof-generator-single-threaded/proof-generator-single-threaded"
     if [ "$use_multithreaded" = true ]; then
         proof_generator_binary="${script_dir}/../build/bin/proof-generator-multi-threaded/proof-generator-multi-threaded"
     fi
 
     if [ -f "$crct_file" ]; then
         mkdir -p "$proof_dir"  # Ensure the output directory exists
-        echo -n "Processing $tbl_file and $crct_file (proof will be at $relative_path): "
+        echo -n "Processing $tbl_file and $crct_file (proof will be at $proof_dir): "
         if $proof_generator_binary -t "$tbl_file" --circuit "$crct_file" --proof "$proof_dir/proof.bin"; then
             color_green "success"
         else
@@ -72,13 +71,11 @@ script_dir=$(get_script_dir)
 clean=false
 parse_args "$@"
 
-# Validate output directory
 if [ -z "$output_dir" ]; then
     echo "Output directory not specified. Using default."
     output_dir=$base_dir
 fi
 
-# Function to process files in a directory
 process_directory() {
     local dir=$1
     while read tbl_file; do
@@ -94,7 +91,6 @@ if [ ${#targets[@]} -gt 0 ]; then
         process_directory "$base_dir/$target"
     done
 else
-    # Process the base directory
     process_directory "$base_dir"
 fi