From 344e4d2099546d0bdd8f712d3994edb39846263e Mon Sep 17 00:00:00 2001 From: Mamy Ratsimbazafy Date: Sat, 25 Jan 2025 13:14:32 +0100 Subject: [PATCH 01/12] CI: add Linux ARM64 --- .github/workflows/ci.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8267e437..2cddebc4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,6 +31,14 @@ jobs: cpu: amd64 TEST_LANG: c BACKEND: ASM + - os: linux + cpu: arm64 + TEST_LANG: c + BACKEND: NO_ASM + - os: linux + cpu: arm64 + TEST_LANG: c + BACKEND: ASM - os: windows cpu: amd64 From ab5b82a8353654abb9e19e80ae0b0d25dc8d95ec Mon Sep 17 00:00:00 2001 From: Mamy Ratsimbazafy Date: Sat, 25 Jan 2025 13:49:09 +0100 Subject: [PATCH 02/12] CI: explicit permissions --- .github/workflows/ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2cddebc4..7b470fdf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,12 @@ name: Constantine CI on: [push, pull_request] +# https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#permissions +permissions: + actions: write + checks: write + contents: read # list commits, change to write to create a release + jobs: build: strategy: From b26bafaeb2b1bd06083c3c5287289a44477acca8 Mon Sep 17 00:00:00 2001 From: Mamy Ratsimbazafy Date: Sat, 25 Jan 2025 14:19:40 +0100 Subject: [PATCH 03/12] CI: use actions/cache@v4 (v2 is deprecated on Feb 1st, and v3 was supposed to be as well) --- .github/workflows/ci.yml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7b470fdf..49dd88c1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ jobs: # version-2-0 is crashing due to https://github.com/mratsim/constantine/issues/471 # version-1-6 and version-2-0 have issues with templates in typedef operating on a generic for Fp12 tower in https://github.com/mratsim/constantine/pull/485 # Hence we only test and officially sypport 2.2.0, though 99% of Constantine should work on older compilers - nim_version: [version-2-2] + nim_version: ['2.2.x'] rust_toolchain: [stable] # [beta, nightly] go_toolchain: [stable] target: @@ -24,10 +24,12 @@ jobs: cpu: i386 TEST_LANG: c BACKEND: NO_ASM - - os: linux - cpu: i386 - TEST_LANG: c - BACKEND: ASM + + # We only need to test 32-bit for targets like RISC-V that don't have ASM. + # - os: linux + # cpu: i386 + # TEST_LANG: c + # BACKEND: ASM - os: linux cpu: amd64 @@ -104,14 +106,14 @@ jobs: - name: Restore MinGW-W64 (Windows) from cache if: runner.os == 'Windows' id: windows-mingw-cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: external/mingw-${{ matrix.target.cpu }} key: 'mingw-${{ matrix.target.cpu }}' - name: Restore Nim DLLs dependencies (Windows) from cache if: runner.os == 'Windows' id: windows-dlls-cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: external/dlls-${{ matrix.target.cpu }} key: 'dlls-${{ matrix.target.cpu }}' @@ -152,7 +154,7 @@ jobs: - name: Restore Nim from cache if: matrix.nim_version != 'devel' id: nim-compiler-cache - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: '${{ github.workspace }}/nim-${{ matrix.nim_version }}-${{ matrix.target.cpu }}' key: 'nim-${{ matrix.target.cpu }}-${{ matrix.nim_version }}' From dc902cf059aa42d4aeabdca6aa3db1e4ef26f719 Mon Sep 17 00:00:00 2001 From: Mamy Ratsimbazafy Date: Sat, 25 Jan 2025 15:10:41 +0100 Subject: [PATCH 04/12] CI: use ARM builder images revert version spec change (was working on something in parallel) --- .github/workflows/ci.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 49dd88c1..f2bc52fa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,13 @@ jobs: # version-2-0 is crashing due to https://github.com/mratsim/constantine/issues/471 # version-1-6 and version-2-0 have issues with templates in typedef operating on a generic for Fp12 tower in https://github.com/mratsim/constantine/pull/485 # Hence we only test and officially sypport 2.2.0, though 99% of Constantine should work on older compilers - nim_version: ['2.2.x'] + + # For ARM support on Linux we need to set a specific builder + # either ubuntu-22.04-arm + # or ubuntu-24.04-arm + # We might as well test on 22.04 for ARM + + nim_version: ['version-2-2'] rust_toolchain: [stable] # [beta, nightly] go_toolchain: [stable] target: @@ -41,10 +47,12 @@ jobs: BACKEND: ASM - os: linux cpu: arm64 + builder: ubuntu-22.04-arm TEST_LANG: c BACKEND: NO_ASM - os: linux cpu: arm64 + builder: ubuntu-22.04-arm TEST_LANG: c BACKEND: ASM From 4a1b5bcfe3335dcb4a0d5118361e441ae9a2dbfe Mon Sep 17 00:00:00 2001 From: Mamy Ratsimbazafy Date: Sat, 25 Jan 2025 20:32:02 +0100 Subject: [PATCH 05/12] CI: remove TEST_LANG option to build with cpp --- .github/workflows/ci.yml | 14 ++------------ constantine.nimble | 7 +------ 2 files changed, 3 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f2bc52fa..3041764c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,57 +22,47 @@ jobs: # or ubuntu-24.04-arm # We might as well test on 22.04 for ARM - nim_version: ['version-2-2'] + nim_version: [version-2-2] rust_toolchain: [stable] # [beta, nightly] go_toolchain: [stable] target: - os: linux cpu: i386 - TEST_LANG: c BACKEND: NO_ASM # We only need to test 32-bit for targets like RISC-V that don't have ASM. # - os: linux # cpu: i386 - # TEST_LANG: c # BACKEND: ASM - os: linux cpu: amd64 - TEST_LANG: c BACKEND: NO_ASM - os: linux cpu: amd64 - TEST_LANG: c BACKEND: ASM - os: linux cpu: arm64 builder: ubuntu-22.04-arm - TEST_LANG: c BACKEND: NO_ASM - os: linux cpu: arm64 builder: ubuntu-22.04-arm - TEST_LANG: c BACKEND: ASM - os: windows cpu: amd64 - TEST_LANG: c BACKEND: NO_ASM - os: windows cpu: amd64 - TEST_LANG: c BACKEND: ASM # MacOS Github agents are now using ARM and we need Nim ARM nightlies: https://github.com/mratsim/constantine/issues/372 # - os: macos # cpu: amd64 - # TEST_LANG: c # BACKEND: NO_ASM # - os: macos # cpu: amd64 - # TEST_LANG: c # BACKEND: ASM include: - target: @@ -84,7 +74,7 @@ jobs: - target: os: windows builder: windows-latest - name: '${{ matrix.target.os }}-${{ matrix.target.cpu }}-${{ matrix.target.TEST_LANG }}-${{ matrix.target.BACKEND }} (${{ matrix.nim_version }})' + name: '${{ matrix.target.os }}-${{ matrix.target.cpu }}-${{ matrix.target.BACKEND }} (${{ matrix.nim_version }})' runs-on: ${{ matrix.builder }} steps: - name: Get branch name diff --git a/constantine.nimble b/constantine.nimble index 5b5c3c43..fee1e80e 100644 --- a/constantine.nimble +++ b/constantine.nimble @@ -764,12 +764,7 @@ proc clearParallelBuild() = rmFile(buildParallel) proc setupTestCommand(flags, path: string): string = - var lang = "c" - if existsEnv"TEST_LANG": - lang = getEnv"TEST_LANG" - - return "nim " & lang & - " -r " & + return "nim c -r" & flags & releaseBuildOptions() & " --outdir:build/test_suite " & From c529ab642baebdc33b47614faeb64d2bd7b6705c Mon Sep 17 00:00:00 2001 From: Mamy Ratsimbazafy Date: Sat, 25 Jan 2025 23:48:08 +0100 Subject: [PATCH 06/12] CI: revamp installer to support ARM --- .github/workflows/ci.yml | 168 +++++++-------- .github/workflows/install_nim.sh | 354 +++++++++++++++++++++++++++++++ constantine/platforms/config.nim | 2 +- 3 files changed, 440 insertions(+), 84 deletions(-) create mode 100644 .github/workflows/install_nim.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3041764c..0f629f15 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,7 @@ permissions: contents: read # list commits, change to write to create a release jobs: - build: + test: strategy: fail-fast: false max-parallel: 20 @@ -17,7 +17,7 @@ jobs: # version-1-6 and version-2-0 have issues with templates in typedef operating on a generic for Fp12 tower in https://github.com/mratsim/constantine/pull/485 # Hence we only test and officially sypport 2.2.0, though 99% of Constantine should work on older compilers - # For ARM support on Linux we need to set a specific builder + # For ARM support on Linux we need to set a specific host # either ubuntu-22.04-arm # or ubuntu-24.04-arm # We might as well test on 22.04 for ARM @@ -25,57 +25,66 @@ jobs: nim_version: [version-2-2] rust_toolchain: [stable] # [beta, nightly] go_toolchain: [stable] + target: + # We only need to test 32-bit for targets like RISC-V that don't have ASM. - os: linux cpu: i386 - BACKEND: NO_ASM - - # We only need to test 32-bit for targets like RISC-V that don't have ASM. - # - os: linux - # cpu: i386 - # BACKEND: ASM + ctt_backend: NO_ASM + host: ubuntu-latest - os: linux cpu: amd64 - BACKEND: NO_ASM + ctt_backend: NO_ASM + host: ubuntu-latest - os: linux cpu: amd64 - BACKEND: ASM - - os: linux - cpu: arm64 - builder: ubuntu-22.04-arm - BACKEND: NO_ASM + ctt_backend: ASM + host: ubuntu-latest + + # ASM code for ARM Linux is not available yet - os: linux cpu: arm64 - builder: ubuntu-22.04-arm - BACKEND: ASM + ctt_backend: NO_ASM + host: ubuntu-22.04-arm - os: windows cpu: amd64 - BACKEND: NO_ASM + ctt_backend: NO_ASM + host: windows-latest - os: windows cpu: amd64 - BACKEND: ASM - - # MacOS Github agents are now using ARM and we need Nim ARM nightlies: https://github.com/mratsim/constantine/issues/372 - # - os: macos - # cpu: amd64 - # BACKEND: NO_ASM - # - os: macos - # cpu: amd64 - # BACKEND: ASM + ctt_backend: ASM + host: windows-latest + + - os: macos + cpu: arm64 + ctt_backend: NO_ASM + host: macos-latest + - os: macos + cpu: arm64 + ctt_backend: ASM + host: macos-latest + include: - target: os: linux - builder: ubuntu-latest - # - target: - # os: macos - # builder: macos-latest + nim_channel: nightly + - target: + os: linux + nim_channel: nightly + - target: + os: macos + # MacOS Github agents are now using ARM and there are no ARM nightlies: + # https://github.com/mratsim/constantine/issues/372 + # So we build from source + nim_channel: source - target: os: windows - builder: windows-latest - name: '${{ matrix.target.os }}-${{ matrix.target.cpu }}-${{ matrix.target.BACKEND }} (${{ matrix.nim_version }})' - runs-on: ${{ matrix.builder }} + nim_channel: nightly + + name: '${{ matrix.target.os }}-${{ matrix.target.cpu }}-${{ matrix.target.ctt_backend }} (${{ matrix.nim_version }})' + runs-on: ${{ matrix.target.host }} steps: - name: Get branch name shell: bash @@ -149,31 +158,46 @@ jobs: echo '${{ github.workspace }}'"/external/mingw-${{ matrix.target.cpu }}/bin" >> $GITHUB_PATH echo '${{ github.workspace }}'"/external/dlls-${{ matrix.target.cpu }}" >> $GITHUB_PATH - - name: Restore Nim from cache - if: matrix.nim_version != 'devel' + - name: Get latest Nim commit hash & install dir + id: versions + shell: bash + run: | + getHash() { + git ls-remote "https://github.com/$1" "${2:-HEAD}" | cut -f 1 + } + nimHash=$(getHash nim-lang/Nim '${{ matrix.branch }}') + installdir="nim-${{ matrix.target.os }}-${{ matrix.target.cpu }}-${{ matrix.nim_version }}-$nimHash" + echo "##[set-output name=installdir;]$(echo $installdir)" + echo "Nim installation directory: $installdir" + + # Nightlies are already on Github + - name: Restore Nim from cache (except nightlies) + if: matrix.nim_channel != 'nightly' id: nim-compiler-cache uses: actions/cache@v4 with: - path: '${{ github.workspace }}/nim-${{ matrix.nim_version }}-${{ matrix.target.cpu }}' - key: 'nim-${{ matrix.target.cpu }}-${{ matrix.nim_version }}' + path: '${{ github.workspace }}/${{ steps.versions.outputs.installdir }}' + key: '${{ steps.versions.outputs.installdir }}' - name: Setup Nim if: steps.nim-compiler-cache.outputs.cache-hit != 'true' - uses: alaviss/setup-nim@0.1.1 - with: - path: 'nim-${{ matrix.nim_version }}-${{ matrix.target.cpu }}' - version: ${{ matrix.nim_version }} - architecture: ${{ matrix.target.cpu }} - add-to-path: false - - - name: Path to cached Nim shell: bash run: | - echo '${{ github.workspace }}'"/nim-${{ matrix.nim_version }}-${{ matrix.target.cpu }}/bin" >> $GITHUB_PATH + bash constantine/.github/workflows/install_nim.sh \ + --nim-channel ${{ matrix.nim_channel }} \ + --nim-version ${{ matrix.nim_version }} \ + --os ${{ matrix.target.os }} \ + --arch ${{ matrix.target.cpu }} \ + --install-dir ${{ steps.versions.outputs.installdir }} + + - name: Path to Nim + shell: bash + run: | + echo '${{ github.workspace }}'/"${{ steps.versions.outputs.installdir }}"/bin >> $GITHUB_PATH echo '${{ github.workspace }}'"/.nimble/bin" >> $GITHUB_PATH - name: Install test dependencies (Linux amd64) - if: runner.os == 'Linux' && matrix.target.cpu == 'amd64' + if: runner.os == 'Linux' && matrix.target.cpu != 'i386' run: | sudo DEBIAN_FRONTEND='noninteractive' apt-get install \ --no-install-recommends -yq \ @@ -211,29 +235,6 @@ jobs: if: runner.os == 'macOS' run: | brew install gmp - - name: Install Clang with Intel Assembly support (macOS) - if: runner.os == 'macOS' - # Apple Clang does not support Intel syntax due to missing - # commit: https://github.com/llvm/llvm-project/commit/ae98182cf7341181e4aa815c372a072dec82779f - # Revision: https://reviews.llvm.org/D113707 - # Apple upstream: FB12137688 - - # Furthermore, Apple Clang can delete symbols when building a static library - # in particular the hasAdxImpl bool for CPU runtime detection. - - # run: | - # mkdir -p external/bin - # cat << EOF > external/bin/clang - # #!/bin/bash - # exec $(brew --prefix llvm@15)/bin/clang "\$@" - # EOF - # cat << EOF > external/bin/clang++ - # #!/bin/bash - # exec $(brew --prefix llvm@15)/bin/clang++ "\$@" - # EOF - # chmod 755 external/bin/{clang,clang++} - # echo '${{ github.workspace }}/external/bin' >> $GITHUB_PATH - run: echo "$(brew --prefix llvm@15)/bin" >> $GITHUB_PATH - name: Setup MSYS2 (Windows) if: runner.os == 'Windows' @@ -257,11 +258,12 @@ jobs: # gcc is an alias to Apple Clang on MacOS run: | nim -v + nimble -v gcc -v clang -v go version rustup --version - if [[ '${{ matrix.target.cpu }}' != 'i386' && '${{ runner.os }}' != 'Windows' ]]; then + if [[ '${{ matrix.target.cpu }}' != 'i386' && '${{ runner.os }}' == 'Linux' ]]; then llvm-config --version fi if [[ '${{ runner.os }}' == 'Linux' ]]; then @@ -295,7 +297,7 @@ jobs: go mod download -modfile=go_test.mod - name: Run Constantine as C library tests (UNIX with Assembly) - if: runner.os != 'Windows' && matrix.target.BACKEND == 'ASM' + if: runner.os != 'Windows' && matrix.target.ctt_backend == 'ASM' shell: bash run: | cd constantine @@ -303,7 +305,7 @@ jobs: nimble make_headers --verbose nimble test_lib --verbose - name: Run Constantine as C library tests (UNIX no Assembly) - if: runner.os != 'Windows' && matrix.target.BACKEND == 'NO_ASM' + if: runner.os != 'Windows' && matrix.target.ctt_backend == 'NO_ASM' shell: bash run: | cd constantine @@ -313,7 +315,7 @@ jobs: - name: Run Constantine as C library tests (Windows with Assembly) # So "test_bindings" uses C and can find GMP # but nim-gmp cannot find GMP on Windows CI - if: runner.os == 'Windows' && matrix.target.BACKEND == 'ASM' + if: runner.os == 'Windows' && matrix.target.ctt_backend == 'ASM' shell: msys2 {0} run: | cd constantine @@ -323,7 +325,7 @@ jobs: - name: Run Constantine as C library tests (Windows no Assembly) # So "test_bindings" uses C and can find GMP # but nim-gmp cannot find GMP on Windows CI - if: runner.os == 'Windows' && matrix.target.BACKEND == 'NO_ASM' + if: runner.os == 'Windows' && matrix.target.ctt_backend == 'NO_ASM' shell: msys2 {0} run: | cd constantine @@ -340,14 +342,14 @@ jobs: go test -modfile=../go_test.mod - name: Run Constantine as Rust library tests (with Assembly) - if: matrix.target.BACKEND == 'ASM' && matrix.target.cpu != 'i386' + if: matrix.target.ctt_backend == 'ASM' && matrix.target.cpu != 'i386' shell: bash # We need to deactivate the assembly (used by default for benches) run: | cd constantine cargo test -- --nocapture - name: Run Constantine as Rust library tests (NO Assembly) - if: matrix.target.BACKEND == 'NO_ASM' && matrix.target.cpu != 'i386' + if: matrix.target.ctt_backend == 'NO_ASM' && matrix.target.cpu != 'i386' shell: bash # We need to deactivate the assembly (used by default for benches) run: | @@ -357,7 +359,7 @@ jobs: - name: Compile Constantine Zkalc benchmark (no assembly) # Skip 32-bit as that would need clang-multilib or -m32 # Skip Windows as clang throws fatal error LNK1107 - if: matrix.target.BACKEND == 'NO_ASM' && matrix.target.cpu != 'i386' && runner.os != 'Windows' + if: matrix.target.ctt_backend == 'NO_ASM' && matrix.target.cpu != 'i386' && runner.os != 'Windows' shell: bash run: | cd constantine @@ -366,20 +368,20 @@ jobs: - name: Compile Constantine Zkalc benchmark (with assembly) # Skip 32-bit as that would need clang-multilib or -m32 # Skip Windows as clang throws fatal error LNK1107 - if: matrix.target.BACKEND == 'ASM' && matrix.target.cpu != 'i386' && runner.os != 'Windows' + if: matrix.target.ctt_backend == 'ASM' && matrix.target.cpu != 'i386' && runner.os != 'Windows' shell: bash run: | cd constantine nimble make_zkalc - name: Run Constantine in-depth tests (Unix - with GMP, with Assembly) - if: runner.os != 'Windows' && matrix.target.BACKEND == 'ASM' + if: runner.os != 'Windows' && matrix.target.ctt_backend == 'ASM' shell: bash run: | cd constantine nimble test_parallel --verbose - name: Run Constantine in-depth tests (Unix - with GMP, no Assembly) - if: runner.os != 'Windows' && matrix.target.BACKEND == 'NO_ASM' + if: runner.os != 'Windows' && matrix.target.ctt_backend == 'NO_ASM' shell: bash run: | cd constantine @@ -390,7 +392,7 @@ jobs: # but nim-gmp cannot find GMP on Windows CI # Also need to workaround asynctools not being able to create pipes https://github.com/nim-lang/Nim/issues/23118 # And LTO impossible constraint in the deneb_kzg test (but not MSM for some reason) - if: runner.os == 'Windows' && matrix.target.BACKEND == 'ASM' + if: runner.os == 'Windows' && matrix.target.ctt_backend == 'ASM' shell: msys2 {0} run: | cd constantine @@ -403,7 +405,7 @@ jobs: # So "test_bindings" uses C and can find GMP # but nim-gmp cannot find GMP on Windows CI # Also need to workaround asynctools not being able to create pipes https://github.com/nim-lang/Nim/issues/23118 - if: runner.os == 'Windows' && matrix.target.BACKEND == 'NO_ASM' + if: runner.os == 'Windows' && matrix.target.ctt_backend == 'NO_ASM' shell: msys2 {0} run: | cd constantine diff --git a/.github/workflows/install_nim.sh b/.github/workflows/install_nim.sh new file mode 100644 index 00000000..bdfc524c --- /dev/null +++ b/.github/workflows/install_nim.sh @@ -0,0 +1,354 @@ +#!/usr/bin/env bash + +# Constantine +# Copyright (c) 2018-2019 Status Research & Development GmbH +# Copyright (c) 2020-Present Mamy André-Ratsimbazafy +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at http://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +########################################## +# +# Nim installer +# Constantine CI Flavor +# +########################################## + +# This script aims to cover all the needs of Constantine CI +# and replaces: +# - Constantine old script in YAML: +# https://github.com/mratsim/constantine/blob/dc73c71/.github/workflows/ci.yml +# - https://github.com/alaviss/setup-nim +# which fails in CI for Linux ARM with "cannot execute binary file: Exec format error" +# unsure why, the same binary works on a Raspberry Pi 4 +# +# The official https://github.com/nim-lang/setup-nimble-action +# only deals with nimble. +# and the repo it forked from https://github.com/jiro4989/setup-nim-action +# doesn't support nightly branches only devel +# +# Note: we don't rm temp files in the script to prevent any risk of data loss. +# Those are scary stories: +# - https://github.com/ValveSoftware/steam-for-linux/issues/3671 +# "Moved ~/.local/share/steam. Ran steam. It deleted everything on system owned by user." +# - https://github.com/MrMEEE/bumblebee-Old-and-abbandoned/issues/123 +# "install script does rm -rf /usr for ubuntu" + +set -eu +set -o pipefail +shopt -s extglob + +# Global variables +# ------------------------- +DATE_FORMAT="%Y-%m-%d %H:%M:%S" +_nightlies_url=https://github.com/nim-lang/nightlies/releases + +# UI +# ------------------------- + +info() { + echo "$(date +"$DATE_FORMAT")" $'\e[1m\e[36m[INF]\e[0m' "$@" +} + +ok() { + echo "$(date +"$DATE_FORMAT")" $'\e[1m\e[32m[ OK]\e[0m' "$@" +} + +err() { + if [[ -n ${GITHUB_ACTION:+z} ]]; then + echo "::error::" "$@" + else + echo "$(date +"$DATE_FORMAT")" $'\e[1m\e[31m[ERR]\e[0m' "$@" + echo $'\e[1m\e[31mError:\e[0m' "$@" >&2 + fi +} + +warn() { + if [[ -n ${GITHUB_ACTION:+z} ]]; then + echo "::warning::" "$@" + else + echo "$(date +"$DATE_FORMAT")" $'\e[1m\e[33m[WRN]\e[0m' "$@" + echo $'\e[1m\e[33mWarning:\e[0m' "$@" >&2 + fi +} + +# CLI parse +# ------------------------- +declare cli_nim_version # 2.2.0 for release, a branch for nightly or source +declare cli_nim_channel # release|nightly|source +declare cli_install_dir +declare cli_os +declare cli_arch + +while ((0 < $#)); do + opt="$1" + case $opt in + --nim-version) cli_nim_version="$2"; shift 2;; + --nim-channel) cli_nim_channel="$2"; shift 2;; + --install-dir) cli_install_dir="$2"; shift 2;; + --os) cli_os="$2"; shift 2;; + --arch) cli_arch="$2"; shift 2;; + *) + err "Unknown option '$opt'" + exit 1 + ;; + esac +done + + +# ENV +# ------------------------- + +fetch_tags() { + # https://docs.github.com/ja/rest/git/refs?apiVersion=2022-11-28 + curl \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${REPO_GITHUB_TOKEN}" \ + -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/repos/nim-lang/nim/git/refs/tags | + jq -r '.[].ref' | + sed -E 's:^refs/tags/v::' +} + +tag_regexp() { + version=$1 + echo "$version" | + sed -E \ + -e 's/\./\\./g' \ + -e 's/^/^/' \ + -e 's/x$//' +} + +latest_version() { + sort -V | tail -n 1 +} + +function get-version() { + ## Get the latest released version from 2.x or 2.2.x + ## Do not modify branch names for nightlies like version-2-2 + local nim_version="$1" + if [[ "${nim_version}" == "stable" ]]; then + nim_version=$(curl -sSL https://nim-lang.org/channels/stable) + elif [[ "$nim_version" =~ ^[0-9]+\.[0-9]+\.x$ ]] || [[ "$nim_version" =~ ^[0-9]+\.x$ ]]; then + nim_version="$(fetch_tags | grep -E "$(tag_regexp "$nim_version")" | latest_version)" + fi + + # If using a specific version, but building from source, we need a v prefix + if [[ "${NIM_CHANNEL}" == "source" ]] && [[ "$nim_version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + nim_version="v${nim_version}" + fi + + echo $nim_version +} + +function get-cpu-arch() { + local ucpu=${1:-$(uname -m)} # `uname -m` Returns arm64 on MacOS and aarch64 on Linux + local arch + case "$ucpu" in + arm64|aarch64) arch=arm64;; + arm|armv7l) arch=arm;; + x32|i*86) arch=x32;; + amd64|x64|x86_64) arch=x64;; + *) + warn "Unknown architecture '$arch'" + ;; + esac + echo $arch +} + +function get-os() { + local uos=${1:-$(uname -s)} + local os + case "$uos" in + linux|Linux*) os=Linux;; + macos|Darwin*) os=MacOS;; + windows|WindowsNT|CYGWIN|MINGW*) os=Windows;; + FreeBSD*) os=FreeBSD;; + OpenBSD*) os=OpenBSD;; + NetBSD*) os=NetBSD;; + SunOS*) os=Solaris;; + *) + warn "Unknown OS '$uos'" + ;; + esac + echo $os +} + +function get-channel() { + # Nim releases are prebuilt for Linux and Windows only (x86-64) + # Nim nightlies are prebuilt for Linux (x86-64 and arm64), Windows x86-64 and MacOS x86-64 + local default=$([[ "$1" =~ ^(Linux|Windows)$ ]] && echo "stable" || echo "source") + if [[ "$2" != @(|stable|nightly|source) ]]; then + # Note: @(|stable) means empty OR stable + err "Invalid channel '$2'. Expected stable, nightly or source." + exit 1 + fi + echo "${2:-$default}" +} + +CPU_ARCH=$(get-cpu-arch "${cli_arch}") +OS=$(get-os "${cli_os}") +NIM_CHANNEL=$(get-channel "${OS}" "${cli_nim_channel}") +NIM_VERSION=$(get-version "${cli_nim_version:-"stable"}") # Depends on channel to add the `v` +NIM_INSTALL_DIR=${cli_install_dir:="$(pwd)/nim-binaries"} +REPO_GITHUB_TOKEN="" # Placeholder + +info "✔ CPU architecture: ${CPU_ARCH}" +info "✔ OS: ${OS}" +info "✔ Nim version: ${NIM_VERSION}" +info "✔ Nim channel: ${NIM_CHANNEL}" +info "✔ Install dir: ${NIM_INSTALL_DIR}" + +# Note: +# We can't just move the binaries +# nimble also depends on lib/system.nim due to nimscript + +# Building Nim from sources +# ------------------------- + +function get-hash() { + ## Get the latest hash of a branch + ## Unused in this script, but useful as a cache invalidation key + git ls-remote "https://github.com/$1" "${2:-HEAD}" | cut -f 1 +} + +function build-nim() { + builddir="nim-build-${NIM_VERSION}-${OS}-${CPU_ARCH}" + + mkdir -p "${builddir}" + pushd "${builddir}" + info "Cloning Nim (${NIM_VERSION})" + git clone -b "${NIM_VERSION}" --depth 1 https://github.com/nim-lang/Nim + cd Nim + info "Building Nim (${NIM_VERSION}) from source." + ./build_all.sh + info "Finished building Nim." + + # clean up to save cache space + rm koch + rm -rf nimcache + rm -rf dist + rm -rf .git + + popd + + cp -a "${builddir}/Nim" "${NIM_INSTALL_DIR}" + info "Source install finished, binaries are available at \"${NIM_INSTALL_DIR}/bin\"" +} + +# Download Nim from website +# ------------------------- + +function download() { + local url="$1" + local dest="${2+z}" + info "Downloading ${url}" + if [[ -z "${dest}" ]]; then + # On empty destination reuse the filename + if ! curl -fsSLO "${url}"; then + err "Download failure." + exit 1 + fi + else + if ! curl -fsSL "${url}" -o "${dest}"; then + err "Download failure." + exit 1 + fi + fi + info "Downloaded successfully: ${url}" +} + +function download-release() { + ## Only valid for Windows and Linux, both either 32-bit x86 or x86-64 + local downloaddir="nim-download-${NIM_VERSION}-${OS}-${CPU_ARCH}" + + mkdir -p "${downloaddir}" + pushd "${downloaddir}" + case "${OS}" in + Linux) + url="https://nim-lang.org/download/nim-${NIM_VERSION}-linux_${CPU_ARCH}.tar.xz" + download "${url}" nim.zip + unzip -q nim.zip + ;; + Windows) + url="https://nim-lang.org/download/nim-${NIM_VERSION}_${CPU_ARCH}.zip" + download "${url}" nim.tar.xz + tar xJf nim.tar.xz + ;; + *) + err "Invalid OS, prebuilt binaries are not available for '${OS}'" + exit 1 + ;; + esac + popd + cp -a "${downloaddir}/nim-${NIM_VERSION}" "${NIM_INSTALL_DIR}" + info "Release install finished, binaries are available at \"${NIM_INSTALL_DIR}/bin\"" +} + +# Download Nim from nightlies +# --------------------------- + +function get-archive-name() { + local ext=$([[ "${OS}" == Windows ]] && echo ".zip" || echo ".tar.xz") + local os=$([[ "${OS}" == MacOS ]] && echo "macosx" || echo "$(tr '[:upper:]' '[:lower:]' <<< "${OS}")") + local arch=$([[ "${CPU_ARCH}" == arm ]] && echo "armv7l" || echo "${CPU_ARCH}") + echo "${os}_${arch}${ext}" +} + +function has-release() { + local tag=$1 + curl -f -I "${_nightlies_url}/${tag}" >/dev/null 2>&1 +} + +function extract() { + local file=$1 + + # The nightly builds are stored in a subdir `nim-v2.2.1` or for devel `nim-v2.3.1` + # We have no easy way to access the version from the name within the nightly repo + # so we skip it with --strip-components + # Note that bsdtar might not set executable with the exec flag on unix + if [[ $file == *.zip ]]; then + # Windows uses BSD tar: https://github.com/actions/cache/pull/126 + # but it might be overwritten by Git Bash tar! + # So we use fully qualified path + /c/Windows/System32/tar.exe -xf "$file" --strip-components=1 + else + tar -xJf "$file" --strip-components 1 + fi +} + +function download-nightly() { + local downloaddir="nim-download-${NIM_VERSION}-${OS}-${CPU_ARCH}" + tag=latest-${NIM_VERSION} + + mkdir -p "${downloaddir}" + pushd "${downloaddir}" + if has-release "${tag}"; then + archive=$(get-archive-name) + local url="${_nightlies_url}/download/${tag}/${archive}" + + download "${url}" + extract "${archive}" + else + err "No nightly release named '$tag'. The provided branch (${NIM_VERSION}) might not be tracked by nightlies, or is being updated." + fi + popd + cp -a "${downloaddir}" "${NIM_INSTALL_DIR}" + info "Nightly install finished, binaries are available at \"${NIM_INSTALL_DIR}/bin\"" +} + +# Main +# --------------------------- + +case ${NIM_CHANNEL} in + source) build-nim;; + nightly) download-nightly;; + release) download-release;; + *) + err "Invalid channel '$2'. Expected stable, nightly or source." + exit 1 + ;; +esac + +ok "Successfully installed Nim." +exit 0 diff --git a/constantine/platforms/config.nim b/constantine/platforms/config.nim index e137c786..5fa4e820 100644 --- a/constantine/platforms/config.nim +++ b/constantine/platforms/config.nim @@ -30,7 +30,7 @@ const UseASM_ARM64* = CTT_ASM and ARM64 and GCC_Compatible when UseASM_X86_64: static: doAssert bool(sizeof(pointer)*8 == 64), "Only 32-bit and 64-bit platforms are supported" -const UseAsmSyntaxIntel* {.booldefine.} = defined(lto) or defined(lto_incremental) +const UseAsmSyntaxIntel* {.booldefine.} = UseASM_X86_32 and (defined(lto) or defined(lto_incremental)) ## When using LTO with AT&T syntax ## - GCC will give spurious "Warning: missing operand; zero assumed" with AT&T syntax ## - Clang will wrongly combine memory offset and constant propagated address of constants From 114b6f487ea8b939dff946636d23e10564cdca26 Mon Sep 17 00:00:00 2001 From: Mamy Ratsimbazafy Date: Sun, 26 Jan 2025 00:18:33 +0100 Subject: [PATCH 07/12] CI: GMP not in PATH on newer MacOS --- .github/workflows/ci.yml | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0f629f15..0c63a64c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -374,25 +374,32 @@ jobs: cd constantine nimble make_zkalc - - name: Run Constantine in-depth tests (Unix - with GMP, with Assembly) - if: runner.os != 'Windows' && matrix.target.ctt_backend == 'ASM' + - name: Run Constantine in-depth tests (Linux - with GMP, with Assembly) + if: runner.os == 'Linux' && matrix.target.ctt_backend == 'ASM' shell: bash run: | cd constantine nimble test_parallel --verbose - name: Run Constantine in-depth tests (Unix - with GMP, no Assembly) - if: runner.os != 'Windows' && matrix.target.ctt_backend == 'NO_ASM' + if: runner.os == 'Linux' && matrix.target.ctt_backend == 'NO_ASM' shell: bash run: | cd constantine CTT_ASM=0 nimble test_parallel --verbose - - name: Run Constantine in-depth tests (Windows - no GMP, with Assembly) - # So "test_bindings" uses C and can find GMP - # but nim-gmp cannot find GMP on Windows CI - # Also need to workaround asynctools not being able to create pipes https://github.com/nim-lang/Nim/issues/23118 - # And LTO impossible constraint in the deneb_kzg test (but not MSM for some reason) - if: runner.os == 'Windows' && matrix.target.ctt_backend == 'ASM' + # On Windows + # So "test_bindings" uses C and can find GMP + # but nim-gmp cannot find GMP on Windows CI + # Also need to workaround asynctools not being able to create pipes https://github.com/nim-lang/Nim/issues/23118 + # And LTO impossible constraint in the deneb_kzg test (but not MSM for some reason) + # + # On mac + # Homebrew install is not in default PATH + # this can be workaround with LD_LIBRARY_PATH for libraries + # but headers would need a + # --passC:-I$(brew --prefix)/include (-I/opt/homebrew/include on latest MacOS versions) + - name: Run Constantine in-depth tests (MacOS/Windows - no GMP, with Assembly)= + if: runner.os != 'Linux' && matrix.target.ctt_backend == 'ASM' shell: msys2 {0} run: | cd constantine @@ -401,11 +408,8 @@ jobs: else nimble test_parallel_no_gmp --verbose fi - - name: Run Constantine in-depth tests (Windows - no GMP, no Assembly) - # So "test_bindings" uses C and can find GMP - # but nim-gmp cannot find GMP on Windows CI - # Also need to workaround asynctools not being able to create pipes https://github.com/nim-lang/Nim/issues/23118 - if: runner.os == 'Windows' && matrix.target.ctt_backend == 'NO_ASM' + - name: Run Constantine in-depth tests (MacOS/Windows - no GMP, no Assembly) + if: runner.os != 'Linux' && matrix.target.ctt_backend == 'NO_ASM' shell: msys2 {0} run: | cd constantine From d887aa7fe5dc7df0100c3a59952f40088126e046 Mon Sep 17 00:00:00 2001 From: Mamy Ratsimbazafy Date: Sun, 26 Jan 2025 01:14:02 +0100 Subject: [PATCH 08/12] CI: GMP not in MacOS path, manually do the C integration tests --- .github/workflows/ci.yml | 63 ++++++++++++++++++++++++++++++---------- 1 file changed, 48 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c63a64c..da328c9b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -296,16 +296,59 @@ jobs: cd constantine go mod download -modfile=go_test.mod - - name: Run Constantine as C library tests (UNIX with Assembly) - if: runner.os != 'Windows' && matrix.target.ctt_backend == 'ASM' + - name: Run Constantine as C library tests (MacOS with Assembly) + # We manually do the tests due to Homebrew lib/headers not being in standard PATH + if: runner.os == 'macOS' && matrix.target.ctt_backend == 'ASM' + shell: bash + run: | + cd constantine + nimble make_lib --verbose + nimble make_headers --verbose + clang -Iinclude -I$(brew --prefix)/include \ + -Llib -L$(brew --prefix)/lib \ + -o build/test_lib/t_libctt_bls12_381_staticlink.exe \ + examples-c/t_libctt_bls12_381.c \ + lib/libconstantine.a \ + -lgmp + clang -Iinclude -I$(brew --prefix)/include \ + -Llib -L$(brew --prefix)/lib \ + -o build/test_lib/t_libctt_bls12_381_dynlink.exe \ + examples-c/t_libctt_bls12_381.c \ + -lconstantine \ + -lgmp + clang -Iinclude -I$(brew --prefix)/include \ + -Llib -L$(brew --prefix)/lib \ + -o build/test_lib/t_libctt_banderwagon_staticlink.exe \ + examples-c/t_libctt_banderwagon.c \ + lib/libconstantine.a \ + -lgmp + clang -Iinclude -I$(brew --prefix)/include \ + -Llib -L$(brew --prefix)/lib \ + -o build/test_lib/t_libctt_banderwagon_dynlink.exe \ + examples-c/t_libctt_banderwagon.c \ + -lconstantine \ + -lgmp + clang -Iinclude -I$(brew --prefix)/include \ + -Llib -L$(brew --prefix)/lib \ + -o build/test_lib/ethereum_bls_signatures_staticlink.exe \ + examples-c/ethereum_bls_signatures.c \ + lib/libconstantine.a + clang -Iinclude -I$(brew --prefix)/include \ + -Llib -L$(brew --prefix)/lib \ + -o build/test_lib/ethereum_bls_signatures_dynlink.exe \ + examples-c/ethereum_bls_signatures.c \ + -lconstantine + + - name: Run Constantine as C library tests (Linux with Assembly) + if: runner.os == 'Linux' && matrix.target.ctt_backend == 'ASM' shell: bash run: | cd constantine nimble make_lib --verbose nimble make_headers --verbose nimble test_lib --verbose - - name: Run Constantine as C library tests (UNIX no Assembly) - if: runner.os != 'Windows' && matrix.target.ctt_backend == 'NO_ASM' + - name: Run Constantine as C library tests (Linux no Assembly) + if: runner.os == 'Linux' && matrix.target.ctt_backend == 'NO_ASM' shell: bash run: | cd constantine @@ -313,7 +356,7 @@ jobs: nimble make_headers --verbose nimble test_lib --verbose - name: Run Constantine as C library tests (Windows with Assembly) - # So "test_bindings" uses C and can find GMP + # So "test_lib" uses C and can find GMP # but nim-gmp cannot find GMP on Windows CI if: runner.os == 'Windows' && matrix.target.ctt_backend == 'ASM' shell: msys2 {0} @@ -322,16 +365,6 @@ jobs: nimble make_lib --verbose nimble make_headers --verbose nimble test_lib --verbose - - name: Run Constantine as C library tests (Windows no Assembly) - # So "test_bindings" uses C and can find GMP - # but nim-gmp cannot find GMP on Windows CI - if: runner.os == 'Windows' && matrix.target.ctt_backend == 'NO_ASM' - shell: msys2 {0} - run: | - cd constantine - CTT_ASM=0 nimble make_lib --verbose - nimble make_headers --verbose - nimble test_lib --verbose - name: Run Constantine as Go library tests # This reuses the static library built with `nimble make_lib` From 39536021ee1a408a696344e50bcbf58199eb32cc Mon Sep 17 00:00:00 2001 From: Mamy Ratsimbazafy Date: Sun, 26 Jan 2025 01:14:08 +0100 Subject: [PATCH 09/12] CI: run 10% less tests in CI --- constantine.nimble | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/constantine.nimble b/constantine.nimble index fee1e80e..2e338e73 100644 --- a/constantine.nimble +++ b/constantine.nimble @@ -445,7 +445,7 @@ const testDesc: seq[tuple[path: string, useGMP: bool]] = @[ ("tests/math_elliptic_curves/t_ec_shortw_prj_g1_add_double.nim", false), # ("tests/math_elliptic_curves/t_ec_shortw_prj_g1_mul_sanity.nim", false), # ("tests/math_elliptic_curves/t_ec_shortw_prj_g1_mul_distri.nim", false), - ("tests/math_elliptic_curves/t_ec_shortw_prj_g1_mul_vs_ref.nim", false), + # ("tests/math_elliptic_curves/t_ec_shortw_prj_g1_mul_vs_ref.nim", false), # ("tests/math_elliptic_curves/t_ec_shortw_prj_g1_mixed_add.nim", false), ("tests/math_elliptic_curves/t_ec_shortw_jac_g1_add_double.nim", false), @@ -473,25 +473,25 @@ const testDesc: seq[tuple[path: string, useGMP: bool]] = @[ # ("tests/math_elliptic_curves/t_ec_shortw_prj_g2_add_double_bn254_snarks.nim", false), # ("tests/math_elliptic_curves/t_ec_shortw_prj_g2_mul_sanity_bn254_snarks.nim", false), # ("tests/math_elliptic_curves/t_ec_shortw_prj_g2_mul_distri_bn254_snarks.nim", false), - ("tests/math_elliptic_curves/t_ec_shortw_prj_g2_mul_vs_ref_bn254_snarks.nim", false), + # ("tests/math_elliptic_curves/t_ec_shortw_prj_g2_mul_vs_ref_bn254_snarks.nim", false), # ("tests/math_elliptic_curves/t_ec_shortw_prj_g2_mixed_add_bn254_snarks.nim", false), # ("tests/math_elliptic_curves/t_ec_shortw_prj_g2_add_double_bls12_381.nim", false), # ("tests/math_elliptic_curves/t_ec_shortw_prj_g2_mul_sanity_bls12_381.nim", false), # ("tests/math_elliptic_curves/t_ec_shortw_prj_g2_mul_distri_bls12_381.nim", false), - ("tests/math_elliptic_curves/t_ec_shortw_prj_g2_mul_vs_ref_bls12_381.nim", false), + # ("tests/math_elliptic_curves/t_ec_shortw_prj_g2_mul_vs_ref_bls12_381.nim", false), # ("tests/math_elliptic_curves/t_ec_shortw_prj_g2_mixed_add_bls12_381.nim", false), # ("tests/math_elliptic_curves/t_ec_shortw_prj_g2_add_double_bls12_377.nim", false), # ("tests/math_elliptic_curves/t_ec_shortw_prj_g2_mul_sanity_bls12_377.nim", false), # ("tests/math_elliptic_curves/t_ec_shortw_prj_g2_mul_distri_bls12_377.nim", false), - ("tests/math_elliptic_curves/t_ec_shortw_prj_g2_mul_vs_ref_bls12_377.nim", false), + # ("tests/math_elliptic_curves/t_ec_shortw_prj_g2_mul_vs_ref_bls12_377.nim", false), # ("tests/math_elliptic_curves/t_ec_shortw_prj_g2_mixed_add_bls12_377.nim", false), # ("tests/math_elliptic_curves/t_ec_shortw_prj_g2_add_double_bw6_761.nim", false), # ("tests/math_elliptic_curves/t_ec_shortw_prj_g2_mul_sanity_bw6_761.nim", false), # ("tests/math_elliptic_curves/t_ec_shortw_prj_g2_mul_distri_bw6_761.nim", false), - ("tests/math_elliptic_curves/t_ec_shortw_prj_g2_mul_vs_ref_bw6_761.nim", false), + # ("tests/math_elliptic_curves/t_ec_shortw_prj_g2_mul_vs_ref_bw6_761.nim", false), # ("tests/math_elliptic_curves/t_ec_shortw_prj_g2_mixed_add_bw6_761.nim", false), # ("tests/math_elliptic_curves/t_ec_shortw_jac_g2_add_double_bn254_snarks.nim", false), @@ -509,13 +509,13 @@ const testDesc: seq[tuple[path: string, useGMP: bool]] = @[ # ("tests/math_elliptic_curves/t_ec_shortw_jac_g2_add_double_bls12_377.nim", false), # ("tests/math_elliptic_curves/t_ec_shortw_jac_g2_mul_sanity_bls12_377.nim", false), # ("tests/math_elliptic_curves/t_ec_shortw_jac_g2_mul_distri_bls12_377.nim", false), - ("tests/math_elliptic_curves/t_ec_shortw_jac_g2_mul_vs_ref_bls12_377.nim", false), + # ("tests/math_elliptic_curves/t_ec_shortw_jac_g2_mul_vs_ref_bls12_377.nim", false), # ("tests/math_elliptic_curves/t_ec_shortw_jac_g2_mixed_add_bls12_377.nim", false), # ("tests/math_elliptic_curves/t_ec_shortw_jac_g2_add_double_bw6_761.nim", false), # ("tests/math_elliptic_curves/t_ec_shortw_jac_g2_mul_sanity_bw6_761.nim", false), # ("tests/math_elliptic_curves/t_ec_shortw_jac_g2_mul_distri_bw6_761.nim", false), - ("tests/math_elliptic_curves/t_ec_shortw_jac_g2_mul_vs_ref_bw6_761.nim", false), + # ("tests/math_elliptic_curves/t_ec_shortw_jac_g2_mul_vs_ref_bw6_761.nim", false), # ("tests/math_elliptic_curves/t_ec_shortw_jac_g2_mixed_add_bw6_761.nim", false), ("tests/math_elliptic_curves/t_ec_shortw_jac_g2_mul_endomorphism_bls12_381", false), From 242fd204a8b57e992ffcce65d5aa2d815b514ea6 Mon Sep 17 00:00:00 2001 From: Mamy Ratsimbazafy Date: Sun, 26 Jan 2025 01:23:05 +0100 Subject: [PATCH 10/12] CI: Go/Rust uses default (with assembly) --- .github/workflows/ci.yml | 48 ++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index da328c9b..e4dea375 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -296,6 +296,24 @@ jobs: cd constantine go mod download -modfile=go_test.mod + - name: Run Constantine as C library tests (Linux with Assembly) + if: runner.os == 'Linux' && matrix.target.ctt_backend == 'ASM' + shell: bash + run: | + cd constantine + nimble make_lib --verbose + nimble make_headers --verbose + nimble test_lib --verbose + - name: Run Constantine as C library tests (Linux no Assembly) + if: runner.os == 'Linux' && matrix.target.ctt_backend == 'NO_ASM' + shell: bash + run: | + cd constantine + CTT_ASM=0 nimble make_lib --verbose + nimble make_headers --verbose + nimble test_lib --verbose + + - name: Run Constantine as C library tests (MacOS with Assembly) # We manually do the tests due to Homebrew lib/headers not being in standard PATH if: runner.os == 'macOS' && matrix.target.ctt_backend == 'ASM' @@ -339,22 +357,7 @@ jobs: examples-c/ethereum_bls_signatures.c \ -lconstantine - - name: Run Constantine as C library tests (Linux with Assembly) - if: runner.os == 'Linux' && matrix.target.ctt_backend == 'ASM' - shell: bash - run: | - cd constantine - nimble make_lib --verbose - nimble make_headers --verbose - nimble test_lib --verbose - - name: Run Constantine as C library tests (Linux no Assembly) - if: runner.os == 'Linux' && matrix.target.ctt_backend == 'NO_ASM' - shell: bash - run: | - cd constantine - CTT_ASM=0 nimble make_lib --verbose - nimble make_headers --verbose - nimble test_lib --verbose + - name: Run Constantine as C library tests (Windows with Assembly) # So "test_lib" uses C and can find GMP # but nim-gmp cannot find GMP on Windows CI @@ -366,9 +369,9 @@ jobs: nimble make_headers --verbose nimble test_lib --verbose - - name: Run Constantine as Go library tests + - name: Run Constantine as Go library tests (with Assembly) # This reuses the static library built with `nimble make_lib` - if: matrix.target.cpu != 'i386' + if: matrix.target.ctt_backend == 'ASM' && matrix.target.cpu != 'i386' shell: bash run: | cd constantine/constantine-go @@ -381,13 +384,6 @@ jobs: run: | cd constantine cargo test -- --nocapture - - name: Run Constantine as Rust library tests (NO Assembly) - if: matrix.target.ctt_backend == 'NO_ASM' && matrix.target.cpu != 'i386' - shell: bash - # We need to deactivate the assembly (used by default for benches) - run: | - cd constantine - CTT_ASM=0 cargo test -- --nocapture - name: Compile Constantine Zkalc benchmark (no assembly) # Skip 32-bit as that would need clang-multilib or -m32 @@ -413,7 +409,7 @@ jobs: run: | cd constantine nimble test_parallel --verbose - - name: Run Constantine in-depth tests (Unix - with GMP, no Assembly) + - name: Run Constantine in-depth tests (Linux - with GMP, no Assembly) if: runner.os == 'Linux' && matrix.target.ctt_backend == 'NO_ASM' shell: bash run: | From a2fa8e4be7b0ebf9546af631bfc55b963f6132ae Mon Sep 17 00:00:00 2001 From: Mamy Ratsimbazafy Date: Sun, 26 Jan 2025 01:35:22 +0100 Subject: [PATCH 11/12] CI: MacOS run C tests --- .github/workflows/ci.yml | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e4dea375..72ea28ff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -322,41 +322,47 @@ jobs: cd constantine nimble make_lib --verbose nimble make_headers --verbose - clang -Iinclude -I$(brew --prefix)/include \ - -Llib -L$(brew --prefix)/lib \ + mkdir -p build/test_lib + clang -Iinclude -I"$(brew --prefix)/include" \ + -Llib -L"$(brew --prefix)/lib" \ -o build/test_lib/t_libctt_bls12_381_staticlink.exe \ examples-c/t_libctt_bls12_381.c \ lib/libconstantine.a \ -lgmp - clang -Iinclude -I$(brew --prefix)/include \ - -Llib -L$(brew --prefix)/lib \ + ./build/test_lib/t_libctt_bls12_381_staticlink.exe + clang -Iinclude -I"$(brew --prefix)/include" \ + -Llib -L"$(brew --prefix)/lib" \ -o build/test_lib/t_libctt_bls12_381_dynlink.exe \ examples-c/t_libctt_bls12_381.c \ -lconstantine \ -lgmp - clang -Iinclude -I$(brew --prefix)/include \ - -Llib -L$(brew --prefix)/lib \ + ./build/test_lib/t_libctt_bls12_381_dynlink.exe + clang -Iinclude -I"$(brew --prefix)/include" \ + -Llib -L"$(brew --prefix)/lib" \ -o build/test_lib/t_libctt_banderwagon_staticlink.exe \ examples-c/t_libctt_banderwagon.c \ lib/libconstantine.a \ -lgmp - clang -Iinclude -I$(brew --prefix)/include \ - -Llib -L$(brew --prefix)/lib \ + ./build/test_lib/t_libctt_banderwagon_staticlink.exe + clang -Iinclude -I"$(brew --prefix)/include" \ + -Llib -L"$(brew --prefix)/lib" \ -o build/test_lib/t_libctt_banderwagon_dynlink.exe \ examples-c/t_libctt_banderwagon.c \ -lconstantine \ -lgmp - clang -Iinclude -I$(brew --prefix)/include \ - -Llib -L$(brew --prefix)/lib \ + ./build/test_lib/t_libctt_banderwagon_dynlink.exe + clang -Iinclude -I"$(brew --prefix)/include" \ + -Llib -L"$(brew --prefix)/lib" \ -o build/test_lib/ethereum_bls_signatures_staticlink.exe \ examples-c/ethereum_bls_signatures.c \ lib/libconstantine.a - clang -Iinclude -I$(brew --prefix)/include \ - -Llib -L$(brew --prefix)/lib \ + ./build/test_lib/ethereum_bls_signatures_staticlink.exe + clang -Iinclude -I"$(brew --prefix)/include" \ + -Llib -L"$(brew --prefix)/lib" \ -o build/test_lib/ethereum_bls_signatures_dynlink.exe \ examples-c/ethereum_bls_signatures.c \ -lconstantine - + ./build/test_lib/ethereum_bls_signatures_dynlink.exe - name: Run Constantine as C library tests (Windows with Assembly) # So "test_lib" uses C and can find GMP From a3228eaee7d4c69e41ae219733e3572d14ba9682 Mon Sep 17 00:00:00 2001 From: Mamy Ratsimbazafy Date: Sun, 26 Jan 2025 02:03:35 +0100 Subject: [PATCH 12/12] CI: move and polish install_nim.sh, fix MacOS using msys2 --- .github/workflows/ci.yml | 32 ++++++++++------ {.github/workflows => scripts}/install_nim.sh | 38 ++++++++++++++++++- 2 files changed, 58 insertions(+), 12 deletions(-) rename {.github/workflows => scripts}/install_nim.sh (88%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72ea28ff..b9553b5b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -183,7 +183,7 @@ jobs: if: steps.nim-compiler-cache.outputs.cache-hit != 'true' shell: bash run: | - bash constantine/.github/workflows/install_nim.sh \ + bash constantine/scripts/install_nim.sh \ --nim-channel ${{ matrix.nim_channel }} \ --nim-version ${{ matrix.nim_version }} \ --os ${{ matrix.target.os }} \ @@ -422,19 +422,29 @@ jobs: cd constantine CTT_ASM=0 nimble test_parallel --verbose - # On Windows - # So "test_bindings" uses C and can find GMP - # but nim-gmp cannot find GMP on Windows CI - # Also need to workaround asynctools not being able to create pipes https://github.com/nim-lang/Nim/issues/23118 - # And LTO impossible constraint in the deneb_kzg test (but not MSM for some reason) - # # On mac # Homebrew install is not in default PATH # this can be workaround with LD_LIBRARY_PATH for libraries # but headers would need a # --passC:-I$(brew --prefix)/include (-I/opt/homebrew/include on latest MacOS versions) - - name: Run Constantine in-depth tests (MacOS/Windows - no GMP, with Assembly)= - if: runner.os != 'Linux' && matrix.target.ctt_backend == 'ASM' + - name: Run Constantine in-depth tests (MacOS - no GMP, with Assembly)= + if: runner.os == 'macOS' && matrix.target.ctt_backend == 'ASM' + run: | + cd constantine + nimble test_parallel_no_gmp --verbose + - name: Run Constantine in-depth tests (MacOS - no GMP, no Assembly) + if: runner.os == 'macOS' && matrix.target.ctt_backend == 'NO_ASM' + run: | + cd constantine + CTT_ASM=0 nimble test_parallel_no_gmp --verbose + + # On Windows + # So "test_bindings" uses C and can find GMP + # but nim-gmp cannot find GMP on Windows CI + # Also need to workaround asynctools not being able to create pipes https://github.com/nim-lang/Nim/issues/23118 + # And LTO impossible constraint in the deneb_kzg test (but not MSM for some reason) + - name: Run Constantine in-depth tests (Windows - no GMP, with Assembly)= + if: runner.os == 'Windows' && matrix.target.ctt_backend == 'ASM' shell: msys2 {0} run: | cd constantine @@ -443,8 +453,8 @@ jobs: else nimble test_parallel_no_gmp --verbose fi - - name: Run Constantine in-depth tests (MacOS/Windows - no GMP, no Assembly) - if: runner.os != 'Linux' && matrix.target.ctt_backend == 'NO_ASM' + - name: Run Constantine in-depth tests (Windows - no GMP, no Assembly) + if: runner.os == 'Windows' && matrix.target.ctt_backend == 'NO_ASM' shell: msys2 {0} run: | cd constantine diff --git a/.github/workflows/install_nim.sh b/scripts/install_nim.sh similarity index 88% rename from .github/workflows/install_nim.sh rename to scripts/install_nim.sh index bdfc524c..3690b3f9 100644 --- a/.github/workflows/install_nim.sh +++ b/scripts/install_nim.sh @@ -47,6 +47,40 @@ _nightlies_url=https://github.com/nim-lang/nightlies/releases # UI # ------------------------- +print-help() { + cat < + defaults to "stable". + VERSION can be + - a version number like '2.2.0'. + - a version number with wildcard like '2.x' or '2.2.x', + which will be replaced by the latest of the specified serie. + - a branch like 'devel' or 'version-2-2' + --nim-channel + defaults to "stable". + The channel specifies the download source of the Nim compiler + - "stable" downloads from Nim website, for Linux and Windows (x86 and x86-64) only + - "nightly" downloads from https://github.com/nim-lang/nightlies/releases + use a branch name like 'devel' or 'version-2-2'. + For Linux (x86, x86-64, arm64), MacOS (x86-64 only), Windows (x86, x86-64) + - "source" builds the compiler from source https://github.com/nim-lang/Nim + --install-dir + defaults to "nim-install". + --os + defaults to host OS + --arch + defaults to host CPU architecture + -h, --help + prints this help +EOF +} + info() { echo "$(date +"$DATE_FORMAT")" $'\e[1m\e[36m[INF]\e[0m' "$@" } @@ -89,8 +123,10 @@ while ((0 < $#)); do --install-dir) cli_install_dir="$2"; shift 2;; --os) cli_os="$2"; shift 2;; --arch) cli_arch="$2"; shift 2;; + -h|--help) print-help; exit 0;; *) err "Unknown option '$opt'" + print-help exit 1 ;; esac @@ -190,7 +226,7 @@ CPU_ARCH=$(get-cpu-arch "${cli_arch}") OS=$(get-os "${cli_os}") NIM_CHANNEL=$(get-channel "${OS}" "${cli_nim_channel}") NIM_VERSION=$(get-version "${cli_nim_version:-"stable"}") # Depends on channel to add the `v` -NIM_INSTALL_DIR=${cli_install_dir:="$(pwd)/nim-binaries"} +NIM_INSTALL_DIR=${cli_install_dir:="$(pwd)/nim-install"} REPO_GITHUB_TOKEN="" # Placeholder info "✔ CPU architecture: ${CPU_ARCH}"