zama-ai · BourgerieQuentin · Nov 14, 2024 · Nov 12, 2024
diff --git a/.github/actionlint.yml b/.github/actionlint.yml
@@ -0,0 +1,5 @@
+self-hosted-runner:
+  labels:
+    - distributed-ci
+    - aws-mac1-metal
+    - aws-mac2-metal
diff --git a/.github/workflows/optimizer_setup/action.yml → .github/actions/optimizer_setup/action.yml b/.github/workflows/optimizer_setup/action.yml → .github/actions/optimizer_setup/action.yml
@@ -1,3 +1,7 @@
+name: Setup rust
+
+description: Setup rust for optimizer
+
 runs:
   using: "composite"
   steps:

diff --git a/...ust_toolchain_for_concrete_cpu/action.yml → ...ust_toolchain_for_concrete_cpu/action.yml b/...ust_toolchain_for_concrete_cpu/action.yml → ...ust_toolchain_for_concrete_cpu/action.yml
@@ -1,3 +1,7 @@
+name: Setup rust toolchain for concrete-cpu
+
+description: Setup rust toolchain for concrete-cpu
+
 runs:
   using: "composite"
   steps:

diff --git a/.github/workflows/concrete_compiler_benchmark.yml b/.github/workflows/concrete_compiler_benchmark.yml
@@ -57,15 +57,17 @@ jobs:
       - name: Ouput benchmark details
         id: benchmark-details
         run: |
-          echo "bench_date=$(date --iso-8601=seconds)" >> "$GITHUB_OUTPUT"
-          echo "commit_date=$(git --no-pager show -s --format=%cd --date=iso8601-strict ${{ github.sha }})" >> "$GITHUB_OUTPUT"
-          echo "commit_hash=$(git describe --tags --dirty)" >> "$GITHUB_OUTPUT"
+          {
+            echo "bench_date=$(date --iso-8601=seconds)";
+            echo "commit_date=$(git --no-pager show -s --format=%cd --date=iso8601-strict ${{ github.sha }})";
+            echo "commit_hash=$(git describe --tags --dirty)";
+          } >> "$GITHUB_OUTPUT"
       - name: Set up home
         # "Install rust" step require root user to have a HOME directory which is not set.
         run: |
           echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}"
       - name: Setup rust toolchain for concrete-cpu
-        uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu
+        uses: ./.github/actions/setup_rust_toolchain_for_concrete_cpu
       - name: Build compiler benchmarks
         run: |
           set -e
@@ -126,7 +128,7 @@ jobs:
         shell: bash
         run: |
           echo "Computing HMac on downloaded artifact"
-          SIGNATURE="$(slab/scripts/hmac_calculator.sh $parsed_benchmark_results.json '${{ secrets.JOB_SECRET }}')"
+          SIGNATURE="$(slab/scripts/hmac_calculator.sh parsed_benchmark_results.json ${{ secrets.JOB_SECRET }})"
           echo "Sending results to Slab..."
           curl -v -k \
           -H "Content-Type: application/json" \

diff --git a/.github/workflows/concrete_compiler_test_cpu.yml b/.github/workflows/concrete_compiler_test_cpu.yml
@@ -127,10 +127,10 @@ jobs:
             for i in `ls *Dialect.md`; do diff $i /concrete/docs/explanations/$i; done;
       - name: Enable complete tests on push to main
         if: github.ref == 'refs/heads/main'
-        run: echo "MINIMAL_TESTS=OFF" >> ${GITHUB_ENV}
+        run: echo "MINIMAL_TESTS=OFF" >> "${GITHUB_ENV}"
       - name: Enable minimal tests otherwise
         if: github.ref != 'refs/heads/main'
-        run: echo "MINIMAL_TESTS=ON" >> ${GITHUB_ENV}
+        run: echo "MINIMAL_TESTS=ON" >> "${GITHUB_ENV}"
       - name: Run compiler tests
         uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3
         with:

diff --git a/.github/workflows/concrete_compiler_test_cpu_distributed.yml b/.github/workflows/concrete_compiler_test_cpu_distributed.yml
@@ -63,7 +63,7 @@ jobs:
           echo "HOME=/shared" >> "${GITHUB_ENV}"
 
       - name: Setup rust toolchain for concrete-cpu
-        uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu
+        uses: ./.github/actions/setup_rust_toolchain_for_concrete_cpu
 
       - name: Build end-to-end distributed test
         run: |

diff --git a/.github/workflows/concrete_compiler_test_macos_cpu.yml b/.github/workflows/concrete_compiler_test_macos_cpu.yml
@@ -39,13 +39,14 @@ jobs:
           submodules: recursive
           fetch-depth: 0
       - name: Setup rust toolchain for concrete-cpu
-        uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu
+        uses: ./.github/actions/setup_rust_toolchain_for_concrete_cpu
       - name: Set python variables
         run: |
-          set -e
-          echo "PIP=${{ format('pip{0}', matrix.python-version) }}" >> "${GITHUB_ENV}"
-          echo "PYTHON=${{ format('python{0}', matrix.python-version) }}" >> "${GITHUB_ENV}"
-          echo "PYTHON_EXEC=$(which ${{ format('python{0}', matrix.python-version) }})" >> "${GITHUB_ENV}"
+          {
+            echo "PIP=${{ format('pip{0}', matrix.python-version) }}"
+            echo "PYTHON=${{ format('python{0}', matrix.python-version) }}"
+            echo "PYTHON_EXEC=$(which ${{ format('python{0}', matrix.python-version) }})"
+          } >> "${GITHUB_ENV}"
       - name: Install dependencies
         run: |
           brew install ninja ccache
@@ -56,30 +57,31 @@ jobs:
           cd compilers/concrete-compiler/compiler
           echo "Debug: ccache statistics (prior to the build):"
           ccache -s
-          make Python3_EXECUTABLE=$PYTHON_EXEC all
+          make Python3_EXECUTABLE="${PYTHON_EXEC}" all
           echo "Debug: ccache statistics (after the build):"
           ccache -s
       - name: Enable complete tests on push to main
         if: github.ref == 'refs/heads/main'
-        run: echo "MINIMAL_TESTS=OFF" >> $GITHUB_ENV
+        run: echo "MINIMAL_TESTS=OFF" >> "${GITHUB_ENV}"
       - name: Enable minimal tests otherwise
         if: github.ref != 'refs/heads/main'
-        run: echo "MINIMAL_TESTS=ON" >> $GITHUB_ENV
+        run: echo "MINIMAL_TESTS=ON" >> "${GITHUB_ENV}"
       - name: Create keyset cache directory
         run: |
-          export KEY_CACHE_DIRECTORY=$(mktemp -d)/KeySetCache
-          echo "KEY_CACHE_DIRECTORY=$KEY_CACHE_DIRECTORY" >> "${GITHUB_ENV}"
-          mkdir $KEY_CACHE_DIRECTORY          
+          KEY_CACHE_DIRECTORY=$(mktemp -d)/KeySetCache
+          export KEY_CACHE_DIRECTORY
+          echo "KEY_CACHE_DIRECTORY=${KEY_CACHE_DIRECTORY}" >> "${GITHUB_ENV}"
+          mkdir "${KEY_CACHE_DIRECTORY}"
       - name: Test
         run: |
           set -e
           cd compilers/concrete-compiler/compiler
           export CONCRETE_COMPILER_DATAFLOW_EXECUTION_ENABLED=OFF
-          make MINIMAL_TESTS=${{ env.MINIMAL_TESTS }} Python3_EXECUTABLE=$PYTHON_EXEC run-tests
+          make MINIMAL_TESTS=${{ env.MINIMAL_TESTS }} Python3_EXECUTABLE="${PYTHON_EXEC}" run-tests
       - name: Cleanup host
         if: success() || failure()
         run: |
-         rm -rf $KEY_CACHE_DIRECTORY
+         rm -rf "${KEY_CACHE_DIRECTORY}"
       - name: Slack Notification
         if: ${{ failure() && github.ref == 'refs/heads/main' }}
         continue-on-error: true

diff --git a/.github/workflows/concrete_ml_test.yml b/.github/workflows/concrete_ml_test.yml
@@ -101,11 +101,11 @@ jobs:
       - name: ML Tests
         run: |
           export HOME="/home/ubuntu"
-          export CONCRETE_PYTHON_WHEEL=$(pwd)/frontends/concrete-python/dist/*manylinux*.whl
+          CONCRETE_PYTHON_WHEEL="$(pwd)/frontends/concrete-python/dist/*manylinux*.whl"
           apt update
           apt install git git-lfs -y
           pip install poetry==1.7.1
-          ./ci/scripts/test_cml.sh --use-wheel $CONCRETE_PYTHON_WHEEL --verbose
+          ./ci/scripts/test_cml.sh --use-wheel "${CONCRETE_PYTHON_WHEEL}" --verbose
       - name: Slack Notification
         if: ${{ failure() && github.ref == 'refs/heads/main' }}
         continue-on-error: true

diff --git a/.github/workflows/concrete_optimizer.yml b/.github/workflows/concrete_optimizer.yml
@@ -36,7 +36,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Setup
-        uses: ./.github/workflows/optimizer_setup
+        uses: ./.github/actions/optimizer_setup
       - name: Formatting
         run: |
           cargo --version
@@ -62,7 +62,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Setup
-        uses: ./.github/workflows/optimizer_setup
+        uses: ./.github/actions/optimizer_setup
       - name: Run benchmark
         run: |
           cd compilers/concrete-optimizer

diff --git a/.github/workflows/concrete_python_benchmark.yml b/.github/workflows/concrete_python_benchmark.yml
@@ -61,8 +61,7 @@ jobs:
             set -e
             rm -rf /build/*
 
-            export PYTHON=${{ format('python{0}', matrix.python-version) }}
-            echo "Using $PYTHON"
+            export PYTHON=python
 
             cd /concrete/frontends/concrete-python
             make PYTHON=$PYTHON venv

diff --git a/.github/workflows/concrete_python_finalize_release.yml b/.github/workflows/concrete_python_finalize_release.yml
@@ -21,7 +21,7 @@ jobs:
           S3_BUCKET_NAME: ${{ secrets.AWS_S3_PYPI_BUCKET_NAME }}
         run: |
           mkdir wheels
-          aws s3 cp s3://${S3_BUCKET_NAME}/cpu/concrete-python/ ./wheels/ --recursive --exclude "*" --include "concrete_python-${{ inputs.version }}-*"
+          aws s3 cp "s3://${S3_BUCKET_NAME}/cpu/concrete-python/" ./wheels/ --recursive --exclude "*" --include "concrete_python-${{ inputs.version }}-*"
           echo "============== Downloaded wheels ==============="
           ls -la ./wheels/
       - name: Push wheels to public PyPI (public)
@@ -37,7 +37,7 @@ jobs:
         uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
         env:
           SLACK_COLOR: ${{ job.status }}
-          SLACK_MESSAGE: "test-linux-x86 (${{ matrix.python-version }}) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" 
+          SLACK_MESSAGE: "publish-to-pypi finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
 
   publish-to-dockerhub:
     runs-on: ubuntu-latest
@@ -50,9 +50,11 @@ jobs:
       - name: Get version from tag
         run: |
           # remove leading 'v' and '-' from tag
-          export VERSION=`echo ${{ inputs.tag }} | sed "s/^v*//g" | sed "s/-//g"`
-          echo "VERSION=$VERSION" >> "${GITHUB_ENV}"
-          echo "NAME_TAG=${{ env.DOCKER_IMAGE_NAME }}:v$VERSION" >> "${GITHUB_ENV}"
+          VERSION=$(echo ${{ inputs.version }} | sed "s/^v*//g" | sed "s/-//g")
+          {
+            echo "VERSION=$VERSION"
+            echo "NAME_TAG=${{ env.DOCKER_IMAGE_NAME }}:v$VERSION"
+          } >> "${GITHUB_ENV}"
       - name: Build image
         run: |
           mkdir empty_context

diff --git a/.github/workflows/concrete_python_release_cpu.yml b/.github/workflows/concrete_python_release_cpu.yml
@@ -77,13 +77,13 @@ jobs:
         run: |
           NIGHTLY_VERSION=$(date +"%Y.%m.%d")
           NIGHTLY_VERSION_ONE_NUMBER=$(date +"%Y%m%d")
-          LATEST_RELEASE_VERSION=`git tag -l |grep "v.*" |sort |tail -n 1 | grep -e '[0-9].*' -o`
+          LATEST_RELEASE_VERSION=$(git tag -l |grep "v.*" |sort |tail -n 1 | grep -e '[0-9].*' -o)
           echo "__version__ = \"${LATEST_RELEASE_VERSION}-dev${NIGHTLY_VERSION_ONE_NUMBER}\"" >| frontends/concrete-python/version.txt
-          git tag nightly-$NIGHTLY_VERSION || true
-          git push origin nightly-$NIGHTLY_VERSION || true
+          git tag "nightly-${NIGHTLY_VERSION}" || true
+          git push origin "nightly-${NIGHTLY_VERSION}" || true
       - name: Set release version (public)
         if: ${{ env.RELEASE_TYPE == 'public' }}
-        run: echo "__version__ = \"`git describe --tags --abbrev=0 | grep -e '[0-9].*' -o`\"" >| frontends/concrete-python/version.txt
+        run: echo "__version__ = \"$(git describe --tags --abbrev=0 | grep -e '[0-9].*' -o)\"" >| frontends/concrete-python/version.txt
       - name: Expose release version from Python
         run: cp frontends/concrete-python/version.txt frontends/concrete-python/concrete/fhe/version.py
       - name: Build wheel
@@ -154,23 +154,22 @@ jobs:
         run: |
           brew install ninja ccache
       - name: Setup rust toolchain for concrete-cpu
-        uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu
+        uses: ./.github/actions/setup_rust_toolchain_for_concrete_cpu
       - name: Set release version (nightly)
         if: ${{ env.RELEASE_TYPE == 'nightly' }}
         run: |
-          NIGHTLY_VERSION=$(date +"%Y.%m.%d")
-          NIGHTLY_VERSION_ONE_NUMBER=$(date +"%Y%m%d")
-          LATEST_RELEASE_VERSION=`git tag -l |grep "v.*" |sort |tail -n 1 | grep -e '[0-9].*' -o`
-          echo "__version__ = \"${LATEST_RELEASE_VERSION}-dev${NIGHTLY_VERSION_ONE_NUMBER}\"" >| frontends/concrete-python/version.txt
+          NIGHTLY_VERSION=$(date +"%Y%m%d")
+          LATEST_RELEASE_VERSION=$(git tag -l |grep "v.*" |sort |tail -n 1 | grep -e '[0-9].*' -o)
+          echo "__version__ = \"${LATEST_RELEASE_VERSION}-dev${NIGHTLY_VERSION}\"" >| frontends/concrete-python/version.txt
       - name: Set release version (public)
         if: ${{ env.RELEASE_TYPE == 'public' }}
-        run: echo "__version__ = \"`git describe --tags --abbrev=0 | grep -e '[0-9].*' -o`\"" >| frontends/concrete-python/version.txt
+        run: echo "__version__ = \"$(git describe --tags --abbrev=0 | grep -e '[0-9].*' -o)\"" >| frontends/concrete-python/version.txt
       - name: Expose release version from Python
         run: cp frontends/concrete-python/version.txt frontends/concrete-python/concrete/fhe/version.py
       - name: Build wheel
         run: |
-          export CONCRETE_PYTHON=$(pwd)/frontends/concrete-python
-          export CONCRETE_COMPILER=$(pwd)/compilers/concrete-compiler/compiler
+          CONCRETE_PYTHON=$(pwd)/frontends/concrete-python
+          CONCRETE_COMPILER=$(pwd)/compilers/concrete-compiler/compiler
           export COMPILER_BUILD_DIRECTORY=$CONCRETE_COMPILER/build
           export PYTHON=${{ format('python{0}', matrix.python-version) }}
           echo "Using $PYTHON"
@@ -183,17 +182,17 @@ jobs:
           $PYTHON -m venv .venv && . .venv/bin/activate
 
           # Install requirements
-          pip install -r $CONCRETE_PYTHON/requirements.txt
-          pip install -r $CONCRETE_PYTHON/requirements.dev.txt
+          pip install -r "${CONCRETE_PYTHON}"/requirements.txt
+          pip install -r "${CONCRETE_PYTHON}"/requirements.dev.txt
 
           # Build python bindings of concrete compiler
-          cd $CONCRETE_COMPILER
+          cd "${CONCRETE_COMPILER}" || exit
           echo "Debug: ccache statistics (prior to the build):" && ccache -s
-          make Python3_EXECUTABLE=$(which python) python-bindings
+          make Python3_EXECUTABLE="$(which python)" python-bindings
           echo "Debug: ccache statistics (after the build):" && ccache -s
 
           # Build wheel
-          cd $CONCRETE_PYTHON
+          cd "${CONCRETE_PYTHON}" || exit
           rm -rf dist
           mkdir -p dist
           pip wheel -v --no-deps -w dist .
@@ -228,7 +227,7 @@ jobs:
           merge-multiple: true
       - name: generate hash
         id: hash
-        run: cd frontends/concrete-python/dist && echo "hash=$(sha256sum *.whl | base64 -w0)" >> $GITHUB_OUTPUT
+        run: cd frontends/concrete-python/dist && echo "hash=$(sha256sum ./*.whl | base64 -w0)" >> "${GITHUB_OUTPUT}"
 
   provenance:
     needs: [hash]
@@ -260,12 +259,13 @@ jobs:
       - name: create draft release
         if: ${{ env.RELEASE_TYPE == 'public'}}
         run: |
-          export TAG=$(git describe --tags --abbrev=0)
-          echo $TAG
+          TAG=$(git describe --tags --abbrev=0)
+          export TAG
+          echo "${TAG}"
           gh release create --draft --repo ${{ github.repository }} \
-          --verify-tag $TAG \
-          --title $TAG \
-          wheels/* *.intoto.jsonl/*
+          --verify-tag "${TAG}" \
+          --title "${TAG}" \
+          wheels/* ./*.intoto.jsonl/*
         env:
           GH_TOKEN: ${{ github.token }}
       - name: Upload wheels to S3
@@ -279,7 +279,7 @@ jobs:
         run: |
           pip install boto3 bigtree
           # upload wheels
-          aws s3 sync ./wheels/ s3://${S3_BUCKET_NAME}/cpu/concrete-python
+          aws s3 sync ./wheels/ "s3://${S3_BUCKET_NAME}/cpu/concrete-python"
           # update indexes and invalidate cloudfront cache
           python .github/workflows/scripts/s3_update_html_indexes.py
       - name: Slack Notification
@@ -323,24 +323,24 @@ jobs:
           CONCRETE_PYTHON=$(pwd)/repo/frontends/concrete-python
 
           # Initialize an empty test environment
-          cd $(mktemp -d)
+          cd "$(mktemp -d)"
           python -m venv .testenv && source .testenv/bin/activate
 
           # Install the concrete-python wheel
-          pip install $WHEEL_DIR/*.whl
+          pip install "${WHEEL_DIR}"/*.whl
 
           # Install extra requirements for tests
           sudo apt update -y
           sudo apt install -y graphviz libgraphviz-dev
-          pip install -r $CONCRETE_PYTHON/requirements.extra-full.txt
-          pip install -r $CONCRETE_PYTHON/requirements.dev.txt
+          pip install -r "${CONCRETE_PYTHON}"/requirements.extra-full.txt
+          pip install -r "${CONCRETE_PYTHON}"/requirements.dev.txt
 
           # TODO - check for version
 
           # Copy test files
-          cp -R $CONCRETE_PYTHON/tests .
-          cp -R $CONCRETE_PYTHON/pytest.ini .
-          cp $CONCRETE_PYTHON/Makefile .
+          cp -R "${CONCRETE_PYTHON}"/tests .
+          cp -R "${CONCRETE_PYTHON}"/pytest.ini .
+          cp "${CONCRETE_PYTHON}"/Makefile .
 
           # Running tests
           make tfhers-utils
@@ -402,27 +402,27 @@ jobs:
           PYTHON=${{ format('python{0}', matrix.python-version) }}
 
           # Initialize an empty test environment
-          export TEST_TMP_DIR=$(mktemp -d)
-          echo "TEST_TMP_DIR=$TEST_TMP_DIR" >> "${GITHUB_ENV}"
-          cd $TEST_TMP_DIR
+          TEST_TMP_DIR=$(mktemp -d)
+          echo "TEST_TMP_DIR=${TEST_TMP_DIR}" >> "${GITHUB_ENV}"
+          cd "${TEST_TMP_DIR}" || exit
 
           # Activate virtual environment
           $PYTHON -m venv .testenv && source .testenv/bin/activate
 
           # Install extra requirements for tests
-          pip install $WHEEL_DIR/*macos*.whl
-          pip install -r $CONCRETE_PYTHON/requirements.dev.txt
+          pip install "${WHEEL_DIR}"/*macos*.whl
+          pip install -r "${CONCRETE_PYTHON}"/requirements.dev.txt
 
           # MacOS x86 have conflict between our OpenMP library, and one from torch
           # we fix it by using a single one (from torch)
           # see discussion: https://discuss.python.org/t/conflicting-binary-extensions-in-different-packages/25332/8
-          export SITE_PACKAGES=$(pwd)/.testenv/lib/$PYTHON_VERSION/site-packages/
-          find $SITE_PACKAGES -not \( -path $SITE_PACKAGES/concrete -prune \) -name 'lib*omp5.dylib' -or -name 'lib*omp.dylib' | xargs -n 1  ln -f -s $SITE_PACKAGES/concrete/.dylibs/libomp.dylib
+          SITE_PACKAGES=$(pwd)/.testenv/lib/$PYTHON_VERSION/site-packages/
+          find . "${SITE_PACKAGES}" \( -not \( -path "${SITE_PACKAGES}"/concrete -prune \) -name 'lib*omp5.dylib' -or -name 'lib*omp.dylib' \) -exec -n 1  ln -f -s "${SITE_PACKAGES}"/concrete/.dylibs/libomp.dylib {} \;
 
           # Copy test files
-          cp -R $CONCRETE_PYTHON/tests .
-          cp -R $CONCRETE_PYTHON/pytest.ini .
-          cp $CONCRETE_PYTHON/Makefile .
+          cp -R "${CONCRETE_PYTHON}"/tests .
+          cp -R "${CONCRETE_PYTHON}"/pytest.ini .
+          cp "${CONCRETE_PYTHON}"/Makefile .
 
           # Running tests
           make tfhers-utils
@@ -431,7 +431,7 @@ jobs:
       - name: Cleanup host
         if: success() || failure()
         run: |
-          rm -rf $TEST_TMP_DIR
+          rm -rf "${TEST_TMP_DIR}"
       - name: Slack Notification
         if: ${{ failure() }}
         continue-on-error: true