From 4a7b2f1530669bdaca1951c70afef4e26ff993a9 Mon Sep 17 00:00:00 2001 From: Quentin Bourgerie Date: Tue, 12 Nov 2024 14:48:48 +0100 Subject: [PATCH] chore(ci): Fix actionlint --- .github/actionlint.yml | 5 ++ .../optimizer_setup/action.yml | 4 + .../action.yml | 4 + .../workflows/concrete_compiler_benchmark.yml | 12 +-- .../workflows/concrete_compiler_test_cpu.yml | 4 +- ...concrete_compiler_test_cpu_distributed.yml | 2 +- .../concrete_compiler_test_macos_cpu.yml | 28 ++++--- .github/workflows/concrete_ml_test.yml | 4 +- .github/workflows/concrete_optimizer.yml | 4 +- .../workflows/concrete_python_benchmark.yml | 3 +- .../concrete_python_finalize_release.yml | 12 +-- .../workflows/concrete_python_release_cpu.yml | 84 +++++++++---------- .../workflows/concrete_python_release_gpu.yml | 18 ++-- .../concrete_python_test_gpu_wheel.yml | 6 +- .../workflows/concrete_python_test_macos.yml | 40 ++++----- .github/workflows/docker_compliance.yml | 2 +- .github/workflows/ml_benchmark_subset.yml | 6 +- 17 files changed, 128 insertions(+), 110 deletions(-) create mode 100644 .github/actionlint.yml rename .github/{workflows => actions}/optimizer_setup/action.yml (85%) rename .github/{workflows => actions}/setup_rust_toolchain_for_concrete_cpu/action.yml (69%) diff --git a/.github/actionlint.yml b/.github/actionlint.yml new file mode 100644 index 0000000000..b811871fe2 --- /dev/null +++ b/.github/actionlint.yml @@ -0,0 +1,5 @@ +self-hosted-runner: + labels: + - distributed-ci + - aws-mac1-metal + - aws-mac2-metal diff --git a/.github/workflows/optimizer_setup/action.yml b/.github/actions/optimizer_setup/action.yml similarity index 85% rename from .github/workflows/optimizer_setup/action.yml rename to .github/actions/optimizer_setup/action.yml index b4048c82d2..3edbaa5235 100644 --- a/.github/workflows/optimizer_setup/action.yml +++ b/.github/actions/optimizer_setup/action.yml @@ -1,3 +1,7 @@ +name: Setup rust + +description: Setup rust for optimizer + runs: using: "composite" steps: diff --git a/.github/workflows/setup_rust_toolchain_for_concrete_cpu/action.yml b/.github/actions/setup_rust_toolchain_for_concrete_cpu/action.yml similarity index 69% rename from .github/workflows/setup_rust_toolchain_for_concrete_cpu/action.yml rename to .github/actions/setup_rust_toolchain_for_concrete_cpu/action.yml index 2f24806e96..1d44a0d7ad 100644 --- a/.github/workflows/setup_rust_toolchain_for_concrete_cpu/action.yml +++ b/.github/actions/setup_rust_toolchain_for_concrete_cpu/action.yml @@ -1,3 +1,7 @@ +name: Setup rust toolchain for concrete-cpu + +description: Setup rust toolchain for concrete-cpu + runs: using: "composite" steps: diff --git a/.github/workflows/concrete_compiler_benchmark.yml b/.github/workflows/concrete_compiler_benchmark.yml index cd660a016f..7c484c6892 100644 --- a/.github/workflows/concrete_compiler_benchmark.yml +++ b/.github/workflows/concrete_compiler_benchmark.yml @@ -57,15 +57,17 @@ jobs: - name: Ouput benchmark details id: benchmark-details run: | - echo "bench_date=$(date --iso-8601=seconds)" >> "$GITHUB_OUTPUT" - echo "commit_date=$(git --no-pager show -s --format=%cd --date=iso8601-strict ${{ github.sha }})" >> "$GITHUB_OUTPUT" - echo "commit_hash=$(git describe --tags --dirty)" >> "$GITHUB_OUTPUT" + { + echo "bench_date=$(date --iso-8601=seconds)"; + echo "commit_date=$(git --no-pager show -s --format=%cd --date=iso8601-strict ${{ github.sha }})"; + echo "commit_hash=$(git describe --tags --dirty)"; + } >> "$GITHUB_OUTPUT" - name: Set up home # "Install rust" step require root user to have a HOME directory which is not set. run: | echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" - name: Setup rust toolchain for concrete-cpu - uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu + uses: ./.github/actions/setup_rust_toolchain_for_concrete_cpu - name: Build compiler benchmarks run: | set -e @@ -126,7 +128,7 @@ jobs: shell: bash run: | echo "Computing HMac on downloaded artifact" - SIGNATURE="$(slab/scripts/hmac_calculator.sh $parsed_benchmark_results.json '${{ secrets.JOB_SECRET }}')" + SIGNATURE="$(slab/scripts/hmac_calculator.sh parsed_benchmark_results.json ${{ secrets.JOB_SECRET }})" echo "Sending results to Slab..." curl -v -k \ -H "Content-Type: application/json" \ diff --git a/.github/workflows/concrete_compiler_test_cpu.yml b/.github/workflows/concrete_compiler_test_cpu.yml index ad2e658eaa..4f99dac2cd 100644 --- a/.github/workflows/concrete_compiler_test_cpu.yml +++ b/.github/workflows/concrete_compiler_test_cpu.yml @@ -127,10 +127,10 @@ jobs: for i in `ls *Dialect.md`; do diff $i /concrete/docs/explanations/$i; done; - name: Enable complete tests on push to main if: github.ref == 'refs/heads/main' - run: echo "MINIMAL_TESTS=OFF" >> ${GITHUB_ENV} + run: echo "MINIMAL_TESTS=OFF" >> "${GITHUB_ENV}" - name: Enable minimal tests otherwise if: github.ref != 'refs/heads/main' - run: echo "MINIMAL_TESTS=ON" >> ${GITHUB_ENV} + run: echo "MINIMAL_TESTS=ON" >> "${GITHUB_ENV}" - name: Run compiler tests uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 with: diff --git a/.github/workflows/concrete_compiler_test_cpu_distributed.yml b/.github/workflows/concrete_compiler_test_cpu_distributed.yml index e836142979..fd5eab9c18 100644 --- a/.github/workflows/concrete_compiler_test_cpu_distributed.yml +++ b/.github/workflows/concrete_compiler_test_cpu_distributed.yml @@ -63,7 +63,7 @@ jobs: echo "HOME=/shared" >> "${GITHUB_ENV}" - name: Setup rust toolchain for concrete-cpu - uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu + uses: ./.github/actions/setup_rust_toolchain_for_concrete_cpu - name: Build end-to-end distributed test run: | diff --git a/.github/workflows/concrete_compiler_test_macos_cpu.yml b/.github/workflows/concrete_compiler_test_macos_cpu.yml index b5d9c5c30e..30434d28b5 100644 --- a/.github/workflows/concrete_compiler_test_macos_cpu.yml +++ b/.github/workflows/concrete_compiler_test_macos_cpu.yml @@ -39,13 +39,14 @@ jobs: submodules: recursive fetch-depth: 0 - name: Setup rust toolchain for concrete-cpu - uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu + uses: ./.github/actions/setup_rust_toolchain_for_concrete_cpu - name: Set python variables run: | - set -e - echo "PIP=${{ format('pip{0}', matrix.python-version) }}" >> "${GITHUB_ENV}" - echo "PYTHON=${{ format('python{0}', matrix.python-version) }}" >> "${GITHUB_ENV}" - echo "PYTHON_EXEC=$(which ${{ format('python{0}', matrix.python-version) }})" >> "${GITHUB_ENV}" + { + echo "PIP=${{ format('pip{0}', matrix.python-version) }}" + echo "PYTHON=${{ format('python{0}', matrix.python-version) }}" + echo "PYTHON_EXEC=$(which ${{ format('python{0}', matrix.python-version) }})" + } >> "${GITHUB_ENV}" - name: Install dependencies run: | brew install ninja ccache @@ -56,30 +57,31 @@ jobs: cd compilers/concrete-compiler/compiler echo "Debug: ccache statistics (prior to the build):" ccache -s - make Python3_EXECUTABLE=$PYTHON_EXEC all + make Python3_EXECUTABLE="${PYTHON_EXEC}" all echo "Debug: ccache statistics (after the build):" ccache -s - name: Enable complete tests on push to main if: github.ref == 'refs/heads/main' - run: echo "MINIMAL_TESTS=OFF" >> $GITHUB_ENV + run: echo "MINIMAL_TESTS=OFF" >> "${GITHUB_ENV}" - name: Enable minimal tests otherwise if: github.ref != 'refs/heads/main' - run: echo "MINIMAL_TESTS=ON" >> $GITHUB_ENV + run: echo "MINIMAL_TESTS=ON" >> "${GITHUB_ENV}" - name: Create keyset cache directory run: | - export KEY_CACHE_DIRECTORY=$(mktemp -d)/KeySetCache - echo "KEY_CACHE_DIRECTORY=$KEY_CACHE_DIRECTORY" >> "${GITHUB_ENV}" - mkdir $KEY_CACHE_DIRECTORY + KEY_CACHE_DIRECTORY=$(mktemp -d)/KeySetCache + export KEY_CACHE_DIRECTORY + echo "KEY_CACHE_DIRECTORY=${KEY_CACHE_DIRECTORY}" >> "${GITHUB_ENV}" + mkdir "${KEY_CACHE_DIRECTORY}" - name: Test run: | set -e cd compilers/concrete-compiler/compiler export CONCRETE_COMPILER_DATAFLOW_EXECUTION_ENABLED=OFF - make MINIMAL_TESTS=${{ env.MINIMAL_TESTS }} Python3_EXECUTABLE=$PYTHON_EXEC run-tests + make MINIMAL_TESTS=${{ env.MINIMAL_TESTS }} Python3_EXECUTABLE="${PYTHON_EXEC}" run-tests - name: Cleanup host if: success() || failure() run: | - rm -rf $KEY_CACHE_DIRECTORY + rm -rf "${KEY_CACHE_DIRECTORY}" - name: Slack Notification if: ${{ failure() && github.ref == 'refs/heads/main' }} continue-on-error: true diff --git a/.github/workflows/concrete_ml_test.yml b/.github/workflows/concrete_ml_test.yml index 26a8ea37e6..ae09ce1eb8 100644 --- a/.github/workflows/concrete_ml_test.yml +++ b/.github/workflows/concrete_ml_test.yml @@ -101,11 +101,11 @@ jobs: - name: ML Tests run: | export HOME="/home/ubuntu" - export CONCRETE_PYTHON_WHEEL=$(pwd)/frontends/concrete-python/dist/*manylinux*.whl + CONCRETE_PYTHON_WHEEL="$(pwd)/frontends/concrete-python/dist/*manylinux*.whl" apt update apt install git git-lfs -y pip install poetry==1.7.1 - ./ci/scripts/test_cml.sh --use-wheel $CONCRETE_PYTHON_WHEEL --verbose + ./ci/scripts/test_cml.sh --use-wheel "${CONCRETE_PYTHON_WHEEL}" --verbose - name: Slack Notification if: ${{ failure() && github.ref == 'refs/heads/main' }} continue-on-error: true diff --git a/.github/workflows/concrete_optimizer.yml b/.github/workflows/concrete_optimizer.yml index f74229d1cd..eee82b8325 100644 --- a/.github/workflows/concrete_optimizer.yml +++ b/.github/workflows/concrete_optimizer.yml @@ -36,7 +36,7 @@ jobs: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Setup - uses: ./.github/workflows/optimizer_setup + uses: ./.github/actions/optimizer_setup - name: Formatting run: | cargo --version @@ -62,7 +62,7 @@ jobs: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Setup - uses: ./.github/workflows/optimizer_setup + uses: ./.github/actions/optimizer_setup - name: Run benchmark run: | cd compilers/concrete-optimizer diff --git a/.github/workflows/concrete_python_benchmark.yml b/.github/workflows/concrete_python_benchmark.yml index 5142d117c2..5ce8af0e55 100644 --- a/.github/workflows/concrete_python_benchmark.yml +++ b/.github/workflows/concrete_python_benchmark.yml @@ -61,8 +61,7 @@ jobs: set -e rm -rf /build/* - export PYTHON=${{ format('python{0}', matrix.python-version) }} - echo "Using $PYTHON" + export PYTHON=python cd /concrete/frontends/concrete-python make PYTHON=$PYTHON venv diff --git a/.github/workflows/concrete_python_finalize_release.yml b/.github/workflows/concrete_python_finalize_release.yml index 482c6f60c6..ac38089a58 100644 --- a/.github/workflows/concrete_python_finalize_release.yml +++ b/.github/workflows/concrete_python_finalize_release.yml @@ -21,7 +21,7 @@ jobs: S3_BUCKET_NAME: ${{ secrets.AWS_S3_PYPI_BUCKET_NAME }} run: | mkdir wheels - aws s3 cp s3://${S3_BUCKET_NAME}/cpu/concrete-python/ ./wheels/ --recursive --exclude "*" --include "concrete_python-${{ inputs.version }}-*" + aws s3 cp "s3://${S3_BUCKET_NAME}/cpu/concrete-python/" ./wheels/ --recursive --exclude "*" --include "concrete_python-${{ inputs.version }}-*" echo "============== Downloaded wheels ===============" ls -la ./wheels/ - name: Push wheels to public PyPI (public) @@ -37,7 +37,7 @@ jobs: uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 env: SLACK_COLOR: ${{ job.status }} - SLACK_MESSAGE: "test-linux-x86 (${{ matrix.python-version }}) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + SLACK_MESSAGE: "publish-to-pypi finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" publish-to-dockerhub: runs-on: ubuntu-latest @@ -50,9 +50,11 @@ jobs: - name: Get version from tag run: | # remove leading 'v' and '-' from tag - export VERSION=`echo ${{ inputs.tag }} | sed "s/^v*//g" | sed "s/-//g"` - echo "VERSION=$VERSION" >> "${GITHUB_ENV}" - echo "NAME_TAG=${{ env.DOCKER_IMAGE_NAME }}:v$VERSION" >> "${GITHUB_ENV}" + VERSION=$(echo ${{ inputs.version }} | sed "s/^v*//g" | sed "s/-//g") + { + echo "VERSION=$VERSION" + echo "NAME_TAG=${{ env.DOCKER_IMAGE_NAME }}:v$VERSION" + } >> "${GITHUB_ENV}" - name: Build image run: | mkdir empty_context diff --git a/.github/workflows/concrete_python_release_cpu.yml b/.github/workflows/concrete_python_release_cpu.yml index 638d1271d7..4c124dace8 100644 --- a/.github/workflows/concrete_python_release_cpu.yml +++ b/.github/workflows/concrete_python_release_cpu.yml @@ -77,13 +77,13 @@ jobs: run: | NIGHTLY_VERSION=$(date +"%Y.%m.%d") NIGHTLY_VERSION_ONE_NUMBER=$(date +"%Y%m%d") - LATEST_RELEASE_VERSION=`git tag -l |grep "v.*" |sort |tail -n 1 | grep -e '[0-9].*' -o` + LATEST_RELEASE_VERSION=$(git tag -l |grep "v.*" |sort |tail -n 1 | grep -e '[0-9].*' -o) echo "__version__ = \"${LATEST_RELEASE_VERSION}-dev${NIGHTLY_VERSION_ONE_NUMBER}\"" >| frontends/concrete-python/version.txt - git tag nightly-$NIGHTLY_VERSION || true - git push origin nightly-$NIGHTLY_VERSION || true + git tag "nightly-${NIGHTLY_VERSION}" || true + git push origin "nightly-${NIGHTLY_VERSION}" || true - name: Set release version (public) if: ${{ env.RELEASE_TYPE == 'public' }} - run: echo "__version__ = \"`git describe --tags --abbrev=0 | grep -e '[0-9].*' -o`\"" >| frontends/concrete-python/version.txt + run: echo "__version__ = \"$(git describe --tags --abbrev=0 | grep -e '[0-9].*' -o)\"" >| frontends/concrete-python/version.txt - name: Expose release version from Python run: cp frontends/concrete-python/version.txt frontends/concrete-python/concrete/fhe/version.py - name: Build wheel @@ -154,23 +154,22 @@ jobs: run: | brew install ninja ccache - name: Setup rust toolchain for concrete-cpu - uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu + uses: ./.github/actions/setup_rust_toolchain_for_concrete_cpu - name: Set release version (nightly) if: ${{ env.RELEASE_TYPE == 'nightly' }} run: | - NIGHTLY_VERSION=$(date +"%Y.%m.%d") - NIGHTLY_VERSION_ONE_NUMBER=$(date +"%Y%m%d") - LATEST_RELEASE_VERSION=`git tag -l |grep "v.*" |sort |tail -n 1 | grep -e '[0-9].*' -o` - echo "__version__ = \"${LATEST_RELEASE_VERSION}-dev${NIGHTLY_VERSION_ONE_NUMBER}\"" >| frontends/concrete-python/version.txt + NIGHTLY_VERSION=$(date +"%Y%m%d") + LATEST_RELEASE_VERSION=$(git tag -l |grep "v.*" |sort |tail -n 1 | grep -e '[0-9].*' -o) + echo "__version__ = \"${LATEST_RELEASE_VERSION}-dev${NIGHTLY_VERSION}\"" >| frontends/concrete-python/version.txt - name: Set release version (public) if: ${{ env.RELEASE_TYPE == 'public' }} - run: echo "__version__ = \"`git describe --tags --abbrev=0 | grep -e '[0-9].*' -o`\"" >| frontends/concrete-python/version.txt + run: echo "__version__ = \"$(git describe --tags --abbrev=0 | grep -e '[0-9].*' -o)\"" >| frontends/concrete-python/version.txt - name: Expose release version from Python run: cp frontends/concrete-python/version.txt frontends/concrete-python/concrete/fhe/version.py - name: Build wheel run: | - export CONCRETE_PYTHON=$(pwd)/frontends/concrete-python - export CONCRETE_COMPILER=$(pwd)/compilers/concrete-compiler/compiler + CONCRETE_PYTHON=$(pwd)/frontends/concrete-python + CONCRETE_COMPILER=$(pwd)/compilers/concrete-compiler/compiler export COMPILER_BUILD_DIRECTORY=$CONCRETE_COMPILER/build export PYTHON=${{ format('python{0}', matrix.python-version) }} echo "Using $PYTHON" @@ -183,17 +182,17 @@ jobs: $PYTHON -m venv .venv && . .venv/bin/activate # Install requirements - pip install -r $CONCRETE_PYTHON/requirements.txt - pip install -r $CONCRETE_PYTHON/requirements.dev.txt + pip install -r "${CONCRETE_PYTHON}"/requirements.txt + pip install -r "${CONCRETE_PYTHON}"/requirements.dev.txt # Build python bindings of concrete compiler - cd $CONCRETE_COMPILER + cd "${CONCRETE_COMPILER}" || exit echo "Debug: ccache statistics (prior to the build):" && ccache -s - make Python3_EXECUTABLE=$(which python) python-bindings + make Python3_EXECUTABLE="$(which python)" python-bindings echo "Debug: ccache statistics (after the build):" && ccache -s # Build wheel - cd $CONCRETE_PYTHON + cd "${CONCRETE_PYTHON}" || exit rm -rf dist mkdir -p dist pip wheel -v --no-deps -w dist . @@ -228,7 +227,7 @@ jobs: merge-multiple: true - name: generate hash id: hash - run: cd frontends/concrete-python/dist && echo "hash=$(sha256sum *.whl | base64 -w0)" >> $GITHUB_OUTPUT + run: cd frontends/concrete-python/dist && echo "hash=$(sha256sum ./*.whl | base64 -w0)" >> "${GITHUB_OUTPUT}" provenance: needs: [hash] @@ -260,12 +259,13 @@ jobs: - name: create draft release if: ${{ env.RELEASE_TYPE == 'public'}} run: | - export TAG=$(git describe --tags --abbrev=0) - echo $TAG + TAG=$(git describe --tags --abbrev=0) + export TAG + echo "${TAG}" gh release create --draft --repo ${{ github.repository }} \ - --verify-tag $TAG \ - --title $TAG \ - wheels/* *.intoto.jsonl/* + --verify-tag "${TAG}" \ + --title "${TAG}" \ + wheels/* ./*.intoto.jsonl/* env: GH_TOKEN: ${{ github.token }} - name: Upload wheels to S3 @@ -279,7 +279,7 @@ jobs: run: | pip install boto3 bigtree # upload wheels - aws s3 sync ./wheels/ s3://${S3_BUCKET_NAME}/cpu/concrete-python + aws s3 sync ./wheels/ "s3://${S3_BUCKET_NAME}/cpu/concrete-python" # update indexes and invalidate cloudfront cache python .github/workflows/scripts/s3_update_html_indexes.py - name: Slack Notification @@ -323,24 +323,24 @@ jobs: CONCRETE_PYTHON=$(pwd)/repo/frontends/concrete-python # Initialize an empty test environment - cd $(mktemp -d) + cd "$(mktemp -d)" python -m venv .testenv && source .testenv/bin/activate # Install the concrete-python wheel - pip install $WHEEL_DIR/*.whl + pip install "${WHEEL_DIR}"/*.whl # Install extra requirements for tests sudo apt update -y sudo apt install -y graphviz libgraphviz-dev - pip install -r $CONCRETE_PYTHON/requirements.extra-full.txt - pip install -r $CONCRETE_PYTHON/requirements.dev.txt + pip install -r "${CONCRETE_PYTHON}"/requirements.extra-full.txt + pip install -r "${CONCRETE_PYTHON}"/requirements.dev.txt # TODO - check for version # Copy test files - cp -R $CONCRETE_PYTHON/tests . - cp -R $CONCRETE_PYTHON/pytest.ini . - cp $CONCRETE_PYTHON/Makefile . + cp -R "${CONCRETE_PYTHON}"/tests . + cp -R "${CONCRETE_PYTHON}"/pytest.ini . + cp "${CONCRETE_PYTHON}"/Makefile . # Running tests make tfhers-utils @@ -402,27 +402,27 @@ jobs: PYTHON=${{ format('python{0}', matrix.python-version) }} # Initialize an empty test environment - export TEST_TMP_DIR=$(mktemp -d) - echo "TEST_TMP_DIR=$TEST_TMP_DIR" >> "${GITHUB_ENV}" - cd $TEST_TMP_DIR + TEST_TMP_DIR=$(mktemp -d) + echo "TEST_TMP_DIR=${TEST_TMP_DIR}" >> "${GITHUB_ENV}" + cd "${TEST_TMP_DIR}" || exit # Activate virtual environment $PYTHON -m venv .testenv && source .testenv/bin/activate # Install extra requirements for tests - pip install $WHEEL_DIR/*macos*.whl - pip install -r $CONCRETE_PYTHON/requirements.dev.txt + pip install "${WHEEL_DIR}"/*macos*.whl + pip install -r "${CONCRETE_PYTHON}"/requirements.dev.txt # MacOS x86 have conflict between our OpenMP library, and one from torch # we fix it by using a single one (from torch) # see discussion: https://discuss.python.org/t/conflicting-binary-extensions-in-different-packages/25332/8 - export SITE_PACKAGES=$(pwd)/.testenv/lib/$PYTHON_VERSION/site-packages/ - find $SITE_PACKAGES -not \( -path $SITE_PACKAGES/concrete -prune \) -name 'lib*omp5.dylib' -or -name 'lib*omp.dylib' | xargs -n 1 ln -f -s $SITE_PACKAGES/concrete/.dylibs/libomp.dylib + SITE_PACKAGES=$(pwd)/.testenv/lib/$PYTHON_VERSION/site-packages/ + find . "${SITE_PACKAGES}" \( -not \( -path "${SITE_PACKAGES}"/concrete -prune \) -name 'lib*omp5.dylib' -or -name 'lib*omp.dylib' \) -exec -n 1 ln -f -s "${SITE_PACKAGES}"/concrete/.dylibs/libomp.dylib {} \; # Copy test files - cp -R $CONCRETE_PYTHON/tests . - cp -R $CONCRETE_PYTHON/pytest.ini . - cp $CONCRETE_PYTHON/Makefile . + cp -R "${CONCRETE_PYTHON}"/tests . + cp -R "${CONCRETE_PYTHON}"/pytest.ini . + cp "${CONCRETE_PYTHON}"/Makefile . # Running tests make tfhers-utils @@ -431,7 +431,7 @@ jobs: - name: Cleanup host if: success() || failure() run: | - rm -rf $TEST_TMP_DIR + rm -rf "${TEST_TMP_DIR}" - name: Slack Notification if: ${{ failure() }} continue-on-error: true diff --git a/.github/workflows/concrete_python_release_gpu.yml b/.github/workflows/concrete_python_release_gpu.yml index 231ab27061..b93e101e45 100644 --- a/.github/workflows/concrete_python_release_gpu.yml +++ b/.github/workflows/concrete_python_release_gpu.yml @@ -58,13 +58,13 @@ jobs: run: | NIGHTLY_VERSION=$(date +"%Y.%m.%d") NIGHTLY_VERSION_ONE_NUMBER=$(date +"%Y%m%d") - LATEST_RELEASE_VERSION=`git tag -l |grep "v.*" |sort |tail -n 1 | grep -e '[0-9].*' -o` + LATEST_RELEASE_VERSION=$(git tag -l |grep "v.*" |sort |tail -n 1 | grep -e '[0-9].*' -o) echo "__version__ = \"${LATEST_RELEASE_VERSION}-dev${NIGHTLY_VERSION_ONE_NUMBER}\"" >| frontends/concrete-python/version.txt - git tag nightly-$NIGHTLY_VERSION || true - git push origin nightly-$NIGHTLY_VERSION || true + git tag nightly-"${NIGHTLY_VERSION}" || true + git push origin nightly-"${NIGHTLY_VERSION}" || true - name: Set release version (public) if: ${{ env.RELEASE_TYPE == 'public' }} - run: echo "__version__ = \"`git describe --tags --abbrev=0 | grep -e '[0-9].*' -o`\"" >| frontends/concrete-python/version.txt + run: echo "__version__ = \"$(git describe --tags --abbrev=0 | grep -e '[0-9].*' -o)\"" >| frontends/concrete-python/version.txt - name: Expose release version from Python run: cp frontends/concrete-python/version.txt frontends/concrete-python/concrete/fhe/version.py - name: Build wheel @@ -167,13 +167,13 @@ jobs: run: | pip install boto3 bigtree # upload wheels - aws s3 sync ./wheels/ s3://${S3_BUCKET_NAME}/gpu/concrete-python + aws s3 sync ./wheels/ s3://"${S3_BUCKET_NAME}"/gpu/concrete-python # update indexes and invalidate cloudfront cache python .github/workflows/scripts/s3_update_html_indexes.py - name: Output Wheel Version id: version run: | - export VERSION=`ls ./wheels/*manylinux* | head -n1 | cut -d "-" -f2` + VERSION=$(find ./wheels/ -name '*manylinux*' | head -n1 | cut -d "-" -f2) echo "VERSION=$VERSION" echo "wheel_version=$VERSION" >> "$GITHUB_OUTPUT" - name: Slack Notification @@ -229,11 +229,11 @@ jobs: # Install extra requirements for tests sudo apt update -y sudo apt install -y graphviz libgraphviz-dev - pip install -r $CONCRETE_PYTHON/requirements.extra-full.txt - pip install -r $CONCRETE_PYTHON/requirements.dev.txt + pip install -r "${CONCRETE_PYTHON}"/requirements.extra-full.txt + pip install -r "${CONCRETE_PYTHON}"/requirements.dev.txt # Running tests - cd $CONCRETE_PYTHON + cd "${CONCRETE_PYTHON}" make pytest-gpu diff --git a/.github/workflows/concrete_python_test_gpu_wheel.yml b/.github/workflows/concrete_python_test_gpu_wheel.yml index 96cb1a63f6..6cd0e37e1b 100644 --- a/.github/workflows/concrete_python_test_gpu_wheel.yml +++ b/.github/workflows/concrete_python_test_gpu_wheel.yml @@ -54,9 +54,9 @@ jobs: # Install extra requirements for tests sudo apt update -y sudo apt install -y graphviz libgraphviz-dev - pip install -r $CONCRETE_PYTHON/requirements.extra-full.txt - pip install -r $CONCRETE_PYTHON/requirements.dev.txt + pip install -r "${CONCRETE_PYTHON}"/requirements.extra-full.txt + pip install -r "${CONCRETE_PYTHON}"/requirements.dev.txt # Running tests - cd $CONCRETE_PYTHON + cd "${CONCRETE_PYTHON}" make pytest-gpu diff --git a/.github/workflows/concrete_python_test_macos.yml b/.github/workflows/concrete_python_test_macos.yml index 9c12d56386..c297c72a30 100644 --- a/.github/workflows/concrete_python_test_macos.yml +++ b/.github/workflows/concrete_python_test_macos.yml @@ -30,7 +30,7 @@ jobs: run: | brew install ninja ccache - name: Setup rust toolchain for concrete-cpu - uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu + uses: ./.github/actions/setup_rust_toolchain_for_concrete_cpu - name: Cache compilation (push) if: github.event_name == 'push' uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 @@ -50,7 +50,7 @@ jobs: - name: Prepare build environment run: | set -e - cd $GITHUB_WORKSPACE/frontends/concrete-python + cd ${{ github.workspace }}/frontends/concrete-python # Setup pkg-config to find OpenBLAS (scipy need it) export PKG_CONFIG_PATH="/opt/homebrew/opt/openblas/lib/pkgconfig" @@ -58,23 +58,23 @@ jobs: rm -rf .venv python3.10 -m venv .venv - . $GITHUB_WORKSPACE/frontends/concrete-python/.venv/bin/activate + . ${{ github.workspace }}/frontends/concrete-python/.venv/bin/activate pip install -r requirements.dev.txt pip install -r requirements.txt - name: Build concrete-compiler python-bindings run: | - $GITHUB_WORKSPACE/frontends/concrete-python .venv/bin/activate - cd $GITHUB_WORKSPACE/compilers/concrete-compiler/compiler + ${{ github.workspace }}/frontends/concrete-python .venv/bin/activate + cd ${{ github.workspace }}/compilers/concrete-compiler/compiler ccache -z - make Python3_EXECUTABLE=$(which python) python-bindings + make Python3_EXECUTABLE="$(which python)" python-bindings ccache -s - name: Create wheels run: | - $GITHUB_WORKSPACE/frontends/concrete-python .venv/bin/activate - cd $GITHUB_WORKSPACE/frontends/concrete-python + ${{ github.workspace }}/frontends/concrete-python .venv/bin/activate + cd ${{ github.workspace }}/frontends/concrete-python - export COMPILER_BUILD_DIRECTORY=$GITHUB_WORKSPACE/compilers/concrete-compiler/compiler + export COMPILER_BUILD_DIRECTORY=${{ github.workspace }}/compilers/concrete-compiler/compiler rm -rf dist && mkdir -p dist pip wheel -v --no-deps -w dist . delocate-wheel -v dist/*macos*.whl @@ -83,37 +83,37 @@ jobs: - name: Prepare test environment run: | set -e - export TEST_TMP_DIR=$(mktemp -d) + TEST_TMP_DIR=$(mktemp -d) echo "TEST_TMP_DIR=$TEST_TMP_DIR" >> "${GITHUB_ENV}" - cd $TEST_TMP_DIR + cd "${TEST_TMP_DIR}" python3.10 -m venv .testenv . .testenv/bin/activate - pip install $GITHUB_WORKSPACE/frontends/concrete-python/dist/*macos*.whl - pip install -r $GITHUB_WORKSPACE/frontends/concrete-python/requirements.dev.txt + pip install ${{ github.workspace }}/frontends/concrete-python/dist/*macos*.whl + pip install -r ${{ github.workspace }}/frontends/concrete-python/requirements.dev.txt # MacOS x86 have conflict between our OpenMP library, and one from torch # we fix it by using a single one (from torch) # see discussion: https://discuss.python.org/t/conflicting-binary-extensions-in-different-packages/25332/8 - find .testenv/lib/python3.10/site-packages -not \( -path .testenv/lib/python3.10/site-packages/concrete -prune \) -name 'lib*omp5.dylib' -or -name 'lib*omp.dylib' | xargs -n 1 ln -f -s $(pwd)/.testenv/lib/python3.10/site-packages/concrete/.dylibs/libomp.dylib + find .testenv/lib/python3.10/site-packages \( -not \( -path .testenv/lib/python3.10/site-packages/concrete -prune \) -name 'lib*omp5.dylib' -or -name 'lib*omp.dylib' \) -exec -n 1 ln -f -s "$(pwd)/.testenv/lib/python3.10/site-packages/concrete/.dylibs/libomp.dylib" {} \; - cp -R $GITHUB_WORKSPACE/frontends/concrete-python/examples ./examples - cp -R $GITHUB_WORKSPACE/frontends/concrete-python/tests ./tests + cp -R ${{ github.workspace }}/frontends/concrete-python/examples ./examples + cp -R ${{ github.workspace }}/frontends/concrete-python/tests ./tests - cp $GITHUB_WORKSPACE/frontends/concrete-python/Makefile . + cp ${{ github.workspace }}/frontends/concrete-python/Makefile . - name: Run pytest-macos run: | set -e - cd $TEST_TMP_DIR && . .testenv/bin/activate + cd "${TEST_TMP_DIR}" && . .testenv/bin/activate KEY_CACHE_DIRECTORY=./KeySetCache PYTEST_MARKERS="not dataflow and not graphviz" make pytest-macos - name: Run test-notebooks run: | set -e - cd $TEST_TMP_DIR && . .testenv/bin/activate + cd "${TEST_TMP_DIR}" && . .testenv/bin/activate make test-notebooks - name: Cleanup host if: success() || failure() run: | - rm -rf $TEST_TMP_DIR + rm -rf "${TEST_TMP_DIR}" diff --git a/.github/workflows/docker_compliance.yml b/.github/workflows/docker_compliance.yml index d5da1eda40..832b5fab76 100644 --- a/.github/workflows/docker_compliance.yml +++ b/.github/workflows/docker_compliance.yml @@ -19,7 +19,7 @@ jobs: - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - name: Lint All Dockerfiles (except third_party) - run: hadolint -V `find -name "*Dockerfile*" -not -path "./third_party/*" |xargs ` + run: find . -name "*Dockerfile*" -not -path "./third_party/*" -exec hadolint -V {} \; - name: Slack Notification if: ${{ failure() && github.ref == 'refs/heads/main' }} continue-on-error: true diff --git a/.github/workflows/ml_benchmark_subset.yml b/.github/workflows/ml_benchmark_subset.yml index 860525e21d..3440349ba2 100644 --- a/.github/workflows/ml_benchmark_subset.yml +++ b/.github/workflows/ml_benchmark_subset.yml @@ -59,7 +59,7 @@ jobs: echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" - name: Setup rust toolchain for concrete-cpu - uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu + uses: ./.github/actions/setup_rust_toolchain_for_concrete_cpu - name: Build compiler and ML benchmarks run: | @@ -94,9 +94,9 @@ jobs: python3 ./ci/benchmark_parser.py compiler/benchmarks_results.json ${{ env.RESULTS_FILENAME }} \ --database compiler_benchmarks \ --hardware ${{ inputs.instance_type }} \ - --project-version ${COMMIT_HASH} \ + --project-version "${COMMIT_HASH}" \ --branch ${{ github.ref_name }} \ - --commit-date ${COMMIT_DATE} \ + --commit-date "${COMMIT_DATE}" \ --bench-date "${{ env.BENCH_DATE }}" - name: Upload parsed results artifact