From 1adf9ee236cc64783cb376d6afec3b026eb2b7ac Mon Sep 17 00:00:00 2001 From: Quentin Bourgerie Date: Thu, 14 Nov 2024 09:39:29 +0100 Subject: [PATCH 1/2] fix(ci): Macos python test --- .../workflows/concrete_python_release_cpu.yml | 6 +- .../workflows/concrete_python_test_macos.yml | 57 +++++++------------ .../scripts/fix_multi_omp_bug_macos.sh | 10 ++++ 3 files changed, 32 insertions(+), 41 deletions(-) create mode 100755 .github/workflows/scripts/fix_multi_omp_bug_macos.sh diff --git a/.github/workflows/concrete_python_release_cpu.yml b/.github/workflows/concrete_python_release_cpu.yml index 60f827935b..87f0dcdb2c 100644 --- a/.github/workflows/concrete_python_release_cpu.yml +++ b/.github/workflows/concrete_python_release_cpu.yml @@ -416,11 +416,7 @@ jobs: pip install "${WHEEL_DIR}"/*macos*.whl pip install -r "${CONCRETE_PYTHON}"/requirements.dev.txt - # MacOS x86 have conflict between our OpenMP library, and one from torch - # we fix it by using a single one (from torch) - # see discussion: https://discuss.python.org/t/conflicting-binary-extensions-in-different-packages/25332/8 - SITE_PACKAGES=$(pwd)/.testenv/lib/$PYTHON/site-packages/ - find . "${SITE_PACKAGES}" \( -not \( -path "${SITE_PACKAGES}"/concrete -prune \) -name 'lib*omp5.dylib' -or -name 'lib*omp.dylib' \) -exec -n 1 ln -f -s "${SITE_PACKAGES}"/concrete/.dylibs/libomp.dylib {} \; + ${{ github.workspace }}/.github/workflows/scripts/fix_multi_omp_bug_macos.sh # Copy test files cp -R "${CONCRETE_PYTHON}"/tests . diff --git a/.github/workflows/concrete_python_test_macos.yml b/.github/workflows/concrete_python_test_macos.yml index c297c72a30..4be70ed3de 100644 --- a/.github/workflows/concrete_python_test_macos.yml +++ b/.github/workflows/concrete_python_test_macos.yml @@ -18,8 +18,13 @@ jobs: strategy: fail-fast: false matrix: + python-version: ["3.10"] machine: ["aws-mac1-metal", "aws-mac2-metal"] runs-on: ${{ matrix.machine }} + env: + python: python${{matrix.python-version}} + concrete-python-dir: ${{ github.workspace }}/frontends/concrete-python + concrete-compiler-dir: ${{ github.workspace }}/compilers/concrete-compiler/compiler steps: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -31,50 +36,34 @@ jobs: brew install ninja ccache - name: Setup rust toolchain for concrete-cpu uses: ./.github/actions/setup_rust_toolchain_for_concrete_cpu - - name: Cache compilation (push) - if: github.event_name == 'push' - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 - with: - path: /Users/runner/Library/Caches/ccache - key: ${{ runner.os }}-${{ runner.arch }}-compilation-cache-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-${{ runner.arch }}-compilation-cache- - - name: Cache compilation (pull_request) - if: github.event_name == 'pull_request' - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 - with: - path: /Users/runner/Library/Caches/ccache - key: ${{ runner.os }}-${{ runner.arch }}-compilation-cache-${{ github.event.pull_request.base.sha }} - restore-keys: | - ${{ runner.os }}-${{ runner.arch }}-compilation-cache- - name: Prepare build environment run: | set -e - cd ${{ github.workspace }}/frontends/concrete-python + cd ${{ env.concrete-python-dir }} # Setup pkg-config to find OpenBLAS (scipy need it) export PKG_CONFIG_PATH="/opt/homebrew/opt/openblas/lib/pkgconfig" rm -rf .venv - python3.10 -m venv .venv + ${{ env.python }} -m venv .venv - . ${{ github.workspace }}/frontends/concrete-python/.venv/bin/activate + . ${{ env.concrete-python-dir }}/.venv/bin/activate pip install -r requirements.dev.txt pip install -r requirements.txt - name: Build concrete-compiler python-bindings run: | - ${{ github.workspace }}/frontends/concrete-python .venv/bin/activate - cd ${{ github.workspace }}/compilers/concrete-compiler/compiler + . ${{ env.concrete-python-dir }}/.venv/bin/activate + cd ${{ env.concrete-compiler-dir }} ccache -z - make Python3_EXECUTABLE="$(which python)" python-bindings + make Python3_EXECUTABLE="$(which ${{ env.python }})" python-bindings ccache -s - name: Create wheels run: | - ${{ github.workspace }}/frontends/concrete-python .venv/bin/activate - cd ${{ github.workspace }}/frontends/concrete-python + . ${{ env.concrete-python-dir }}/.venv/bin/activate + cd ${{ env.concrete-python-dir }} - export COMPILER_BUILD_DIRECTORY=${{ github.workspace }}/compilers/concrete-compiler/compiler + export COMPILER_BUILD_DIRECTORY=${{ env.concrete-compiler-dir }}/build rm -rf dist && mkdir -p dist pip wheel -v --no-deps -w dist . delocate-wheel -v dist/*macos*.whl @@ -87,22 +76,18 @@ jobs: echo "TEST_TMP_DIR=$TEST_TMP_DIR" >> "${GITHUB_ENV}" cd "${TEST_TMP_DIR}" - python3.10 -m venv .testenv + ${{ env.python }} -m venv .testenv . .testenv/bin/activate - pip install ${{ github.workspace }}/frontends/concrete-python/dist/*macos*.whl - pip install -r ${{ github.workspace }}/frontends/concrete-python/requirements.dev.txt - - # MacOS x86 have conflict between our OpenMP library, and one from torch - # we fix it by using a single one (from torch) - # see discussion: https://discuss.python.org/t/conflicting-binary-extensions-in-different-packages/25332/8 + pip install ${{ env.concrete-python-dir }}/dist/*macos*.whl + pip install -r ${{ env.concrete-python-dir }}/requirements.dev.txt - find .testenv/lib/python3.10/site-packages \( -not \( -path .testenv/lib/python3.10/site-packages/concrete -prune \) -name 'lib*omp5.dylib' -or -name 'lib*omp.dylib' \) -exec -n 1 ln -f -s "$(pwd)/.testenv/lib/python3.10/site-packages/concrete/.dylibs/libomp.dylib" {} \; + ${{ github.workspace }}/.github/workflows/scripts/fix_multi_omp_bug_macos.sh - cp -R ${{ github.workspace }}/frontends/concrete-python/examples ./examples - cp -R ${{ github.workspace }}/frontends/concrete-python/tests ./tests + cp -R ${{ env.concrete-python-dir }}/examples ./examples + cp -R ${{ env.concrete-python-dir }}/tests ./tests - cp ${{ github.workspace }}/frontends/concrete-python/Makefile . + cp ${{ env.concrete-python-dir }}/Makefile . - name: Run pytest-macos run: | set -e diff --git a/.github/workflows/scripts/fix_multi_omp_bug_macos.sh b/.github/workflows/scripts/fix_multi_omp_bug_macos.sh new file mode 100755 index 0000000000..7b536c3a6f --- /dev/null +++ b/.github/workflows/scripts/fix_multi_omp_bug_macos.sh @@ -0,0 +1,10 @@ +#!/bin/bash -e +# Link all omp lib to concrete one to avoid load of different omp lib. + +SITE_PACKAGES=$(python -c "import site; print(site.getsitepackages()[0])") +if [ $(basename "$SITE_PACKAGES") != "site-packages" ]; then + echo "python site packages($SITE_PACKAGES) dirname is not equals to 'site-packages', you probably not execute this script in a venv" + exit 1 +fi + +find "$SITE_PACKAGES" \( -not \( -path "$SITE_PACKAGES/concrete" -prune \) -name 'lib*omp5.dylib' -or -name 'lib*omp.dylib' \) -exec ln -f -s "$SITE_PACKAGES/concrete/.dylibs/libomp.dylib" {} \; From dd65e8ff184f2e1f374d78a14c0b620bf67acb85 Mon Sep 17 00:00:00 2001 From: Quentin Bourgerie Date: Fri, 15 Nov 2024 14:06:11 +0100 Subject: [PATCH 2/2] chore(ci): Remove zsh debugging --- .github/workflows/concrete_python_release_cpu.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/concrete_python_release_cpu.yml b/.github/workflows/concrete_python_release_cpu.yml index 87f0dcdb2c..f05e87c0b5 100644 --- a/.github/workflows/concrete_python_release_cpu.yml +++ b/.github/workflows/concrete_python_release_cpu.yml @@ -384,9 +384,6 @@ jobs: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] runs-on: ["aws-mac1-metal", "aws-mac2-metal"] runs-on: ${{ matrix.runs-on }} - defaults: - run: - shell: zsh steps: - name: Download wheels uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8