Merge pull request #1140 from zama-ai/fix-test-macos

Fix test macos
zama-ai · Nov 15, 2024 · de1e246 · de1e246
2 parents b796dc8 + dd65e8f
commit de1e246
Show file tree

Hide file tree

Showing 3 changed files with 32 additions and 44 deletions.
diff --git a/.github/workflows/concrete_python_release_cpu.yml b/.github/workflows/concrete_python_release_cpu.yml
@@ -384,9 +384,6 @@ jobs:
         python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
         runs-on: ["aws-mac1-metal", "aws-mac2-metal"]
     runs-on: ${{ matrix.runs-on }}
-    defaults:
-      run:
-        shell: zsh
     steps:
       - name: Download wheels
         uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
@@ -416,11 +413,7 @@ jobs:
           pip install "${WHEEL_DIR}"/*macos*.whl
           pip install -r "${CONCRETE_PYTHON}"/requirements.dev.txt
 
-          # MacOS x86 have conflict between our OpenMP library, and one from torch
-          # we fix it by using a single one (from torch)
-          # see discussion: https://discuss.python.org/t/conflicting-binary-extensions-in-different-packages/25332/8
-          SITE_PACKAGES=$(pwd)/.testenv/lib/$PYTHON/site-packages/
-          find . "${SITE_PACKAGES}" \( -not \( -path "${SITE_PACKAGES}"/concrete -prune \) -name 'lib*omp5.dylib' -or -name 'lib*omp.dylib' \) -exec -n 1  ln -f -s "${SITE_PACKAGES}"/concrete/.dylibs/libomp.dylib {} \;
+          ${{ github.workspace }}/.github/workflows/scripts/fix_multi_omp_bug_macos.sh
 
           # Copy test files
           cp -R "${CONCRETE_PYTHON}"/tests .

diff --git a/.github/workflows/concrete_python_test_macos.yml b/.github/workflows/concrete_python_test_macos.yml
@@ -18,8 +18,13 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
+        python-version: ["3.10"]
         machine: ["aws-mac1-metal", "aws-mac2-metal"]
     runs-on: ${{ matrix.machine }}
+    env:
+      python: python${{matrix.python-version}}
+      concrete-python-dir: ${{ github.workspace }}/frontends/concrete-python
+      concrete-compiler-dir: ${{ github.workspace }}/compilers/concrete-compiler/compiler
     steps:
       - name: Checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -31,50 +36,34 @@ jobs:
           brew install ninja ccache
       - name: Setup rust toolchain for concrete-cpu
         uses: ./.github/actions/setup_rust_toolchain_for_concrete_cpu
-      - name: Cache compilation (push)
-        if: github.event_name == 'push'
-        uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
-        with:
-          path: /Users/runner/Library/Caches/ccache
-          key: ${{ runner.os }}-${{ runner.arch }}-compilation-cache-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-${{ runner.arch }}-compilation-cache-
-      - name: Cache compilation (pull_request)
-        if: github.event_name == 'pull_request'
-        uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
-        with:
-          path: /Users/runner/Library/Caches/ccache
-          key: ${{ runner.os }}-${{ runner.arch }}-compilation-cache-${{ github.event.pull_request.base.sha }}
-          restore-keys: |
-            ${{ runner.os }}-${{ runner.arch }}-compilation-cache-
       - name: Prepare build environment
         run: |
           set -e
-          cd ${{ github.workspace }}/frontends/concrete-python
+          cd ${{ env.concrete-python-dir }}
 
           # Setup pkg-config to find OpenBLAS (scipy need it)
           export PKG_CONFIG_PATH="/opt/homebrew/opt/openblas/lib/pkgconfig"
 
           rm -rf .venv
-          python3.10 -m venv .venv
+          ${{ env.python }} -m venv .venv
 
-          . ${{ github.workspace }}/frontends/concrete-python/.venv/bin/activate
+          . ${{ env.concrete-python-dir }}/.venv/bin/activate
 
           pip install -r requirements.dev.txt
           pip install -r requirements.txt
       - name: Build concrete-compiler python-bindings
         run: |
-          ${{ github.workspace }}/frontends/concrete-python .venv/bin/activate
-          cd ${{ github.workspace }}/compilers/concrete-compiler/compiler
+          . ${{ env.concrete-python-dir }}/.venv/bin/activate
+          cd ${{ env.concrete-compiler-dir }}
           ccache -z
-          make Python3_EXECUTABLE="$(which python)" python-bindings
+          make Python3_EXECUTABLE="$(which ${{ env.python }})" python-bindings
           ccache -s
       - name: Create wheels
         run: |
-          ${{ github.workspace }}/frontends/concrete-python .venv/bin/activate
-          cd ${{ github.workspace }}/frontends/concrete-python
+          . ${{ env.concrete-python-dir }}/.venv/bin/activate
+          cd ${{ env.concrete-python-dir }}
 
-          export COMPILER_BUILD_DIRECTORY=${{ github.workspace }}/compilers/concrete-compiler/compiler
+          export COMPILER_BUILD_DIRECTORY=${{ env.concrete-compiler-dir }}/build
           rm -rf dist && mkdir -p dist
           pip wheel -v --no-deps -w dist .
           delocate-wheel -v dist/*macos*.whl
@@ -87,22 +76,18 @@ jobs:
           echo "TEST_TMP_DIR=$TEST_TMP_DIR" >> "${GITHUB_ENV}"
           cd "${TEST_TMP_DIR}"
 
-          python3.10 -m venv .testenv
+          ${{ env.python }} -m venv .testenv
           . .testenv/bin/activate
 
-          pip install ${{ github.workspace }}/frontends/concrete-python/dist/*macos*.whl
-          pip install -r ${{ github.workspace }}/frontends/concrete-python/requirements.dev.txt
-
-          # MacOS x86 have conflict between our OpenMP library, and one from torch
-          # we fix it by using a single one (from torch)
-          # see discussion: https://discuss.python.org/t/conflicting-binary-extensions-in-different-packages/25332/8
+          pip install ${{ env.concrete-python-dir }}/dist/*macos*.whl
+          pip install -r ${{ env.concrete-python-dir }}/requirements.dev.txt
 
-          find .testenv/lib/python3.10/site-packages \( -not \( -path .testenv/lib/python3.10/site-packages/concrete -prune \) -name 'lib*omp5.dylib' -or -name 'lib*omp.dylib' \) -exec -n 1  ln -f -s "$(pwd)/.testenv/lib/python3.10/site-packages/concrete/.dylibs/libomp.dylib" {} \;
+          ${{ github.workspace }}/.github/workflows/scripts/fix_multi_omp_bug_macos.sh
 
-          cp -R ${{ github.workspace }}/frontends/concrete-python/examples ./examples
-          cp -R ${{ github.workspace }}/frontends/concrete-python/tests ./tests
+          cp -R ${{ env.concrete-python-dir }}/examples ./examples
+          cp -R ${{ env.concrete-python-dir }}/tests ./tests
 
-          cp ${{ github.workspace }}/frontends/concrete-python/Makefile .
+          cp ${{ env.concrete-python-dir }}/Makefile .
       - name: Run pytest-macos
         run: |
           set -e

diff --git a/.github/workflows/scripts/fix_multi_omp_bug_macos.sh b/.github/workflows/scripts/fix_multi_omp_bug_macos.sh
@@ -0,0 +1,10 @@
+#!/bin/bash -e
+# Link all omp lib to concrete one to avoid load of different omp lib.
+
+SITE_PACKAGES=$(python -c "import site; print(site.getsitepackages()[0])")
+if [ $(basename "$SITE_PACKAGES") != "site-packages" ]; then
+    echo "python site packages($SITE_PACKAGES) dirname is not equals to 'site-packages', you probably not execute this script in a venv"
+    exit 1
+fi
+
+find "$SITE_PACKAGES" \( -not \( -path "$SITE_PACKAGES/concrete" -prune \) -name 'lib*omp5.dylib' -or -name 'lib*omp.dylib' \) -exec ln -f -s "$SITE_PACKAGES/concrete/.dylibs/libomp.dylib" {} \;