Concrete Python Release #335
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Concrete Python Release | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| instance_id: | |
| description: 'Instance ID' | |
| type: string | |
| instance_image_id: | |
| description: 'Instance AMI ID' | |
| type: string | |
| instance_type: | |
| description: 'Instance product type' | |
| type: string | |
| runner_name: | |
| description: 'Action runner name' | |
| type: string | |
| request_id: | |
| description: 'Slab request ID' | |
| type: string | |
| user_inputs: | |
| description: 'either "nightly" or "public" or "private" to specify the release type' | |
| required: true | |
| default: 'nightly' | |
| type: string | |
| env: | |
| DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler | |
| GLIB_VER: 2_28 | |
| RELEASE_TYPE: ${{ inputs.user_inputs }} | |
| jobs: | |
| build-linux-x86: | |
| continue-on-error: true | |
| strategy: | |
| matrix: | |
| python-version: ["3.8", "3.9", "3.10", "3.11"] | |
| runs-on: ${{ github.event.inputs.runner_name }} | |
| steps: | |
| - name: Log instance configuration | |
| run: | | |
| echo "IDs: ${{ inputs.instance_id }}" | |
| echo "AMI: ${{ inputs.instance_image_id }}" | |
| echo "Type: ${{ inputs.instance_type }}" | |
| echo "Request ID: ${{ inputs.request_id }}" | |
| echo "User Inputs: ${{ inputs.user_inputs }}" | |
| - name: Set up GitHub environment | |
| run: | | |
| echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| - name: Set release version (nightly) | |
| if: ${{ env.RELEASE_TYPE == 'nightly' }} | |
| run: | | |
| VERSION=$(date +"%Y.%m.%d") | |
| echo "__version__ = \"$VERSION\"" >| frontends/concrete-python/version.txt | |
| git tag nightly-$VERSION || true | |
| git push origin nightly-$VERSION || true | |
| - name: Set release version (public) | |
| if: ${{ env.RELEASE_TYPE == 'public' }} | |
| run: echo "__version__ = \"`git describe --tags --abbrev=0 | grep -e '[0-9].*' -o`\"" >| frontends/concrete-python/version.txt | |
| - name: Expose release version from Python | |
| run: cp frontends/concrete-python/version.txt frontends/concrete-python/concrete/fhe/version.py | |
| - name: Build wheel | |
| uses: addnab/docker-run-action@v3 | |
| id: build-compiler-bindings | |
| with: | |
| registry: ghcr.io | |
| image: ${{ env.DOCKER_IMAGE_TEST }} | |
| username: ${{ secrets.GHCR_LOGIN }} | |
| password: ${{ secrets.GHCR_PASSWORD }} | |
| options: >- | |
| -v ${{ github.workspace }}:/concrete | |
| -v ${{ github.workspace }}/build:/build | |
| -v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket | |
| -e SSH_AUTH_SOCK=/ssh.socket | |
| ${{ env.DOCKER_GPU_OPTION }} | |
| shell: bash | |
| run: | | |
| rustup toolchain install nightly-2024-01-31 | |
| rm -rf /build/* | |
| export PYTHON=${{ format('python{0}', matrix.python-version) }} | |
| echo "Using $PYTHON" | |
| dnf -y install graphviz graphviz-devel | |
| cd /concrete/frontends/concrete-python | |
| make PYTHON=$PYTHON venv | |
| source .venv/bin/activate | |
| cd /concrete/compilers/concrete-compiler/compiler | |
| make BUILD_DIR=/build CCACHE=ON DATAFLOW_EXECUTION_ENABLED=ON Python3_EXECUTABLE=$(which python) python-bindings | |
| echo "Debug: ccache statistics (after the build):" | |
| ccache -s | |
| cd /concrete/frontends/concrete-python | |
| export COMPILER_BUILD_DIRECTORY="/build" | |
| make whl | |
| deactivate | |
| - name: Upload wheel | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ format('wheel-{0}-linux-x86', matrix.python-version) }} | |
| path: frontends/concrete-python/dist/*manylinux*.whl | |
| retention-days: 3 | |
| build-macos: | |
| strategy: | |
| matrix: | |
| python-version: ["3.8", "3.9", "3.10", "3.11"] | |
| runs-on: ["aws-mac1-metal", "aws-mac2-metal"] | |
| runs-on: ${{ matrix.runs-on }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| - name: Install OS Dependencies | |
| run: | | |
| brew install ninja ccache | |
| - name: Setup rust toolchain for concrete-cpu | |
| uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu | |
| - name: Set release version (nightly) | |
| if: ${{ env.RELEASE_TYPE == 'nightly' }} | |
| run: echo "__version__ = \"$(date +"%Y.%m.%d")\"" >| frontends/concrete-python/version.txt | |
| - name: Set release version (public) | |
| if: ${{ env.RELEASE_TYPE == 'public' }} | |
| run: echo "__version__ = \"`git describe --tags --abbrev=0 | grep -e '[0-9].*' -o`\"" >| frontends/concrete-python/version.txt | |
| - name: Expose release version from Python | |
| run: cp frontends/concrete-python/version.txt frontends/concrete-python/concrete/fhe/version.py | |
| - name: Build wheel | |
| run: | | |
| export CONCRETE_PYTHON=$(pwd)/frontends/concrete-python | |
| export CONCRETE_COMPILER=$(pwd)/compilers/concrete-compiler/compiler | |
| export COMPILER_BUILD_DIRECTORY=$CONCRETE_COMPILER/build | |
| export PYTHON=${{ format('python{0}', matrix.python-version) }} | |
| echo "Using $PYTHON" | |
| # Setup pkg-config to find OpenBLAS (scipy need it) | |
| export PKG_CONFIG_PATH="/opt/homebrew/opt/openblas/lib/pkgconfig" | |
| # Setup vitual environment | |
| rm -rf .venv | |
| $PYTHON -m venv .venv && . .venv/bin/activate | |
| # Install requirements | |
| pip install -r $CONCRETE_PYTHON/requirements.txt | |
| pip install -r $CONCRETE_PYTHON/requirements.dev.txt | |
| # Build python bindings of concrete compiler | |
| cd $CONCRETE_COMPILER | |
| echo "Debug: ccache statistics (prior to the build):" && ccache -s | |
| make Python3_EXECUTABLE=$(which python) python-bindings | |
| echo "Debug: ccache statistics (after the build):" && ccache -s | |
| # Build wheel | |
| cd $CONCRETE_PYTHON | |
| rm -rf dist | |
| mkdir -p dist | |
| pip wheel -v --no-deps -w dist . | |
| delocate-wheel -v dist/*macos*.whl | |
| deactivate | |
| - name: Upload wheel | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ format('wheel-{0}-{1}', matrix.python-version, matrix.runs-on) }} | |
| path: frontends/concrete-python/dist/*macos*.whl | |
| retention-days: 3 | |
| push: | |
| needs: [build-linux-x86, build-macos] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| path: wheels | |
| merge-multiple: true | |
| - name: Push wheels to internal PyPI (nightly) | |
| if: ${{ env.RELEASE_TYPE == 'nightly' }} | |
| run: | | |
| pip install twine==4.0.2 | |
| twine upload wheels/*.whl \ | |
| -u "${{ secrets.INTERNAL_PYPI_USER }}" \ | |
| -p "${{ secrets.INTERNAL_PYPI_PASSWORD }}" \ | |
| --repository-url "${{ secrets.INTERNAL_PYPI_URL }}" | |
| - name: Push wheels to public PyPI (public) | |
| if: ${{ env.RELEASE_TYPE == 'public' }} | |
| run: | | |
| pip install twine==4.0.2 | |
| twine upload wheels/*.whl \ | |
| -u "${{ secrets.PUBLIC_PYPI_USER }}" \ | |
| -p "${{ secrets.PUBLIC_PYPI_PASSWORD }}" \ | |
| -r pypi | |
| - name: Start pushing Docker images | |
| if: ${{ env.RELEASE_TYPE == 'public' }} | |
| run: | | |
| export TAG=$(git describe --tags --abbrev=0) | |
| curl -L \ | |
| -X POST \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "Authorization: Bearer ${{ secrets.CONCRETE_ACTIONS_TOKEN }}" \ | |
| -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| https://api.github.com/repos/zama-ai/concrete/actions/workflows/concrete_python_push_docker_image.yml/dispatches \ | |
| -d '{"ref":"$TAG","inputs":{"tag":"$TAG"}}' | |
| test-linux-x86: | |
| needs: [build-linux-x86] | |
| continue-on-error: true | |
| strategy: | |
| matrix: | |
| python-version: ["3.8", "3.9", "3.10", "3.11"] | |
| runs-on: ${{ github.event.inputs.runner_name }} | |
| steps: | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Download wheels | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: ${{ format('wheel-{0}-linux-x86', matrix.python-version) }} | |
| path: ${{ format('wheel-{0}-linux-x86', matrix.python-version) }} | |
| - name: Checkout the repository | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| path: repo | |
| - name: Test wheel | |
| run: | | |
| WHEEL_DIR=$(pwd)/${{ format('wheel-{0}-linux-x86', matrix.python-version) }} | |
| CONCRETE_PYTHON=$(pwd)/repo/frontends/concrete-python | |
| # Initialize an empty test environment | |
| cd $(mktemp -d) | |
| python -m venv .testenv && source .testenv/bin/activate | |
| # Install the concrete-python wheel | |
| pip install $WHEEL_DIR/*.whl | |
| # Install extra requirements for tests | |
| sudo apt update -y | |
| sudo apt install -y graphviz libgraphviz-dev | |
| pip install -r $CONCRETE_PYTHON/requirements.extra-full.txt | |
| pip install -r $CONCRETE_PYTHON/requirements.dev.txt | |
| # TODO - check for version | |
| # Copy test files | |
| cp -R $CONCRETE_PYTHON/tests . | |
| cp -R $CONCRETE_PYTHON/pytest.ini . | |
| # Running tests | |
| pytest tests -svv -n auto | |
| test-macos: | |
| needs: [build-macos] | |
| continue-on-error: true | |
| strategy: | |
| matrix: | |
| python-version: ["3.8", "3.9", "3.10", "3.11"] | |
| runs-on: ["aws-mac1-metal", "aws-mac2-metal"] | |
| runs-on: ${{ matrix.runs-on }} | |
| steps: | |
| - name: Download wheels | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: ${{ format('wheel-{0}-{1}', matrix.python-version, matrix.runs-on) }} | |
| path: ${{ format('wheel-{0}-{1}', matrix.python-version, matrix.runs-on) }} | |
| - name: Checkout the repository | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| path: repo | |
| - name: Test wheel | |
| run: | | |
| WHEEL_DIR=$(pwd)/${{ format('wheel-{0}-{1}', matrix.python-version, matrix.runs-on) }} | |
| CONCRETE_PYTHON=$(pwd)/repo/frontends/concrete-python | |
| PYTHON=${{ format('python{0}', matrix.python-version) }} | |
| # Initialize an empty test environment | |
| export TEST_TMP_DIR=$(mktemp -d) | |
| echo "TEST_TMP_DIR=$TEST_TMP_DIR" >> "${GITHUB_ENV}" | |
| cd $TEST_TMP_DIR | |
| # Activate virtual environment | |
| $PYTHON -m venv .testenv && source .testenv/bin/activate | |
| # Install extra requirements for tests | |
| pip install $WHEEL_DIR/*macos*.whl | |
| pip install -r $CONCRETE_PYTHON/requirements.dev.txt | |
| # MacOS x86 have conflict between our OpenMP library, and one from torch | |
| # we fix it by using a single one (from torch) | |
| # see discussion: https://discuss.python.org/t/conflicting-binary-extensions-in-different-packages/25332/8 | |
| export SITE_PACKAGES=$(pwd)/.testenv/lib/$PYTHON_VERSION/site-packages/ | |
| find $SITE_PACKAGES -not \( -path $SITE_PACKAGES/concrete -prune \) -name 'lib*omp5.dylib' -or -name 'lib*omp.dylib' | xargs -n 1 ln -f -s $SITE_PACKAGES/concrete/.dylibs/libomp.dylib | |
| # Copy test files | |
| cp -R $CONCRETE_PYTHON/tests . | |
| cp -R $CONCRETE_PYTHON/pytest.ini . | |
| cp $CONCRETE_PYTHON/Makefile . | |
| # Running tests | |
| mkdir ./KeySetCache | |
| pytest tests -svv -n auto --key-cache "./KeySetCache" -m "not dataflow and not graphviz" | |
| - name: Cleanup host | |
| if: success() || failure() | |
| run: | | |
| rm -rf $TEST_TMP_DIR |