From bc2393eec1624d1d3b77ba52a512aeec1b47f1b9 Mon Sep 17 00:00:00 2001 From: Benoit Chevallier-Mames Date: Tue, 28 May 2024 09:49:45 +0200 Subject: [PATCH] feat: adding support for python3.11 --- .github/workflows/continuous-integration.yaml | 52 +++++++++++++++++-- README.md | 2 +- deps_licenses/licenses_linux_user.txt.md5 | 4 ++ deps_licenses/licenses_mac_intel_user.txt.md5 | 4 ++ .../licenses_mac_silicon_user.txt.md5 | 4 ++ docs/getting-started/pip_installing.md | 2 +- poetry.lock | 17 +++--- pyproject.toml | 9 ++-- script/actions_utils/generate_test_matrix.py | 1 + .../check_installation_with_all_python.sh | 4 +- script/make_utils/licenses.sh | 2 + 11 files changed, 80 insertions(+), 21 deletions(-) diff --git a/.github/workflows/continuous-integration.yaml b/.github/workflows/continuous-integration.yaml index f2f39055b..193255874 100644 --- a/.github/workflows/continuous-integration.yaml +++ b/.github/workflows/continuous-integration.yaml @@ -21,13 +21,13 @@ on: - pr linux_python_versions: - description: "Space separated list of python versions (3.8, 3.9, 3.10 are supported) to launch on linux" + description: "Space separated list of python versions (3.8, 3.9, 3.10, 3.11 are supported) to launch on linux" required: false type: string default: "3.8" macos_python_versions: - description: "Space separated list of python versions (3.8, 3.9, 3.10 are supported) to launch on macos (intel)" + description: "Space separated list of python versions (3.8, 3.9, 3.10, 3.11 are supported) to launch on macos (intel)" required: false type: string default: "3.8" @@ -109,6 +109,7 @@ jobs: needs-38-linux-runner: ${{ steps.set-matrix.outputs.needs-38-linux-runner }} needs-39-linux-runner: ${{ steps.set-matrix.outputs.needs-39-linux-runner }} needs-310-linux-runner: ${{ steps.set-matrix.outputs.needs-310-linux-runner }} + needs-311-linux-runner: ${{ steps.set-matrix.outputs.needs-311-linux-runner }} instance-type: ${{ steps.set-matrix.outputs.instance-type }} linux-python-versions: ${{ steps.set-matrix.outputs.linux-python-versions }} macos-python-versions: ${{ steps.set-matrix.outputs.macos-python-versions }} @@ -157,10 +158,10 @@ jobs: LINUX_PYTHON_VERSIONS="3.8" MACOS_PYTHON_VERSIONS="" elif [[ "${BUILD_TYPE}" == "weekly" ]]; then - LINUX_PYTHON_VERSIONS="3.8 3.9 3.10" + LINUX_PYTHON_VERSIONS="3.8 3.9 3.10 3.11" MACOS_PYTHON_VERSIONS="3.9" elif [[ "${BUILD_TYPE}" == "release" ]] || [[ "${BUILD_TYPE}" == "published_release" ]]; then - LINUX_PYTHON_VERSIONS="3.8 3.9 3.10" + LINUX_PYTHON_VERSIONS="3.8 3.9 3.10 3.11" MACOS_PYTHON_VERSIONS="" elif [[ "${BUILD_TYPE}" == "push_to_main" ]]; then LINUX_PYTHON_VERSIONS="3.8" @@ -208,6 +209,8 @@ jobs: jq -rc '. | map(select(.os_kind=="linux" and .python_version=="3.9")) | length > 0') NEEDS_LINUX_310_RUNNER=$(echo "${LINUX_MATRIX}" | \ jq -rc '. | map(select(.os_kind=="linux" and .python_version=="3.10")) | length > 0') + NEEDS_LINUX_311_RUNNER=$(echo "${LINUX_MATRIX}" | \ + jq -rc '. | map(select(.os_kind=="linux" and .python_version=="3.11")) | length > 0') echo "Needs Linux 3.8 runner:" echo "${NEEDS_LINUX_38_RUNNER}" @@ -218,9 +221,13 @@ jobs: echo "Needs Linux 3.10 runner:" echo "${NEEDS_LINUX_310_RUNNER}" + echo "Needs Linux 3.11 runner:" + echo "${NEEDS_LINUX_311_RUNNER}" + echo "needs-38-linux-runner=${NEEDS_LINUX_38_RUNNER}" >> $GITHUB_OUTPUT echo "needs-39-linux-runner=${NEEDS_LINUX_39_RUNNER}" >> $GITHUB_OUTPUT echo "needs-310-linux-runner=${NEEDS_LINUX_310_RUNNER}" >> $GITHUB_OUTPUT + echo "needs-311-linux-runner=${NEEDS_LINUX_311_RUNNER}" >> $GITHUB_OUTPUT echo "instance-type=${INSTANCE_TYPE}" >> $GITHUB_OUTPUT start-runner-linux: @@ -235,6 +242,8 @@ jobs: ec2-instance-id-39: ${{ steps.start-ec2-runner-39.outputs.ec2-instance-id || '' }} label-310: ${{ steps.start-ec2-runner-310.outputs.label }} ec2-instance-id-310: ${{ steps.start-ec2-runner-310.outputs.ec2-instance-id || '' }} + label-311: ${{ steps.start-ec2-runner-311.outputs.label }} + ec2-instance-id-311: ${{ steps.start-ec2-runner-311.outputs.ec2-instance-id || '' }} matrix: ${{ steps.update-linux-matrix.outputs.linux-matrix }} steps: - name: Checkout Code @@ -319,6 +328,30 @@ jobs: {"Key": "Team", "Value": "CML"} ] + - name: Start EC2 runner python 3.11 + id: start-ec2-runner-311 + if: ${{ !cancelled() && fromJSON(needs.matrix-preparation.outputs.needs-311-linux-runner) }} + uses: machulav/ec2-github-runner@fcfb31a5760dad1314a64a0e172b78ec6fc8a17e + with: + mode: start + github-token: ${{ secrets.EC2_RUNNER_BOT_TOKEN }} + ec2-image-id: ${{ secrets.AWS_EC2_AMI }} + ec2-instance-type: ${{ needs.matrix-preparation.outputs.instance-type }} + subnet-id: ${{ secrets.AWS_EC2_SUBNET_ID }} + security-group-id: ${{ secrets.AWS_EC2_SECURITY_GROUP_ID }} + aws-resource-tags: > + [ + {"Key": "Name", "Value": "cml-ci-ec2-github-runner-py311"}, + {"Key": "GitHubRepository", "Value": "${{ github.repository }}"}, + {"Key": "Python version", "Value": "3.11"}, + {"Key": "Actor", "Value": "${{ github.actor }}"}, + {"Key": "Action", "Value": "${{ github.action }}"}, + {"Key": "GitHash", "Value": "${{ github.sha }}"}, + {"Key": "RefName", "Value": "${{ github.ref_name }}"}, + {"Key": "RunId", "Value": "${{ github.run_id }}"}, + {"Key": "Team", "Value": "CML"} + ] + - name: Update Linux runs_on Matrix id: update-linux-matrix @@ -331,6 +364,8 @@ jobs: '(. | map(select(.os_kind=="linux" and .python_version=="3.9") |= . + {"runs_on": "${{ steps.start-ec2-runner-39.outputs.label }}"}) )') MATRIX=$(echo "${MATRIX}" | jq -rc \ '(. | map(select(.os_kind=="linux" and .python_version=="3.10") |= . + {"runs_on": "${{ steps.start-ec2-runner-310.outputs.label }}"}) )') + MATRIX=$(echo "${MATRIX}" | jq -rc \ + '(. | map(select(.os_kind=="linux" and .python_version=="3.11") |= . + {"runs_on": "${{ steps.start-ec2-runner-311.outputs.label }}"}) )') echo "Updated matrix:" echo "${MATRIX}" @@ -1002,6 +1037,15 @@ jobs: ec2-instance-id: ${{ needs.start-runner-linux.outputs.ec2-instance-id-310 }} mode: stop + - name: Stop EC2 runner python 3.11 + uses: machulav/ec2-github-runner@fcfb31a5760dad1314a64a0e172b78ec6fc8a17e + if: ${{ always() && needs.start-runner-linux.outputs.ec2-instance-id-311 }} + with: + github-token: ${{ secrets.EC2_RUNNER_BOT_TOKEN }} + label: ${{ needs.start-runner-linux.outputs.label-311 }} + ec2-instance-id: ${{ needs.start-runner-linux.outputs.ec2-instance-id-311 }} + mode: stop + build-macos-intel: name: Python ${{ matrix.python_version }} (macOS, intel) needs: [matrix-preparation] diff --git a/README.md b/README.md index 698d0bc8b..e650f0973 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,7 @@ Depending on your OS, Concrete ML may be installed with Docker or with pip: | macOS 11+ (Intel) | Yes | Yes | | macOS 11+ (Apple Silicon: M1, M2, etc.) | Coming soon | Yes | -Note: Concrete ML only supports Python `3.8`, `3.9` and `3.10`. +Note: Concrete ML only supports Python `3.8`, `3.9`, `3.10` and `3.11`. Concrete ML can be installed on Kaggle ([see this question on the community for more details](https://community.zama.ai/t/how-do-we-use-concrete-ml-on-kaggle/332)) and on Google Colab. #### Docker diff --git a/deps_licenses/licenses_linux_user.txt.md5 b/deps_licenses/licenses_linux_user.txt.md5 index a8d8b21ec..a5a1e90f2 100644 --- a/deps_licenses/licenses_linux_user.txt.md5 +++ b/deps_licenses/licenses_linux_user.txt.md5 @@ -1 +1,5 @@ +<<<<<<< HEAD f07036c8072f32a26aa79e5e4bbd9c32 +======= +ead3663b936227b3d224dc480933b084 +>>>>>>> 8de97ea7 (feat: adding support for python3.11) diff --git a/deps_licenses/licenses_mac_intel_user.txt.md5 b/deps_licenses/licenses_mac_intel_user.txt.md5 index a8d8b21ec..a5a1e90f2 100644 --- a/deps_licenses/licenses_mac_intel_user.txt.md5 +++ b/deps_licenses/licenses_mac_intel_user.txt.md5 @@ -1 +1,5 @@ +<<<<<<< HEAD f07036c8072f32a26aa79e5e4bbd9c32 +======= +ead3663b936227b3d224dc480933b084 +>>>>>>> 8de97ea7 (feat: adding support for python3.11) diff --git a/deps_licenses/licenses_mac_silicon_user.txt.md5 b/deps_licenses/licenses_mac_silicon_user.txt.md5 index a8d8b21ec..a5a1e90f2 100644 --- a/deps_licenses/licenses_mac_silicon_user.txt.md5 +++ b/deps_licenses/licenses_mac_silicon_user.txt.md5 @@ -1 +1,5 @@ +<<<<<<< HEAD f07036c8072f32a26aa79e5e4bbd9c32 +======= +ead3663b936227b3d224dc480933b084 +>>>>>>> 8de97ea7 (feat: adding support for python3.11) diff --git a/docs/getting-started/pip_installing.md b/docs/getting-started/pip_installing.md index 6852af622..433be6f9d 100644 --- a/docs/getting-started/pip_installing.md +++ b/docs/getting-started/pip_installing.md @@ -24,7 +24,7 @@ Depending on your OS/HW, Concrete ML may be installed with Docker or with pip: ### Python support -- **Version**: In the current release, Concrete ML supports only `3.8`, `3.9` and `3.10` versions of `python`. +- **Version**: In the current release, Concrete ML supports only `3.8`, `3.9`, `3.10` and `3.11` versions of `python`. - **Linux requirement**: The Concrete ML Python package requires `glibc >= 2.28`. On Linux, you can check your `glibc` version by running `ldd --version`. - **Kaggle installation**: Concrete ML can be installed on Kaggle ([see question on community for more details](https://community.zama.ai/t/how-do-we-use-concrete-ml-on-kaggle/332)) and on Google Colab. diff --git a/poetry.lock b/poetry.lock index f1cd90bf9..f6e0be3b2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3098,6 +3098,7 @@ files = [ {file = "msgpack-1.0.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5fbb160554e319f7b22ecf530a80a3ff496d38e8e07ae763b9e82fadfe96f273"}, {file = "msgpack-1.0.8-cp39-cp39-win32.whl", hash = "sha256:f9af38a89b6a5c04b7d18c492c8ccf2aee7048aff1ce8437c4683bb5a1df893d"}, {file = "msgpack-1.0.8-cp39-cp39-win_amd64.whl", hash = "sha256:ed59dd52075f8fc91da6053b12e8c89e37aa043f8986efd89e61fae69dc1b011"}, + {file = "msgpack-1.0.8-py3-none-any.whl", hash = "sha256:24f727df1e20b9876fa6e95f840a2a2651e34c0ad147676356f4bf5fbb0206ca"}, {file = "msgpack-1.0.8.tar.gz", hash = "sha256:95c02b0e27e706e48d0e5426d1710ca78e0f0628d6e89d5b5a5b91a5f12274f3"}, ] @@ -3660,7 +3661,6 @@ description = "Nvidia JIT LTO Library" optional = false python-versions = ">=3" files = [ - {file = "nvidia_nvjitlink_cu12-12.5.82-py3-none-manylinux2014_aarch64.whl", hash = "sha256:98103729cc5226e13ca319a10bbf9433bbbd44ef64fe72f45f067cacc14b8d27"}, {file = "nvidia_nvjitlink_cu12-12.5.82-py3-none-manylinux2014_x86_64.whl", hash = "sha256:f9b37bc5c8cf7509665cb6ada5aaa0ce65618f2332b7d3e78e9790511f111212"}, {file = "nvidia_nvjitlink_cu12-12.5.82-py3-none-win_amd64.whl", hash = "sha256:e782564d705ff0bf61ac3e1bf730166da66dd2fe9012f111ede5fc49b64ae697"}, ] @@ -3904,7 +3904,8 @@ files = [ [package.dependencies] numpy = [ {version = ">=1.20.3", markers = "python_version < \"3.10\""}, - {version = ">=1.21.0", markers = "python_version >= \"3.10\""}, + {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, + {version = ">=1.21.0", markers = "python_version >= \"3.10\" and python_version < \"3.11\""}, ] python-dateutil = ">=2.8.2" pytz = ">=2020.1" @@ -4348,13 +4349,13 @@ files = [ [[package]] name = "py-progress-tracker" -version = "0.6.0" +version = "0.7.0" description = "A simple benchmarking library" optional = false -python-versions = ">=3.7,<3.11" +python-versions = ">=3.7" files = [ - {file = "py-progress-tracker-0.6.0.tar.gz", hash = "sha256:04e25e9596cd4a47af310041f75abe64b2d13b8d09ff6582c1644fccc3524226"}, - {file = "py_progress_tracker-0.6.0-py3-none-any.whl", hash = "sha256:05bf573d413fa28e3c224c26da9508a87b490376fced499eb141fc51e640e3a0"}, + {file = "py_progress_tracker-0.7.0-py3-none-any.whl", hash = "sha256:02c08b5799f4473cf7d1a23f2c6f10a7e735b85008595ff33c6e0d80442a57fc"}, + {file = "py_progress_tracker-0.7.0.tar.gz", hash = "sha256:4bf9e908ed89481459a24824505904a2c7e8348221f0a478722ceae37f6f2af0"}, ] [package.dependencies] @@ -7069,5 +7070,5 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", [metadata] lock-version = "2.0" -python-versions = ">=3.8.1,<3.11" -content-hash = "c8700a1a81483292d4e1933f2d1fc1fe0a3f6c94d92bf359b7f1bd3132d6904e" +python-versions = ">=3.8.1,<3.12" +content-hash = "2a37109db34ea85621294cef66e418855e749d257d1d7f4492a1b6cf8dddb390" diff --git a/pyproject.toml b/pyproject.toml index 907aa46e3..2dfb83baf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,9 +31,9 @@ readme = "README.md" "Bug Tracker" = "https://github.com/zama-ai/concrete-ml/issues" [tool.poetry.dependencies] -# Investigate if it is better to fix specific versions or use lower and upper bounds +# Investigate if it is better to fix specific versions or use lower and upper bounds # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/2665 -python = ">=3.8.1,<3.11" +python = ">=3.8.1,<3.12" # Based on the following link, explicitly indicating the source makes poetry only look for that # source, so we should only use this for Concrete Python's non-public releases # https://python-poetry.org/docs/1.7/repositories#project-configuration @@ -52,7 +52,7 @@ typing-extensions = "^4.5.0" brevitas = "0.10.2" onnx = "1.16.1" onnxoptimizer = "0.3.13" -# onnxruntime versions supported by onnx versions and opsets can be found here : +# onnxruntime versions supported by onnx versions and opsets can be found here : # https://onnxruntime.ai/docs/reference/compatibility.html#onnx-opset-support onnxruntime = "1.18" hummingbird-ml = {version="0.4.11", extras = ["onnx"]} @@ -98,7 +98,7 @@ pip-audit = "^2.1.0" types-requests = "^2.32.0" requests="^2.32.1" jupyter = "^1.0.0" -py-progress-tracker = "0.6.0" +py-progress-tracker = "0.7.0" nbqa = "^1.3.1" darglint = "^1.8.1" linkcheckmd = "^1.4.0" @@ -167,4 +167,3 @@ line-length = 100 # Add a setuptools_scm section to avoid warnings [tool.setuptools_scm] - diff --git a/script/actions_utils/generate_test_matrix.py b/script/actions_utils/generate_test_matrix.py index 2dfea3b42..9810c91b7 100644 --- a/script/actions_utils/generate_test_matrix.py +++ b/script/actions_utils/generate_test_matrix.py @@ -21,6 +21,7 @@ class PythonVersion(enum.Enum): V_3_8 = "3.8" V_3_9 = "3.9" V_3_10 = "3.10" + V_3_11 = "3.11" class OS(enum.Enum): diff --git a/script/make_utils/check_installation_with_all_python.sh b/script/make_utils/check_installation_with_all_python.sh index 4fb307926..8aca76dfe 100755 --- a/script/make_utils/check_installation_with_all_python.sh +++ b/script/make_utils/check_installation_with_all_python.sh @@ -25,7 +25,7 @@ do ;; "--all" ) - VERSION_LIST="3.8 3.9 3.10" + VERSION_LIST="3.8 3.9 3.10 3.11" ;; "--version" ) @@ -44,7 +44,7 @@ done if [ "${VERSION_LIST}" == "" ] then - VERSION_LIST="3.8 3.9 3.10" + VERSION_LIST="3.8 3.9 3.10 3.11" fi for VERSION in $VERSION_LIST diff --git a/script/make_utils/licenses.sh b/script/make_utils/licenses.sh index 5167de510..a334cbe8a 100755 --- a/script/make_utils/licenses.sh +++ b/script/make_utils/licenses.sh @@ -96,6 +96,8 @@ then IS_CORRECT_PYTHON=1 elif [[ "${CHECK_VERSION}" == *"3.10"* ]]; then IS_CORRECT_PYTHON=1 + elif [[ "${CHECK_VERSION}" == *"3.11"* ]]; then + IS_CORRECT_PYTHON=1 fi if [ $IS_CORRECT_PYTHON -eq 0 ]