feat: Support 3.12 python (#847)

zama-ai · Dec 4, 2024 · 0edf038 · 0edf038
1 parent 2ff770c
commit 0edf038
Show file tree

Hide file tree

Showing 43 changed files with 2,025 additions and 1,282 deletions.
diff --git a/.github/workflows/continuous-integration.yaml b/.github/workflows/continuous-integration.yaml
@@ -21,13 +21,13 @@ on:
           - pr
 
       linux_python_versions:
-        description: "Space separated list of python versions (3.8, 3.9, 3.10, 3.11 are supported) to launch on linux"
+        description: "Space separated list of python versions (3.8, 3.9, 3.10, 3.11, 3.12 are supported) to launch on linux"
         required: false
         type: string
         default: "3.8"
 
       macos_python_versions:
-        description: "Space separated list of python versions (3.8, 3.9, 3.10, 3.11 are supported) to launch on macos (intel)"
+        description: "Space separated list of python versions (3.8, 3.9, 3.10, 3.11, 3.12 are supported) to launch on macos (intel)"
         required: false
         type: string
         default: "3.8"
@@ -165,6 +165,7 @@ jobs:
       needs-39-linux-runner: ${{ steps.set-matrix.outputs.needs-39-linux-runner }}
       needs-310-linux-runner: ${{ steps.set-matrix.outputs.needs-310-linux-runner }}
       needs-311-linux-runner: ${{ steps.set-matrix.outputs.needs-311-linux-runner }}
+      needs-312-linux-runner: ${{ steps.set-matrix.outputs.needs-312-linux-runner }}
       instance-type: ${{ steps.set-matrix.outputs.instance-type }}
       linux-python-versions: ${{ steps.set-matrix.outputs.linux-python-versions }}
       macos-python-versions: ${{ steps.set-matrix.outputs.macos-python-versions }}
@@ -213,10 +214,10 @@ jobs:
             LINUX_PYTHON_VERSIONS="3.8"
             MACOS_PYTHON_VERSIONS=""
           elif [[ "${BUILD_TYPE}" == "weekly" ]]; then
-            LINUX_PYTHON_VERSIONS="3.8 3.9 3.10 3.11"
+            LINUX_PYTHON_VERSIONS="3.8 3.9 3.10 3.11 3.12"
             MACOS_PYTHON_VERSIONS="3.9"
           elif [[ "${BUILD_TYPE}" == "release" ]] || [[ "${BUILD_TYPE}" == "published_release" ]]; then
-            LINUX_PYTHON_VERSIONS="3.8 3.9 3.10 3.11"
+            LINUX_PYTHON_VERSIONS="3.8 3.9 3.10 3.11 3.12"
             MACOS_PYTHON_VERSIONS=""
           elif [[ "${BUILD_TYPE}" == "push_to_main" ]]; then
             LINUX_PYTHON_VERSIONS="3.8"
@@ -266,6 +267,8 @@ jobs:
             jq -rc '. | map(select(.os_kind=="linux" and .python_version=="3.10")) | length > 0')
           NEEDS_LINUX_311_RUNNER=$(echo "${LINUX_MATRIX}" | \
             jq -rc '. | map(select(.os_kind=="linux" and .python_version=="3.11")) | length > 0')
+          NEEDS_LINUX_312_RUNNER=$(echo "${LINUX_MATRIX}" | \
+            jq -rc '. | map(select(.os_kind=="linux" and .python_version=="3.12")) | length > 0')
 
           echo "Needs Linux 3.8 runner:"
           echo "${NEEDS_LINUX_38_RUNNER}"
@@ -279,10 +282,14 @@ jobs:
           echo "Needs Linux 3.11 runner:"
           echo "${NEEDS_LINUX_311_RUNNER}"
 
+          echo "Needs Linux 3.12 runner:"
+          echo "${NEEDS_LINUX_312_RUNNER}"
+
           echo "needs-38-linux-runner=${NEEDS_LINUX_38_RUNNER}" >> $GITHUB_OUTPUT
           echo "needs-39-linux-runner=${NEEDS_LINUX_39_RUNNER}" >> $GITHUB_OUTPUT
           echo "needs-310-linux-runner=${NEEDS_LINUX_310_RUNNER}" >> $GITHUB_OUTPUT
           echo "needs-311-linux-runner=${NEEDS_LINUX_311_RUNNER}" >> $GITHUB_OUTPUT
+          echo "needs-312-linux-runner=${NEEDS_LINUX_312_RUNNER}" >> $GITHUB_OUTPUT
           echo "instance-type=${INSTANCE_TYPE}" >> $GITHUB_OUTPUT
 
   start-runner-linux:
@@ -299,6 +306,8 @@ jobs:
       ec2-instance-id-310: ${{ steps.start-ec2-runner-310.outputs.ec2-instance-id || '' }}
       label-311: ${{ steps.start-ec2-runner-311.outputs.label }}
       ec2-instance-id-311: ${{ steps.start-ec2-runner-311.outputs.ec2-instance-id || '' }}
+      label-312: ${{ steps.start-ec2-runner-312.outputs.label }}
+      ec2-instance-id-312: ${{ steps.start-ec2-runner-312.outputs.ec2-instance-id || '' }}
       matrix: ${{ steps.update-linux-matrix.outputs.linux-matrix }}
     steps:
       - name: Checkout Code
@@ -407,6 +416,29 @@ jobs:
               {"Key": "Team", "Value": "CML"}
             ]
 
+      - name: Start EC2 runner python 3.12
+        id: start-ec2-runner-312
+        if: ${{ !cancelled() && fromJSON(needs.matrix-preparation.outputs.needs-312-linux-runner) }}
+        uses: machulav/ec2-github-runner@fcfb31a5760dad1314a64a0e172b78ec6fc8a17e
+        with:
+          mode: start
+          github-token: ${{ secrets.EC2_RUNNER_BOT_TOKEN }}
+          ec2-image-id: ${{ secrets.AWS_EC2_AMI }}
+          ec2-instance-type: ${{ needs.matrix-preparation.outputs.instance-type }}
+          subnet-id: ${{ secrets.AWS_EC2_SUBNET_ID }}
+          security-group-id: ${{ secrets.AWS_EC2_SECURITY_GROUP_ID }}
+          aws-resource-tags: >
+            [
+              {"Key": "Name", "Value": "cml-ci-ec2-github-runner-py312"},
+              {"Key": "GitHubRepository", "Value": "${{ github.repository }}"},
+              {"Key": "Python version", "Value": "3.12"},
+              {"Key": "Actor", "Value": "${{ github.actor }}"},
+              {"Key": "Action", "Value": "${{ github.action }}"},
+              {"Key": "GitHash", "Value": "${{ github.sha }}"},
+              {"Key": "RefName", "Value": "${{ github.ref_name }}"},
+              {"Key": "RunId", "Value": "${{ github.run_id }}"},
+              {"Key": "Team", "Value": "CML"}
+            ]
 
       - name: Update Linux runs_on Matrix
         id: update-linux-matrix
@@ -421,6 +453,8 @@ jobs:
           '(. | map(select(.os_kind=="linux" and .python_version=="3.10") |= . + {"runs_on": "${{ steps.start-ec2-runner-310.outputs.label }}"}) )')
           MATRIX=$(echo "${MATRIX}" | jq -rc \
           '(. | map(select(.os_kind=="linux" and .python_version=="3.11") |= . + {"runs_on": "${{ steps.start-ec2-runner-311.outputs.label }}"}) )')
+          MATRIX=$(echo "${MATRIX}" | jq -rc \
+          '(. | map(select(.os_kind=="linux" and .python_version=="3.12") |= . + {"runs_on": "${{ steps.start-ec2-runner-312.outputs.label }}"}) )')
 
           echo "Updated matrix:"
           echo "${MATRIX}"
@@ -1058,6 +1092,15 @@ jobs:
           ec2-instance-id: ${{ needs.start-runner-linux.outputs.ec2-instance-id-311 }}
           mode: stop
 
+      - name: Stop EC2 runner python 3.12
+        uses: machulav/ec2-github-runner@fcfb31a5760dad1314a64a0e172b78ec6fc8a17e
+        if: ${{ always() && needs.start-runner-linux.outputs.ec2-instance-id-312 }}
+        with:
+          github-token: ${{ secrets.EC2_RUNNER_BOT_TOKEN }}
+          label: ${{ needs.start-runner-linux.outputs.label-312 }}
+          ec2-instance-id: ${{ needs.start-runner-linux.outputs.ec2-instance-id-312 }}
+          mode: stop
+
   build-macos-intel:
     name: Python ${{ matrix.python_version }} (macOS, intel)
     needs: [matrix-preparation]

diff --git a/Makefile b/Makefile
@@ -1,9 +1,16 @@
 SHELL:=$(shell /usr/bin/env which bash)
 
+DEV_DOCKER_PYTHON?=py38
+ifeq ($(DEV_DOCKER_PYTHON),py312)
+UBUNTU_BASE=24.10
+else 
+UBUNTU_BASE=20.04
+endif 
+
 DEV_DOCKER_IMG:=concrete-ml-dev
 DEV_DOCKERFILE:=docker/Dockerfile.dev
-DEV_CONTAINER_VENV_VOLUME:=concrete-ml-venv
-DEV_CONTAINER_CACHE_VOLUME:=concrete-ml-cache
+DEV_CONTAINER_VENV_VOLUME:=concrete-ml-venv-$(DEV_DOCKER_PYTHON)
+DEV_CONTAINER_CACHE_VOLUME:=concrete-ml-cache-$(DEV_DOCKER_PYTHON)
 DOCKER_VENV_PATH:="$${HOME}"/dev_venv/
 SRC_DIR:=src
 TEST?=tests
@@ -207,7 +214,7 @@ spcc_internal: $(SPCC_DEPS)
 .PHONY: pytest_internal # Run pytest
 pytest_internal:
 	poetry run pytest --version
-	poetry run pytest $(TEST) \
+	MKL_NUM_THREADS=4 OMP_NUM_THREADS=4 poetry run pytest $(TEST) \
 	-svv \
 	--count=$(COUNT) \
 	--randomly-dont-reorganize \
@@ -342,12 +349,12 @@ mypy_ci:
 
 .PHONY: docker_build # Build dev docker
 docker_build:
-	BUILD_ARGS=; \
+	BUILD_ARGS="--build-arg UBUNTU_BASE=$(UBUNTU_BASE) ";\
 	if [[ $$(uname) == "Linux" ]]; then \
-		BUILD_ARGS="--build-arg BUILD_UID=$$(id -u) --build-arg BUILD_GID=$$(id -g)"; \
+		BUILD_ARGS+="--build-arg BUILD_UID=$$(id -u) --build-arg BUILD_GID=$$(id -g)"; \
 	fi; \
 	DOCKER_BUILDKIT=1 docker build $${BUILD_ARGS:+$$BUILD_ARGS} \
-	--pull -t $(DEV_DOCKER_IMG) -f $(DEV_DOCKERFILE) .
+	--pull -t $(DEV_DOCKER_IMG):$(DEV_DOCKER_PYTHON) -f $(DEV_DOCKERFILE) .
 
 .PHONY: docker_rebuild # Rebuild docker
 docker_rebuild: docker_clean_volumes
@@ -356,7 +363,7 @@ docker_rebuild: docker_clean_volumes
 		BUILD_ARGS="--build-arg BUILD_UID=$$(id -u) --build-arg BUILD_GID=$$(id -g)"; \
 	fi; \
 	DOCKER_BUILDKIT=1 docker build $${BUILD_ARGS:+$$BUILD_ARGS} \
-	--pull --no-cache -t $(DEV_DOCKER_IMG) -f $(DEV_DOCKERFILE) .
+	--pull --no-cache -t $(DEV_DOCKER_IMG):$(DEV_DOCKER_PYTHON) -f $(DEV_DOCKERFILE) .
 
 .PHONY: docker_start # Launch docker
 docker_start:
@@ -371,7 +378,7 @@ docker_start:
 	--volume /"$$(pwd)":/src \
 	--volume $(DEV_CONTAINER_VENV_VOLUME):/home/dev_user/dev_venv \
 	--volume $(DEV_CONTAINER_CACHE_VOLUME):/home/dev_user/.cache \
-	$(DEV_DOCKER_IMG) || rm -f "$${EV_FILE}"
+	$(DEV_DOCKER_IMG):$(DEV_DOCKER_PYTHON) || rm -f "$${EV_FILE}"
 
 .PHONY: docker_build_and_start # Docker build and start
 docker_build_and_start: docker_build docker_start
@@ -642,7 +649,7 @@ docker_publish_measurements: docker_rebuild
 	docker run --volume /"$$(pwd)":/src \
 	--volume $(DEV_CONTAINER_VENV_VOLUME):/home/dev_user/dev_venv \
 	--volume $(DEV_CONTAINER_CACHE_VOLUME):/home/dev_user/.cache \
-	$(DEV_DOCKER_IMG) \
+	$(DEV_DOCKER_IMG):$(DEV_DOCKER_PYTHON) \
 	/bin/bash ./script/progress_tracker_utils/benchmark_and_publish_findings_in_docker.sh \
 	"$${LAUNCH_COMMAND}"
 

diff --git a/README.md b/README.md
@@ -83,7 +83,7 @@ Depending on your OS, Concrete ML may be installed with Docker or with pip:
 |            macOS 11+ (Intel)            |         Yes         |       Yes        |
 | macOS 11+ (Apple Silicon: M1, M2, etc.) |     Coming soon     |       Yes        |
 
-Note: Concrete ML only supports Python `3.8`, `3.9`, `3.10` and `3.11`.
+Note: Concrete ML only supports Python `3.8`, `3.9`, `3.10`, `3.11` and `3.12`.
 Concrete ML can be installed on Kaggle ([see this question on the community for more details](https://community.zama.ai/t/how-do-we-use-concrete-ml-on-kaggle/332)) and on Google Colab.
 
 #### Docker

diff --git a/benchmarks/common.py b/benchmarks/common.py
@@ -757,6 +757,8 @@ def benchmark_name_generator(
 
     elif model_name in DEEP_LEARNING_NAMES:
         config_str = f"_{config['n_bits']}"
+    else:
+        config_str = "UNKNOWN"
 
     # We remove underscores to make sure to not have any conflict when splitting
     return model_name.replace("_", "-") + config_str + joiner + dataset_name.replace("_", "-")

diff --git a/benchmarks/glm.py b/benchmarks/glm.py
@@ -8,6 +8,7 @@
 import numpy as np
 import pandas
 import py_progress_tracker as progress
+import sklearn
 from common import (
     BENCHMARK_CONFIGURATION,
     GLMS_STRING_TO_CLASS,
@@ -29,6 +30,10 @@
     StandardScaler,
 )
 
+sklearn_sparse_arg = (
+    {"sparse": False} if "1.1." in sklearn.__version__ else {"sparse_output": False}
+)
+
 
 # pylint: disable=redefined-outer-name
 def get_data() -> Tuple[pandas.DataFrame, str]:
@@ -102,7 +107,7 @@ def get_preprocessor() -> ColumnTransformer:
             ("log_scaled_numeric", log_scale_transformer, ["Density"]),
             (
                 "onehot_categorical",
-                OneHotEncoder(sparse=False),
+                OneHotEncoder(**sklearn_sparse_arg),
                 ["VehBrand", "VehPower", "VehGas", "Region", "Area"],
             ),
         ],

diff --git a/deps_licenses/licenses_linux_user.txt b/deps_licenses/licenses_linux_user.txt
@@ -7,22 +7,22 @@ certifi, 2024.8.30, Mozilla Public License 2.0 (MPL 2.0)
 charset-normalizer, 3.4.0, MIT License
 coloredlogs, 15.0.1, MIT License
 concrete-ml-extensions, 0.1.2, BSD-3-Clause-Clear
-concrete-python, 2.8.1, BSD-3-Clause
+concrete-python, 2.9.0rc1.dev20241130, BSD-3-Clause
 dependencies, 2.0.1, BSD License
 dill, 0.3.9, BSD License
 filelock, 3.16.1, The Unlicense (Unlicense)
-flatbuffers, 2.0.7, Apache Software License
-fsspec, 2024.9.0, BSD License
-huggingface-hub, 0.25.2, Apache Software License
+flatbuffers, 24.3.25, Apache Software License
+fsspec, 2024.10.0, BSD License
+huggingface-hub, 0.26.3, Apache Software License
 humanfriendly, 10.0, MIT License
 hummingbird-ml, 0.4.11, MIT License
 idna, 3.10, BSD License
 importlib_resources, 6.4.5, Apache Software License
 joblib, 1.4.2, BSD License
-jsonpickle, 3.3.0, BSD License
+jsonpickle, 4.0.0, BSD License
 mpmath, 1.3.0, BSD License
 networkx, 3.1, BSD License
-numpy, 1.24.2, BSD License
+numpy, 1.26.4, BSD License
 nvidia-cublas-cu12, 12.1.3.1, Other/Proprietary License
 nvidia-cuda-cupti-cu12, 12.1.105, Other/Proprietary License
 nvidia-cuda-nvrtc-cu12, 12.1.105, Other/Proprietary License
@@ -33,22 +33,22 @@ nvidia-curand-cu12, 10.3.2.106, Other/Proprietary License
 nvidia-cusolver-cu12, 11.4.5.107, Other/Proprietary License
 nvidia-cusparse-cu12, 12.1.0.106, Other/Proprietary License
 nvidia-nccl-cu12, 2.20.5, Other/Proprietary License
-nvidia-nvjitlink-cu12, 12.6.77, Other/Proprietary License
+nvidia-nvjitlink-cu12, 12.6.85, Other/Proprietary License
 nvidia-nvtx-cu12, 12.1.105, Other/Proprietary License
-onnx, 1.16.1, Apache License v2.0
+onnx, 1.17.0, Apache License v2.0
 onnxconverter-common, 1.13.0, MIT License
 onnxmltools, 1.12.0, Apache Software License
 onnxoptimizer, 0.3.13, Apache License v2.0
 onnxruntime, 1.18.0, MIT License
-packaging, 24.1, Apache Software License; BSD License
+packaging, 24.2, Apache Software License; BSD License
 pandas, 2.0.3, BSD License
-protobuf, 4.25.5, 3-Clause BSD License
+protobuf, 3.20.3, BSD-3-Clause
 psutil, 5.9.8, BSD License
 python-dateutil, 2.9.0.post0, Apache Software License; BSD License
 pytz, 2024.2, MIT License
 requests, 2.32.3, Apache Software License
-scikit-learn, 1.1.3, BSD License
-scipy, 1.10.1, BSD License
+scikit-learn, 1.4.0, BSD License
+scipy, 1.13.1, BSD License
 six, 1.16.0, MIT License
 skl2onnx, 1.17.0, Apache Software License
 skops, 0.5.0, MIT
@@ -57,7 +57,7 @@ sympy, 1.13.3, BSD License
 tabulate, 0.8.10, MIT License
 threadpoolctl, 3.5.0, BSD License
 torch, 2.3.1, BSD License
-tqdm, 4.66.5, MIT License; Mozilla Public License 2.0 (MPL 2.0)
+tqdm, 4.67.1, MIT License; Mozilla Public License 2.0 (MPL 2.0)
 triton, 2.3.1, MIT License
 typing_extensions, 4.12.2, Python Software Foundation License
 tzdata, 2024.2, Apache Software License

diff --git a/deps_licenses/licenses_linux_user.txt.md5 b/deps_licenses/licenses_linux_user.txt.md5
@@ -1 +1 @@
-8ea8aec4f5aac03565c2dcb9f3f8a1da
+85a519f8389929d8e87aa0367decff53
diff --git a/deps_licenses/licenses_mac_intel_user.txt b/deps_licenses/licenses_mac_intel_user.txt
@@ -6,36 +6,36 @@ brevitas, 0.10.2, UNKNOWN
 certifi, 2024.8.30, Mozilla Public License 2.0 (MPL 2.0)
 charset-normalizer, 3.4.0, MIT License
 coloredlogs, 15.0.1, MIT License
-concrete-python, 2.8.1, BSD-3-Clause
+concrete-python, 2.9.0rc1.dev20241130, BSD-3-Clause
 dependencies, 2.0.1, BSD License
 dill, 0.3.9, BSD License
 filelock, 3.16.1, The Unlicense (Unlicense)
-flatbuffers, 2.0.7, Apache Software License
-fsspec, 2024.9.0, BSD License
-huggingface-hub, 0.25.2, Apache Software License
+flatbuffers, 24.3.25, Apache Software License
+fsspec, 2024.10.0, BSD License
+huggingface-hub, 0.26.3, Apache Software License
 humanfriendly, 10.0, MIT License
 hummingbird-ml, 0.4.11, MIT License
 idna, 3.10, BSD License
 importlib_resources, 6.4.5, Apache Software License
 joblib, 1.4.2, BSD License
-jsonpickle, 3.3.0, BSD License
+jsonpickle, 4.0.0, BSD License
 mpmath, 1.3.0, BSD License
 networkx, 3.1, BSD License
-numpy, 1.24.2, BSD License
-onnx, 1.16.1, Apache License v2.0
+numpy, 1.26.4, BSD License
+onnx, 1.17.0, Apache License v2.0
 onnxconverter-common, 1.13.0, MIT License
 onnxmltools, 1.12.0, Apache Software License
 onnxoptimizer, 0.3.13, Apache License v2.0
 onnxruntime, 1.18.0, MIT License
-packaging, 24.1, Apache Software License; BSD License
+packaging, 24.2, Apache Software License; BSD License
 pandas, 2.0.3, BSD License
-protobuf, 4.25.5, 3-Clause BSD License
+protobuf, 3.20.3, BSD-3-Clause
 psutil, 5.9.8, BSD License
 python-dateutil, 2.9.0.post0, Apache Software License; BSD License
 pytz, 2024.2, MIT License
 requests, 2.32.3, Apache Software License
-scikit-learn, 1.1.3, BSD License
-scipy, 1.10.1, BSD License
+scikit-learn, 1.4.0, BSD License
+scipy, 1.13.1, BSD License
 six, 1.16.0, MIT License
 skl2onnx, 1.17.0, Apache Software License
 skops, 0.5.0, MIT
@@ -44,7 +44,7 @@ sympy, 1.13.3, BSD License
 tabulate, 0.8.10, MIT License
 threadpoolctl, 3.5.0, BSD License
 torch, 2.2.2, BSD License
-tqdm, 4.66.5, MIT License; Mozilla Public License 2.0 (MPL 2.0)
+tqdm, 4.67.1, MIT License; Mozilla Public License 2.0 (MPL 2.0)
 typing_extensions, 4.12.2, Python Software Foundation License
 tzdata, 2024.2, Apache Software License
 unfoldNd, 0.2.2, MIT License

diff --git a/deps_licenses/licenses_mac_intel_user.txt.md5 b/deps_licenses/licenses_mac_intel_user.txt.md5
@@ -1 +1 @@
-8ea8aec4f5aac03565c2dcb9f3f8a1da
+85a519f8389929d8e87aa0367decff53
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		8ea8aec4f5aac03565c2dcb9f3f8a1da
		85a519f8389929d8e87aa0367decff53