From 1297c04a53cb0c7766f6d94071c95d98258d85a2 Mon Sep 17 00:00:00 2001 From: jfrery Date: Wed, 11 Dec 2024 16:38:46 +0100 Subject: [PATCH 1/5] chore: switch aws region weekly release --- .github/workflows/continuous-integration.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/continuous-integration.yaml b/.github/workflows/continuous-integration.yaml index 0c7554e55..41eb0d405 100644 --- a/.github/workflows/continuous-integration.yaml +++ b/.github/workflows/continuous-integration.yaml @@ -201,9 +201,9 @@ jobs: # Manage instance type INSTANCE_TYPE="c5.4xlarge" if [[ "${BUILD_TYPE}" == "weekly" ]]; then - INSTANCE_TYPE="c6i.16xlarge" + INSTANCE_TYPE="hpc7a.96xlarge" elif [[ "${BUILD_TYPE}" == "release" ]]; then - INSTANCE_TYPE="c6i.16xlarge" + INSTANCE_TYPE="hpc7a.96xlarge" fi # Manage python versions From c944efa96756d69ff9539ce7fad8977b10fe7bd6 Mon Sep 17 00:00:00 2001 From: jfrery Date: Fri, 13 Dec 2024 09:42:08 +0100 Subject: [PATCH 2/5] chore: dynamic core number for pytest / numpy / fhe --- Makefile | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 50e38c9fb..22e6d4ecc 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,16 @@ POETRY_VERSION:=1.8.4 APIDOCS_OUTPUT?="./docs/references/api" OPEN_PR="true" +# Check the total number of CPU cores and use min(4, TOTAL_CPUS/4) for pytest +UNAME_S := $(shell uname -s) +ifeq ($(UNAME_S),Darwin) # macOS + TOTAL_CPUS := $(shell sysctl -n hw.ncpu) +else # Assume Linux + TOTAL_CPUS := $(shell nproc) +endif +PYTEST_CORES := $(shell if [ `expr $(TOTAL_CPUS) / 4` -lt 4 ]; then expr $(TOTAL_CPUS) / 4; else echo 4; fi) +FHE_NUMPY_CORES := $(shell expr $(TOTAL_CPUS) - $(PYTEST_CORES)) + # At the end of the command, we currently need to force an 'import skorch' in Python in order to # avoid an obscure bug that led to all pytest commands to fail when installing dependencies with # Poetry >= 1.3. It is however not very clear how this import fixes the issue, as the bug was @@ -214,7 +224,7 @@ spcc_internal: $(SPCC_DEPS) .PHONY: pytest_internal # Run pytest pytest_internal: poetry run pytest --version - MKL_NUM_THREADS=4 OMP_NUM_THREADS=4 poetry run pytest $(TEST) \ + MKL_NUM_THREADS=$(FHE_NUMPY_CORES) OMP_NUM_THREADS=$(FHE_NUMPY_CORES) poetry run pytest $(TEST) \ -svv \ --count=$(COUNT) \ --randomly-dont-reorganize \ @@ -229,7 +239,11 @@ pytest_internal: # --durations=10 is to show the 10 slowest tests .PHONY: pytest_internal_parallel # Run pytest with multiple CPUs pytest_internal_parallel: - "$(MAKE)" pytest_internal PYTEST_OPTIONS="-n $(N_CPU) --durations=10 ${PYTEST_OPTIONS}" + @echo "Total CPUs: $(TOTAL_CPUS)" + @echo "Assigning $(PYTEST_CORES) cores to pytest" + @echo "Leaving $(FHE_NUMPY_CORES) cores for FHE/Numpy (OMP_NUM_THREADS and MKL_NUM_THREADS)" + MKL_NUM_THREADS=$(FHE_NUMPY_CORES) OMP_NUM_THREADS=$(FHE_NUMPY_CORES) \ + "$(MAKE)" pytest_internal PYTEST_OPTIONS="-n $(PYTEST_CORES) --durations=10 ${PYTEST_OPTIONS}" # --global-coverage-infos-json=global-coverage-infos.json is to dump the coverage report in the file # --cov PATH is the directory PATH to consider for coverage. Default to SRC_DIR=src From 4f03aad7b212caeb4db8898c560553a2474153e4 Mon Sep 17 00:00:00 2001 From: jfrery Date: Fri, 13 Dec 2024 10:54:20 +0100 Subject: [PATCH 3/5] chore: specific N cores per pytest worker --- Makefile | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 22e6d4ecc..70ee19e9f 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,7 @@ else # Assume Linux TOTAL_CPUS := $(shell nproc) endif PYTEST_CORES := $(shell if [ `expr $(TOTAL_CPUS) / 4` -lt 4 ]; then expr $(TOTAL_CPUS) / 4; else echo 4; fi) -FHE_NUMPY_CORES := $(shell expr $(TOTAL_CPUS) - $(PYTEST_CORES)) +FHE_NUMPY_CORES := $(shell expr \( $(TOTAL_CPUS) - $(PYTEST_CORES) \) / $(PYTEST_CORES) ) # At the end of the command, we currently need to force an 'import skorch' in Python in order to # avoid an obscure bug that led to all pytest commands to fail when installing dependencies with @@ -224,7 +224,12 @@ spcc_internal: $(SPCC_DEPS) .PHONY: pytest_internal # Run pytest pytest_internal: poetry run pytest --version - MKL_NUM_THREADS=$(FHE_NUMPY_CORES) OMP_NUM_THREADS=$(FHE_NUMPY_CORES) poetry run pytest $(TEST) \ + MKL_NUM_THREADS=$(FHE_NUMPY_CORES) \ + OMP_NUM_THREADS=$(FHE_NUMPY_CORES) \ + OPENBLAS_NUM_THREADS=$(FHE_NUMPY_CORES) \ + VECLIB_MAXIMUM_THREADS=$(FHE_NUMPY_CORES) \ + NUMEXPR_NUM_THREADS=$(FHE_NUMPY_CORES) \ + poetry run pytest $(TEST) \ -svv \ --count=$(COUNT) \ --randomly-dont-reorganize \ @@ -241,8 +246,12 @@ pytest_internal: pytest_internal_parallel: @echo "Total CPUs: $(TOTAL_CPUS)" @echo "Assigning $(PYTEST_CORES) cores to pytest" - @echo "Leaving $(FHE_NUMPY_CORES) cores for FHE/Numpy (OMP_NUM_THREADS and MKL_NUM_THREADS)" - MKL_NUM_THREADS=$(FHE_NUMPY_CORES) OMP_NUM_THREADS=$(FHE_NUMPY_CORES) \ + @echo "Leaving $(FHE_NUMPY_CORES) cores per pytest worker for FHE/Numpy/sklearn" + MKL_NUM_THREADS=$(FHE_NUMPY_CORES) \ + OMP_NUM_THREADS=$(FHE_NUMPY_CORES) \ + OPENBLAS_NUM_THREADS=$(FHE_NUMPY_CORES) \ + VECLIB_MAXIMUM_THREADS=$(FHE_NUMPY_CORES) \ + NUMEXPR_NUM_THREADS=$(FHE_NUMPY_CORES) \ "$(MAKE)" pytest_internal PYTEST_OPTIONS="-n $(PYTEST_CORES) --durations=10 ${PYTEST_OPTIONS}" # --global-coverage-infos-json=global-coverage-infos.json is to dump the coverage report in the file From ec2ae5a8a509ab2f43ede90f324040f7f83f4f6f Mon Sep 17 00:00:00 2001 From: jfrery Date: Fri, 13 Dec 2024 11:30:43 +0100 Subject: [PATCH 4/5] chore: no need to subtract pytest cores as it will be also used in the test execution --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 70ee19e9f..1b471327a 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,8 @@ else # Assume Linux TOTAL_CPUS := $(shell nproc) endif PYTEST_CORES := $(shell if [ `expr $(TOTAL_CPUS) / 4` -lt 4 ]; then expr $(TOTAL_CPUS) / 4; else echo 4; fi) -FHE_NUMPY_CORES := $(shell expr \( $(TOTAL_CPUS) - $(PYTEST_CORES) \) / $(PYTEST_CORES) ) +# Calculate cores per pytest worker: total_cores / pytest_cores +FHE_NUMPY_CORES := $(shell expr $(TOTAL_CPUS) / $(PYTEST_CORES)) # At the end of the command, we currently need to force an 'import skorch' in Python in order to # avoid an obscure bug that led to all pytest commands to fail when installing dependencies with From de6b29d02c2cd621d0a8058ff170473da2c5c47d Mon Sep 17 00:00:00 2001 From: jfrery Date: Fri, 13 Dec 2024 13:54:22 +0100 Subject: [PATCH 5/5] chore: update machine to c6i 32xlarge --- .github/workflows/continuous-integration.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/continuous-integration.yaml b/.github/workflows/continuous-integration.yaml index 41eb0d405..7ad20ec62 100644 --- a/.github/workflows/continuous-integration.yaml +++ b/.github/workflows/continuous-integration.yaml @@ -201,9 +201,9 @@ jobs: # Manage instance type INSTANCE_TYPE="c5.4xlarge" if [[ "${BUILD_TYPE}" == "weekly" ]]; then - INSTANCE_TYPE="hpc7a.96xlarge" + INSTANCE_TYPE="c6i.32xlarge" elif [[ "${BUILD_TYPE}" == "release" ]]; then - INSTANCE_TYPE="hpc7a.96xlarge" + INSTANCE_TYPE="c6i.32xlarge" fi # Manage python versions