chore: integrate input compression

.gitignore

@@ -154,7 +154,7 @@ use_case_examples/llm/gpt2*
 use_case_examples/cifar_brevitas_training/.datasets/
 
 # Key cache
-ConcreteNumpyKeyCache
+ConcretePythonKeyCache
 
 # This index file is autogenerated
 docs/index.rst

Makefile

@@ -19,7 +19,7 @@ OPEN_PR="true"
 # Force the installation of a Concrete Python version, which is very useful with nightly versions
 # /!\ WARNING /!\: This version should NEVER be a wildcard as it might create some
 # issues when trying to run it in the future.
-CONCRETE_PYTHON_VERSION="concrete-python==2.6.0-rc1"
+CONCRETE_PYTHON_VERSION="concrete-python==2024.4.19"
 
 # Force the installation of Concrete Python's latest version, release-candidates included
 # CONCRETE_PYTHON_VERSION="$$(poetry run python \

benchmarks/common.py

@@ -237,15 +237,14 @@
 
 
 # This is only for benchmarks to speed up compilation times
-# Jit compiler is now deprecated and will soon be removed, it is thus forced to False by default
 # Parameter `enable_unsafe_features` and `use_insecure_key_cache` are needed in order to be able to
 # cache generated keys through `insecure_key_cache_location`. As the name suggests, these
 # parameters are unsafe and should only be used for debugging in development
 BENCHMARK_CONFIGURATION = fhe.Configuration(
     dump_artifacts_on_unexpected_failures=True,
     enable_unsafe_features=True,
     use_insecure_key_cache=True,
-    insecure_key_cache_location="ConcreteNumpyKeyCache",
+    insecure_key_cache_location="ConcretePythonKeyCache",
 )
 
 

conftest.py

@@ -2,6 +2,7 @@
 
 import hashlib
 import json
+import os
 import random
 import re
 from pathlib import Path
@@ -96,7 +97,7 @@ def monkeypatched_compilation_configuration_init_for_codeblocks(
     self.enable_unsafe_features = True
     self.treat_warnings_as_errors = True
     self.use_insecure_key_cache = True
-    self.insecure_key_cache_location = "ConcreteNumpyKeyCache"
+    self.insecure_key_cache_location = "ConcretePythonKeyCache"
 
 
 def pytest_sessionstart(session: pytest.Session):
@@ -139,25 +140,27 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus):  # pylint: disabl
 
 @pytest.fixture
 def default_configuration():
-    """Return the default test compilation configuration."""
+    """Return the compilation configuration for tests that can execute in FHE."""
 
     # Parameter `enable_unsafe_features` and `use_insecure_key_cache` are needed in order to be
     # able to cache generated keys through `insecure_key_cache_location`. As the name suggests,
     # these parameters are unsafe and should only be used for debugging in development
+    # Simulation compilation is done lazily when calling circuit.simulate, so we do not need to
+    # set it by default
     return Configuration(
         dump_artifacts_on_unexpected_failures=False,
         enable_unsafe_features=True,
         use_insecure_key_cache=True,
-        insecure_key_cache_location="ConcreteNumpyKeyCache",
-        # Simulation compilation is done lazily on circuit.simulate
+        insecure_key_cache_location="ConcretePythonKeyCache",
         fhe_simulation=False,
         fhe_execution=True,
+        compress_input_ciphertexts=os.environ.get("USE_INPUT_COMPRESSION", "1") == "1",
     )
 
 
 @pytest.fixture
 def simulation_configuration():
-    """Return the simulation test compilation configuration for simulation."""
+    """Return the compilation configuration for tests that only simulate."""
 
     # Parameter `enable_unsafe_features` and `use_insecure_key_cache` are needed in order to be
     # able to cache generated keys through `insecure_key_cache_location`. As the name suggests,
@@ -166,9 +169,10 @@ def simulation_configuration():
         dump_artifacts_on_unexpected_failures=False,
         enable_unsafe_features=True,
         use_insecure_key_cache=True,
-        insecure_key_cache_location="ConcreteNumpyKeyCache",
+        insecure_key_cache_location="ConcretePythonKeyCache",
         fhe_simulation=True,
         fhe_execution=False,
+        compress_input_ciphertexts=os.environ.get("USE_INPUT_COMPRESSION", "1") == "1",
     )
 
 

deps_licenses/licenses_linux_user.txt

@@ -9,7 +9,7 @@ certifi, 2023.7.22, Mozilla Public License 2.0 (MPL 2.0)
 charset-normalizer, 3.3.2, MIT License
 click, 8.1.7, BSD License
 coloredlogs, 15.0.1, MIT License
-concrete-python, 2.6.0rc1, BSD-3-Clause
+concrete-python, 2024.4.19, BSD-3-Clause
 dependencies, 2.0.1, BSD License
 dill, 0.3.8, BSD License
 exceptiongroup, 1.2.1, MIT License
@@ -74,8 +74,7 @@ tqdm, 4.66.2, MIT License; Mozilla Public License 2.0 (MPL 2.0)
 transformers, 4.40.0, Apache Software License
 typing_extensions, 4.5.0, Python Software Foundation License
 tzdata, 2024.1, Apache Software License
-urllib3, 1.26.18, MIT License
+urllib3, 2.2.1, MIT License
 uvicorn, 0.21.1, BSD License
 xgboost, 1.6.2, Apache Software License
 z3-solver, 4.13.0.0, MIT License
-zipp, 3.18.1, MIT License

deps_licenses/licenses_linux_user.txt.md5

@@ -1 +1 @@
-e672d77021d3214dea6c135c6e33e9a4
+ae40a75edf0bfc21ffa11c3d59deaf7c

deps_licenses/licenses_mac_intel_user.txt

@@ -2,17 +2,17 @@ Name, Version, License
 GitPython, 3.1.41, BSD License
 PyYAML, 6.0.1, MIT License
 anyio, 3.7.1, MIT License
-boto3, 1.34.84, Apache Software License
-botocore, 1.34.84, Apache Software License
+boto3, 1.34.88, Apache Software License
+botocore, 1.34.88, Apache Software License
 brevitas, 0.8.0, UNKNOWN
 certifi, 2023.7.22, Mozilla Public License 2.0 (MPL 2.0)
 charset-normalizer, 3.3.2, MIT License
 click, 8.1.7, BSD License
 coloredlogs, 15.0.1, MIT License
-concrete-python, 2.6.0rc1, BSD-3-Clause
+concrete-python, 2024.4.19, BSD-3-Clause
 dependencies, 2.0.1, BSD License
 dill, 0.3.8, BSD License
-exceptiongroup, 1.2.0, MIT License
+exceptiongroup, 1.2.1, MIT License
 fastapi, 0.103.2, MIT License
 filelock, 3.13.4, The Unlicense (Unlicense)
 flatbuffers, 24.3.25, Apache Software License
@@ -38,7 +38,7 @@ onnxoptimizer, 0.3.13, Apache License v2.0
 onnxruntime, 1.17.3, MIT License
 packaging, 24.0, Apache Software License; BSD License
 pandas, 2.0.3, BSD License
-pluggy, 1.4.0, MIT License
+pluggy, 1.5.0, MIT License
 protobuf, 3.20.3, BSD-3-Clause
 psutil, 5.9.8, BSD License
 pydantic, 1.10.15, MIT License
@@ -47,7 +47,7 @@ pytest-json-report, 1.5.0, MIT
 pytest-metadata, 3.1.1, Mozilla Public License 2.0 (MPL 2.0)
 python-dateutil, 2.9.0.post0, Apache Software License; BSD License
 pytz, 2024.1, MIT License
-regex, 2023.12.25, Apache Software License
+regex, 2024.4.16, Apache Software License
 requests, 2.31.0, Apache Software License
 s3transfer, 0.10.1, Apache Software License
 safetensors, 0.4.3, Apache Software License
@@ -63,11 +63,11 @@ starlette, 0.27.0, BSD License
 sympy, 1.12, BSD License
 tabulate, 0.8.10, MIT License
 threadpoolctl, 3.4.0, BSD License
-tokenizers, 0.15.2, Apache Software License
+tokenizers, 0.19.1, Apache Software License
 tomli, 2.0.1, MIT License
 torch, 1.13.1, BSD License
 tqdm, 4.66.2, MIT License; Mozilla Public License 2.0 (MPL 2.0)
-transformers, 4.39.3, Apache Software License
+transformers, 4.40.0, Apache Software License
 typing_extensions, 4.5.0, Python Software Foundation License
 tzdata, 2024.1, Apache Software License
 urllib3, 2.2.1, MIT License

deps_licenses/licenses_mac_intel_user.txt.md5

@@ -1 +1 @@
-180ea2b3232d89fc71be20d1707620b4
+ae40a75edf0bfc21ffa11c3d59deaf7c

deps_licenses/licenses_mac_silicon_user.txt

@@ -2,17 +2,17 @@ Name, Version, License
 GitPython, 3.1.41, BSD License
 PyYAML, 6.0.1, MIT License
 anyio, 3.7.1, MIT License
-boto3, 1.34.84, Apache Software License
-botocore, 1.34.84, Apache Software License
+boto3, 1.34.88, Apache Software License
+botocore, 1.34.88, Apache Software License
 brevitas, 0.8.0, UNKNOWN
 certifi, 2023.7.22, Mozilla Public License 2.0 (MPL 2.0)
 charset-normalizer, 3.3.2, MIT License
 click, 8.1.7, BSD License
 coloredlogs, 15.0.1, MIT License
-concrete-python, 2.6.0rc1, BSD-3-Clause
+concrete-python, 2024.4.19, BSD-3-Clause
 dependencies, 2.0.1, BSD License
 dill, 0.3.8, BSD License
-exceptiongroup, 1.2.0, MIT License
+exceptiongroup, 1.2.1, MIT License
 fastapi, 0.103.2, MIT License
 filelock, 3.13.4, The Unlicense (Unlicense)
 flatbuffers, 24.3.25, Apache Software License
@@ -38,7 +38,7 @@ onnxoptimizer, 0.3.13, Apache License v2.0
 onnxruntime, 1.17.3, MIT License
 packaging, 24.0, Apache Software License; BSD License
 pandas, 2.0.3, BSD License
-pluggy, 1.4.0, MIT License
+pluggy, 1.5.0, MIT License
 protobuf, 3.20.3, BSD-3-Clause
 psutil, 5.9.8, BSD License
 pydantic, 1.10.15, MIT License
@@ -47,7 +47,7 @@ pytest-json-report, 1.5.0, MIT
 pytest-metadata, 3.1.1, Mozilla Public License 2.0 (MPL 2.0)
 python-dateutil, 2.9.0.post0, Apache Software License; BSD License
 pytz, 2024.1, MIT License
-regex, 2023.12.25, Apache Software License
+regex, 2024.4.16, Apache Software License
 requests, 2.31.0, Apache Software License
 s3transfer, 0.10.1, Apache Software License
 safetensors, 0.4.3, Apache Software License
@@ -63,11 +63,11 @@ starlette, 0.27.0, BSD License
 sympy, 1.12, BSD License
 tabulate, 0.8.10, MIT License
 threadpoolctl, 3.4.0, BSD License
-tokenizers, 0.15.2, Apache Software License
+tokenizers, 0.19.1, Apache Software License
 tomli, 2.0.1, MIT License
 torch, 1.13.1, BSD License
 tqdm, 4.66.2, MIT License; Mozilla Public License 2.0 (MPL 2.0)
-transformers, 4.39.3, Apache Software License
+transformers, 4.40.0, Apache Software License
 typing_extensions, 4.5.0, Python Software Foundation License
 tzdata, 2024.1, Apache Software License
 urllib3, 2.2.1, MIT License

deps_licenses/licenses_mac_silicon_user.txt.md5

@@ -1 +1 @@
-180ea2b3232d89fc71be20d1707620b4
+ae40a75edf0bfc21ffa11c3d59deaf7c

docker/release_resources/sanity_check.py

@@ -154,7 +154,7 @@ def main(args):
 
     keyring_dir_as_str = None
     if is_fast:
-        keyring_dir = Path.home().resolve() / "ConcreteNumpyKeyCache"
+        keyring_dir = Path.home().resolve() / "ConcretePythonKeyCache"
         keyring_dir.mkdir(parents=True, exist_ok=True)
         keyring_dir_as_str = str(keyring_dir)
         print(f"Using {keyring_dir_as_str} as key cache dir")
@@ -163,7 +163,7 @@ def main(args):
     cn_check(args, keyring_dir_as_str)
 
     if is_fast:
-        keyring_dir = Path.home().resolve() / "ConcreteNumpyKeyCache"
+        keyring_dir = Path.home().resolve() / "ConcretePythonKeyCache"
         if keyring_dir is not None:
             # Remove incomplete keys
             for incomplete_keys in keyring_dir.glob("**/*incomplete*"):

docs/advanced_examples/ClientServer.ipynb

@@ -146,9 +146,9 @@
      "output_type": "stream",
      "text": [
       "total 76K\r\n",
-      "-rw-r--r-- 1 root root  59K Apr 25 20:00 client.zip\r\n",
-      "-rw-r--r-- 1 root root 9.5K Apr 25 20:00 server.zip\r\n",
-      "-rw-r--r-- 1 root root   75 Apr 25 20:00 versions.json\r\n"
+      "-rw-r--r-- 1 root root  59K Apr 26 12:58 client.zip\r\n",
+      "-rw-r--r-- 1 root root 9.5K Apr 26 12:58 server.zip\r\n",
+      "-rw-r--r-- 1 root root   76 Apr 26 12:58 versions.json\r\n"
      ]
     }
    ],
@@ -177,7 +177,7 @@
      "output_type": "stream",
      "text": [
       "total 12K\r\n",
-      "-rw-r--r-- 1 root root 9.5K Apr 25 20:00 server.zip\r\n"
+      "-rw-r--r-- 1 root root 9.5K Apr 26 12:58 server.zip\r\n"
      ]
     }
    ],
@@ -197,7 +197,7 @@
      "output_type": "stream",
      "text": [
       "total 60K\r\n",
-      "-rw-r--r-- 1 root root 59K Apr 25 20:00 client.zip\r\n"
+      "-rw-r--r-- 1 root root 59K Apr 26 12:58 client.zip\r\n"
      ]
     }
    ],
@@ -223,7 +223,7 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "KeySetCache: miss, regenerating /tmp/tmp61ls254t/6318749875182965247\n"
+      "KeySetCache: miss, regenerating /tmp/tmpwu_1zoqc/6318749875182965247\n"
      ]
     }
    ],
@@ -270,8 +270,8 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Encrypted data is 1538.10 times larger than the clear data\n",
-      "The average execution time is 0.71 seconds per sample.\n"
+      "Encrypted data is 4.10 times larger than the clear data\n",
+      "The average execution time is 0.70 seconds per sample.\n"
      ]
     }
    ],