feat: encrypted data-frame

zama-ai · Apr 2, 2024 · d2d6250 · d2d6250
1 parent e5661c1
commit d2d6250
Show file tree

Hide file tree

Showing 28 changed files with 4,191 additions and 932 deletions.
diff --git a/.github/workflows/continuous-integration.yaml b/.github/workflows/continuous-integration.yaml
@@ -388,7 +388,7 @@ jobs:
       # benchmarks and use cases 
       - name: Pull LFS test files
         run: |
-          git lfs pull --include "tests/data" --exclude  ""
+          git lfs pull --include "tests/data/**, src/concrete/ml/pandas/_client_server_files/**" --exclude  ""
 
       - name: Set up Python ${{ matrix.python_version }}
         uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c
@@ -1036,7 +1036,7 @@ jobs:
       # benchmarks and use cases 
       - name: Pull LFS test files
         run: |
-          git lfs pull --include "tests/data" --exclude  ""
+          git lfs pull --include "tests/data/**, src/concrete/ml/pandas/_client_server_files/**" --exclude  ""
 
       - name: Set up Python ${{ matrix.python_version }}
         uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c

diff --git a/.github/workflows/refresh-one-notebook.yaml b/.github/workflows/refresh-one-notebook.yaml
@@ -16,6 +16,7 @@ on:
         - DecisionTreeClassifier \n
         - DecisionTreeRegressor \n
         - Deployment \n
+        - encrypted_pandas \n
         - ExperimentPrivacyTreePaper \n
         - FromImageNetToCifar \n
         - FullyConnectedNeuralNetwork \n
@@ -60,6 +61,7 @@ env:
   DecisionTreeClassifier: "docs/advanced_examples/DecisionTreeClassifier.ipynb" 
   DecisionTreeRegressor: "docs/advanced_examples/DecisionTreeRegressor.ipynb" 
   Deployment: "docs/advanced_examples/Deployment.ipynb" 
+  encrypted_pandas: "use_case_examples/dataframe/encrypted_pandas.ipynb" 
   ExperimentPrivacyTreePaper: "docs/advanced_examples/ExperimentPrivacyTreePaper.ipynb" 
   FromImageNetToCifar: "use_case_examples/cifar/cifar_brevitas_finetuning/FromImageNetToCifar.ipynb" 
   FullyConnectedNeuralNetwork: "docs/advanced_examples/FullyConnectedNeuralNetwork.ipynb" 

diff --git a/.gitignore b/.gitignore
@@ -96,6 +96,9 @@ ipython_config.py
 # VScode
 .vscode
 
+# macOS
+.DS_Store
+
 # pipenv
 #   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
 #   However, in case of collaboration, if having platform-specific dependencies or dependencies

diff --git a/ci/aws_ami_build_component.yaml b/ci/aws_ami_build_component.yaml
@@ -77,7 +77,7 @@ phases:
             - apt install -y git-lfs
             - git lfs install
             - source venv/bin/activate
-            - python -m pip install pytest==7.1.1 pandas==1.3.0 tensorflow==2.12.0 tf2onnx==1.13.0 torchvision==0.14.1
+            - python -m pip install pytest==7.1.1 pandas==2.0.3 tensorflow==2.12.0 tf2onnx==1.13.0 torchvision==0.14.1
 
         # We disable tests for test_deploy file because the instance does not have AWS CLI setup
       - name: RunTests

diff --git a/deps_licenses/licenses_linux_user.txt b/deps_licenses/licenses_linux_user.txt
@@ -2,8 +2,8 @@ Name, Version, License
 GitPython, 3.1.41, BSD License
 PyYAML, 6.0.1, MIT License
 anyio, 3.7.1, MIT License
-boto3, 1.34.38, Apache Software License
-botocore, 1.34.38, Apache Software License
+boto3, 1.34.72, Apache Software License
+botocore, 1.34.72, Apache Software License
 brevitas, 0.8.0, UNKNOWN
 certifi, 2023.7.22, Mozilla Public License 2.0 (MPL 2.0)
 charset-normalizer, 3.3.2, MIT License
@@ -14,16 +14,16 @@ dependencies, 2.0.1, BSD License
 dill, 0.3.8, BSD License
 exceptiongroup, 1.2.0, MIT License
 fastapi, 0.103.2, MIT License
-filelock, 3.13.1, The Unlicense (Unlicense)
-flatbuffers, 23.5.26, Apache Software License
-fsspec, 2024.2.0, BSD License
+filelock, 3.13.3, The Unlicense (Unlicense)
+flatbuffers, 24.3.25, Apache Software License
+fsspec, 2024.3.1, BSD License
 gitdb, 4.0.11, BSD License
 h11, 0.14.0, MIT License
-huggingface-hub, 0.20.3, Apache Software License
+huggingface-hub, 0.22.1, Apache Software License
 humanfriendly, 10.0, MIT License
 hummingbird-ml, 0.4.8, MIT License
 idna, 3.6, BSD License
-importlib-resources, 6.1.1, Apache Software License
+importlib_resources, 6.4.0, Apache Software License
 iniconfig, 2.0.0, MIT License
 jmespath, 1.0.1, MIT License
 joblib, 1.3.2, BSD License
@@ -40,18 +40,20 @@ onnxconverter-common, 1.13.0, MIT License
 onnxmltools, 1.11.0, Apache Software License
 onnxoptimizer, 0.3.10, Apache License v2.0
 onnxruntime, 1.13.1, MIT License
-packaging, 23.2, Apache Software License; BSD License
+packaging, 24.0, Apache Software License; BSD License
+pandas, 2.0.3, BSD License
 pluggy, 1.4.0, MIT License
 protobuf, 3.20.3, BSD-3-Clause
 psutil, 5.9.8, BSD License
 pydantic, 1.10.14, MIT License
 pytest, 7.4.1, MIT License
 pytest-json-report, 1.5.0, MIT
-pytest-metadata, 3.1.0, Mozilla Public License 2.0 (MPL 2.0)
-python-dateutil, 2.8.2, Apache Software License; BSD License
+pytest-metadata, 3.1.1, Mozilla Public License 2.0 (MPL 2.0)
+python-dateutil, 2.9.0.post0, Apache Software License; BSD License
+pytz, 2024.1, MIT License
 regex, 2023.12.25, Apache Software License
 requests, 2.31.0, Apache Software License
-s3transfer, 0.10.0, Apache Software License
+s3transfer, 0.10.1, Apache Software License
 safetensors, 0.4.2, Apache Software License
 scikit-learn, 1.1.3, BSD License
 scipy, 1.10.1, BSD License
@@ -60,18 +62,20 @@ skl2onnx, 1.12, Apache Software License
 skops, 0.5.0, MIT
 skorch, 0.11.0, new BSD 3-Clause
 smmap, 5.0.1, BSD License
-sniffio, 1.3.0, Apache Software License; MIT License
+sniffio, 1.3.1, Apache Software License; MIT License
 starlette, 0.27.0, BSD License
 sympy, 1.12, BSD License
 tabulate, 0.8.10, MIT License
-threadpoolctl, 3.2.0, BSD License
-tokenizers, 0.15.1, Apache Software License
+threadpoolctl, 3.4.0, BSD License
+tokenizers, 0.15.2, Apache Software License
 tomli, 2.0.1, MIT License
 torch, 1.13.1, BSD License
-tqdm, 4.66.1, MIT License; Mozilla Public License 2.0 (MPL 2.0)
-transformers, 4.37.2, Apache Software License
+tqdm, 4.66.2, MIT License; Mozilla Public License 2.0 (MPL 2.0)
+transformers, 4.39.1, Apache Software License
 typing_extensions, 4.5.0, Python Software Foundation License
-urllib3, 2.0.7, MIT License
+tzdata, 2024.1, Apache Software License
+urllib3, 1.26.18, MIT License
 uvicorn, 0.21.1, BSD License
 xgboost, 1.6.2, Apache Software License
-z3-solver, 4.12.5.0, MIT License
+z3-solver, 4.13.0.0, MIT License
+zipp, 3.18.1, MIT License
diff --git a/deps_licenses/licenses_linux_user.txt.md5 b/deps_licenses/licenses_linux_user.txt.md5
@@ -1 +1 @@
-2e50e53583434ebc1c2a34c502c81875
+8de2e8c13fe9a1fe80d9cce43dee7493
diff --git a/deps_licenses/licenses_mac_intel_user.txt b/deps_licenses/licenses_mac_intel_user.txt
@@ -2,31 +2,32 @@ Name, Version, License
 GitPython, 3.1.41, BSD License
 PyYAML, 6.0.1, MIT License
 anyio, 3.7.1, MIT License
-boto3, 1.34.38, Apache Software License
-botocore, 1.34.38, Apache Software License
+boto3, 1.34.72, Apache Software License
+botocore, 1.34.72, Apache Software License
 brevitas, 0.8.0, UNKNOWN
 certifi, 2023.7.22, Mozilla Public License 2.0 (MPL 2.0)
 charset-normalizer, 3.3.2, MIT License
 click, 8.1.7, BSD License
 coloredlogs, 15.0.1, MIT License
-concrete-python, 2.5.1, BSD-3-Clause
+concrete-python, 2024.3.27, BSD-3-Clause
 dependencies, 2.0.1, BSD License
 dill, 0.3.8, BSD License
 exceptiongroup, 1.2.0, MIT License
 fastapi, 0.103.2, MIT License
-filelock, 3.13.1, The Unlicense (Unlicense)
-flatbuffers, 23.5.26, Apache Software License
-fsspec, 2024.2.0, BSD License
+filelock, 3.13.3, The Unlicense (Unlicense)
+flatbuffers, 24.3.25, Apache Software License
+fsspec, 2024.3.1, BSD License
 gitdb, 4.0.11, BSD License
 h11, 0.14.0, MIT License
-huggingface-hub, 0.20.3, Apache Software License
+huggingface-hub, 0.22.1, Apache Software License
 humanfriendly, 10.0, MIT License
 hummingbird-ml, 0.4.8, MIT License
 idna, 3.6, BSD License
-importlib-resources, 6.1.1, Apache Software License
+importlib_resources, 6.4.0, Apache Software License
 iniconfig, 2.0.0, MIT License
 jmespath, 1.0.1, MIT License
 joblib, 1.3.2, BSD License
+jsonpickle, 3.0.3, BSD License
 mpmath, 1.3.0, BSD License
 networkx, 3.1, BSD License
 numpy, 1.23.5, BSD License
@@ -35,18 +36,20 @@ onnxconverter-common, 1.13.0, MIT License
 onnxmltools, 1.11.0, Apache Software License
 onnxoptimizer, 0.3.10, Apache License v2.0
 onnxruntime, 1.13.1, MIT License
-packaging, 23.2, Apache Software License; BSD License
+packaging, 24.0, Apache Software License; BSD License
+pandas, 2.0.3, BSD License
 pluggy, 1.4.0, MIT License
 protobuf, 3.20.3, BSD-3-Clause
 psutil, 5.9.8, BSD License
 pydantic, 1.10.14, MIT License
 pytest, 7.4.1, MIT License
 pytest-json-report, 1.5.0, MIT
-pytest-metadata, 3.1.0, Mozilla Public License 2.0 (MPL 2.0)
-python-dateutil, 2.8.2, Apache Software License; BSD License
+pytest-metadata, 3.1.1, Mozilla Public License 2.0 (MPL 2.0)
+python-dateutil, 2.9.0.post0, Apache Software License; BSD License
+pytz, 2024.1, MIT License
 regex, 2023.12.25, Apache Software License
 requests, 2.31.0, Apache Software License
-s3transfer, 0.10.0, Apache Software License
+s3transfer, 0.10.1, Apache Software License
 safetensors, 0.4.2, Apache Software License
 scikit-learn, 1.1.3, BSD License
 scipy, 1.10.1, BSD License
@@ -55,19 +58,20 @@ skl2onnx, 1.12, Apache Software License
 skops, 0.5.0, MIT
 skorch, 0.11.0, new BSD 3-Clause
 smmap, 5.0.1, BSD License
-sniffio, 1.3.0, Apache Software License; MIT License
+sniffio, 1.3.1, Apache Software License; MIT License
 starlette, 0.27.0, BSD License
 sympy, 1.12, BSD License
 tabulate, 0.8.10, MIT License
-threadpoolctl, 3.2.0, BSD License
-tokenizers, 0.15.1, Apache Software License
+threadpoolctl, 3.4.0, BSD License
+tokenizers, 0.15.2, Apache Software License
 tomli, 2.0.1, MIT License
 torch, 1.13.1, BSD License
-tqdm, 4.66.1, MIT License; Mozilla Public License 2.0 (MPL 2.0)
-transformers, 4.37.2, Apache Software License
+tqdm, 4.66.2, MIT License; Mozilla Public License 2.0 (MPL 2.0)
+transformers, 4.39.1, Apache Software License
 typing_extensions, 4.5.0, Python Software Foundation License
+tzdata, 2024.1, Apache Software License
 urllib3, 1.26.18, MIT License
 uvicorn, 0.21.1, BSD License
 xgboost, 1.6.2, Apache Software License
-z3-solver, 4.12.5.0, MIT License
-zipp, 3.17.0, MIT License
+z3-solver, 4.13.0.0, MIT License
+zipp, 3.18.1, MIT License
diff --git a/deps_licenses/licenses_mac_intel_user.txt.md5 b/deps_licenses/licenses_mac_intel_user.txt.md5
@@ -1 +1 @@
-a923947bfb17b658ab8efe61d5cafe96
+8de2e8c13fe9a1fe80d9cce43dee7493
diff --git a/deps_licenses/licenses_mac_silicon_user.txt b/deps_licenses/licenses_mac_silicon_user.txt
@@ -2,31 +2,32 @@ Name, Version, License
 GitPython, 3.1.41, BSD License
 PyYAML, 6.0.1, MIT License
 anyio, 3.7.1, MIT License
-boto3, 1.34.38, Apache Software License
-botocore, 1.34.38, Apache Software License
+boto3, 1.34.72, Apache Software License
+botocore, 1.34.72, Apache Software License
 brevitas, 0.8.0, UNKNOWN
 certifi, 2023.7.22, Mozilla Public License 2.0 (MPL 2.0)
 charset-normalizer, 3.3.2, MIT License
 click, 8.1.7, BSD License
 coloredlogs, 15.0.1, MIT License
-concrete-python, 2.5.1, BSD-3-Clause
+concrete-python, 2024.3.27, BSD-3-Clause
 dependencies, 2.0.1, BSD License
 dill, 0.3.8, BSD License
 exceptiongroup, 1.2.0, MIT License
 fastapi, 0.103.2, MIT License
-filelock, 3.13.1, The Unlicense (Unlicense)
-flatbuffers, 23.5.26, Apache Software License
-fsspec, 2024.2.0, BSD License
+filelock, 3.13.3, The Unlicense (Unlicense)
+flatbuffers, 24.3.25, Apache Software License
+fsspec, 2024.3.1, BSD License
 gitdb, 4.0.11, BSD License
 h11, 0.14.0, MIT License
-huggingface-hub, 0.20.3, Apache Software License
+huggingface-hub, 0.22.1, Apache Software License
 humanfriendly, 10.0, MIT License
 hummingbird-ml, 0.4.8, MIT License
 idna, 3.6, BSD License
-importlib-resources, 6.1.1, Apache Software License
+importlib_resources, 6.4.0, Apache Software License
 iniconfig, 2.0.0, MIT License
 jmespath, 1.0.1, MIT License
 joblib, 1.3.2, BSD License
+jsonpickle, 3.0.3, BSD License
 mpmath, 1.3.0, BSD License
 networkx, 3.1, BSD License
 numpy, 1.23.5, BSD License
@@ -35,18 +36,20 @@ onnxconverter-common, 1.13.0, MIT License
 onnxmltools, 1.11.0, Apache Software License
 onnxoptimizer, 0.3.10, Apache License v2.0
 onnxruntime, 1.13.1, MIT License
-packaging, 23.2, Apache Software License; BSD License
+packaging, 24.0, Apache Software License; BSD License
+pandas, 2.0.3, BSD License
 pluggy, 1.4.0, MIT License
 protobuf, 3.20.3, BSD-3-Clause
 psutil, 5.9.8, BSD License
 pydantic, 1.10.14, MIT License
 pytest, 7.4.1, MIT License
 pytest-json-report, 1.5.0, MIT
-pytest-metadata, 3.1.0, Mozilla Public License 2.0 (MPL 2.0)
-python-dateutil, 2.8.2, Apache Software License; BSD License
+pytest-metadata, 3.1.1, Mozilla Public License 2.0 (MPL 2.0)
+python-dateutil, 2.9.0.post0, Apache Software License; BSD License
+pytz, 2024.1, MIT License
 regex, 2023.12.25, Apache Software License
 requests, 2.31.0, Apache Software License
-s3transfer, 0.10.0, Apache Software License
+s3transfer, 0.10.1, Apache Software License
 safetensors, 0.4.2, Apache Software License
 scikit-learn, 1.1.3, BSD License
 scipy, 1.10.1, BSD License
@@ -55,19 +58,19 @@ skl2onnx, 1.12, Apache Software License
 skops, 0.5.0, MIT
 skorch, 0.11.0, new BSD 3-Clause
 smmap, 5.0.1, BSD License
-sniffio, 1.3.0, Apache Software License; MIT License
+sniffio, 1.3.1, Apache Software License; MIT License
 starlette, 0.27.0, BSD License
 sympy, 1.12, BSD License
 tabulate, 0.8.10, MIT License
-threadpoolctl, 3.2.0, BSD License
-tokenizers, 0.15.1, Apache Software License
+threadpoolctl, 3.4.0, BSD License
+tokenizers, 0.15.2, Apache Software License
 tomli, 2.0.1, MIT License
 torch, 1.13.1, BSD License
-tqdm, 4.66.1, MIT License; Mozilla Public License 2.0 (MPL 2.0)
-transformers, 4.37.2, Apache Software License
+tqdm, 4.66.2, MIT License; Mozilla Public License 2.0 (MPL 2.0)
+transformers, 4.39.1, Apache Software License
 typing_extensions, 4.5.0, Python Software Foundation License
-urllib3, 1.26.18, MIT License
+tzdata, 2024.1, Apache Software License
+urllib3, 2.2.1, MIT License
 uvicorn, 0.21.1, BSD License
 xgboost, 1.6.2, Apache Software License
-z3-solver, 4.12.5.0, MIT License
-zipp, 3.17.0, MIT License
+z3-solver, 4.13.0.0, MIT License
diff --git a/deps_licenses/licenses_mac_silicon_user.txt.md5 b/deps_licenses/licenses_mac_silicon_user.txt.md5
@@ -1 +1 @@
-a923947bfb17b658ab8efe61d5cafe96
+8de2e8c13fe9a1fe80d9cce43dee7493
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		2e50e53583434ebc1c2a34c502c81875
		8de2e8c13fe9a1fe80d9cce43dee7493
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		a923947bfb17b658ab8efe61d5cafe96
		8de2e8c13fe9a1fe80d9cce43dee7493