From 7178816818ac7a2656e49d9ea764e3a5baa0fc59 Mon Sep 17 00:00:00 2001 From: kcelia Date: Mon, 18 Sep 2023 10:56:08 +0200 Subject: [PATCH] chore: force the configuration of KNN to run under MONO settings --- conftest.py | 2 + .../ml/search_parameters/p_error_search.py | 20 +-- src/concrete/ml/sklearn/base.py | 144 +++++++++--------- src/concrete/ml/sklearn/neighbors.py | 15 +- .../test_pbs_error_probability_settings.py | 21 +-- tests/deployment/test_client_server.py | 15 +- tests/sklearn/test_dump_onnx.py | 10 +- tests/sklearn/test_sklearn_models.py | 81 ++++------ .../credit_scoring/CreditScoring.ipynb | 10 +- 9 files changed, 135 insertions(+), 183 deletions(-) diff --git a/conftest.py b/conftest.py index c4fa713c72..32ba7bae00 100644 --- a/conftest.py +++ b/conftest.py @@ -499,6 +499,8 @@ def check_is_good_execution_for_cml_vs_circuit_impl( # `check_subfunctions_in_fhe` if is_classifier_or_partial_classifier(model): if isinstance(model, SklearnKNeighborsMixin): + # For KNN `predict_proba` is not supported for now + # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3962 results_cnp_circuit = model.predict(*inputs, fhe=fhe_mode) results_model = model.predict(*inputs, fhe="disable") else: diff --git a/src/concrete/ml/search_parameters/p_error_search.py b/src/concrete/ml/search_parameters/p_error_search.py index eec213001e..dbed2c1f7a 100644 --- a/src/concrete/ml/search_parameters/p_error_search.py +++ b/src/concrete/ml/search_parameters/p_error_search.py @@ -58,11 +58,9 @@ import numpy import torch -from concrete.fhe import ParameterSelectionStrategy -from concrete.fhe.compilation import Configuration from tqdm import tqdm -from ..common.utils import get_model_name, is_brevitas_model, is_model_class_in_a_list +from ..common.utils import is_brevitas_model, is_model_class_in_a_list from ..sklearn import ( get_sklearn_neighbors_models, get_sklearn_neural_net_models, @@ -110,16 +108,6 @@ def compile_and_simulated_fhe_inference( """ compile_params: Dict = {} - - default_configuration = Configuration( - dump_artifacts_on_unexpected_failures=False, - enable_unsafe_features=True, - use_insecure_key_cache=True, - insecure_key_cache_location="ConcreteNumpyKeyCache", - parameter_selection_strategy=ParameterSelectionStrategy.MONO - if get_model_name(estimator) == "KNeighborsClassifier" - else ParameterSelectionStrategy.MULTI, - ) compile_function: Callable[..., Any] dequantized_output: numpy.ndarray @@ -150,11 +138,7 @@ def compile_and_simulated_fhe_inference( if not estimator.is_fitted: estimator.fit(calibration_data, ground_truth) - estimator.compile( - calibration_data, - p_error=p_error, - configuration=default_configuration, - ) + estimator.compile(calibration_data, p_error=p_error) predict_method = getattr(estimator, predict) dequantized_output = predict_method(calibration_data, fhe="simulate") diff --git a/src/concrete/ml/sklearn/base.py b/src/concrete/ml/sklearn/base.py index 83f40bf1f1..97fb8149ba 100644 --- a/src/concrete/ml/sklearn/base.py +++ b/src/concrete/ml/sklearn/base.py @@ -632,7 +632,6 @@ def predict(self, X: Data, fhe: Union[FheMode, str] = FheMode.DISABLE) -> numpy. for q_X_i in q_X: # Expected encrypt_run_decrypt output shape is (1, n_features) while q_X_i # is of shape (n_features,) - q_X_i = numpy.expand_dims(q_X_i, 0) # For mypy, even though we already check this with self.check_model_is_compiled() @@ -1697,7 +1696,7 @@ def predict_proba(self, X: Data, fhe: Union[FheMode, str] = FheMode.DISABLE) -> return y_proba -# pylint: disable=invalid-name,too-many-instance-attributes +# pylint: disable-next=invalid-name,too-many-instance-attributes class SklearnKNeighborsMixin(BaseEstimator, sklearn.base.BaseEstimator, ABC): """A Mixin class for sklearn KNeighbors models with FHE. @@ -1712,24 +1711,22 @@ def __init_subclass__(cls): _NEIGHBORS_MODELS.add(cls) _ALL_SKLEARN_MODELS.add(cls) - def __init__(self, n_bits: Union[int, Dict[str, int]] = 3): + def __init__(self, n_bits: int = 3): """Initialize the FHE knn model. Args: - n_bits (int, Dict[str, int]): Number of bits to quantize the model. If an int is passed - for n_bits, the value will be used for quantizing inputs and weights. If a dict is - passed, then it should contain "op_inputs" and "op_weights" as keys with - corresponding number of quantization bits so that: - - op_inputs : number of bits to quantize the input values - - op_weights: number of bits to quantize the learned parameters - Default to 3. + n_bits (int): Number of bits to quantize the model. IThe value will be used for + quantizing inputs and X_fit. Default to 3. """ - self.n_bits: Union[int, Dict[str, int]] = n_bits - - #: The quantizer to use for quantizing the model's weights - self._weight_quantizer: Optional[UniformQuantizer] = None - self._q_X_fit_quantizer: Optional[UniformQuantizer] = None + self.n_bits: int = n_bits + # _q_X_fit: In distance metric algorithms, `_q_X_fit` stores the training set to compute + # the similarity or distance measures. There is no `weights` attribute because there isn't + # a training phase self._q_X_fit: numpy.ndarray + # _y: Labels of `_q_X_fit` + self._y: numpy.ndarray + # _q_X_fit_quantizer: The quantizer to use for quantizing the model's training set + self._q_X_fit_quantizer: Optional[UniformQuantizer] = None BaseEstimator.__init__(self) @@ -1748,7 +1745,7 @@ def _set_onnx_model(self, test_input: numpy.ndarray) -> None: test_input=test_input, extra_config={ "onnx_target_opset": OPSET_VERSION_FOR_ONNX_EXPORT, - # pylint: disable=protected-access, no-member + # pylint: disable-next=protected-access, no-member constants.BATCH_SIZE: self.sklearn_model._fit_X.shape[0], }, ).model @@ -1765,6 +1762,8 @@ def _clean_graph(self) -> None: def fit(self, X: Data, y: Target, **fit_parameters): # Reset for double fit self._is_fitted = False + self.input_quantizers = [] + self.output_quantizers = [] # KNeighbors handles multi-labels data X, y = check_X_y_and_assert_multi_output(X, y) @@ -1780,31 +1779,23 @@ def fit(self, X: Data, y: Target, **fit_parameters): # Retrieve the ONNX graph self._set_onnx_model(X) - # Convert the n_bits attribute into a proper dictionary - n_bits = get_n_bits_dict(self.n_bits) - - input_n_bits = n_bits["op_inputs"] - input_options = QuantizationOptions(n_bits=input_n_bits, is_signed=True) - # Quantize the inputs and store the associated quantizer - q_inputs = QuantizedArray(n_bits=input_n_bits, values=X, options=input_options) + input_options = QuantizationOptions(n_bits=self.n_bits, is_signed=True) + q_inputs = QuantizedArray(n_bits=self.n_bits, values=X, options=input_options) input_quantizer = q_inputs.quantizer self.input_quantizers.append(input_quantizer) - weights_n_bits = n_bits["op_weights"] - weight_options = QuantizationOptions(n_bits=weights_n_bits, is_signed=True) - # Quantize the _X_fit and store the associated quantizer - # Weights in KNN algorithms are the train data points - # pylint: disable=protected-access + # pylint: disable-next=protected-access _X_fit = self.sklearn_model._fit_X + # We assume that the inputs have the same distribution as the _X_fit q_X_fit = QuantizedArray( - n_bits=n_bits["op_weights"], + n_bits=self.n_bits, values=numpy.expand_dims(_X_fit, axis=1) if len(_X_fit.shape) == 1 else _X_fit, - options=weight_options, + options=input_options, ) self._q_X_fit = q_X_fit.qvalues - self._q_X_fit_quantizer = self._weight_quantizer = q_X_fit.quantizer + self._q_X_fit_quantizer = q_X_fit.quantizer # mypy assert self._q_X_fit_quantizer.scale is not None @@ -1821,9 +1812,6 @@ def fit(self, X: Data, y: Target, **fit_parameters): output_quantizer = UniformQuantizer(params=self.output_quant_params, no_clipping=True) - # Since the matmul and the bias both use the same scale and zero-points, we obtain that - # y = S*(q_y - 2*Z) when de-quantizing the values. We therefore need to multiply the initial - # output zero_point by 2 assert output_quantizer.zero_point is not None self.output_quantizers.append(output_quantizer) @@ -1843,14 +1831,8 @@ def quantize_input(self, X: numpy.ndarray) -> numpy.ndarray: def dequantize_output(self, q_y_preds: numpy.ndarray) -> numpy.ndarray: self.check_model_is_fitted() - # We compute the sorted argmax in FHE, which are integers. # No need to de-quantize the output values - - assert q_y_preds[0].shape[-1] == self.n_neighbors, ( - f"Shape error: `q_y_preds` must be shape of ({self.n_neighbors},) and got:" - f"`{q_y_preds.shape}`" - ) return q_y_preds def _get_module_to_compile(self) -> Union[Compiler, QuantizedModule]: @@ -1911,6 +1893,8 @@ def pairwise_euclidean_distance(q_X): def topk_sorting(x): """Argsort in FHE. + Time complexity: O(nlog²(k)) + Args: x (numpy.ndarray): The quantized input values. @@ -1951,68 +1935,70 @@ def scatter1d(x, v, indices): x[i] = v[idx] return x - def mul_tlu(a, b): - """Matrix multiplication. - - Args: - a (numpy.ndarray): An encrypted array - b (numpy.ndarray): An encrypted array - - Returns: - numpy.ndarray: The result of a * b - """ - return a * b - comparisons = numpy.zeros(x.shape) idx = numpy.arange(x.size) + fhe_zeros(x.shape) n, k = x.size, self.n_neighbors ln2n = int(numpy.ceil(numpy.log2(n))) + # Number of stages for t in range(ln2n - 1, -1, -1): p = 2**t r = 0 + # d: Length of the bitonic sequence d = p for bq in range(ln2n - 1, t - 1, -1): q = 2**bq + # Determine the range of indexes to be compared range_i = numpy.array( [i for i in range(0, n - d) if i & p == r and comparisons[i] < k] ) if len(range_i) == 0: + # Edge case, for k=1 continue - a = gather1d(x, range_i) # x[range_i] - a_i = gather1d(idx, range_i) # idx[range_i] - b = gather1d(x, range_i + d) # x[range_i + d] - b_i = gather1d(idx, range_i + d) # idx[range_i + d] + # Select 2 bitonic sequences `a` and `b` of length `d` + # a = x[range_i]: first bitonic sequence + a = gather1d(x, range_i) + a_i = gather1d(idx, range_i) + # b = x[range_i + d]: Second bitonic sequence + # b_i = idx[range_i]: Indexes of a_i elements in the original x + b = gather1d(x, range_i + d) + b_i = gather1d(idx, range_i + d) + # Select max(a, b) diff = a - b - sign = diff < 0 - max_x = a + numpy.maximum(0, b - a) - x = scatter1d(x, a + b - max_x, range_i) # x[range_i] = a + b - max_x - x = scatter1d(x, max_x, range_i + d) # x[range_i + d] = max_x - max_idx = a_i + mul_tlu((b_i - a_i), sign) + # Swap if a > b + # x[range_i] = max_x(a, b): First bitonic sequence gets min(a, b) + x = scatter1d(x, a + b - max_x, range_i) + # x[range_i + d] = min(a, b): Second bitonic sequence gets max(a, b) + x = scatter1d(x, max_x, range_i + d) + + # Max index selection + sign = diff < 0 + max_idx = a_i + (b_i - a_i) * sign - # idx[range_i] = a_i + b_i - max_idx + # Update indexes array according to the max items + # idx[range_i] = a_i + b_i - max_idx <=> min_idx idx = scatter1d(idx, a_i + b_i - max_idx, range_i) - idx = scatter1d(idx, max_idx, range_i + d) # idx[range_i + d] = max_idx + # idx[range_i + d] = max_idx + idx = scatter1d(idx, max_idx, range_i + d) + # Update comparisons[range_i + d] = comparisons[range_i + d] + 1 - d = q - p r = p + # Return only the topk indexes topk_indexes = [] for i in range((self.n_neighbors)): topk_indexes.append(idx[i]) topk_indexes = fhe_array(topk_indexes) - assert topk_indexes.shape[0] == self.n_neighbors - return topk_indexes # 1. Pairwise_euclidiean distance @@ -2020,9 +2006,10 @@ def mul_tlu(a, b): # with fhe.tag(f"distance_matrix"): distance_matrix = pairwise_euclidean_distance(q_X) - # The square root in the Euclidean distance calculation is not applied. + # The square root in the Euclidean distance calculation is not applied to speed up FHE + # computations. # Being a monotonic function, it does not affect the logic of the calculation, notably for - # for the argsort + # the argsort. # 2. Sorting args # with fhe.tag(f"sorted_args"): @@ -2031,6 +2018,25 @@ def mul_tlu(a, b): return numpy.expand_dims(sorted_args, axis=0) + # KNN works only for MONO in the latest concrete Python version + # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3978 + def compile(self, *args, **kwargs) -> Circuit: + # If a configuration instance is given as a positional parameter, set the strategy to + # multi-parameter + if len(args) >= 2: + configuration = force_mono_parameter_in_configuration(args[1]) + args_list = list(args) + args_list[1] = configuration + args = tuple(args_list) + + # Else, retrieve the configuration in kwargs if it exists, or create a new one, and set the + # strategy to multi-parameter + else: + configuration = kwargs.get("configuration", None) + kwargs["configuration"] = force_mono_parameter_in_configuration(configuration) + + return BaseEstimator.compile(self, *args, **kwargs) + def predict(self, X: Data, fhe: Union[FheMode, str] = FheMode.DISABLE) -> numpy.ndarray: X = check_array_and_assert(X) @@ -2040,7 +2046,7 @@ def predict(self, X: Data, fhe: Union[FheMode, str] = FheMode.DISABLE) -> numpy. # Argsort arg_sort = super().predict(query[None], fhe) # Majority vote - # pylint: disable=protected-access + # pylint: disable-next=protected-access label_indices = self._y[arg_sort.flatten()] y_pred = self.majority_vote(label_indices) y_preds.append(y_pred) diff --git a/src/concrete/ml/sklearn/neighbors.py b/src/concrete/ml/sklearn/neighbors.py index d7dad8639e..eb50f4312d 100644 --- a/src/concrete/ml/sklearn/neighbors.py +++ b/src/concrete/ml/sklearn/neighbors.py @@ -1,6 +1,7 @@ """Implement sklearn linear model.""" from typing import Any, Dict +import numpy import sklearn.linear_model from .base import SklearnKNeighborsClassifierMixin @@ -28,7 +29,7 @@ class KNeighborsClassifier(SklearnKNeighborsClassifierMixin): def __init__( self, - n_bits=3, + n_bits=2, n_neighbors=3, *, weights="uniform", @@ -42,6 +43,13 @@ def __init__( # Call SklearnKNeighborsClassifierMixin's __init__ method super().__init__(n_bits=n_bits) + assert algorithm in ["brute", "auto"], f"Algorithm = `{algorithm}` is supported in FHE." + assert not callable(metric), "`metric` should not be a callable object" + assert ( + p == 2 and metric == "minkowski" + ), "Only `L2` norm is supported with `p=2` and `metric = 'minkowski'`" + + self._y: numpy.ndarray self.n_neighbors = n_neighbors self.algorithm = algorithm self.leaf_size = leaf_size @@ -50,10 +58,9 @@ def __init__( self.metric_params = metric_params self.n_jobs = n_jobs self.weights = weights - self._y = None def dump_dict(self) -> Dict[str, Any]: - assert self._weight_quantizer is not None, self._is_not_fitted_error_message() + assert self._q_X_fit_quantizer is not None, self._is_not_fitted_error_message() metadata: Dict[str, Any] = {} @@ -63,7 +70,6 @@ def dump_dict(self) -> Dict[str, Any]: metadata["_is_fitted"] = self._is_fitted metadata["_is_compiled"] = self._is_compiled metadata["input_quantizers"] = self.input_quantizers - metadata["_weight_quantizer"] = self._weight_quantizer metadata["_q_X_fit_quantizer"] = self._q_X_fit_quantizer metadata["_q_X_fit"] = self._q_X_fit metadata["_y"] = self._y @@ -99,7 +105,6 @@ def load_dict(cls, metadata: Dict): obj._is_compiled = metadata["_is_compiled"] obj.input_quantizers = metadata["input_quantizers"] obj.output_quantizers = metadata["output_quantizers"] - obj._weight_quantizer = metadata["_weight_quantizer"] obj._q_X_fit_quantizer = metadata["_q_X_fit_quantizer"] obj._q_X_fit = metadata["_q_X_fit"] obj._y = metadata["_y"] diff --git a/tests/common/test_pbs_error_probability_settings.py b/tests/common/test_pbs_error_probability_settings.py index 4066119eb9..31aad3aea9 100644 --- a/tests/common/test_pbs_error_probability_settings.py +++ b/tests/common/test_pbs_error_probability_settings.py @@ -4,12 +4,9 @@ import numpy import pytest -from concrete.fhe.compilation import Configuration from sklearn.exceptions import ConvergenceWarning from torch import nn -from concrete import fhe -from concrete.ml.common.utils import get_model_name from concrete.ml.pytest.torch_models import FCSmall from concrete.ml.pytest.utils import sklearn_models_and_datasets from concrete.ml.torch.compile import compile_torch_model @@ -29,7 +26,7 @@ {"global_p_error": 0.038, "p_error": 0.39}, ], ) -def test_config_sklearn(model_class, parameters, kwargs, load_data, default_configuration): +def test_config_sklearn(model_class, parameters, kwargs, load_data): """Testing with p_error and global_p_error configs with sklearn models.""" x, y = load_data(model_class, **parameters) @@ -41,24 +38,12 @@ def test_config_sklearn(model_class, parameters, kwargs, load_data, default_conf # Fit the model model.fit(x, y) - if get_model_name(model_class) == "KNeighborsClassifier": - - default_configuration = Configuration( - dump_artifacts_on_unexpected_failures=False, - enable_unsafe_features=True, - use_insecure_key_cache=True, - insecure_key_cache_location="ConcreteNumpyKeyCache", - parameter_selection_strategy=fhe.ParameterSelectionStrategy.MONO, - single_precision=True, - ) - if kwargs.get("p_error", None) is not None and kwargs.get("global_p_error", None) is not None: with pytest.raises(ValueError) as excinfo: - model.compile(x, default_configuration, verbose=True, **kwargs) + model.compile(x, verbose=True, **kwargs) assert "Please only set one of (p_error, global_p_error) values" in str(excinfo.value) else: - - model.compile(x, default_configuration, verbose=True, **kwargs) + model.compile(x, verbose=True, **kwargs) # We still need to check that we have the expected probabilities # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/2206 diff --git a/tests/deployment/test_client_server.py b/tests/deployment/test_client_server.py index f5e4a8e438..7df681a1a5 100644 --- a/tests/deployment/test_client_server.py +++ b/tests/deployment/test_client_server.py @@ -9,12 +9,9 @@ import numpy import pytest -from concrete.fhe.compilation import Configuration from sklearn.exceptions import ConvergenceWarning from torch import nn -from concrete import fhe -from concrete.ml.common.utils import get_model_name from concrete.ml.deployment.fhe_client_server import FHEModelClient, FHEModelDev, FHEModelServer from concrete.ml.pytest.torch_models import FCSmall from concrete.ml.pytest.utils import instantiate_model_generic, sklearn_models_and_datasets @@ -98,20 +95,10 @@ def test_client_server_sklearn( # Compile extra_params = {"global_p_error": 1 / 100_000} - if get_model_name(model_class) == "KNeighborsClassifier": - - default_configuration = Configuration( - dump_artifacts_on_unexpected_failures=False, - enable_unsafe_features=True, - use_insecure_key_cache=True, - insecure_key_cache_location="ConcreteNumpyKeyCache", - parameter_selection_strategy=fhe.ParameterSelectionStrategy.MONO, - single_precision=True, - ) - # Running the simulation using a model that is not compiled should not be possible with pytest.raises(AttributeError, match=".* model is not compiled.*"): client_server_simulation(x_train, x_test, model, default_configuration) + # With n_bits = 3, KNN is not compilable fhe_circuit = model.compile( x_train, default_configuration, **extra_params, show_mlir=(n_bits <= 8) diff --git a/tests/sklearn/test_dump_onnx.py b/tests/sklearn/test_dump_onnx.py index f1949a6ca3..e2957788f1 100644 --- a/tests/sklearn/test_dump_onnx.py +++ b/tests/sklearn/test_dump_onnx.py @@ -9,7 +9,6 @@ import pytest from sklearn.exceptions import ConvergenceWarning -from concrete import fhe from concrete.ml.common.utils import is_model_class_in_a_list from concrete.ml.pytest.utils import get_model_name, sklearn_models_and_datasets from concrete.ml.sklearn import get_sklearn_tree_models @@ -37,9 +36,9 @@ def check_onnx_file_dump(model_class, parameters, load_data, str_expected, defau model.set_params(**model_params) if get_model_name(model) == "KNeighborsClassifier": - model.n_bits = 4 - default_configuration.parameter_selection_strategy = fhe.ParameterSelectionStrategy.MONO - default_configuration.single_precision = True + # KNN works only for small quantization bits + # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3979 + model.n_bits = 2 with warnings.catch_warnings(): # Sometimes, we miss convergence, which is not a problem for our test @@ -50,6 +49,7 @@ def check_onnx_file_dump(model_class, parameters, load_data, str_expected, defau with warnings.catch_warnings(): # Use FHE simulation to not have issues with precision model.compile(x, default_configuration) + # Get ONNX model onnx_model = model.onnx_model @@ -423,7 +423,7 @@ def test_dump( return %variable }""", "KNeighborsClassifier": """graph torch_jit ( - %input_0[DOUBLE, symx3] + %input_0[DOUBLE, symx2] ) { %/_operators.0/Constant_output_0 = Constant[value = ]() %/_operators.0/Unsqueeze_output_0 = Unsqueeze(%input_0, %/_operators.0/Constant_output_0) diff --git a/tests/sklearn/test_sklearn_models.py b/tests/sklearn/test_sklearn_models.py index 931d0c3222..816ae45535 100644 --- a/tests/sklearn/test_sklearn_models.py +++ b/tests/sklearn/test_sklearn_models.py @@ -225,6 +225,7 @@ def check_correctness_with_sklearn( def check_double_fit(model_class, n_bits, x_1, x_2, y_1, y_2): """Check double fit.""" + model = instantiate_model_generic(model_class, n_bits=n_bits) # Sometimes, we miss convergence, which is not a problem for our test @@ -280,17 +281,10 @@ def check_double_fit(model_class, n_bits, x_1, x_2, y_1, y_2): # Check that the new quantizers are different from the first ones. This is because we # currently expect all quantizers to be re-computed when re-fitting a model - # For now, in KNN, we compute the pairwise Euclidean distance between the encrypted - # X and each element in the database. - # Then, we return the indices of the k closest distances to this point. - # The exact precision of computation of the quantization and dequantization parameters - # is not relevant in this case. That's why the assertion test is being ignored - # for now in the context of the KNN algorithm. - if get_model_name(model) != "KNeighborsClassifier": - assert all( - quantizer_1 != quantizer_2 - for (quantizer_1, quantizer_2) in zip(quantizers_1, quantizers_2) - ) + assert all( + quantizer_1 != quantizer_2 + for (quantizer_1, quantizer_2) in zip(quantizers_1, quantizers_2) + ) # Set the same torch seed manually before re-fitting the neural network if is_model_class_in_a_list(model_class, get_sklearn_neural_net_models()): @@ -311,20 +305,13 @@ def check_double_fit(model_class, n_bits, x_1, x_2, y_1, y_2): # quantizers to be re-computed when re-fitting. Since we used the same dataset as the first # fit, we also expect these quantizers to be the same. - # For now, in KNN, we compute the pairwise Euclidean distance between the encrypted - # X and each element in the database. - # Then, we return the indices of the k closest distances to this point. - # The exact precision of computation of the quantization and dequantization parameters - # is not relevant in this case. That's why the assertion test is being ignored - # for now in the context of the KNN algorithm. - if get_model_name(model) != "KNeighborsClassifier": - assert all( - quantizer_1 == quantizer_3 - for (quantizer_1, quantizer_3) in zip( - input_quantizers_1 + output_quantizers_1, - input_quantizers_3 + output_quantizers_3, - ) + assert all( + quantizer_1 == quantizer_3 + for (quantizer_1, quantizer_3) in zip( + input_quantizers_1 + output_quantizers_1, + input_quantizers_3 + output_quantizers_3, ) + ) def check_serialization(model, x, use_dump_method): @@ -585,7 +572,6 @@ def cast_input(x, y, input_type): # Sometimes, we miss convergence, which is not a problem for our test with warnings.catch_warnings(): warnings.simplefilter("ignore", category=ConvergenceWarning) - model.fit(x, y) # Make sure `predict` is working when FHE is disabled @@ -656,8 +642,8 @@ def check_pipeline(model_class, x, y): param_grid = { "model__n_bits": [2, 3], } - - grid_search = GridSearchCV(pipe_cv, param_grid, error_score="raise", cv=3) + # Since the data-set is really small for KNN, we have to decrease the number of splits + grid_search = GridSearchCV(pipe_cv, param_grid, error_score="raise", cv=2) # Sometimes, we miss convergence, which is not a problem for our test with warnings.catch_warnings(): @@ -686,9 +672,7 @@ def check_grid_search(model_class, x, y, scoring): "n_jobs": [1], } elif model_class in get_sklearn_neighbors_models(): - param_grid = { - "n_bits": [3], - } + param_grid = {"n_bits": [2], "n_neighbors": [2]} else: param_grid = { "n_bits": [20], @@ -707,7 +691,7 @@ def check_grid_search(model_class, x, y, scoring): pytest.skip("Skipping predict_proba for KNN, doesn't work for now") _ = GridSearchCV( - model_class(), param_grid, cv=5, scoring=scoring, error_score="raise", n_jobs=1 + model_class(), param_grid, cv=2, scoring=scoring, error_score="raise", n_jobs=1 ).fit(x, y) @@ -807,7 +791,8 @@ def get_hyper_param_combinations(model_class): "base_score": [0.5, None], } elif model_class in get_sklearn_neighbors_models(): - hyper_param_combinations = {"n_neighbors": [2, 4]} + # Use small `n_neighbors` values for KNN, because the data-set is too small for now + hyper_param_combinations = {"n_neighbors": [1, 2]} else: assert is_model_class_in_a_list( @@ -1350,6 +1335,7 @@ def test_input_support( ): """Test all models with Pandas, List or Torch inputs.""" x, y = get_dataset(model_class, parameters, n_bits, load_data, is_weekly_option) + if verbose: print("Run input_support") @@ -1452,7 +1438,8 @@ def test_predict_correctness( "Inference in the clear (with " f"number_of_tests_in_non_fhe = {number_of_tests_in_non_fhe})" ) - + # KNN works only for smaller quantization bits + # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3979 if n_bits > 5 and get_model_name(model) == "KNeighborsClassifier": pytest.skip("Use less than 5 bits with KNN.") @@ -1475,11 +1462,6 @@ def test_predict_correctness( print("Compile the model") with warnings.catch_warnings(): - - if get_model_name(model) == "KNeighborsClassifier": - default_configuration.parameter_selection_strategy = ( - ParameterSelectionStrategy.MONO - ) fhe_circuit = model.compile( x, default_configuration, @@ -1553,7 +1535,6 @@ def test_p_error_global_p_error_simulation( parameters, error_param, load_data, - default_configuration, is_weekly_option, ): """Test p_error and global_p_error simulation. @@ -1567,23 +1548,24 @@ def test_p_error_global_p_error_simulation( if "global_p_error" in error_param: pytest.skip("global_p_error behave very differently depending on the type of model.") - # Get data-set - n_bits = min(N_BITS_REGULAR_BUILDS) if get_model_name(model_class) == "KNeighborsClassifier": - n_bits = min(n_bits, 2) - default_configuration.parameter_selection_strategy = ParameterSelectionStrategy.MONO + # KNN works only for smaller quantization bits + # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3979 + n_bits = min([2] + N_BITS_REGULAR_BUILDS) + else: + n_bits = min(N_BITS_REGULAR_BUILDS) - # Initialize and fit the model + # Get data-set, initialize and fit the model model, x = preamble(model_class, parameters, n_bits, load_data, is_weekly_option) # Check if model is linear is_linear_model = is_model_class_in_a_list(model_class, get_sklearn_linear_models()) - # Check if model is linear + # Check if model is a distance metrics model is_knn_model = is_model_class_in_a_list(model_class, get_sklearn_neighbors_models()) # Compile with a large p_error to be sure the result is random. - model.compile(x, default_configuration, **error_param) + model.compile(x, **error_param) def check_for_divergent_predictions(x, model, fhe, max_iterations=N_ALLOWED_FHE_RUN): """Detect divergence between simulated/FHE execution and clear run.""" @@ -1595,7 +1577,6 @@ def check_for_divergent_predictions(x, model, fhe, max_iterations=N_ALLOWED_FHE_ else model.predict ) y_expected = predict_function(x, fhe="disable") - for i in range(max_iterations): y_pred = predict_function(x[i : i + 1], fhe=fhe).ravel() if not numpy.array_equal(y_pred, y_expected[i : i + 1].ravel()): @@ -1617,6 +1598,7 @@ def check_for_divergent_predictions(x, model, fhe, max_iterations=N_ALLOWED_FHE_ simulation_diff_found = check_for_divergent_predictions(x, model, fhe="simulate") fhe_diff_found = check_for_divergent_predictions(x, model, fhe="execute") + # Check for differences in predictions # Remark that, with the old VL, linear models (or, more generally, circuits without PBS) were # badly simulated. It has been fixed in the new simulation. @@ -1720,9 +1702,10 @@ def test_mono_parameter_warnings( if is_model_class_in_a_list(model_class, get_sklearn_linear_models()): return - # KNN works only for ParameterSelectionStrategy.MULTI + # KNN is manually forced to use mono-parameter + # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3978 if is_model_class_in_a_list(model_class, get_sklearn_neighbors_models()): - pytest.skip("Skipping predict_proba for KNN, doesn't work for now") + return n_bits = min(N_BITS_REGULAR_BUILDS) diff --git a/use_case_examples/credit_scoring/CreditScoring.ipynb b/use_case_examples/credit_scoring/CreditScoring.ipynb index b5af7d35c1..c4ce77f6cc 100644 --- a/use_case_examples/credit_scoring/CreditScoring.ipynb +++ b/use_case_examples/credit_scoring/CreditScoring.ipynb @@ -20,11 +20,7 @@ "from functools import partial\n", "\n", "import numpy as np\n", - "import pandas as pd\n", - "from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score\n", - "from sklearn.model_selection import train_test_split\n", - "from sklearn.pipeline import Pipeline\n", - "from sklearn.preprocessing import StandardScaler" + "import pandas as pd" ] }, { @@ -36,6 +32,10 @@ "# Importing the models, from both scikit-learn and Concrete ML\n", "from sklearn.ensemble import RandomForestClassifier as SklearnRandomForestClassifier\n", "from sklearn.linear_model import LogisticRegression as SklearnLogisticRegression\n", + "from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.pipeline import Pipeline\n", + "from sklearn.preprocessing import StandardScaler\n", "from sklearn.tree import DecisionTreeClassifier as SklearnDecisionTreeClassifier\n", "from xgboost import XGBClassifier as SklearnXGBoostClassifier\n", "\n",