From a3649b6e4565141bcf6cc3fce742a7e4e1120507 Mon Sep 17 00:00:00 2001 From: kcelia Date: Wed, 25 Oct 2023 18:10:00 +0200 Subject: [PATCH 01/73] chore: Version 1 - Set manually number of lsb to be removed in ops_impl.py for comparison functions - Confirm that rounding improves the execution time for tree-based models (see experiements in fhenet-experiments --- src/concrete/ml/onnx/convert.py | 1 + src/concrete/ml/onnx/ops_impl.py | 66 +++++++++++++++++--- src/concrete/ml/sklearn/base.py | 26 ++++++++ src/concrete/ml/sklearn/test_test.py | 90 ++++++++++++++++++++++++++++ 4 files changed, 174 insertions(+), 9 deletions(-) create mode 100644 src/concrete/ml/sklearn/test_test.py diff --git a/src/concrete/ml/onnx/convert.py b/src/concrete/ml/onnx/convert.py index 9be80500c..0dec34009 100644 --- a/src/concrete/ml/onnx/convert.py +++ b/src/concrete/ml/onnx/convert.py @@ -200,6 +200,7 @@ def get_equivalent_numpy_forward_from_onnx( # Check supported operators required_onnx_operators = set(get_op_type(node) for node in equivalent_onnx_model.graph.node) + print(f"{required_onnx_operators=}") unsupported_operators = required_onnx_operators - IMPLEMENTED_ONNX_OPS if len(unsupported_operators) > 0: raise ValueError( diff --git a/src/concrete/ml/onnx/ops_impl.py b/src/concrete/ml/onnx/ops_impl.py index d06489a49..97320cb79 100644 --- a/src/concrete/ml/onnx/ops_impl.py +++ b/src/concrete/ml/onnx/ops_impl.py @@ -14,14 +14,19 @@ from scipy import special from typing_extensions import SupportsIndex -from ..common import utils -from ..common.debugging import assert_false, assert_true -from .onnx_impl_utils import ( +from concrete import fhe +from concrete.ml.common import utils +from concrete.ml.common.debugging import assert_false, assert_true +from concrete.ml.onnx.onnx_impl_utils import ( compute_onnx_pool_padding, numpy_onnx_pad, onnx_avgpool_compute_norm_const, ) +ROUNDING = -1 +ACTIVATE_MULT = False +VERBOSE = False + class RawOpOutput(numpy.ndarray): """Type construct that marks an ndarray as a raw output of a quantized op.""" @@ -294,6 +299,10 @@ def numpy_gemm( # the compiler here y = numpy.matmul(a_prime, b_prime) + if VERBOSE: + print("ops_impl - gemm") + if ROUNDING >= 1 and ACTIVATE_MULT: + y = fhe.round_bit_pattern(y, lsbs_to_remove=ROUNDING) if processed_alpha != 1: y = y * processed_alpha @@ -322,7 +331,15 @@ def numpy_matmul( Returns: Tuple[numpy.ndarray]: Matrix multiply results from A * B """ - return (numpy.matmul(a, b),) + if VERBOSE: + print("ops_impl - matmul") + + op = numpy.matmul(a, b) + if ROUNDING >= 1 and ACTIVATE_MULT: + y = fhe.round_bit_pattern(op, lsbs_to_remove=ROUNDING) + return y + else: + return (numpy.matmul(a, b),) def numpy_relu( @@ -897,7 +914,7 @@ def numpy_equal( Returns: Tuple[numpy.ndarray]: Output tensor """ - + # for this operation, rounding is not equivalent return (numpy.equal(x, y),) @@ -951,7 +968,14 @@ def numpy_greater( Tuple[numpy.ndarray]: Output tensor """ - return (numpy.greater(x, y),) + if VERBOSE: + print(f"ops_impl - greater - {type(x)=}, {type(y)=}") + if ROUNDING >= 1 and not ACTIVATE_MULT: + half = 1 << ROUNDING - 1 + op = fhe.round_bit_pattern((y - x) - half, lsbs_to_remove=ROUNDING) + return (op < 0,) + else: + return (numpy.greater(x, y),) def numpy_greater_float( @@ -988,8 +1012,16 @@ def numpy_greater_or_equal( Returns: Tuple[numpy.ndarray]: Output tensor """ + # For this opporation, the rounding should work + if VERBOSE: + print(f"ops_impl - greater_or_equal - {type(x)=}, {type(y)=}") - return (numpy.greater_equal(x, y),) + if ROUNDING >= 1 and not ACTIVATE_MULT: + half = 1 << ROUNDING - 1 + op = fhe.round_bit_pattern((x - y) - half, lsbs_to_remove=ROUNDING) + return (op >= 0,) + else: + return (numpy.greater_equal(x, y),) def numpy_greater_or_equal_float( @@ -1026,8 +1058,16 @@ def numpy_less( Returns: Tuple[numpy.ndarray]: Output tensor """ + # For this operation, rounding should work + if VERBOSE: + print(f"ops_impl - less - {type(x)=}, {type(y)=}") - return (numpy.less(x, y),) + if ROUNDING >= 1 and not ACTIVATE_MULT: + half = 1 << ROUNDING - 1 + op = fhe.round_bit_pattern(x - y - half, lsbs_to_remove=ROUNDING) + return (op < 0,) + else: + return (numpy.less(x, y),) def numpy_less_float( @@ -1065,7 +1105,14 @@ def numpy_less_or_equal( Tuple[numpy.ndarray]: Output tensor """ - return (numpy.less_equal(x, y),) + if VERBOSE: + print("ops_impl - less_or_equal") + if ROUNDING >= 1 and not ACTIVATE_MULT: + half = 1 << (ROUNDING - 1) + op = fhe.round_bit_pattern((y - x) - half, lsbs_to_remove=ROUNDING) + return (op >= 0,) + else: + return (numpy.less_equal(x, y),) def numpy_less_or_equal_float( @@ -1519,6 +1566,7 @@ def numpy_or_float( Returns: Tuple[numpy.ndarray]: Output tensor """ + return cast_to_float(numpy_or(a, b)) diff --git a/src/concrete/ml/sklearn/base.py b/src/concrete/ml/sklearn/base.py index 37d045f5c..c5c1dd77a 100644 --- a/src/concrete/ml/sklearn/base.py +++ b/src/concrete/ml/sklearn/base.py @@ -1282,6 +1282,7 @@ def __init__(self, n_bits: int): #: The model's inference function. Is None if the model is not fitted. self._tree_inference: Optional[Callable] = None + self._rounder = cnp.AutoRounder(target_msbs=5) BaseEstimator.__init__(self) @@ -1320,6 +1321,11 @@ def fit(self, X: Data, y: Target, **fit_parameters): self._is_fitted = True + # inputset = numpy.array(list(_get_inputset_generator(q_X))) + # self._rounder.adjust(self._tree_inference, inputset) + + self._tree_inference(q_X.astype("int")) + return self def quantize_input(self, X: numpy.ndarray) -> numpy.ndarray: @@ -1357,6 +1363,26 @@ def _get_module_to_compile(self) -> Union[Compiler, QuantizedModule]: return compiler def compile(self, *args, **kwargs) -> Circuit: + def force_auto_adjust_rounder_in_configuration(configuration): + if configuration is None: + configuration = Configuration(auto_adjust_rounders=True, **kwargs) + else: + configuration.auto_adjust_rounders = True + return configuration + + # If a configuration instance is given as a positional parameter, set auto_adjust_rounder + if len(args) >= 2: + configuration = force_auto_adjust_rounder_in_configuration(args[1]) + args_list = list(args) + args_list[1] = configuration + args = tuple(args_list) + + # Else, retrieve the configuration in kwargs if it exists, or create a new one, and set the + # auto_adjust_rounder + else: + configuration = kwargs.get("configuration", None) + kwargs["configuration"] = force_auto_adjust_rounder_in_configuration(configuration) + BaseEstimator.compile(self, *args, **kwargs) # Check that the graph only has a single output diff --git a/src/concrete/ml/sklearn/test_test.py b/src/concrete/ml/sklearn/test_test.py new file mode 100644 index 000000000..57ab47321 --- /dev/null +++ b/src/concrete/ml/sklearn/test_test.py @@ -0,0 +1,90 @@ +# General imports +import argparse +import warnings +from time import time + +from sklearn.datasets import make_classification +from sklearn.metrics import accuracy_score, r2_score +from sklearn.model_selection import train_test_split +from tqdm import tqdm + +from concrete.ml.sklearn import _get_sklearn_tree_models, is_classifier_or_partial_classifier + +# pylint: disable=E0611 +from concrete.ml.sklearn.utils_test_tree_rounding import ( + check_if_file_exists, + set_lstr, + set_seed, + write, +) + +warnings.filterwarnings("ignore") + +# Parse command line arguments +parser = argparse.ArgumentParser(description="Update the ROUNDING variable in X.py.") +parser.add_argument("n_bits", type=int, help="x") +parser.add_argument("lsbr", type=int, help="x") +parser.add_argument("--seed", type=int, help="random seed", default=42) +parser.add_argument("--file_name", type=str, default="impact_rounding_on_tree_based_models.txt") + +args = parser.parse_args() + +lsbr = args.lsbr +seed = args.seed +n_bits = args.n_bits +file_name = args.file_name +models = _get_sklearn_tree_models() + +set_seed(seed) +set_lstr(lsbr) +check_if_file_exists(file_name) + +print(f"{n_bits=} | {lsbr=} | {seed=}") + +for model_class in tqdm(models): + for ds in [1500, 5000]: + for features in [5, 100, 200]: + model = model_class(n_bits=n_bits) + X, y = make_classification(n_samples=ds, n_features=features, random_state=seed) + X_train, X_test, y_train, y_test = train_test_split( + X, y, test_size=0.1, random_state=seed + ) + model, sklearn_model = model.fit_benchmark(X_train, y_train) + + write(file_name, f"{model.__class__.__name__},{ds},{features},") + + circuit = model.compile(X_train) + max_bit = circuit.graph.maximum_integer_bit_width() + + write(file_name, f"{n_bits},{lsbr},{max_bit},") + + start_time = time() + circuit.keygen() + delta_time_key = time() - start_time + + write(file_name, f"{delta_time_key:.5f},") + + y_pred_sk = sklearn_model.predict(X_test) + y_pred_simulate = model.predict(X_test, fhe="simulate") + y_pred_disable = model.predict(X_test, fhe="disable") + + metric = accuracy_score if is_classifier_or_partial_classifier(model) else r2_score + + score_sk = metric(y_true=y_test, y_pred=y_pred_sk) + score_disable = metric(y_true=y_test, y_pred=y_pred_disable) + score_simulate = metric(y_true=y_test, y_pred=y_pred_simulate) + + write(file_name, f"{score_disable:.5f},{score_simulate:.5f},") + + start_time = time() + y_pred_execute = model.predict(X_test[0, None], fhe="execute") + delta_time_inf = time() - start_time + score_execute = (y_pred_execute == y_test[0]).mean() + + write(file_name, f"{delta_time_inf:.5f},{score_sk:.5f},{seed}\n") + + print( + f"{model.__class__.__name__}, {ds=}, dim={features}, {n_bits=}, {lsbr=}, {max_bit=}," + f"sk={score_sk:.3f}, disable={score_disable:.3f}, simulate={score_simulate:.3f}, " + f"key_gen={delta_time_key:.2f}, {delta_time_inf=:.2f}\n" + ) From e12f13a8a6b1ec8ba93044bea0bc18b95944fa44 Mon Sep 17 00:00:00 2001 From: kcelia Date: Mon, 11 Dec 2023 16:56:12 +0100 Subject: [PATCH 02/73] chore: rebase on main --- src/concrete/ml/onnx/convert.py | 31 +++++- src/concrete/ml/onnx/onnx_impl_utils.py | 46 ++++++++- src/concrete/ml/onnx/onnx_utils.py | 116 ++++++++++++++++++++++- src/concrete/ml/onnx/ops_impl.py | 98 +++++++++---------- src/concrete/ml/sklearn/base.py | 53 ++++------- src/concrete/ml/sklearn/test_test.py | 90 ------------------ src/concrete/ml/sklearn/tree_to_numpy.py | 19 +++- tests/sklearn/test_sklearn_models.py | 75 ++++++++++++++- 8 files changed, 337 insertions(+), 191 deletions(-) delete mode 100644 src/concrete/ml/sklearn/test_test.py diff --git a/src/concrete/ml/onnx/convert.py b/src/concrete/ml/onnx/convert.py index 0dec34009..5943f653d 100644 --- a/src/concrete/ml/onnx/convert.py +++ b/src/concrete/ml/onnx/convert.py @@ -2,7 +2,7 @@ import tempfile from pathlib import Path -from typing import Callable, Tuple, Union +from typing import Callable, Optional, Tuple, Union import numpy import onnx @@ -10,7 +10,12 @@ import torch from onnx import checker, helper -from .onnx_utils import IMPLEMENTED_ONNX_OPS, execute_onnx_with_numpy, get_op_type +from .onnx_utils import ( + IMPLEMENTED_ONNX_OPS, + compute_lsb_to_remove_for_trees, + execute_onnx_with_numpy, + get_op_type, +) OPSET_VERSION_FOR_ONNX_EXPORT = 14 @@ -158,15 +163,21 @@ def get_equivalent_numpy_forward_from_torch( def get_equivalent_numpy_forward_from_onnx( onnx_model: onnx.ModelProto, + q_x: Optional[numpy.ndarray] = None, check_model: bool = True, + use_rounding: bool = False, ) -> Tuple[Callable[..., Tuple[numpy.ndarray, ...]], onnx.ModelProto]: """Get the numpy equivalent forward of the provided ONNX model. Args: onnx_model (onnx.ModelProto): the ONNX model for which to get the equivalent numpy forward. + q_x (numpy.ndarray): Quantized input used to compute the LSBs to remove if 'use_rounding'. + Defaults to None if rounding is not used. check_model (bool): set to True to run the onnx checker on the model. Defaults to True. + use_rounding (bool): Use rounding feature or not. + Defaults to False. Raises: ValueError: Raised if there is an unsupported ONNX operator required to convert the torch @@ -176,9 +187,11 @@ def get_equivalent_numpy_forward_from_onnx( Callable[..., Tuple[numpy.ndarray, ...]]: The function that will execute the equivalent numpy function. """ + + lsbs_to_remove: list = [0, 0] + if check_model: checker.check_model(onnx_model) - checker.check_model(onnx_model) # Optimize ONNX graph # List of all currently supported onnx optimizer passes @@ -200,7 +213,6 @@ def get_equivalent_numpy_forward_from_onnx( # Check supported operators required_onnx_operators = set(get_op_type(node) for node in equivalent_onnx_model.graph.node) - print(f"{required_onnx_operators=}") unsupported_operators = required_onnx_operators - IMPLEMENTED_ONNX_OPS if len(unsupported_operators) > 0: raise ValueError( @@ -209,7 +221,16 @@ def get_equivalent_numpy_forward_from_onnx( f"Available ONNX operators: {', '.join(sorted(IMPLEMENTED_ONNX_OPS))}" ) + # Remove this workaround (which computes the LSB to be remove manually) once + # the truncate feature is released + # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4143 + if use_rounding: + assert ( + q_x is not None + ), "A representative quantized input-set is needed when using the rounding feature" + lsbs_to_remove = compute_lsb_to_remove_for_trees(equivalent_onnx_model, q_x) + # Return lambda of numpy equivalent of onnx execution return ( - lambda *args: execute_onnx_with_numpy(equivalent_onnx_model.graph, *args) + lambda *args: execute_onnx_with_numpy(equivalent_onnx_model.graph, lsbs_to_remove, *args) ), equivalent_onnx_model diff --git a/src/concrete/ml/onnx/onnx_impl_utils.py b/src/concrete/ml/onnx/onnx_impl_utils.py index 2cf52b3e8..e897a2cc1 100644 --- a/src/concrete/ml/onnx/onnx_impl_utils.py +++ b/src/concrete/ml/onnx/onnx_impl_utils.py @@ -1,11 +1,12 @@ """Utility functions for onnx operator implementations.""" -from typing import Tuple, Union +from typing import Callable, Tuple, Union import numpy from concrete.fhe import conv as cnp_conv from concrete.fhe import ones as cnp_ones from concrete.fhe.tracing import Tracer +from concrete.fhe import round_bit_pattern from ..common.debugging import assert_true @@ -225,3 +226,46 @@ def onnx_avgpool_compute_norm_const( norm_const = float(numpy.prod(numpy.array(kernel_shape))) return norm_const + + +# This function needs to be updated when the truncate feature is released. +# The following changes should be made: +# - Remove the `half` term +# - Replace `rounding_bit_pattern` with `truncate_bit_pattern` +# - Potentially replace `lsbs_to_remove` with `auto_truncate` +# - Adjust the typing +# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4143 +def rounded_comparison( + x: numpy.ndarray, y: numpy.ndarray, lsbs_to_remove: int, operation: ComparisonOperationType +) -> Tuple[bool]: + """Comparison operation using `round_bit_pattern` function. + + `round_bit_pattern` rounds the bit pattern of an integer to the closer + It also checks for any potential overflow. If so, it readjusts the LSBs accordingly. + + The parameter `lsbs_to_remove` in `round_bit_pattern` can either be an integer specifying the + number of LSBS to remove, or an `AutoRounder` object that determines the required number of LSBs + based on the specified number of MSBs to retain. But in our case, we choose to compute the LSBs + manually. + + Args: + x (numpy.ndarray): Input tensor + y (numpy.ndarray): Input tensor + lsbs_to_remove (int): Number of the least significant bits to remove + operation (ComparisonOperationType): Comparison operation, which can `<`, `<=` and `==` + + Returns: + Tuple[bool]: If x and y satisfy the comparison operator. + """ + + assert isinstance(lsbs_to_remove, int) + + # Workaround: in this context, `round_bit_pattern` is used as a truncate operation. + # Consequently, we subtract a term, called `half` that will subsequently be re-added during the + # `round_bit_pattern` process. + half = 1 << (lsbs_to_remove - 1) + + # To determine if 'x' 'operation' 'y' (operation being <, >, >=, <=), we evaluate 'x - y' + rounded_subtraction = round_bit_pattern((x - y) - half, lsbs_to_remove=lsbs_to_remove) + + return (operation(rounded_subtraction),) diff --git a/src/concrete/ml/onnx/onnx_utils.py b/src/concrete/ml/onnx/onnx_utils.py index 6a08ed567..6a9e12a8a 100644 --- a/src/concrete/ml/onnx/onnx_utils.py +++ b/src/concrete/ml/onnx/onnx_utils.py @@ -214,8 +214,8 @@ # Original file: # https://github.com/google/jax/blob/f6d329b2d9b5f83c6a59e5739aa1ca8d4d1ffa1c/examples/onnx2xla.py - -from typing import Any, Callable, Dict, Tuple +import math +from typing import Any, Callable, Dict, List, Tuple import numpy import onnx @@ -413,6 +413,9 @@ # All numpy operators used for tree-based models ONNX_OPS_TO_NUMPY_IMPL_BOOL = {**ONNX_OPS_TO_NUMPY_IMPL, **ONNX_COMPARISON_OPS_TO_NUMPY_IMPL_BOOL} +# All numpy operators used for tree-based models that support auto rounding +SUPPORTED_ROUNDED_OPERATIONS = ["Less", "LessOrEqual", "Greater", "GreaterOrEqual", "Equal"] + IMPLEMENTED_ONNX_OPS = set(ONNX_OPS_TO_NUMPY_IMPL.keys()) @@ -443,12 +446,14 @@ def get_op_type(node): def execute_onnx_with_numpy( graph: onnx.GraphProto, + lsbs_to_remove: List, *inputs: numpy.ndarray, ) -> Tuple[numpy.ndarray, ...]: """Execute the provided ONNX graph on the given inputs. Args: graph (onnx.GraphProto): The ONNX graph to execute. + lsbs_to_remove (List): The number of least significant bit to be removed in each stage. *inputs: The inputs of the graph. Returns: @@ -461,9 +466,16 @@ def execute_onnx_with_numpy( for initializer in graph.initializer }, ) + for node in graph.node: curr_inputs = (node_results[input_name] for input_name in node.input) attributes = {attribute.name: get_attribute(attribute) for attribute in node.attribute} + + if node.op_type in SUPPORTED_ROUNDED_OPERATIONS: + attributes["lsbs_to_remove"] = ( + lsbs_to_remove[0] if node.op_type != "Equal" else lsbs_to_remove[1] + ) + outputs = ONNX_OPS_TO_NUMPY_IMPL_BOOL[node.op_type](*curr_inputs, **attributes) node_results.update(zip(node.output, outputs)) @@ -495,3 +507,103 @@ def remove_initializer_from_input(model: onnx.ModelProto): # pragma: no cover inputs.remove(name_to_input[initializer.name]) return model + + +# Remove this function once the truncate feature is released +# FIXME: https://github.com/zama-ai/concrete-ml/issues/397 +def compute_lsb_to_remove_for_trees(onnx_model: onnx.ModelProto, q_x: numpy.ndarray) -> List[int]: + """Compute the LSB to remove for the comparison operators in the trees. + + Referring to this paper: https://scnakandala.github.io/papers/TR_2020_Hummingbird.pdf, there are + 2 levels of comparison for trees, one at the level of X.A < B and a second at + the level of I.C == D. + + Args: + onnx_model (onnx.ModelProto): The model to clean + q_x (numpy.ndarray): The quantized inputs + + Returns: + List: the number of LSB to remove for level 1 and level 2 + """ + + def get_bitwidth(array: numpy.ndarray) -> int: + """Compute the bitwidth required to represent the largest value in `array`. + + Args: + array (umpy.ndarray): The array for which the bitwidth needs to be checked. + + Returns: + int: The required bits to represent the array. + """ + + max_val = numpy.max(numpy.abs(array)) + # + 1 is added to include the sign bit + bitwidth = math.ceil(math.log2(max_val + 1)) + 1 + return bitwidth + + def update_lsbs_if_overflow_detected(array: numpy.ndarray, initial_bitwidth: int) -> int: + """Update the number of LSBs to remove based on overflow detection. + + Args: + array (umpy.ndarray): The array for which the bitwidth needs to be checked. + initial_bitwidth (int): The target bitwidth that should not be exceeded. + + Returns: + int: The updated LSB to remove. + """ + + lsbs_to_remove = initial_bitwidth + + if lsbs_to_remove > 0: + half = 1 << (lsbs_to_remove - 1) + if get_bitwidth(array - half) <= initial_bitwidth: + lsbs_to_remove -= 1 + + return lsbs_to_remove + + quant_params = { + onnx_init.name: numpy_helper.to_array(onnx_init) + for onnx_init in onnx_model.graph.initializer + if "weight" in onnx_init.name or "bias" in onnx_init.name + } + + key_mat_1 = [key for key in quant_params.keys() if "_1" in key and "weight" in key][0] + key_bias_1 = [key for key in quant_params.keys() if "_1" in key and "bias" in key][0] + + key_mat_2 = [key for key in quant_params.keys() if "_2" in key and "weight" in key][0] + key_bias_2 = [key for key in quant_params.keys() if "_2" in key and "bias" in key][0] + + # shape: (nodes, features) or (trees * nodes, features) + mat_1 = quant_params[key_mat_1] + # shape: (nodes, 1) or (trees * nodes, 1) + bias_1 = quant_params[key_bias_1] + + # shape: (trees, leaves, nodes) + mat_2 = quant_params[key_mat_2] + # shape: (leaves, 1) or (trees * leaves, 1) + bias_2 = quant_params[key_bias_2] + + n_features = mat_1.shape[1] + n_nodes = mat_2.shape[2] + n_leaves = mat_2.shape[1] + + mat_1 = mat_1.reshape(-1, n_nodes, n_features) + bias_1 = bias_1.reshape(-1, 1, n_nodes) + bias_2 = bias_2.reshape(-1, 1, n_leaves) + + # If <= -> stage = biais_1 - (q_x @ mat_1.transpose(0, 2, 1)) + # If < -> stage = (q_x @ mat_1.transpose(0, 2, 1)) - biais_1 + stage_1 = bias_1 - (q_x @ mat_1.transpose(0, 2, 1)) + + # The matrix I, as referenced in this paper: + # https://scnakandala.github.io/papers/TR_2020_Hummingbird.pdf, results from the condition: + # X.A < B and consists exclusively of binary elements, 1 and 0. + # Given this assumption, we randomly generate it. + matrix_q = numpy.random.randint(0, 2, size=(stage_1.shape)) + + stage_2 = ((matrix_q @ mat_2.transpose(0, 2, 1)) + bias_2).sum(axis=0) + + lsbs_to_remove_1 = update_lsbs_if_overflow_detected(stage_1, get_bitwidth(stage_1)) + lsbs_to_remove_2 = update_lsbs_if_overflow_detected(stage_2, get_bitwidth(stage_2)) + + return [lsbs_to_remove_1, lsbs_to_remove_2] diff --git a/src/concrete/ml/onnx/ops_impl.py b/src/concrete/ml/onnx/ops_impl.py index 97320cb79..991fb5e4e 100644 --- a/src/concrete/ml/onnx/ops_impl.py +++ b/src/concrete/ml/onnx/ops_impl.py @@ -14,19 +14,16 @@ from scipy import special from typing_extensions import SupportsIndex -from concrete import fhe +# pylint: disable=ungrouped-imports from concrete.ml.common import utils from concrete.ml.common.debugging import assert_false, assert_true from concrete.ml.onnx.onnx_impl_utils import ( compute_onnx_pool_padding, numpy_onnx_pad, onnx_avgpool_compute_norm_const, + rounded_comparison, ) -ROUNDING = -1 -ACTIVATE_MULT = False -VERBOSE = False - class RawOpOutput(numpy.ndarray): """Type construct that marks an ndarray as a raw output of a quantized op.""" @@ -299,10 +296,6 @@ def numpy_gemm( # the compiler here y = numpy.matmul(a_prime, b_prime) - if VERBOSE: - print("ops_impl - gemm") - if ROUNDING >= 1 and ACTIVATE_MULT: - y = fhe.round_bit_pattern(y, lsbs_to_remove=ROUNDING) if processed_alpha != 1: y = y * processed_alpha @@ -331,15 +324,7 @@ def numpy_matmul( Returns: Tuple[numpy.ndarray]: Matrix multiply results from A * B """ - if VERBOSE: - print("ops_impl - matmul") - - op = numpy.matmul(a, b) - if ROUNDING >= 1 and ACTIVATE_MULT: - y = fhe.round_bit_pattern(op, lsbs_to_remove=ROUNDING) - return y - else: - return (numpy.matmul(a, b),) + return (numpy.matmul(a, b),) def numpy_relu( @@ -902,6 +887,8 @@ def numpy_exp( def numpy_equal( x: numpy.ndarray, y: numpy.ndarray, + *, + lsbs_to_remove: Optional[int] = None, ) -> Tuple[numpy.ndarray]: """Compute equal in numpy according to ONNX spec. @@ -910,11 +897,16 @@ def numpy_equal( Args: x (numpy.ndarray): Input tensor y (numpy.ndarray): Input tensor + lsbs_to_remove (Optional[int]): The number of the least significant bits to remove Returns: Tuple[numpy.ndarray]: Output tensor """ - # for this operation, rounding is not equivalent + + # In the case of trees, x == y <=> x <= y or x < y - 1, because y is the max sum. + if lsbs_to_remove is not None and lsbs_to_remove > 0: + return rounded_comparison(y, x, lsbs_to_remove, operation=lambda x: x >= 0) + return (numpy.equal(x, y),) @@ -955,6 +947,8 @@ def numpy_not_float( def numpy_greater( x: numpy.ndarray, y: numpy.ndarray, + *, + lsbs_to_remove: Optional[int] = None, ) -> Tuple[numpy.ndarray]: """Compute greater in numpy according to ONNX spec. @@ -963,19 +957,17 @@ def numpy_greater( Args: x (numpy.ndarray): Input tensor y (numpy.ndarray): Input tensor + lsbs_to_remove (Optional[int]): The number of the least significant bits to remove Returns: Tuple[numpy.ndarray]: Output tensor """ - if VERBOSE: - print(f"ops_impl - greater - {type(x)=}, {type(y)=}") - if ROUNDING >= 1 and not ACTIVATE_MULT: - half = 1 << ROUNDING - 1 - op = fhe.round_bit_pattern((y - x) - half, lsbs_to_remove=ROUNDING) - return (op < 0,) - else: - return (numpy.greater(x, y),) + if lsbs_to_remove is not None and lsbs_to_remove > 0: + return rounded_comparison(y, x, lsbs_to_remove, operation=lambda x: x < 0) + + # Else, default numpy greater comparison + return (numpy.greater(x, y),) def numpy_greater_float( @@ -1000,6 +992,8 @@ def numpy_greater_float( def numpy_greater_or_equal( x: numpy.ndarray, y: numpy.ndarray, + *, + lsbs_to_remove: Optional[int] = None, ) -> Tuple[numpy.ndarray]: """Compute greater or equal in numpy according to ONNX spec. @@ -1008,20 +1002,19 @@ def numpy_greater_or_equal( Args: x (numpy.ndarray): Input tensor y (numpy.ndarray): Input tensor + lsbs_to_remove (Optional[int]): The number of the least significant bits to remove Returns: Tuple[numpy.ndarray]: Output tensor """ - # For this opporation, the rounding should work - if VERBOSE: - print(f"ops_impl - greater_or_equal - {type(x)=}, {type(y)=}") - if ROUNDING >= 1 and not ACTIVATE_MULT: - half = 1 << ROUNDING - 1 - op = fhe.round_bit_pattern((x - y) - half, lsbs_to_remove=ROUNDING) - return (op >= 0,) - else: - return (numpy.greater_equal(x, y),) + if lsbs_to_remove is not None and lsbs_to_remove > 0: + return rounded_comparison( + x, y, lsbs_to_remove, operation=lambda x: x >= 0 + ) # pragma: no cover + + # Else, default numpy greater_equal comparison + return (numpy.greater_equal(x, y),) def numpy_greater_or_equal_float( @@ -1046,6 +1039,8 @@ def numpy_greater_or_equal_float( def numpy_less( x: numpy.ndarray, y: numpy.ndarray, + *, + lsbs_to_remove: Optional[int] = None, ) -> Tuple[numpy.ndarray]: """Compute less in numpy according to ONNX spec. @@ -1054,20 +1049,17 @@ def numpy_less( Args: x (numpy.ndarray): Input tensor y (numpy.ndarray): Input tensor + lsbs_to_remove (Optional[int]): The number of the least significant bits to remove Returns: Tuple[numpy.ndarray]: Output tensor """ - # For this operation, rounding should work - if VERBOSE: - print(f"ops_impl - less - {type(x)=}, {type(y)=}") - if ROUNDING >= 1 and not ACTIVATE_MULT: - half = 1 << ROUNDING - 1 - op = fhe.round_bit_pattern(x - y - half, lsbs_to_remove=ROUNDING) - return (op < 0,) - else: - return (numpy.less(x, y),) + if lsbs_to_remove is not None and lsbs_to_remove > 0: + return rounded_comparison(x, y, lsbs_to_remove, operation=lambda x: x < 0) + + # Else, default numpy less comparison + return (numpy.less(x, y),) def numpy_less_float( @@ -1092,6 +1084,8 @@ def numpy_less_float( def numpy_less_or_equal( x: numpy.ndarray, y: numpy.ndarray, + *, + lsbs_to_remove: Optional[int] = None, ) -> Tuple[numpy.ndarray]: """Compute less or equal in numpy according to ONNX spec. @@ -1100,19 +1094,17 @@ def numpy_less_or_equal( Args: x (numpy.ndarray): Input tensor y (numpy.ndarray): Input tensor + lsbs_to_remove (Optional[int]]): The number of the least significant bits to remove Returns: Tuple[numpy.ndarray]: Output tensor """ - if VERBOSE: - print("ops_impl - less_or_equal") - if ROUNDING >= 1 and not ACTIVATE_MULT: - half = 1 << (ROUNDING - 1) - op = fhe.round_bit_pattern((y - x) - half, lsbs_to_remove=ROUNDING) - return (op >= 0,) - else: - return (numpy.less_equal(x, y),) + if lsbs_to_remove is not None and lsbs_to_remove > 0: + return rounded_comparison(y, x, lsbs_to_remove, operation=lambda x: x >= 0) + + # Else, default numpy less_or_equal comparison + return (numpy.less_equal(x, y),) def numpy_less_or_equal_float( diff --git a/src/concrete/ml/sklearn/base.py b/src/concrete/ml/sklearn/base.py index c5c1dd77a..13252648c 100644 --- a/src/concrete/ml/sklearn/base.py +++ b/src/concrete/ml/sklearn/base.py @@ -1280,9 +1280,8 @@ def __init__(self, n_bits: int): """ self.n_bits: int = n_bits - #: The model's inference function. Is None if the model is not fitted. + # _tree_inference: The model's inference function. Is None if the model is not fitted. self._tree_inference: Optional[Callable] = None - self._rounder = cnp.AutoRounder(target_msbs=5) BaseEstimator.__init__(self) @@ -1311,22 +1310,30 @@ def fit(self, X: Data, y: Target, **fit_parameters): # Check that the underlying sklearn model has been set and fit assert self.sklearn_model is not None, self._sklearn_model_is_not_fitted_error_message() + self._is_fitted = True + # Convert the tree inference with Numpy operators - self._tree_inference, self.output_quantizers, self.onnx_model_ = tree_to_numpy( - self.sklearn_model, - q_X[:1], - framework=self.framework, - output_n_bits=self.n_bits, - ) + # Adjust the auto rounder manually + self._convert_tree_to_numpy_and_compute_lsbs_to_remove(q_X, use_rounding=True) - self._is_fitted = True + return self + + def _convert_tree_to_numpy_and_compute_lsbs_to_remove( + self, q_X: numpy.ndarray, use_rounding: bool + ): - # inputset = numpy.array(list(_get_inputset_generator(q_X))) - # self._rounder.adjust(self._tree_inference, inputset) + assert self._is_fitted is True, "Model must be fitted" - self._tree_inference(q_X.astype("int")) + check_array_and_assert(q_X) - return self + # Convert the tree inference with Numpy operators + self._tree_inference, self.output_quantizers, self.onnx_model_ = tree_to_numpy( + self.sklearn_model, # type: ignore[arg-type] + q_X, + use_rounding=use_rounding, + framework=self.framework, + output_n_bits=self.n_bits, + ) def quantize_input(self, X: numpy.ndarray) -> numpy.ndarray: self.check_model_is_fitted() @@ -1363,26 +1370,6 @@ def _get_module_to_compile(self) -> Union[Compiler, QuantizedModule]: return compiler def compile(self, *args, **kwargs) -> Circuit: - def force_auto_adjust_rounder_in_configuration(configuration): - if configuration is None: - configuration = Configuration(auto_adjust_rounders=True, **kwargs) - else: - configuration.auto_adjust_rounders = True - return configuration - - # If a configuration instance is given as a positional parameter, set auto_adjust_rounder - if len(args) >= 2: - configuration = force_auto_adjust_rounder_in_configuration(args[1]) - args_list = list(args) - args_list[1] = configuration - args = tuple(args_list) - - # Else, retrieve the configuration in kwargs if it exists, or create a new one, and set the - # auto_adjust_rounder - else: - configuration = kwargs.get("configuration", None) - kwargs["configuration"] = force_auto_adjust_rounder_in_configuration(configuration) - BaseEstimator.compile(self, *args, **kwargs) # Check that the graph only has a single output diff --git a/src/concrete/ml/sklearn/test_test.py b/src/concrete/ml/sklearn/test_test.py deleted file mode 100644 index 57ab47321..000000000 --- a/src/concrete/ml/sklearn/test_test.py +++ /dev/null @@ -1,90 +0,0 @@ -# General imports -import argparse -import warnings -from time import time - -from sklearn.datasets import make_classification -from sklearn.metrics import accuracy_score, r2_score -from sklearn.model_selection import train_test_split -from tqdm import tqdm - -from concrete.ml.sklearn import _get_sklearn_tree_models, is_classifier_or_partial_classifier - -# pylint: disable=E0611 -from concrete.ml.sklearn.utils_test_tree_rounding import ( - check_if_file_exists, - set_lstr, - set_seed, - write, -) - -warnings.filterwarnings("ignore") - -# Parse command line arguments -parser = argparse.ArgumentParser(description="Update the ROUNDING variable in X.py.") -parser.add_argument("n_bits", type=int, help="x") -parser.add_argument("lsbr", type=int, help="x") -parser.add_argument("--seed", type=int, help="random seed", default=42) -parser.add_argument("--file_name", type=str, default="impact_rounding_on_tree_based_models.txt") - -args = parser.parse_args() - -lsbr = args.lsbr -seed = args.seed -n_bits = args.n_bits -file_name = args.file_name -models = _get_sklearn_tree_models() - -set_seed(seed) -set_lstr(lsbr) -check_if_file_exists(file_name) - -print(f"{n_bits=} | {lsbr=} | {seed=}") - -for model_class in tqdm(models): - for ds in [1500, 5000]: - for features in [5, 100, 200]: - model = model_class(n_bits=n_bits) - X, y = make_classification(n_samples=ds, n_features=features, random_state=seed) - X_train, X_test, y_train, y_test = train_test_split( - X, y, test_size=0.1, random_state=seed - ) - model, sklearn_model = model.fit_benchmark(X_train, y_train) - - write(file_name, f"{model.__class__.__name__},{ds},{features},") - - circuit = model.compile(X_train) - max_bit = circuit.graph.maximum_integer_bit_width() - - write(file_name, f"{n_bits},{lsbr},{max_bit},") - - start_time = time() - circuit.keygen() - delta_time_key = time() - start_time - - write(file_name, f"{delta_time_key:.5f},") - - y_pred_sk = sklearn_model.predict(X_test) - y_pred_simulate = model.predict(X_test, fhe="simulate") - y_pred_disable = model.predict(X_test, fhe="disable") - - metric = accuracy_score if is_classifier_or_partial_classifier(model) else r2_score - - score_sk = metric(y_true=y_test, y_pred=y_pred_sk) - score_disable = metric(y_true=y_test, y_pred=y_pred_disable) - score_simulate = metric(y_true=y_test, y_pred=y_pred_simulate) - - write(file_name, f"{score_disable:.5f},{score_simulate:.5f},") - - start_time = time() - y_pred_execute = model.predict(X_test[0, None], fhe="execute") - delta_time_inf = time() - start_time - score_execute = (y_pred_execute == y_test[0]).mean() - - write(file_name, f"{delta_time_inf:.5f},{score_sk:.5f},{seed}\n") - - print( - f"{model.__class__.__name__}, {ds=}, dim={features}, {n_bits=}, {lsbr=}, {max_bit=}," - f"sk={score_sk:.3f}, disable={score_disable:.3f}, simulate={score_simulate:.3f}, " - f"key_gen={delta_time_key:.2f}, {delta_time_inf=:.2f}\n" - ) diff --git a/src/concrete/ml/sklearn/tree_to_numpy.py b/src/concrete/ml/sklearn/tree_to_numpy.py index 6444eafb1..3cec1ae60 100644 --- a/src/concrete/ml/sklearn/tree_to_numpy.py +++ b/src/concrete/ml/sklearn/tree_to_numpy.py @@ -1,6 +1,6 @@ """Implements the conversion of a tree model to a numpy function.""" import warnings -from typing import Callable, List, Tuple +from typing import Callable, List, Optional, Tuple import numpy import onnx @@ -293,15 +293,17 @@ def tree_values_preprocessing( # pylint: disable=too-many-locals def tree_to_numpy( model: Callable, - x: numpy.ndarray, + q_x: numpy.ndarray, framework: str, + use_rounding: bool = False, output_n_bits: int = MAX_BITWIDTH_BACKWARD_COMPATIBLE, ) -> Tuple[Callable, List[UniformQuantizer], onnx.ModelProto]: """Convert the tree inference to a numpy functions using Hummingbird. Args: model (Callable): The tree model to convert. - x (numpy.ndarray): The input data. + q_x (numpy.ndarray): The quantized input data. + use_rounding (bool): Use rounding feature or not. framework (str): The framework from which the ONNX model is generated. (options: 'xgboost', 'sklearn') output_n_bits (int): The number of bits of the output. Default to 8. @@ -319,7 +321,12 @@ def tree_to_numpy( f"framework={framework} is not supported. It must be either 'xgboost' or 'sklearn'", ) - onnx_model = get_onnx_model(model, x, framework) + # Execute with 1 example for efficiency in large data scenarios to prevent slowdown + onnx_model = get_onnx_model(model, q_x[:1], framework) + + # if use_rounding: + # compute LSB to remove in stage 1 and 2 + # attach LSBs to the ONNX # Get the expected number of ONNX outputs in the sklearn model. expected_number_of_outputs = 1 if is_regressor_or_partial_regressor(model) else 2 @@ -333,6 +340,8 @@ def tree_to_numpy( # but also rounding the threshold such that they are now integers q_y = tree_values_preprocessing(onnx_model, framework, output_n_bits) - _tree_inference, onnx_model = get_equivalent_numpy_forward_from_onnx(onnx_model) + _tree_inference, onnx_model = get_equivalent_numpy_forward_from_onnx( + onnx_model, q_x, use_rounding=use_rounding + ) return (_tree_inference, [q_y.quantizer], onnx_model) diff --git a/tests/sklearn/test_sklearn_models.py b/tests/sklearn/test_sklearn_models.py index dcaa8b899..0eca2b982 100644 --- a/tests/sklearn/test_sklearn_models.py +++ b/tests/sklearn/test_sklearn_models.py @@ -22,10 +22,11 @@ """ import copy - -# pylint: disable=too-many-lines, too-many-arguments import json +import random import tempfile + +# pylint: disable=too-many-lines, too-many-arguments import warnings from typing import Any, Dict, List @@ -1139,6 +1140,38 @@ def check_load_fitted_sklearn_linear_models(model_class, n_bits, x, y, check_flo ) +def check_rounding_consistency( + model, + x, + predict_method, + metric, +): + + """Test that Concrete ML witout and with rounding are 'equivalent'.""" + + random_int = random.randint(0, x.shape[0] - 1) + + model.compile(x) + + rounded_predict_quantized = predict_method(x, fhe="disable") + rounded_predict_simulate = predict_method(x, fhe="simulate") + rounded_predict_fhe = predict_method(x[random_int, None], fhe="execute") + + # pylint: disable=protected-access + quant_x = model.quantize_input(x).astype("float") + model._convert_tree_to_numpy_and_compute_lsbs_to_remove(quant_x, use_rounding=False) + + model.compile(x) + + not_rounded_predict_quantized = predict_method(x, fhe="disable") + not_rounded_predict_simulate = predict_method(x, fhe="simulate") + not_rounded_predict_fhe = predict_method(x[random_int, None], fhe="execute") + + metric(rounded_predict_quantized, not_rounded_predict_quantized) + metric(rounded_predict_simulate, not_rounded_predict_simulate) + metric(rounded_predict_fhe, not_rounded_predict_fhe) + + # Neural network models are skipped for this test # The `fit_benchmark` function of QNNs returns a QAT model and a FP32 model that is similar # in structure but trained from scratch. Furthermore, the `n_bits` setting @@ -1765,3 +1798,41 @@ def test_linear_models_have_no_tlu( # Check that no TLUs are found within the MLIR check_circuit_has_no_tlu(fhe_circuit) + + +# Test only tree-based models +@pytest.mark.parametrize("model_class, parameters", get_sklearn_tree_models_and_datasets()) +@pytest.mark.parametrize("n_bits", [2, 3, 4, 5]) +def test_rounding_consistency( + model_class, + parameters, + n_bits, + load_data, + check_r2_score, + check_accuracy, + is_weekly_option, + verbose=True, +): + """Test that Concrete ML witout and with rounding are 'equivalent'.""" + + if verbose: + print("Run check_rounding_consistency") + + model, x = preamble(model_class, parameters, n_bits, load_data, is_weekly_option) + + # Check `predict_proba` for classifiers + if is_classifier_or_partial_classifier(model): + check_rounding_consistency( + model, + x, + predict_method=model.predict_proba, + metric=check_r2_score, + ) + else: + # Check `predict` for regressors + check_rounding_consistency( + model, + x, + predict_method=model.predict, + metric=check_accuracy, + ) From d88d6033235c7496dacfd2571e42b280240750ef Mon Sep 17 00:00:00 2001 From: kcelia Date: Wed, 29 Nov 2023 13:16:01 +0100 Subject: [PATCH 03/73] chore: add new rounded operators --- src/concrete/ml/onnx/ops_impl.py | 107 +++++++++++++++++++++---------- 1 file changed, 72 insertions(+), 35 deletions(-) diff --git a/src/concrete/ml/onnx/ops_impl.py b/src/concrete/ml/onnx/ops_impl.py index 991fb5e4e..206c80128 100644 --- a/src/concrete/ml/onnx/ops_impl.py +++ b/src/concrete/ml/onnx/ops_impl.py @@ -887,8 +887,6 @@ def numpy_exp( def numpy_equal( x: numpy.ndarray, y: numpy.ndarray, - *, - lsbs_to_remove: Optional[int] = None, ) -> Tuple[numpy.ndarray]: """Compute equal in numpy according to ONNX spec. @@ -897,17 +895,36 @@ def numpy_equal( Args: x (numpy.ndarray): Input tensor y (numpy.ndarray): Input tensor - lsbs_to_remove (Optional[int]): The number of the least significant bits to remove + + Returns: + Tuple[numpy.ndarray]: Output tensor + """ + return (numpy.equal(x, y),) + + +def numpy_rounded_equal( + x: numpy.ndarray, + y: numpy.ndarray, + lsbs_to_remove: int, +) -> Tuple[numpy.ndarray]: + """Compute rounded equal according to ONNX spec. + + See https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Equal-11 + + Args: + x (numpy.ndarray): Input tensor + y (numpy.ndarray): Input tensor + lsbs_to_remove (int): The number of the least significant bits to remove Returns: Tuple[numpy.ndarray]: Output tensor """ # In the case of trees, x == y <=> x <= y or x < y - 1, because y is the max sum. - if lsbs_to_remove is not None and lsbs_to_remove > 0: + if lsbs_to_remove > 0: return rounded_comparison(y, x, lsbs_to_remove, operation=lambda x: x >= 0) - return (numpy.equal(x, y),) + return numpy_equal(x, y) def numpy_not( @@ -947,8 +964,6 @@ def numpy_not_float( def numpy_greater( x: numpy.ndarray, y: numpy.ndarray, - *, - lsbs_to_remove: Optional[int] = None, ) -> Tuple[numpy.ndarray]: """Compute greater in numpy according to ONNX spec. @@ -957,16 +972,11 @@ def numpy_greater( Args: x (numpy.ndarray): Input tensor y (numpy.ndarray): Input tensor - lsbs_to_remove (Optional[int]): The number of the least significant bits to remove Returns: Tuple[numpy.ndarray]: Output tensor """ - if lsbs_to_remove is not None and lsbs_to_remove > 0: - return rounded_comparison(y, x, lsbs_to_remove, operation=lambda x: x < 0) - - # Else, default numpy greater comparison return (numpy.greater(x, y),) @@ -992,8 +1002,6 @@ def numpy_greater_float( def numpy_greater_or_equal( x: numpy.ndarray, y: numpy.ndarray, - *, - lsbs_to_remove: Optional[int] = None, ) -> Tuple[numpy.ndarray]: """Compute greater or equal in numpy according to ONNX spec. @@ -1002,18 +1010,11 @@ def numpy_greater_or_equal( Args: x (numpy.ndarray): Input tensor y (numpy.ndarray): Input tensor - lsbs_to_remove (Optional[int]): The number of the least significant bits to remove Returns: Tuple[numpy.ndarray]: Output tensor """ - if lsbs_to_remove is not None and lsbs_to_remove > 0: - return rounded_comparison( - x, y, lsbs_to_remove, operation=lambda x: x >= 0 - ) # pragma: no cover - - # Else, default numpy greater_equal comparison return (numpy.greater_equal(x, y),) @@ -1039,8 +1040,6 @@ def numpy_greater_or_equal_float( def numpy_less( x: numpy.ndarray, y: numpy.ndarray, - *, - lsbs_to_remove: Optional[int] = None, ) -> Tuple[numpy.ndarray]: """Compute less in numpy according to ONNX spec. @@ -1049,16 +1048,11 @@ def numpy_less( Args: x (numpy.ndarray): Input tensor y (numpy.ndarray): Input tensor - lsbs_to_remove (Optional[int]): The number of the least significant bits to remove Returns: Tuple[numpy.ndarray]: Output tensor """ - if lsbs_to_remove is not None and lsbs_to_remove > 0: - return rounded_comparison(x, y, lsbs_to_remove, operation=lambda x: x < 0) - - # Else, default numpy less comparison return (numpy.less(x, y),) @@ -1081,11 +1075,34 @@ def numpy_less_float( return cast_to_float(numpy_less(x, y)) +def numpy_rounded_less( + x: numpy.ndarray, + y: numpy.ndarray, + lsbs_to_remove: int, +) -> Tuple[numpy.ndarray]: + """Compute rounded less according to ONNX spec. + + See https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Less-13 + + Args: + x (numpy.ndarray): Input tensor + y (numpy.ndarray): Input tensor + lsbs_to_remove (int): The number of the least significant bits to remove + + Returns: + Tuple[numpy.ndarray]: Output tensor + """ + + if lsbs_to_remove > 0: + return rounded_comparison(x, y, lsbs_to_remove, operation=lambda x: x < 0) + + # Else, default numpy less_or_equal comparison + return numpy_less(x, y) + + def numpy_less_or_equal( x: numpy.ndarray, y: numpy.ndarray, - *, - lsbs_to_remove: Optional[int] = None, ) -> Tuple[numpy.ndarray]: """Compute less or equal in numpy according to ONNX spec. @@ -1094,16 +1111,11 @@ def numpy_less_or_equal( Args: x (numpy.ndarray): Input tensor y (numpy.ndarray): Input tensor - lsbs_to_remove (Optional[int]]): The number of the least significant bits to remove Returns: Tuple[numpy.ndarray]: Output tensor """ - if lsbs_to_remove is not None and lsbs_to_remove > 0: - return rounded_comparison(y, x, lsbs_to_remove, operation=lambda x: x >= 0) - - # Else, default numpy less_or_equal comparison return (numpy.less_equal(x, y),) @@ -1126,6 +1138,31 @@ def numpy_less_or_equal_float( return cast_to_float(numpy_less_or_equal(x, y)) +def numpy_rounded_less_or_equal( + x: numpy.ndarray, + y: numpy.ndarray, + lsbs_to_remove: int, +) -> Tuple[numpy.ndarray]: + """Compute rounded less or equal according to ONNX spec. + + See https://github.com/onnx/onnx/blob/main/docs/Changelog.md#LessOrEqual-12 + + Args: + x (numpy.ndarray): Input tensor + y (numpy.ndarray): Input tensor + lsbs_to_remove (int): The number of the least significant bits to remove + + Returns: + Tuple[numpy.ndarray]: Output tensor + """ + + if lsbs_to_remove > 0: + return rounded_comparison(y, x, lsbs_to_remove, operation=lambda x: x >= 0) + + # Else, default numpy less_or_equal comparison + return numpy_less_or_equal(x, y) + + def numpy_identity( x: numpy.ndarray, ) -> Tuple[numpy.ndarray]: From e630ab5676a937275d1c2080e8e06835ea9c0ff6 Mon Sep 17 00:00:00 2001 From: kcelia Date: Wed, 29 Nov 2023 13:21:24 +0100 Subject: [PATCH 04/73] chore: add rounded operators to ONNX_OPS_TO_NUMPY_IMPL_BOOL and remove lsbs to remove --- src/concrete/ml/onnx/onnx_utils.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/concrete/ml/onnx/onnx_utils.py b/src/concrete/ml/onnx/onnx_utils.py index 6a9e12a8a..b6ce3822b 100644 --- a/src/concrete/ml/onnx/onnx_utils.py +++ b/src/concrete/ml/onnx/onnx_utils.py @@ -281,6 +281,9 @@ numpy_relu, numpy_reshape, numpy_round, + numpy_rounded_equal, + numpy_rounded_less, + numpy_rounded_less_or_equal, numpy_selu, numpy_shape, numpy_sigmoid, @@ -350,6 +353,7 @@ "Log": numpy_log, "Exp": numpy_exp, "Equal": numpy_equal, + "Rounded_Equal": numpy_rounded_equal, "Identity": numpy_identity, "Reshape": numpy_reshape, "Transpose": numpy_transpose, @@ -404,7 +408,9 @@ "Greater": numpy_greater, "GreaterOrEqual": numpy_greater_or_equal, "Less": numpy_less, + "Rounded_Less": numpy_rounded_less, "LessOrEqual": numpy_less_or_equal, + "Rounded_LessOrEqual": numpy_rounded_less_or_equal, } # All numpy operators used in QuantizedOps @@ -413,10 +419,6 @@ # All numpy operators used for tree-based models ONNX_OPS_TO_NUMPY_IMPL_BOOL = {**ONNX_OPS_TO_NUMPY_IMPL, **ONNX_COMPARISON_OPS_TO_NUMPY_IMPL_BOOL} -# All numpy operators used for tree-based models that support auto rounding -SUPPORTED_ROUNDED_OPERATIONS = ["Less", "LessOrEqual", "Greater", "GreaterOrEqual", "Equal"] - - IMPLEMENTED_ONNX_OPS = set(ONNX_OPS_TO_NUMPY_IMPL.keys()) @@ -446,14 +448,12 @@ def get_op_type(node): def execute_onnx_with_numpy( graph: onnx.GraphProto, - lsbs_to_remove: List, *inputs: numpy.ndarray, ) -> Tuple[numpy.ndarray, ...]: """Execute the provided ONNX graph on the given inputs. Args: graph (onnx.GraphProto): The ONNX graph to execute. - lsbs_to_remove (List): The number of least significant bit to be removed in each stage. *inputs: The inputs of the graph. Returns: @@ -471,11 +471,6 @@ def execute_onnx_with_numpy( curr_inputs = (node_results[input_name] for input_name in node.input) attributes = {attribute.name: get_attribute(attribute) for attribute in node.attribute} - if node.op_type in SUPPORTED_ROUNDED_OPERATIONS: - attributes["lsbs_to_remove"] = ( - lsbs_to_remove[0] if node.op_type != "Equal" else lsbs_to_remove[1] - ) - outputs = ONNX_OPS_TO_NUMPY_IMPL_BOOL[node.op_type](*curr_inputs, **attributes) node_results.update(zip(node.output, outputs)) From 3ea13c8afd705ed705135d02324a09998ca132d2 Mon Sep 17 00:00:00 2001 From: kcelia Date: Wed, 29 Nov 2023 13:24:13 +0100 Subject: [PATCH 05/73] chore: update convert.py --- docs/deep-learning/onnx_support.md | 185 ----------------------- src/concrete/ml/onnx/convert.py | 23 +-- src/concrete/ml/onnx/onnx_utils.py | 103 +------------ src/concrete/ml/sklearn/tree_to_numpy.py | 115 +++++++++++++- 4 files changed, 112 insertions(+), 314 deletions(-) delete mode 100644 docs/deep-learning/onnx_support.md diff --git a/docs/deep-learning/onnx_support.md b/docs/deep-learning/onnx_support.md deleted file mode 100644 index 69506db34..000000000 --- a/docs/deep-learning/onnx_support.md +++ /dev/null @@ -1,185 +0,0 @@ -# Using ONNX - -In addition to Concrete ML models and [custom models in torch](torch_support.md), it is also possible to directly compile [ONNX](https://onnx.ai/) models. This can be particularly appealing, notably to import models trained with Keras. - -ONNX models can be compiled by directly importing models that are already quantized with Quantization Aware Training (QAT) or by performing Post-Training Quantization (PTQ) with Concrete ML. - -## Simple example - -The following example shows how to compile an ONNX model using PTQ. The model was initially trained using Keras before being exported to ONNX. The training code is not shown here. - -{% hint style="warning" %} -This example uses Post-Training Quantization, i.e., the quantization is not performed during training. This model would not have good performance in FHE. Quantization Aware Training should be added by the model developer. Additionally, importing QAT ONNX models can be done [as shown below](onnx_support.md#quantization-aware-training). -{% endhint %} - -```python -import numpy -import onnx -import tensorflow -import tf2onnx - -from concrete.ml.torch.compile import compile_onnx_model -from concrete.fhe.compilation import Configuration - - -class FC(tensorflow.keras.Model): - """A fully-connected model.""" - - def __init__(self): - super().__init__() - hidden_layer_size = 10 - output_size = 5 - - self.dense1 = tensorflow.keras.layers.Dense( - hidden_layer_size, - activation=tensorflow.nn.relu, - ) - self.dense2 = tensorflow.keras.layers.Dense(output_size, activation=tensorflow.nn.relu6) - self.flatten = tensorflow.keras.layers.Flatten() - - def call(self, inputs): - """Forward function.""" - x = self.flatten(inputs) - x = self.dense1(x) - x = self.dense2(x) - return self.flatten(x) - - -n_bits = 6 -input_output_feature = 2 -input_shape = (input_output_feature,) -num_inputs = 1 -n_examples = 5000 - -# Define the Keras model -keras_model = FC() -keras_model.build((None,) + input_shape) -keras_model.compute_output_shape(input_shape=(None, input_output_feature)) - -# Create random input -input_set = numpy.random.uniform(-100, 100, size=(n_examples, *input_shape)) - -# Convert to ONNX -tf2onnx.convert.from_keras(keras_model, opset=14, output_path="tmp.model.onnx") - -onnx_model = onnx.load("tmp.model.onnx") -onnx.checker.check_model(onnx_model) - -# Compile -quantized_module = compile_onnx_model( - onnx_model, input_set, n_bits=2 -) - -# Create test data from the same distribution and quantize using -# learned quantization parameters during compilation -x_test = tuple(numpy.random.uniform(-100, 100, size=(1, *input_shape)) for _ in range(num_inputs)) - -y_clear = quantized_module.forward(*x_test, fhe="disable") -y_fhe = quantized_module.forward(*x_test, fhe="execute") - -print("Execution in clear: ", y_clear) -print("Execution in FHE: ", y_fhe) -print("Equality: ", numpy.sum(y_clear == y_fhe), "over", numpy.size(y_fhe), "values") -``` - -{% hint style="warning" %} -While Keras was used in this example, it is not officially supported. Additional work is needed to test all of Keras's types of layers and models. -{% endhint %} - -## Quantization Aware Training - -Models trained using [Quantization Aware Training](https://docs.zama.ai/concrete-ml/advanced-topics/quantization) contain quantizers in the ONNX graph. These quantizers ensure that the inputs to the Linear/Dense and Conv layers are quantized. Since these QAT models have quantizers that are configured during training to a specific number of bits, the ONNX graph will need to be imported using the same settings: - - - -```python -# Define the number of bits to use for quantizing weights and activations during training -n_bits_qat = 3 - -quantized_numpy_module = compile_onnx_model( - onnx_model, - input_set, - import_qat=True, - n_bits=n_bits_qat, -) -``` - -## Supported operators - -The following operators are supported for evaluation and conversion to an equivalent FHE circuit. Other operators were not implemented, either due to FHE constraints or because they are rarely used in PyTorch activations or scikit-learn models. - - - - - -- Abs -- Acos -- Acosh -- Add -- Asin -- Asinh -- Atan -- Atanh -- AveragePool -- BatchNormalization -- Cast -- Celu -- Clip -- Concat -- Constant -- ConstantOfShape -- Conv -- Cos -- Cosh -- Div -- Elu -- Equal -- Erf -- Exp -- Flatten -- Floor -- Gather -- Gemm -- Greater -- GreaterOrEqual -- HardSigmoid -- HardSwish -- Identity -- LeakyRelu -- Less -- LessOrEqual -- Log -- MatMul -- Max -- MaxPool -- Min -- Mul -- Neg -- Not -- Or -- PRelu -- Pad -- Pow -- ReduceSum -- Relu -- Reshape -- Round -- Selu -- Shape -- Sigmoid -- Sign -- Sin -- Sinh -- Slice -- Softplus -- Squeeze -- Sub -- Tan -- Tanh -- ThresholdedRelu -- Transpose -- Unsqueeze -- Where -- onnx.brevitas.Quant - - diff --git a/src/concrete/ml/onnx/convert.py b/src/concrete/ml/onnx/convert.py index 5943f653d..2d3d8ce03 100644 --- a/src/concrete/ml/onnx/convert.py +++ b/src/concrete/ml/onnx/convert.py @@ -2,7 +2,7 @@ import tempfile from pathlib import Path -from typing import Callable, Optional, Tuple, Union +from typing import Callable, Tuple, Union import numpy import onnx @@ -12,7 +12,6 @@ from .onnx_utils import ( IMPLEMENTED_ONNX_OPS, - compute_lsb_to_remove_for_trees, execute_onnx_with_numpy, get_op_type, ) @@ -163,21 +162,15 @@ def get_equivalent_numpy_forward_from_torch( def get_equivalent_numpy_forward_from_onnx( onnx_model: onnx.ModelProto, - q_x: Optional[numpy.ndarray] = None, check_model: bool = True, - use_rounding: bool = False, ) -> Tuple[Callable[..., Tuple[numpy.ndarray, ...]], onnx.ModelProto]: """Get the numpy equivalent forward of the provided ONNX model. Args: onnx_model (onnx.ModelProto): the ONNX model for which to get the equivalent numpy forward. - q_x (numpy.ndarray): Quantized input used to compute the LSBs to remove if 'use_rounding'. - Defaults to None if rounding is not used. check_model (bool): set to True to run the onnx checker on the model. Defaults to True. - use_rounding (bool): Use rounding feature or not. - Defaults to False. Raises: ValueError: Raised if there is an unsupported ONNX operator required to convert the torch @@ -188,11 +181,10 @@ def get_equivalent_numpy_forward_from_onnx( the equivalent numpy function. """ - lsbs_to_remove: list = [0, 0] - if check_model: checker.check_model(onnx_model) + # plus pour NN # Optimize ONNX graph # List of all currently supported onnx optimizer passes # From https://github.com/onnx/optimizer/blob/master/onnxoptimizer/pass_registry.h @@ -221,16 +213,7 @@ def get_equivalent_numpy_forward_from_onnx( f"Available ONNX operators: {', '.join(sorted(IMPLEMENTED_ONNX_OPS))}" ) - # Remove this workaround (which computes the LSB to be remove manually) once - # the truncate feature is released - # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4143 - if use_rounding: - assert ( - q_x is not None - ), "A representative quantized input-set is needed when using the rounding feature" - lsbs_to_remove = compute_lsb_to_remove_for_trees(equivalent_onnx_model, q_x) - # Return lambda of numpy equivalent of onnx execution return ( - lambda *args: execute_onnx_with_numpy(equivalent_onnx_model.graph, lsbs_to_remove, *args) + lambda *args: execute_onnx_with_numpy(equivalent_onnx_model.graph, *args) ), equivalent_onnx_model diff --git a/src/concrete/ml/onnx/onnx_utils.py b/src/concrete/ml/onnx/onnx_utils.py index b6ce3822b..7ff812019 100644 --- a/src/concrete/ml/onnx/onnx_utils.py +++ b/src/concrete/ml/onnx/onnx_utils.py @@ -214,8 +214,7 @@ # Original file: # https://github.com/google/jax/blob/f6d329b2d9b5f83c6a59e5739aa1ca8d4d1ffa1c/examples/onnx2xla.py -import math -from typing import Any, Callable, Dict, List, Tuple +from typing import Any, Callable, Dict, Tuple import numpy import onnx @@ -502,103 +501,3 @@ def remove_initializer_from_input(model: onnx.ModelProto): # pragma: no cover inputs.remove(name_to_input[initializer.name]) return model - - -# Remove this function once the truncate feature is released -# FIXME: https://github.com/zama-ai/concrete-ml/issues/397 -def compute_lsb_to_remove_for_trees(onnx_model: onnx.ModelProto, q_x: numpy.ndarray) -> List[int]: - """Compute the LSB to remove for the comparison operators in the trees. - - Referring to this paper: https://scnakandala.github.io/papers/TR_2020_Hummingbird.pdf, there are - 2 levels of comparison for trees, one at the level of X.A < B and a second at - the level of I.C == D. - - Args: - onnx_model (onnx.ModelProto): The model to clean - q_x (numpy.ndarray): The quantized inputs - - Returns: - List: the number of LSB to remove for level 1 and level 2 - """ - - def get_bitwidth(array: numpy.ndarray) -> int: - """Compute the bitwidth required to represent the largest value in `array`. - - Args: - array (umpy.ndarray): The array for which the bitwidth needs to be checked. - - Returns: - int: The required bits to represent the array. - """ - - max_val = numpy.max(numpy.abs(array)) - # + 1 is added to include the sign bit - bitwidth = math.ceil(math.log2(max_val + 1)) + 1 - return bitwidth - - def update_lsbs_if_overflow_detected(array: numpy.ndarray, initial_bitwidth: int) -> int: - """Update the number of LSBs to remove based on overflow detection. - - Args: - array (umpy.ndarray): The array for which the bitwidth needs to be checked. - initial_bitwidth (int): The target bitwidth that should not be exceeded. - - Returns: - int: The updated LSB to remove. - """ - - lsbs_to_remove = initial_bitwidth - - if lsbs_to_remove > 0: - half = 1 << (lsbs_to_remove - 1) - if get_bitwidth(array - half) <= initial_bitwidth: - lsbs_to_remove -= 1 - - return lsbs_to_remove - - quant_params = { - onnx_init.name: numpy_helper.to_array(onnx_init) - for onnx_init in onnx_model.graph.initializer - if "weight" in onnx_init.name or "bias" in onnx_init.name - } - - key_mat_1 = [key for key in quant_params.keys() if "_1" in key and "weight" in key][0] - key_bias_1 = [key for key in quant_params.keys() if "_1" in key and "bias" in key][0] - - key_mat_2 = [key for key in quant_params.keys() if "_2" in key and "weight" in key][0] - key_bias_2 = [key for key in quant_params.keys() if "_2" in key and "bias" in key][0] - - # shape: (nodes, features) or (trees * nodes, features) - mat_1 = quant_params[key_mat_1] - # shape: (nodes, 1) or (trees * nodes, 1) - bias_1 = quant_params[key_bias_1] - - # shape: (trees, leaves, nodes) - mat_2 = quant_params[key_mat_2] - # shape: (leaves, 1) or (trees * leaves, 1) - bias_2 = quant_params[key_bias_2] - - n_features = mat_1.shape[1] - n_nodes = mat_2.shape[2] - n_leaves = mat_2.shape[1] - - mat_1 = mat_1.reshape(-1, n_nodes, n_features) - bias_1 = bias_1.reshape(-1, 1, n_nodes) - bias_2 = bias_2.reshape(-1, 1, n_leaves) - - # If <= -> stage = biais_1 - (q_x @ mat_1.transpose(0, 2, 1)) - # If < -> stage = (q_x @ mat_1.transpose(0, 2, 1)) - biais_1 - stage_1 = bias_1 - (q_x @ mat_1.transpose(0, 2, 1)) - - # The matrix I, as referenced in this paper: - # https://scnakandala.github.io/papers/TR_2020_Hummingbird.pdf, results from the condition: - # X.A < B and consists exclusively of binary elements, 1 and 0. - # Given this assumption, we randomly generate it. - matrix_q = numpy.random.randint(0, 2, size=(stage_1.shape)) - - stage_2 = ((matrix_q @ mat_2.transpose(0, 2, 1)) + bias_2).sum(axis=0) - - lsbs_to_remove_1 = update_lsbs_if_overflow_detected(stage_1, get_bitwidth(stage_1)) - lsbs_to_remove_2 = update_lsbs_if_overflow_detected(stage_2, get_bitwidth(stage_2)) - - return [lsbs_to_remove_1, lsbs_to_remove_2] diff --git a/src/concrete/ml/sklearn/tree_to_numpy.py b/src/concrete/ml/sklearn/tree_to_numpy.py index 3cec1ae60..130cb2bbe 100644 --- a/src/concrete/ml/sklearn/tree_to_numpy.py +++ b/src/concrete/ml/sklearn/tree_to_numpy.py @@ -1,6 +1,7 @@ """Implements the conversion of a tree model to a numpy function.""" +import math import warnings -from typing import Callable, List, Optional, Tuple +from typing import Callable, List, Tuple import numpy import onnx @@ -324,9 +325,11 @@ def tree_to_numpy( # Execute with 1 example for efficiency in large data scenarios to prevent slowdown onnx_model = get_onnx_model(model, q_x[:1], framework) - # if use_rounding: - # compute LSB to remove in stage 1 and 2 - # attach LSBs to the ONNX + if use_rounding: + # compute LSB to remove in stage 1 and 2 + # : List[lsbs_to_remove_stage1, lsbs_to_remove_stage2] + lsbs_to_remove = compute_lsb_to_remove_for_trees(onnx_model, q_x) + # TODO: Jordan's function - attach LSBs to the ONNX # Get the expected number of ONNX outputs in the sklearn model. expected_number_of_outputs = 1 if is_regressor_or_partial_regressor(model) else 2 @@ -340,8 +343,106 @@ def tree_to_numpy( # but also rounding the threshold such that they are now integers q_y = tree_values_preprocessing(onnx_model, framework, output_n_bits) - _tree_inference, onnx_model = get_equivalent_numpy_forward_from_onnx( - onnx_model, q_x, use_rounding=use_rounding - ) + _tree_inference, onnx_model = get_equivalent_numpy_forward_from_onnx(onnx_model) return (_tree_inference, [q_y.quantizer], onnx_model) + + +# Remove this function once the truncate feature is released +# FIXME: https://github.com/zama-ai/concrete-ml/issues/397 +def compute_lsb_to_remove_for_trees(onnx_model: onnx.ModelProto, q_x: numpy.ndarray) -> List[int]: + """Compute the LSB to remove for the comparison operators in the trees. + + Referring to this paper: https://scnakandala.github.io/papers/TR_2020_Hummingbird.pdf, there are + 2 levels of comparison for trees, one at the level of X.A < B and a second at + the level of I.C == D. + + Args: + onnx_model (onnx.ModelProto): The model to clean + q_x (numpy.ndarray): The quantized inputs + + Returns: + List: the number of LSB to remove for level 1 and level 2 + """ + + def get_bitwidth(array: numpy.ndarray) -> int: + """Compute the bitwidth required to represent the largest value in `array`. + + Args: + array (umpy.ndarray): The array for which the bitwidth needs to be checked. + + Returns: + int: The required bits to represent the array. + """ + + max_val = numpy.max(numpy.abs(array)) + # + 1 is added to include the sign bit + bitwidth = math.ceil(math.log2(max_val + 1)) + 1 + return bitwidth + + def update_lsbs_if_overflow_detected(array: numpy.ndarray, initial_bitwidth: int) -> int: + """Update the number of LSBs to remove based on overflow detection. + + Args: + array (umpy.ndarray): The array for which the bitwidth needs to be checked. + initial_bitwidth (int): The target bitwidth that should not be exceeded. + + Returns: + int: The updated LSB to remove. + """ + + lsbs_to_remove = initial_bitwidth + + if lsbs_to_remove > 0: + half = 1 << (lsbs_to_remove - 1) + if get_bitwidth(array - half) <= initial_bitwidth: + lsbs_to_remove -= 1 + + return lsbs_to_remove + + quant_params = { + onnx_init.name: numpy_helper.to_array(onnx_init) + for onnx_init in onnx_model.graph.initializer + if "weight" in onnx_init.name or "bias" in onnx_init.name + } + + key_mat_1 = [key for key in quant_params.keys() if "_1" in key and "weight" in key][0] + key_bias_1 = [key for key in quant_params.keys() if "_1" in key and "bias" in key][0] + + key_mat_2 = [key for key in quant_params.keys() if "_2" in key and "weight" in key][0] + key_bias_2 = [key for key in quant_params.keys() if "_2" in key and "bias" in key][0] + + # shape: (nodes, features) or (trees * nodes, features) + mat_1 = quant_params[key_mat_1] + # shape: (nodes, 1) or (trees * nodes, 1) + bias_1 = quant_params[key_bias_1] + + # shape: (trees, leaves, nodes) + mat_2 = quant_params[key_mat_2] + # shape: (leaves, 1) or (trees * leaves, 1) + bias_2 = quant_params[key_bias_2] + + n_features = mat_1.shape[1] + n_nodes = mat_2.shape[2] + n_leaves = mat_2.shape[1] + + mat_1 = mat_1.reshape(-1, n_nodes, n_features) + bias_1 = bias_1.reshape(-1, 1, n_nodes) + bias_2 = bias_2.reshape(-1, 1, n_leaves) + + # If <= -> stage = biais_1 - (q_x @ mat_1.transpose(0, 2, 1)) + # If < -> stage = (q_x @ mat_1.transpose(0, 2, 1)) - biais_1 + stage_1 = bias_1 - (q_x @ mat_1.transpose(0, 2, 1)) + + # The matrix I, as referenced in this paper: + # https://scnakandala.github.io/papers/TR_2020_Hummingbird.pdf, results from the condition: + # X.A < B and consists exclusively of binary elements, 1 and 0. + # Given this assumption, we randomly generate it. + matrix_q = numpy.random.randint(0, 2, size=(stage_1.shape)) + + stage_2 = ((matrix_q @ mat_2.transpose(0, 2, 1)) + bias_2).sum(axis=0) + + lsbs_to_remove_1 = update_lsbs_if_overflow_detected(stage_1, get_bitwidth(stage_1)) + lsbs_to_remove_2 = update_lsbs_if_overflow_detected(stage_2, get_bitwidth(stage_2)) + + return [lsbs_to_remove_1, lsbs_to_remove_2] From de51201778e096b792f984fc8dd94b3f4d4d5c7b Mon Sep 17 00:00:00 2001 From: jfrery Date: Wed, 29 Nov 2023 13:57:26 +0100 Subject: [PATCH 06/73] chore: add conversion to our rounded ops in onnx graph for trees --- src/concrete/ml/onnx/convert.py | 3 +- src/concrete/ml/sklearn/tree_to_numpy.py | 60 ++++++++++++++++++++++-- 2 files changed, 58 insertions(+), 5 deletions(-) diff --git a/src/concrete/ml/onnx/convert.py b/src/concrete/ml/onnx/convert.py index 2d3d8ce03..376a2f7f8 100644 --- a/src/concrete/ml/onnx/convert.py +++ b/src/concrete/ml/onnx/convert.py @@ -196,12 +196,11 @@ def get_equivalent_numpy_forward_from_onnx( "eliminate_unused_initializer", ] equivalent_onnx_model = onnxoptimizer.optimize(onnx_model, onnx_passes) - checker.check_model(equivalent_onnx_model) + # Custom optimization # ONNX optimizer does not optimize Mat-Mult + Bias pattern into GEMM if the input isn't a matrix # We manually do the optimization for this case equivalent_onnx_model = fuse_matmul_bias_to_gemm(equivalent_onnx_model) - checker.check_model(equivalent_onnx_model) # Check supported operators required_onnx_operators = set(get_op_type(node) for node in equivalent_onnx_model.graph.node) diff --git a/src/concrete/ml/sklearn/tree_to_numpy.py b/src/concrete/ml/sklearn/tree_to_numpy.py index 130cb2bbe..ab9681b60 100644 --- a/src/concrete/ml/sklearn/tree_to_numpy.py +++ b/src/concrete/ml/sklearn/tree_to_numpy.py @@ -22,6 +22,9 @@ # Silence Hummingbird warnings warnings.filterwarnings("ignore") +from typing import List + +import onnx from hummingbird.ml import convert as hb_convert # noqa: E402 # pylint: enable=wrong-import-position,wrong-import-order @@ -194,7 +197,7 @@ def preprocess_tree_predictions( def tree_onnx_graph_preprocessing( - onnx_model: onnx.ModelProto, framework: str, expected_number_of_outputs: int + onnx_model: onnx.ModelProto, framework: str, expected_number_of_outputs: int, use_rounding: bool ): """Apply pre-processing onto the ONNX graph. @@ -203,6 +206,7 @@ def tree_onnx_graph_preprocessing( framework (str): The framework from which the ONNX model is generated. (options: 'xgboost', 'sklearn') expected_number_of_outputs (int): The expected number of outputs in the ONNX model. + use_rounding (bool): Whether to use rounding. """ # Make sure the ONNX version returned by Hummingbird is OPSET_VERSION_FOR_ONNX_EXPORT onnx_version = get_onnx_opset_version(onnx_model) @@ -247,6 +251,54 @@ def tree_onnx_graph_preprocessing( # Cast nodes are not necessary so remove them. remove_node_types(onnx_model, op_types_to_remove=["Cast"]) + # Replace Greater and Less by a rounded op if use_rounding. + if use_rounding: + replace_operator_with_rounded_version(onnx_model, lsbs_to_remove=lsbs_to_remove) + + +def replace_operator_with_rounded_version(onnx_model, lsbs_to_remove): + """ + Replace the first occurrence of Greater/Less/GreaterOrEqual/LessOrEqual + with RoundedGreater/RoundedLess/RoundedGreaterOrEqual/RoundedLessOrEqual. + + Args: + onnx_model (onnx.ModelProto): The ONNX model. + lsbs_to_remove (int): The number of LSBs to remove. + + Returns: + onnx.ModelProto: The modified ONNX model. + """ + # Mapping of original operators to their rounded counterparts + operator_mapping = { + 'Greater': 'RoundedGreater', + 'Less': 'RoundedLess', + 'GreaterOrEqual': 'RoundedGreaterOrEqual', + 'LessOrEqual': 'RoundedLessOrEqual' + } + + new_nodes = [] + replaced = False + + for node in onnx_model.graph.node: + if not replaced and node.op_type in operator_mapping: + # Create a new node with the corresponding rounded operator + rounded_node = onnx.helper.make_node( + operator_mapping[node.op_type], + inputs=node.input, + outputs=node.output, + lsbs_to_remove=lsbs_to_remove + ) + new_nodes.append(rounded_node) + replaced = True + else: + new_nodes.append(node) + + # Replace the graph's node list with the new list + onnx_model.graph.ClearField('node') + onnx_model.graph.node.extend(new_nodes) + + return onnx_model + def tree_values_preprocessing( onnx_model: onnx.ModelProto, @@ -336,14 +388,16 @@ def tree_to_numpy( # ONNX graph pre-processing to make the model FHE friendly # i.e., delete irrelevant nodes and cut the graph before the final ensemble sum) - tree_onnx_graph_preprocessing(onnx_model, framework, expected_number_of_outputs) + tree_onnx_graph_preprocessing(onnx_model, framework, expected_number_of_outputs, use_rounding) # Tree values pre-processing # i.e., mainly predictions quantization # but also rounding the threshold such that they are now integers q_y = tree_values_preprocessing(onnx_model, framework, output_n_bits) - _tree_inference, onnx_model = get_equivalent_numpy_forward_from_onnx(onnx_model) + # Get the numpy inference for the quantized tree (_tree_inference). + # Use check_model = False here since we have custom onnx operator that won't be recognised. + _tree_inference, onnx_model = get_equivalent_numpy_forward_from_onnx(onnx_model, q_x, check_model=False) return (_tree_inference, [q_y.quantizer], onnx_model) From d79127fa355b4265e28a2c4f07ab05b368510e6b Mon Sep 17 00:00:00 2001 From: jfrery Date: Wed, 29 Nov 2023 14:41:14 +0100 Subject: [PATCH 07/73] chore: update conversion to rounded ops to use list of lsbs --- src/concrete/ml/onnx/convert.py | 6 +- src/concrete/ml/sklearn/tree_to_numpy.py | 76 +++++++++++++++--------- 2 files changed, 48 insertions(+), 34 deletions(-) diff --git a/src/concrete/ml/onnx/convert.py b/src/concrete/ml/onnx/convert.py index 376a2f7f8..0380195fe 100644 --- a/src/concrete/ml/onnx/convert.py +++ b/src/concrete/ml/onnx/convert.py @@ -10,11 +10,7 @@ import torch from onnx import checker, helper -from .onnx_utils import ( - IMPLEMENTED_ONNX_OPS, - execute_onnx_with_numpy, - get_op_type, -) +from .onnx_utils import IMPLEMENTED_ONNX_OPS, execute_onnx_with_numpy, get_op_type OPSET_VERSION_FOR_ONNX_EXPORT = 14 diff --git a/src/concrete/ml/sklearn/tree_to_numpy.py b/src/concrete/ml/sklearn/tree_to_numpy.py index ab9681b60..9bd23fc30 100644 --- a/src/concrete/ml/sklearn/tree_to_numpy.py +++ b/src/concrete/ml/sklearn/tree_to_numpy.py @@ -197,7 +197,7 @@ def preprocess_tree_predictions( def tree_onnx_graph_preprocessing( - onnx_model: onnx.ModelProto, framework: str, expected_number_of_outputs: int, use_rounding: bool + onnx_model: onnx.ModelProto, framework: str, expected_number_of_outputs: int ): """Apply pre-processing onto the ONNX graph. @@ -206,7 +206,6 @@ def tree_onnx_graph_preprocessing( framework (str): The framework from which the ONNX model is generated. (options: 'xgboost', 'sklearn') expected_number_of_outputs (int): The expected number of outputs in the ONNX model. - use_rounding (bool): Whether to use rounding. """ # Make sure the ONNX version returned by Hummingbird is OPSET_VERSION_FOR_ONNX_EXPORT onnx_version = get_onnx_opset_version(onnx_model) @@ -251,50 +250,65 @@ def tree_onnx_graph_preprocessing( # Cast nodes are not necessary so remove them. remove_node_types(onnx_model, op_types_to_remove=["Cast"]) - # Replace Greater and Less by a rounded op if use_rounding. - if use_rounding: - replace_operator_with_rounded_version(onnx_model, lsbs_to_remove=lsbs_to_remove) - def replace_operator_with_rounded_version(onnx_model, lsbs_to_remove): - """ - Replace the first occurrence of Greater/Less/GreaterOrEqual/LessOrEqual - with RoundedGreater/RoundedLess/RoundedGreaterOrEqual/RoundedLessOrEqual. + """Replace comparisons with rounded comparisons. Args: onnx_model (onnx.ModelProto): The ONNX model. - lsbs_to_remove (int): The number of LSBs to remove. + lsbs_to_remove (List[int]): A list of two integers specifying the number of LSBs to remove. Returns: onnx.ModelProto: The modified ONNX model. """ + + assert_true(isinstance(lsbs_to_remove, list), "lsbs_to_remove must be a list.") + assert_true(len(lsbs_to_remove) == 2, "lsbs_to_remove must have exactly two values.") + # Mapping of original operators to their rounded counterparts operator_mapping = { - 'Greater': 'RoundedGreater', - 'Less': 'RoundedLess', - 'GreaterOrEqual': 'RoundedGreaterOrEqual', - 'LessOrEqual': 'RoundedLessOrEqual' + "Greater": "RoundedGreater", + "Less": "RoundedLess", + "GreaterOrEqual": "RoundedGreaterOrEqual", + "LessOrEqual": "RoundedLessOrEqual", + "Equal": "RoundedEqual", } + # Track if the required operators have been replaced + comparison_replaced = False + equal_replaced = False + new_nodes = [] - replaced = False for node in onnx_model.graph.node: - if not replaced and node.op_type in operator_mapping: - # Create a new node with the corresponding rounded operator - rounded_node = onnx.helper.make_node( - operator_mapping[node.op_type], - inputs=node.input, - outputs=node.output, - lsbs_to_remove=lsbs_to_remove - ) - new_nodes.append(rounded_node) - replaced = True + if not comparison_replaced and node.op_type in operator_mapping and node.op_type != "Equal": + # Use the first value in lsbs_to_remove for the comparison operator + lsbs = lsbs_to_remove[0] + comparison_replaced = True + elif not equal_replaced and node.op_type == "Equal": + # Use the second value in lsbs_to_remove for the Equal operator + lsbs = lsbs_to_remove[1] + equal_replaced = True else: new_nodes.append(node) + continue + + # Create a new node with the corresponding rounded operator + rounded_node = onnx.helper.make_node( + operator_mapping[node.op_type], + inputs=node.input, + outputs=node.output, + lsbs_to_remove=lsbs, + ) + new_nodes.append(rounded_node) + + # Ensure that both a comparison and an equal operator were replaced + assert_true( + comparison_replaced and equal_replaced, "Required operators not found in the model." + ) # Replace the graph's node list with the new list - onnx_model.graph.ClearField('node') + onnx_model.graph.ClearField("node") onnx_model.graph.node.extend(new_nodes) return onnx_model @@ -381,14 +395,16 @@ def tree_to_numpy( # compute LSB to remove in stage 1 and 2 # : List[lsbs_to_remove_stage1, lsbs_to_remove_stage2] lsbs_to_remove = compute_lsb_to_remove_for_trees(onnx_model, q_x) - # TODO: Jordan's function - attach LSBs to the ONNX + replace_operator_with_rounded_version(onnx_model, lsbs_to_remove) + + replace_operator_with_rounded_version(onnx_model, [3, 3]) # Get the expected number of ONNX outputs in the sklearn model. expected_number_of_outputs = 1 if is_regressor_or_partial_regressor(model) else 2 # ONNX graph pre-processing to make the model FHE friendly # i.e., delete irrelevant nodes and cut the graph before the final ensemble sum) - tree_onnx_graph_preprocessing(onnx_model, framework, expected_number_of_outputs, use_rounding) + tree_onnx_graph_preprocessing(onnx_model, framework, expected_number_of_outputs) # Tree values pre-processing # i.e., mainly predictions quantization @@ -397,7 +413,9 @@ def tree_to_numpy( # Get the numpy inference for the quantized tree (_tree_inference). # Use check_model = False here since we have custom onnx operator that won't be recognised. - _tree_inference, onnx_model = get_equivalent_numpy_forward_from_onnx(onnx_model, q_x, check_model=False) + _tree_inference, onnx_model = get_equivalent_numpy_forward_from_onnx( + onnx_model, q_x, check_model=False + ) return (_tree_inference, [q_y.quantizer], onnx_model) From dddead65705ed71833ff2aab9fd5927b0574d6b7 Mon Sep 17 00:00:00 2001 From: kcelia Date: Wed, 29 Nov 2023 14:25:20 +0100 Subject: [PATCH 08/73] chore: fix lsb to remove according to op comparison --- src/concrete/ml/onnx/onnx_utils.py | 6 +++--- src/concrete/ml/sklearn/tree_to_numpy.py | 27 ++++++++++++++++++------ 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/src/concrete/ml/onnx/onnx_utils.py b/src/concrete/ml/onnx/onnx_utils.py index 7ff812019..c05daa98b 100644 --- a/src/concrete/ml/onnx/onnx_utils.py +++ b/src/concrete/ml/onnx/onnx_utils.py @@ -352,7 +352,7 @@ "Log": numpy_log, "Exp": numpy_exp, "Equal": numpy_equal, - "Rounded_Equal": numpy_rounded_equal, + "RoundedOrEqual": numpy_rounded_equal, "Identity": numpy_identity, "Reshape": numpy_reshape, "Transpose": numpy_transpose, @@ -407,9 +407,9 @@ "Greater": numpy_greater, "GreaterOrEqual": numpy_greater_or_equal, "Less": numpy_less, - "Rounded_Less": numpy_rounded_less, + "RoundedOrLess": numpy_rounded_less, "LessOrEqual": numpy_less_or_equal, - "Rounded_LessOrEqual": numpy_rounded_less_or_equal, + "RoundedOrLessOrEqual": numpy_rounded_less_or_equal, } # All numpy operators used in QuantizedOps diff --git a/src/concrete/ml/sklearn/tree_to_numpy.py b/src/concrete/ml/sklearn/tree_to_numpy.py index 9bd23fc30..4d549ac39 100644 --- a/src/concrete/ml/sklearn/tree_to_numpy.py +++ b/src/concrete/ml/sklearn/tree_to_numpy.py @@ -15,6 +15,7 @@ ) from ..onnx.convert import OPSET_VERSION_FOR_ONNX_EXPORT, get_equivalent_numpy_forward_from_onnx from ..onnx.onnx_model_manipulations import clean_graph_at_node_op_type, remove_node_types +from ..onnx.onnx_utils import get_op_type from ..quantization import QuantizedArray from ..quantization.quantizers import UniformQuantizer @@ -393,7 +394,8 @@ def tree_to_numpy( if use_rounding: # compute LSB to remove in stage 1 and 2 - # : List[lsbs_to_remove_stage1, lsbs_to_remove_stage2] + # <=, <, == + # : List[(lsbs_to_remove_stage1, lsbs_to_remove_stage1), lsbs_to_remove_stage2] lsbs_to_remove = compute_lsb_to_remove_for_trees(onnx_model, q_x) replace_operator_with_rounded_version(onnx_model, lsbs_to_remove) @@ -502,9 +504,20 @@ def update_lsbs_if_overflow_detected(array: numpy.ndarray, initial_bitwidth: int bias_1 = bias_1.reshape(-1, 1, n_nodes) bias_2 = bias_2.reshape(-1, 1, n_leaves) - # If <= -> stage = biais_1 - (q_x @ mat_1.transpose(0, 2, 1)) - # If < -> stage = (q_x @ mat_1.transpose(0, 2, 1)) - biais_1 - stage_1 = bias_1 - (q_x @ mat_1.transpose(0, 2, 1)) + required_onnx_operators = set(get_op_type(node) for node in onnx_model.graph.node) + + # If operator == `<`, np.less(x, y) is equivalent to: + # round_bit_pattern((x - y) - half, lsbs_to_remove=r) < 0. + # Therfore, stage_1 = (q_x @ mat_1.transpose(0, 2, 1)) - bias_1 + if "Less" in required_onnx_operators: + stage_1 = (q_x @ mat_1.transpose(0, 2, 1)) - bias_1 + else: + # If operator == `<=`, np.less_equal(x, y) is equivalent to: + # round_bit_pattern((y - x) - half, lsbs_to_remove=r) >= 0. + # Therfore, stage_1 = bias_1 - (q_x @ mat_1.transpose(0, 2, 1)) + stage_1 = bias_1 - (q_x @ mat_1.transpose(0, 2, 1)) + + lsbs_to_remove_1 = update_lsbs_if_overflow_detected(stage_1, get_bitwidth(stage_1)) # The matrix I, as referenced in this paper: # https://scnakandala.github.io/papers/TR_2020_Hummingbird.pdf, results from the condition: @@ -512,9 +525,11 @@ def update_lsbs_if_overflow_detected(array: numpy.ndarray, initial_bitwidth: int # Given this assumption, we randomly generate it. matrix_q = numpy.random.randint(0, 2, size=(stage_1.shape)) - stage_2 = ((matrix_q @ mat_2.transpose(0, 2, 1)) + bias_2).sum(axis=0) + # If operator == `==`, np.equal(x, y) is equivalent to: + # round_bit_pattern((x - y) - half, lsbs_to_remove=r) >= 0. + # Therfore, stage_2 = bias_1 - (q_x @ mat_2.transpose(0, 2, 1)) + stage_2 = ((bias_2 - matrix_q @ mat_2.transpose(0, 2, 1))).sum(axis=0) - lsbs_to_remove_1 = update_lsbs_if_overflow_detected(stage_1, get_bitwidth(stage_1)) lsbs_to_remove_2 = update_lsbs_if_overflow_detected(stage_2, get_bitwidth(stage_2)) return [lsbs_to_remove_1, lsbs_to_remove_2] From 6a33e62241cafea95a21f7395545fca088a0b6d8 Mon Sep 17 00:00:00 2001 From: kcelia Date: Wed, 29 Nov 2023 15:00:32 +0100 Subject: [PATCH 09/73] chore: reconcial with Jordan's version disable check_model for the moment remove redundant check_model update formulas --- docs/deep-learning/onnx_support.md | 185 +++++++++++++++++++++++ src/concrete/ml/onnx/convert.py | 2 - src/concrete/ml/onnx/onnx_impl_utils.py | 2 + src/concrete/ml/onnx/onnx_utils.py | 23 ++- src/concrete/ml/onnx/ops_impl.py | 7 +- src/concrete/ml/sklearn/base.py | 8 +- src/concrete/ml/sklearn/tree_to_numpy.py | 42 +++-- 7 files changed, 228 insertions(+), 41 deletions(-) create mode 100644 docs/deep-learning/onnx_support.md diff --git a/docs/deep-learning/onnx_support.md b/docs/deep-learning/onnx_support.md new file mode 100644 index 000000000..69506db34 --- /dev/null +++ b/docs/deep-learning/onnx_support.md @@ -0,0 +1,185 @@ +# Using ONNX + +In addition to Concrete ML models and [custom models in torch](torch_support.md), it is also possible to directly compile [ONNX](https://onnx.ai/) models. This can be particularly appealing, notably to import models trained with Keras. + +ONNX models can be compiled by directly importing models that are already quantized with Quantization Aware Training (QAT) or by performing Post-Training Quantization (PTQ) with Concrete ML. + +## Simple example + +The following example shows how to compile an ONNX model using PTQ. The model was initially trained using Keras before being exported to ONNX. The training code is not shown here. + +{% hint style="warning" %} +This example uses Post-Training Quantization, i.e., the quantization is not performed during training. This model would not have good performance in FHE. Quantization Aware Training should be added by the model developer. Additionally, importing QAT ONNX models can be done [as shown below](onnx_support.md#quantization-aware-training). +{% endhint %} + +```python +import numpy +import onnx +import tensorflow +import tf2onnx + +from concrete.ml.torch.compile import compile_onnx_model +from concrete.fhe.compilation import Configuration + + +class FC(tensorflow.keras.Model): + """A fully-connected model.""" + + def __init__(self): + super().__init__() + hidden_layer_size = 10 + output_size = 5 + + self.dense1 = tensorflow.keras.layers.Dense( + hidden_layer_size, + activation=tensorflow.nn.relu, + ) + self.dense2 = tensorflow.keras.layers.Dense(output_size, activation=tensorflow.nn.relu6) + self.flatten = tensorflow.keras.layers.Flatten() + + def call(self, inputs): + """Forward function.""" + x = self.flatten(inputs) + x = self.dense1(x) + x = self.dense2(x) + return self.flatten(x) + + +n_bits = 6 +input_output_feature = 2 +input_shape = (input_output_feature,) +num_inputs = 1 +n_examples = 5000 + +# Define the Keras model +keras_model = FC() +keras_model.build((None,) + input_shape) +keras_model.compute_output_shape(input_shape=(None, input_output_feature)) + +# Create random input +input_set = numpy.random.uniform(-100, 100, size=(n_examples, *input_shape)) + +# Convert to ONNX +tf2onnx.convert.from_keras(keras_model, opset=14, output_path="tmp.model.onnx") + +onnx_model = onnx.load("tmp.model.onnx") +onnx.checker.check_model(onnx_model) + +# Compile +quantized_module = compile_onnx_model( + onnx_model, input_set, n_bits=2 +) + +# Create test data from the same distribution and quantize using +# learned quantization parameters during compilation +x_test = tuple(numpy.random.uniform(-100, 100, size=(1, *input_shape)) for _ in range(num_inputs)) + +y_clear = quantized_module.forward(*x_test, fhe="disable") +y_fhe = quantized_module.forward(*x_test, fhe="execute") + +print("Execution in clear: ", y_clear) +print("Execution in FHE: ", y_fhe) +print("Equality: ", numpy.sum(y_clear == y_fhe), "over", numpy.size(y_fhe), "values") +``` + +{% hint style="warning" %} +While Keras was used in this example, it is not officially supported. Additional work is needed to test all of Keras's types of layers and models. +{% endhint %} + +## Quantization Aware Training + +Models trained using [Quantization Aware Training](https://docs.zama.ai/concrete-ml/advanced-topics/quantization) contain quantizers in the ONNX graph. These quantizers ensure that the inputs to the Linear/Dense and Conv layers are quantized. Since these QAT models have quantizers that are configured during training to a specific number of bits, the ONNX graph will need to be imported using the same settings: + + + +```python +# Define the number of bits to use for quantizing weights and activations during training +n_bits_qat = 3 + +quantized_numpy_module = compile_onnx_model( + onnx_model, + input_set, + import_qat=True, + n_bits=n_bits_qat, +) +``` + +## Supported operators + +The following operators are supported for evaluation and conversion to an equivalent FHE circuit. Other operators were not implemented, either due to FHE constraints or because they are rarely used in PyTorch activations or scikit-learn models. + + + + + +- Abs +- Acos +- Acosh +- Add +- Asin +- Asinh +- Atan +- Atanh +- AveragePool +- BatchNormalization +- Cast +- Celu +- Clip +- Concat +- Constant +- ConstantOfShape +- Conv +- Cos +- Cosh +- Div +- Elu +- Equal +- Erf +- Exp +- Flatten +- Floor +- Gather +- Gemm +- Greater +- GreaterOrEqual +- HardSigmoid +- HardSwish +- Identity +- LeakyRelu +- Less +- LessOrEqual +- Log +- MatMul +- Max +- MaxPool +- Min +- Mul +- Neg +- Not +- Or +- PRelu +- Pad +- Pow +- ReduceSum +- Relu +- Reshape +- Round +- Selu +- Shape +- Sigmoid +- Sign +- Sin +- Sinh +- Slice +- Softplus +- Squeeze +- Sub +- Tan +- Tanh +- ThresholdedRelu +- Transpose +- Unsqueeze +- Where +- onnx.brevitas.Quant + + diff --git a/src/concrete/ml/onnx/convert.py b/src/concrete/ml/onnx/convert.py index 0380195fe..a48ff216d 100644 --- a/src/concrete/ml/onnx/convert.py +++ b/src/concrete/ml/onnx/convert.py @@ -176,11 +176,9 @@ def get_equivalent_numpy_forward_from_onnx( Callable[..., Tuple[numpy.ndarray, ...]]: The function that will execute the equivalent numpy function. """ - if check_model: checker.check_model(onnx_model) - # plus pour NN # Optimize ONNX graph # List of all currently supported onnx optimizer passes # From https://github.com/onnx/optimizer/blob/master/onnxoptimizer/pass_registry.h diff --git a/src/concrete/ml/onnx/onnx_impl_utils.py b/src/concrete/ml/onnx/onnx_impl_utils.py index e897a2cc1..081406fe1 100644 --- a/src/concrete/ml/onnx/onnx_impl_utils.py +++ b/src/concrete/ml/onnx/onnx_impl_utils.py @@ -10,6 +10,8 @@ from ..common.debugging import assert_true +ComparisonOperationType = Callable[[int], bool] + def numpy_onnx_pad( x: numpy.ndarray, diff --git a/src/concrete/ml/onnx/onnx_utils.py b/src/concrete/ml/onnx/onnx_utils.py index c05daa98b..6a0d6c341 100644 --- a/src/concrete/ml/onnx/onnx_utils.py +++ b/src/concrete/ml/onnx/onnx_utils.py @@ -213,7 +213,6 @@ # Original file: # https://github.com/google/jax/blob/f6d329b2d9b5f83c6a59e5739aa1ca8d4d1ffa1c/examples/onnx2xla.py - from typing import Any, Callable, Dict, Tuple import numpy @@ -352,7 +351,6 @@ "Log": numpy_log, "Exp": numpy_exp, "Equal": numpy_equal, - "RoundedOrEqual": numpy_rounded_equal, "Identity": numpy_identity, "Reshape": numpy_reshape, "Transpose": numpy_transpose, @@ -407,18 +405,29 @@ "Greater": numpy_greater, "GreaterOrEqual": numpy_greater_or_equal, "Less": numpy_less, - "RoundedOrLess": numpy_rounded_less, "LessOrEqual": numpy_less_or_equal, - "RoundedOrLessOrEqual": numpy_rounded_less_or_equal, +} + +# Rounded comparison operators used in tree-based models +ONNX_ROUNDED_COMPARISON_OPS_IMPL_BOOL: Dict[str, Callable[..., Tuple[numpy.ndarray, ...]]] = { + "RoundedEqual": numpy_rounded_equal, + "RoundedLess": numpy_rounded_less, + "RoundedLessOrEqual": numpy_rounded_less_or_equal, } # All numpy operators used in QuantizedOps ONNX_OPS_TO_NUMPY_IMPL.update(ONNX_COMPARISON_OPS_TO_NUMPY_IMPL_FLOAT) # All numpy operators used for tree-based models -ONNX_OPS_TO_NUMPY_IMPL_BOOL = {**ONNX_OPS_TO_NUMPY_IMPL, **ONNX_COMPARISON_OPS_TO_NUMPY_IMPL_BOOL} +ONNX_OPS_TO_NUMPY_IMPL_BOOL = { + **ONNX_OPS_TO_NUMPY_IMPL, + **ONNX_COMPARISON_OPS_TO_NUMPY_IMPL_BOOL, + **ONNX_ROUNDED_COMPARISON_OPS_IMPL_BOOL, +} -IMPLEMENTED_ONNX_OPS = set(ONNX_OPS_TO_NUMPY_IMPL.keys()) +IMPLEMENTED_ONNX_OPS = set( + ONNX_OPS_TO_NUMPY_IMPL.keys() | ONNX_ROUNDED_COMPARISON_OPS_IMPL_BOOL.keys() +) def get_attribute(attribute: onnx.AttributeProto) -> Any: @@ -465,11 +474,9 @@ def execute_onnx_with_numpy( for initializer in graph.initializer }, ) - for node in graph.node: curr_inputs = (node_results[input_name] for input_name in node.input) attributes = {attribute.name: get_attribute(attribute) for attribute in node.attribute} - outputs = ONNX_OPS_TO_NUMPY_IMPL_BOOL[node.op_type](*curr_inputs, **attributes) node_results.update(zip(node.output, outputs)) diff --git a/src/concrete/ml/onnx/ops_impl.py b/src/concrete/ml/onnx/ops_impl.py index 206c80128..8f1d6879d 100644 --- a/src/concrete/ml/onnx/ops_impl.py +++ b/src/concrete/ml/onnx/ops_impl.py @@ -899,6 +899,7 @@ def numpy_equal( Returns: Tuple[numpy.ndarray]: Output tensor """ + return (numpy.equal(x, y),) @@ -924,6 +925,7 @@ def numpy_rounded_equal( if lsbs_to_remove > 0: return rounded_comparison(y, x, lsbs_to_remove, operation=lambda x: x >= 0) + # Else, default numpy_equal operator return numpy_equal(x, y) @@ -1096,7 +1098,7 @@ def numpy_rounded_less( if lsbs_to_remove > 0: return rounded_comparison(x, y, lsbs_to_remove, operation=lambda x: x < 0) - # Else, default numpy less_or_equal comparison + # Else, default numpy_less operator return numpy_less(x, y) @@ -1159,7 +1161,7 @@ def numpy_rounded_less_or_equal( if lsbs_to_remove > 0: return rounded_comparison(y, x, lsbs_to_remove, operation=lambda x: x >= 0) - # Else, default numpy less_or_equal comparison + # Else, default numpy_less_or_equal operator return numpy_less_or_equal(x, y) @@ -1595,7 +1597,6 @@ def numpy_or_float( Returns: Tuple[numpy.ndarray]: Output tensor """ - return cast_to_float(numpy_or(a, b)) diff --git a/src/concrete/ml/sklearn/base.py b/src/concrete/ml/sklearn/base.py index 13252648c..0d5406c7c 100644 --- a/src/concrete/ml/sklearn/base.py +++ b/src/concrete/ml/sklearn/base.py @@ -1280,7 +1280,7 @@ def __init__(self, n_bits: int): """ self.n_bits: int = n_bits - # _tree_inference: The model's inference function. Is None if the model is not fitted. + #: The model's inference function. Is None if the model is not fitted. self._tree_inference: Optional[Callable] = None BaseEstimator.__init__(self) @@ -1310,19 +1310,19 @@ def fit(self, X: Data, y: Target, **fit_parameters): # Check that the underlying sklearn model has been set and fit assert self.sklearn_model is not None, self._sklearn_model_is_not_fitted_error_message() - self._is_fitted = True - # Convert the tree inference with Numpy operators # Adjust the auto rounder manually self._convert_tree_to_numpy_and_compute_lsbs_to_remove(q_X, use_rounding=True) + self._is_fitted = True + return self def _convert_tree_to_numpy_and_compute_lsbs_to_remove( self, q_X: numpy.ndarray, use_rounding: bool ): - assert self._is_fitted is True, "Model must be fitted" + assert self.sklearn_model is not None, self._sklearn_model_is_not_fitted_error_message() check_array_and_assert(q_X) diff --git a/src/concrete/ml/sklearn/tree_to_numpy.py b/src/concrete/ml/sklearn/tree_to_numpy.py index 4d549ac39..cefedf584 100644 --- a/src/concrete/ml/sklearn/tree_to_numpy.py +++ b/src/concrete/ml/sklearn/tree_to_numpy.py @@ -23,13 +23,9 @@ # Silence Hummingbird warnings warnings.filterwarnings("ignore") -from typing import List -import onnx from hummingbird.ml import convert as hb_convert # noqa: E402 -# pylint: enable=wrong-import-position,wrong-import-order - # pylint: disable=too-many-branches @@ -268,9 +264,7 @@ def replace_operator_with_rounded_version(onnx_model, lsbs_to_remove): # Mapping of original operators to their rounded counterparts operator_mapping = { - "Greater": "RoundedGreater", "Less": "RoundedLess", - "GreaterOrEqual": "RoundedGreaterOrEqual", "LessOrEqual": "RoundedLessOrEqual", "Equal": "RoundedEqual", } @@ -392,14 +386,12 @@ def tree_to_numpy( # Execute with 1 example for efficiency in large data scenarios to prevent slowdown onnx_model = get_onnx_model(model, q_x[:1], framework) + # compute LSB to remove in stage 1 and stage 2 if use_rounding: - # compute LSB to remove in stage 1 and 2 - # <=, <, == - # : List[(lsbs_to_remove_stage1, lsbs_to_remove_stage1), lsbs_to_remove_stage2] + # First LSB refers to Less or LessOrEqual comparisons + # Second LSB refers to Equal comparison lsbs_to_remove = compute_lsb_to_remove_for_trees(onnx_model, q_x) - replace_operator_with_rounded_version(onnx_model, lsbs_to_remove) - - replace_operator_with_rounded_version(onnx_model, [3, 3]) + onnx_model = replace_operator_with_rounded_version(onnx_model, lsbs_to_remove) # Get the expected number of ONNX outputs in the sklearn model. expected_number_of_outputs = 1 if is_regressor_or_partial_regressor(model) else 2 @@ -416,14 +408,14 @@ def tree_to_numpy( # Get the numpy inference for the quantized tree (_tree_inference). # Use check_model = False here since we have custom onnx operator that won't be recognised. _tree_inference, onnx_model = get_equivalent_numpy_forward_from_onnx( - onnx_model, q_x, check_model=False + onnx_model, check_model=False ) return (_tree_inference, [q_y.quantizer], onnx_model) # Remove this function once the truncate feature is released -# FIXME: https://github.com/zama-ai/concrete-ml/issues/397 +# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4143 def compute_lsb_to_remove_for_trees(onnx_model: onnx.ModelProto, q_x: numpy.ndarray) -> List[int]: """Compute the LSB to remove for the comparison operators in the trees. @@ -454,23 +446,25 @@ def get_bitwidth(array: numpy.ndarray) -> int: bitwidth = math.ceil(math.log2(max_val + 1)) + 1 return bitwidth - def update_lsbs_if_overflow_detected(array: numpy.ndarray, initial_bitwidth: int) -> int: + def get_lsbs_to_remove(array: numpy.ndarray) -> int: """Update the number of LSBs to remove based on overflow detection. Args: array (umpy.ndarray): The array for which the bitwidth needs to be checked. - initial_bitwidth (int): The target bitwidth that should not be exceeded. Returns: int: The updated LSB to remove. """ + initial_bitwidth = get_bitwidth(array) lsbs_to_remove = initial_bitwidth - if lsbs_to_remove > 0: + while lsbs_to_remove > 0: half = 1 << (lsbs_to_remove - 1) if get_bitwidth(array - half) <= initial_bitwidth: lsbs_to_remove -= 1 + else: + break return lsbs_to_remove @@ -506,18 +500,18 @@ def update_lsbs_if_overflow_detected(array: numpy.ndarray, initial_bitwidth: int required_onnx_operators = set(get_op_type(node) for node in onnx_model.graph.node) - # If operator == `<`, np.less(x, y) is equivalent to: + # If operator is `<`, np.less(x, y) is equivalent to: # round_bit_pattern((x - y) - half, lsbs_to_remove=r) < 0. # Therfore, stage_1 = (q_x @ mat_1.transpose(0, 2, 1)) - bias_1 if "Less" in required_onnx_operators: stage_1 = (q_x @ mat_1.transpose(0, 2, 1)) - bias_1 - else: - # If operator == `<=`, np.less_equal(x, y) is equivalent to: + elif "LessOrEqual" in required_onnx_operators: + # If operator is `<=`, np.less_equal(x, y) is equivalent to: # round_bit_pattern((y - x) - half, lsbs_to_remove=r) >= 0. # Therfore, stage_1 = bias_1 - (q_x @ mat_1.transpose(0, 2, 1)) stage_1 = bias_1 - (q_x @ mat_1.transpose(0, 2, 1)) - lsbs_to_remove_1 = update_lsbs_if_overflow_detected(stage_1, get_bitwidth(stage_1)) + lsbs_to_remove_stage_1 = get_lsbs_to_remove(stage_1) # The matrix I, as referenced in this paper: # https://scnakandala.github.io/papers/TR_2020_Hummingbird.pdf, results from the condition: @@ -525,11 +519,11 @@ def update_lsbs_if_overflow_detected(array: numpy.ndarray, initial_bitwidth: int # Given this assumption, we randomly generate it. matrix_q = numpy.random.randint(0, 2, size=(stage_1.shape)) - # If operator == `==`, np.equal(x, y) is equivalent to: + # If operator is `==`, np.equal(x, y) is equivalent to: # round_bit_pattern((x - y) - half, lsbs_to_remove=r) >= 0. # Therfore, stage_2 = bias_1 - (q_x @ mat_2.transpose(0, 2, 1)) stage_2 = ((bias_2 - matrix_q @ mat_2.transpose(0, 2, 1))).sum(axis=0) - lsbs_to_remove_2 = update_lsbs_if_overflow_detected(stage_2, get_bitwidth(stage_2)) + lsbs_to_remove_stage_2 = get_lsbs_to_remove(stage_2) - return [lsbs_to_remove_1, lsbs_to_remove_2] + return [lsbs_to_remove_stage_1, lsbs_to_remove_stage_2] From 9bf30ceeaeb3fbfd7371b436517ea1841bf99151 Mon Sep 17 00:00:00 2001 From: kcelia Date: Thu, 30 Nov 2023 12:51:35 +0100 Subject: [PATCH 10/73] chore: update --- src/concrete/ml/sklearn/base.py | 35 +++++++++++++----------- src/concrete/ml/sklearn/tree_to_numpy.py | 5 +++- tests/sklearn/test_sklearn_models.py | 31 ++++++++++++++++----- 3 files changed, 47 insertions(+), 24 deletions(-) diff --git a/src/concrete/ml/sklearn/base.py b/src/concrete/ml/sklearn/base.py index 0d5406c7c..ae7690fa6 100644 --- a/src/concrete/ml/sklearn/base.py +++ b/src/concrete/ml/sklearn/base.py @@ -1283,6 +1283,9 @@ def __init__(self, n_bits: int): #: The model's inference function. Is None if the model is not fitted. self._tree_inference: Optional[Callable] = None + #: Set to `True` to enable the rounding feature, or `False` to disable it. + self._use_rounding: bool = True + BaseEstimator.__init__(self) def fit(self, X: Data, y: Target, **fit_parameters): @@ -1311,28 +1314,28 @@ def fit(self, X: Data, y: Target, **fit_parameters): assert self.sklearn_model is not None, self._sklearn_model_is_not_fitted_error_message() # Convert the tree inference with Numpy operators - # Adjust the auto rounder manually - self._convert_tree_to_numpy_and_compute_lsbs_to_remove(q_X, use_rounding=True) - + self._tree_inference, self.output_quantizers, self.onnx_model_ = tree_to_numpy( + self.sklearn_model, # type: ignore[arg-type] + q_X, + use_rounding=self._use_rounding, + framework=self.framework, + output_n_bits=self.n_bits, + ) self._is_fitted = True return self - def _convert_tree_to_numpy_and_compute_lsbs_to_remove( - self, q_X: numpy.ndarray, use_rounding: bool - ): - - assert self.sklearn_model is not None, self._sklearn_model_is_not_fitted_error_message() + def disable_rounding(self): + """Disable the rounding feature.""" - check_array_and_assert(q_X) + self.use_rounding = False - # Convert the tree inference with Numpy operators - self._tree_inference, self.output_quantizers, self.onnx_model_ = tree_to_numpy( - self.sklearn_model, # type: ignore[arg-type] - q_X, - use_rounding=use_rounding, - framework=self.framework, - output_n_bits=self.n_bits, + warnings.warn( + "Using tree models without the rounding function is deprecated. " + "Consider setting 'use_rounding' to True for accelerated execution " + "of FHE calculations and key generation.", + category=UserWarning, + stacklevel=2, ) def quantize_input(self, X: numpy.ndarray) -> numpy.ndarray: diff --git a/src/concrete/ml/sklearn/tree_to_numpy.py b/src/concrete/ml/sklearn/tree_to_numpy.py index cefedf584..3c891e60e 100644 --- a/src/concrete/ml/sklearn/tree_to_numpy.py +++ b/src/concrete/ml/sklearn/tree_to_numpy.py @@ -276,12 +276,15 @@ def replace_operator_with_rounded_version(onnx_model, lsbs_to_remove): new_nodes = [] for node in onnx_model.graph.node: + # The first comparison in the tree involves the operators `<` and `<=` + # Assign the first value of LBS in `lsbs_to_remove` to the relevant nodes if not comparison_replaced and node.op_type in operator_mapping and node.op_type != "Equal": # Use the first value in lsbs_to_remove for the comparison operator lsbs = lsbs_to_remove[0] comparison_replaced = True + # The second comparison in the tree involves only the operator `==` + # Assign the second value of LBS in `lsbs_to_remove` to the relevant node elif not equal_replaced and node.op_type == "Equal": - # Use the second value in lsbs_to_remove for the Equal operator lsbs = lsbs_to_remove[1] equal_replaced = True else: diff --git a/tests/sklearn/test_sklearn_models.py b/tests/sklearn/test_sklearn_models.py index 0eca2b982..e1f68ca6e 100644 --- a/tests/sklearn/test_sklearn_models.py +++ b/tests/sklearn/test_sklearn_models.py @@ -152,6 +152,17 @@ def get_n_bits_non_correctness(model_class): return n_bits +def fit_and_compile(model, x, y): + """Fit the model and compile it.""" + + with warnings.catch_warnings(): + # Sometimes, we miss convergence, which is not a problem for our test + warnings.simplefilter("ignore", category=ConvergenceWarning) + model.fit(x, y) + + model.compile(x) + + def check_correctness_with_sklearn( model_class, x, @@ -1143,25 +1154,28 @@ def check_load_fitted_sklearn_linear_models(model_class, n_bits, x, y, check_flo def check_rounding_consistency( model, x, + y, predict_method, metric, ): - """Test that Concrete ML witout and with rounding are 'equivalent'.""" random_int = random.randint(0, x.shape[0] - 1) - model.compile(x) + # Fit and compile with rounding enabled + fit_and_compile(model, x, y) rounded_predict_quantized = predict_method(x, fhe="disable") rounded_predict_simulate = predict_method(x, fhe="simulate") rounded_predict_fhe = predict_method(x[random_int, None], fhe="execute") - # pylint: disable=protected-access - quant_x = model.quantize_input(x).astype("float") - model._convert_tree_to_numpy_and_compute_lsbs_to_remove(quant_x, use_rounding=False) + # Fit and compile without rounding - model.compile(x) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=UserWarning) + model.disable_rounding() + + fit_and_compile(model, x, y) not_rounded_predict_quantized = predict_method(x, fhe="disable") not_rounded_predict_simulate = predict_method(x, fhe="simulate") @@ -1818,13 +1832,15 @@ def test_rounding_consistency( if verbose: print("Run check_rounding_consistency") - model, x = preamble(model_class, parameters, n_bits, load_data, is_weekly_option) + model = instantiate_model_generic(model_class, n_bits=n_bits) + x, y = get_dataset(model_class, parameters, n_bits, load_data, is_weekly_option) # Check `predict_proba` for classifiers if is_classifier_or_partial_classifier(model): check_rounding_consistency( model, x, + y, predict_method=model.predict_proba, metric=check_r2_score, ) @@ -1833,6 +1849,7 @@ def test_rounding_consistency( check_rounding_consistency( model, x, + y, predict_method=model.predict, metric=check_accuracy, ) From c0107a651e4a44ea2bcc3fc21fe9d723210357cd Mon Sep 17 00:00:00 2001 From: kcelia Date: Thu, 30 Nov 2023 15:12:06 +0100 Subject: [PATCH 11/73] chore: fix dump tests --- tests/sklearn/test_dump_onnx.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/sklearn/test_dump_onnx.py b/tests/sklearn/test_dump_onnx.py index 454518d5b..19c53c572 100644 --- a/tests/sklearn/test_dump_onnx.py +++ b/tests/sklearn/test_dump_onnx.py @@ -214,11 +214,11 @@ def test_dump( %input_0[DOUBLE, symx10] ) { %/_operators.0/Gemm_output_0 = Gemm[alpha = 1, beta = 0, transB = 1](%_operators.0.weight_1, %input_0) - %/_operators.0/LessOrEqual_output_0 = LessOrEqual(%/_operators.0/Gemm_output_0, %_operators.0.bias_1) + %/_operators.0/LessOrEqual_output_0 = RoundedLessOrEqual[lsbs_to_remove = 7](%/_operators.0/Gemm_output_0, %_operators.0.bias_1) %/_operators.0/Reshape_output_0 = Reshape[allowzero = 0](%/_operators.0/LessOrEqual_output_0, %/_operators.0/Constant_output_0) %/_operators.0/MatMul_output_0 = MatMul(%_operators.0.weight_2, %/_operators.0/Reshape_output_0) %/_operators.0/Reshape_1_output_0 = Reshape[allowzero = 0](%/_operators.0/MatMul_output_0, %/_operators.0/Constant_1_output_0) - %/_operators.0/Equal_output_0 = Equal(%_operators.0.bias_2, %/_operators.0/Reshape_1_output_0) + %/_operators.0/Equal_output_0 = RoundedEqual[lsbs_to_remove = 5](%_operators.0.bias_2, %/_operators.0/Reshape_1_output_0) %/_operators.0/Reshape_2_output_0 = Reshape[allowzero = 0](%/_operators.0/Equal_output_0, %/_operators.0/Constant_2_output_0) %/_operators.0/MatMul_1_output_0 = MatMul(%_operators.0.weight_3, %/_operators.0/Reshape_2_output_0) %/_operators.0/Reshape_3_output_0 = Reshape[allowzero = 0](%/_operators.0/MatMul_1_output_0, %/_operators.0/Constant_3_output_0) @@ -286,11 +286,11 @@ def test_dump( %input_0[DOUBLE, symx10] ) { %/_operators.0/Gemm_output_0 = Gemm[alpha = 1, beta = 0, transB = 1](%_operators.0.weight_1, %input_0) - %/_operators.0/LessOrEqual_output_0 = LessOrEqual(%/_operators.0/Gemm_output_0, %_operators.0.bias_1) + %/_operators.0/LessOrEqual_output_0 = RoundedLessOrEqual[lsbs_to_remove = 7](%/_operators.0/Gemm_output_0, %_operators.0.bias_1) %/_operators.0/Reshape_output_0 = Reshape[allowzero = 0](%/_operators.0/LessOrEqual_output_0, %/_operators.0/Constant_output_0) %/_operators.0/MatMul_output_0 = MatMul(%_operators.0.weight_2, %/_operators.0/Reshape_output_0) %/_operators.0/Reshape_1_output_0 = Reshape[allowzero = 0](%/_operators.0/MatMul_output_0, %/_operators.0/Constant_1_output_0) - %/_operators.0/Equal_output_0 = Equal(%_operators.0.bias_2, %/_operators.0/Reshape_1_output_0) + %/_operators.0/Equal_output_0 = RoundedEqual[lsbs_to_remove = 5](%_operators.0.bias_2, %/_operators.0/Reshape_1_output_0) %/_operators.0/Reshape_2_output_0 = Reshape[allowzero = 0](%/_operators.0/Equal_output_0, %/_operators.0/Constant_2_output_0) %/_operators.0/MatMul_1_output_0 = MatMul(%_operators.0.weight_3, %/_operators.0/Reshape_2_output_0) %/_operators.0/Reshape_3_output_0 = Reshape[allowzero = 0](%/_operators.0/MatMul_1_output_0, %/_operators.0/Constant_3_output_0) @@ -328,11 +328,11 @@ def test_dump( %input_0[DOUBLE, symx10] ) { %/_operators.0/Gemm_output_0 = Gemm[alpha = 1, beta = 0, transB = 1](%_operators.0.weight_1, %input_0) - %/_operators.0/Less_output_0 = Less(%/_operators.0/Gemm_output_0, %_operators.0.bias_1) + %/_operators.0/Less_output_0 = RoundedLess[lsbs_to_remove = 7](%/_operators.0/Gemm_output_0, %_operators.0.bias_1) %/_operators.0/Reshape_output_0 = Reshape[allowzero = 0](%/_operators.0/Less_output_0, %/_operators.0/Constant_output_0) %/_operators.0/MatMul_output_0 = MatMul(%_operators.0.weight_2, %/_operators.0/Reshape_output_0) %/_operators.0/Reshape_1_output_0 = Reshape[allowzero = 0](%/_operators.0/MatMul_output_0, %/_operators.0/Constant_1_output_0) - %/_operators.0/Equal_output_0 = Equal(%_operators.0.bias_2, %/_operators.0/Reshape_1_output_0) + %/_operators.0/Equal_output_0 = RoundedEqual[lsbs_to_remove = 0](%_operators.0.bias_2, %/_operators.0/Reshape_1_output_0) %/_operators.0/Reshape_2_output_0 = Reshape[allowzero = 0](%/_operators.0/Equal_output_0, %/_operators.0/Constant_2_output_0) %/_operators.0/MatMul_1_output_0 = MatMul(%_operators.0.weight_3, %/_operators.0/Reshape_2_output_0) %/_operators.0/Reshape_3_output_0 = Reshape[allowzero = 0](%/_operators.0/MatMul_1_output_0, %/_operators.0/Constant_3_output_0) @@ -376,11 +376,11 @@ def test_dump( %/_operators.0/Constant_4_output_0[INT64, 3] ) { %/_operators.0/Gemm_output_0 = Gemm[alpha = 1, beta = 0, transB = 1](%_operators.0.weight_1, %input_0) - %/_operators.0/Less_output_0 = Less(%/_operators.0/Gemm_output_0, %_operators.0.bias_1) + %/_operators.0/Less_output_0 = RoundedLess[lsbs_to_remove = 7](%/_operators.0/Gemm_output_0, %_operators.0.bias_1) %/_operators.0/Reshape_output_0 = Reshape[allowzero = 0](%/_operators.0/Less_output_0, %/_operators.0/Constant_output_0) %/_operators.0/MatMul_output_0 = MatMul(%_operators.0.weight_2, %/_operators.0/Reshape_output_0) %/_operators.0/Reshape_1_output_0 = Reshape[allowzero = 0](%/_operators.0/MatMul_output_0, %/_operators.0/Constant_1_output_0) - %/_operators.0/Equal_output_0 = Equal(%_operators.0.bias_2, %/_operators.0/Reshape_1_output_0) + %/_operators.0/Equal_output_0 = RoundedEqual[lsbs_to_remove = 0](%_operators.0.bias_2, %/_operators.0/Reshape_1_output_0) %/_operators.0/Reshape_2_output_0 = Reshape[allowzero = 0](%/_operators.0/Equal_output_0, %/_operators.0/Constant_2_output_0) %/_operators.0/MatMul_1_output_0 = MatMul(%_operators.0.weight_3, %/_operators.0/Reshape_2_output_0) %/_operators.0/Reshape_3_output_0 = Reshape[allowzero = 0](%/_operators.0/MatMul_1_output_0, %/_operators.0/Constant_3_output_0) From 6c9a475094dea7fc23a1f8c5ecbed2c2e8ee23f9 Mon Sep 17 00:00:00 2001 From: kcelia Date: Thu, 30 Nov 2023 17:10:28 +0100 Subject: [PATCH 12/73] chore: rename variable --- src/concrete/ml/sklearn/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/concrete/ml/sklearn/base.py b/src/concrete/ml/sklearn/base.py index ae7690fa6..6df902098 100644 --- a/src/concrete/ml/sklearn/base.py +++ b/src/concrete/ml/sklearn/base.py @@ -1328,7 +1328,7 @@ def fit(self, X: Data, y: Target, **fit_parameters): def disable_rounding(self): """Disable the rounding feature.""" - self.use_rounding = False + self._use_rounding = False warnings.warn( "Using tree models without the rounding function is deprecated. " From 3d858fec48fc17da66024fa7ad68aef879fd0f82 Mon Sep 17 00:00:00 2001 From: kcelia Date: Fri, 1 Dec 2023 16:11:02 +0100 Subject: [PATCH 13/73] chore: restore previous version --- src/concrete/ml/onnx/convert.py | 9 ++- src/concrete/ml/onnx/onnx_utils.py | 34 +++++---- src/concrete/ml/onnx/ops_impl.py | 92 +++++------------------- src/concrete/ml/sklearn/base.py | 1 + src/concrete/ml/sklearn/tree_to_numpy.py | 9 ++- tests/sklearn/test_dump_onnx.py | 16 ++--- 6 files changed, 55 insertions(+), 106 deletions(-) diff --git a/src/concrete/ml/onnx/convert.py b/src/concrete/ml/onnx/convert.py index a48ff216d..bc19f2f52 100644 --- a/src/concrete/ml/onnx/convert.py +++ b/src/concrete/ml/onnx/convert.py @@ -2,7 +2,7 @@ import tempfile from pathlib import Path -from typing import Callable, Tuple, Union +from typing import Callable, List, Tuple, Union import numpy import onnx @@ -158,6 +158,7 @@ def get_equivalent_numpy_forward_from_torch( def get_equivalent_numpy_forward_from_onnx( onnx_model: onnx.ModelProto, + lsbs_to_remove: List[int] = None, check_model: bool = True, ) -> Tuple[Callable[..., Tuple[numpy.ndarray, ...]], onnx.ModelProto]: """Get the numpy equivalent forward of the provided ONNX model. @@ -167,6 +168,10 @@ def get_equivalent_numpy_forward_from_onnx( forward. check_model (bool): set to True to run the onnx checker on the model. Defaults to True. + lsbs_to_remove (List[int]): Contains the values of the least significant bits to + remove during tree traversal. The first value pertains to the first comparison (either + "less" or "less_or_equal"), and the second value relates to the "Equal" comparison + operation. Default value set to None, when the rounding feature is not used. Raises: ValueError: Raised if there is an unsupported ONNX operator required to convert the torch @@ -208,5 +213,5 @@ def get_equivalent_numpy_forward_from_onnx( # Return lambda of numpy equivalent of onnx execution return ( - lambda *args: execute_onnx_with_numpy(equivalent_onnx_model.graph, *args) + lambda *args: execute_onnx_with_numpy(equivalent_onnx_model.graph, lsbs_to_remove, *args) ), equivalent_onnx_model diff --git a/src/concrete/ml/onnx/onnx_utils.py b/src/concrete/ml/onnx/onnx_utils.py index 6a0d6c341..1bcb40c4f 100644 --- a/src/concrete/ml/onnx/onnx_utils.py +++ b/src/concrete/ml/onnx/onnx_utils.py @@ -213,7 +213,7 @@ # Original file: # https://github.com/google/jax/blob/f6d329b2d9b5f83c6a59e5739aa1ca8d4d1ffa1c/examples/onnx2xla.py -from typing import Any, Callable, Dict, Tuple +from typing import Any, Callable, Dict, List, Optional, Tuple import numpy import onnx @@ -279,9 +279,6 @@ numpy_relu, numpy_reshape, numpy_round, - numpy_rounded_equal, - numpy_rounded_less, - numpy_rounded_less_or_equal, numpy_selu, numpy_shape, numpy_sigmoid, @@ -408,26 +405,16 @@ "LessOrEqual": numpy_less_or_equal, } -# Rounded comparison operators used in tree-based models -ONNX_ROUNDED_COMPARISON_OPS_IMPL_BOOL: Dict[str, Callable[..., Tuple[numpy.ndarray, ...]]] = { - "RoundedEqual": numpy_rounded_equal, - "RoundedLess": numpy_rounded_less, - "RoundedLessOrEqual": numpy_rounded_less_or_equal, -} +# All numpy operators used for tree-based models that support auto rounding +SUPPORTED_ROUNDED_OPERATIONS = ["Less", "LessOrEqual", "Equal"] # All numpy operators used in QuantizedOps ONNX_OPS_TO_NUMPY_IMPL.update(ONNX_COMPARISON_OPS_TO_NUMPY_IMPL_FLOAT) # All numpy operators used for tree-based models -ONNX_OPS_TO_NUMPY_IMPL_BOOL = { - **ONNX_OPS_TO_NUMPY_IMPL, - **ONNX_COMPARISON_OPS_TO_NUMPY_IMPL_BOOL, - **ONNX_ROUNDED_COMPARISON_OPS_IMPL_BOOL, -} +ONNX_OPS_TO_NUMPY_IMPL_BOOL = {**ONNX_OPS_TO_NUMPY_IMPL, **ONNX_COMPARISON_OPS_TO_NUMPY_IMPL_BOOL} -IMPLEMENTED_ONNX_OPS = set( - ONNX_OPS_TO_NUMPY_IMPL.keys() | ONNX_ROUNDED_COMPARISON_OPS_IMPL_BOOL.keys() -) +IMPLEMENTED_ONNX_OPS = set(ONNX_OPS_TO_NUMPY_IMPL.keys()) def get_attribute(attribute: onnx.AttributeProto) -> Any: @@ -456,12 +443,17 @@ def get_op_type(node): def execute_onnx_with_numpy( graph: onnx.GraphProto, + lsbs_to_remove: Optional[List[int]], *inputs: numpy.ndarray, ) -> Tuple[numpy.ndarray, ...]: """Execute the provided ONNX graph on the given inputs. Args: graph (onnx.GraphProto): The ONNX graph to execute. + lsbs_to_remove (Optional[List[int]]): Contains the values of the least significant bits to + remove during tree traversal. The first value pertains to the first comparison (either + "less" or "less_or_equal"), and the second value relates to the "Equal" comparison + operation. Default value set to None, when the rounding feature is not used. *inputs: The inputs of the graph. Returns: @@ -477,6 +469,12 @@ def execute_onnx_with_numpy( for node in graph.node: curr_inputs = (node_results[input_name] for input_name in node.input) attributes = {attribute.name: get_attribute(attribute) for attribute in node.attribute} + + if lsbs_to_remove is not None and node.op_type in SUPPORTED_ROUNDED_OPERATIONS: + attributes["lsbs_to_remove"] = ( + lsbs_to_remove[0] if node.op_type != "Equal" else lsbs_to_remove[1] + ) + outputs = ONNX_OPS_TO_NUMPY_IMPL_BOOL[node.op_type](*curr_inputs, **attributes) node_results.update(zip(node.output, outputs)) diff --git a/src/concrete/ml/onnx/ops_impl.py b/src/concrete/ml/onnx/ops_impl.py index 8f1d6879d..1e9fdcbd4 100644 --- a/src/concrete/ml/onnx/ops_impl.py +++ b/src/concrete/ml/onnx/ops_impl.py @@ -887,6 +887,8 @@ def numpy_exp( def numpy_equal( x: numpy.ndarray, y: numpy.ndarray, + *, + lsbs_to_remove: Optional[int] = None, ) -> Tuple[numpy.ndarray]: """Compute equal in numpy according to ONNX spec. @@ -895,38 +897,18 @@ def numpy_equal( Args: x (numpy.ndarray): Input tensor y (numpy.ndarray): Input tensor - - Returns: - Tuple[numpy.ndarray]: Output tensor - """ - - return (numpy.equal(x, y),) - - -def numpy_rounded_equal( - x: numpy.ndarray, - y: numpy.ndarray, - lsbs_to_remove: int, -) -> Tuple[numpy.ndarray]: - """Compute rounded equal according to ONNX spec. - - See https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Equal-11 - - Args: - x (numpy.ndarray): Input tensor - y (numpy.ndarray): Input tensor - lsbs_to_remove (int): The number of the least significant bits to remove + lsbs_to_remove (Optional[int]): The number of the least significant bits to remove. Returns: Tuple[numpy.ndarray]: Output tensor """ # In the case of trees, x == y <=> x <= y or x < y - 1, because y is the max sum. - if lsbs_to_remove > 0: + if lsbs_to_remove is not None and lsbs_to_remove > 0: return rounded_comparison(y, x, lsbs_to_remove, operation=lambda x: x >= 0) # Else, default numpy_equal operator - return numpy_equal(x, y) + return (numpy.equal(x, y),) def numpy_not( @@ -1042,6 +1024,8 @@ def numpy_greater_or_equal_float( def numpy_less( x: numpy.ndarray, y: numpy.ndarray, + *, + lsbs_to_remove: Optional[int] = None, ) -> Tuple[numpy.ndarray]: """Compute less in numpy according to ONNX spec. @@ -1050,11 +1034,16 @@ def numpy_less( Args: x (numpy.ndarray): Input tensor y (numpy.ndarray): Input tensor + lsbs_to_remove (Optional[int]): The number of the least significant bits to remove Returns: Tuple[numpy.ndarray]: Output tensor """ + if lsbs_to_remove is not None and lsbs_to_remove > 0: + return rounded_comparison(x, y, lsbs_to_remove, operation=lambda x: x < 0) + + # Else, default numpy_less operator return (numpy.less(x, y),) @@ -1077,34 +1066,11 @@ def numpy_less_float( return cast_to_float(numpy_less(x, y)) -def numpy_rounded_less( - x: numpy.ndarray, - y: numpy.ndarray, - lsbs_to_remove: int, -) -> Tuple[numpy.ndarray]: - """Compute rounded less according to ONNX spec. - - See https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Less-13 - - Args: - x (numpy.ndarray): Input tensor - y (numpy.ndarray): Input tensor - lsbs_to_remove (int): The number of the least significant bits to remove - - Returns: - Tuple[numpy.ndarray]: Output tensor - """ - - if lsbs_to_remove > 0: - return rounded_comparison(x, y, lsbs_to_remove, operation=lambda x: x < 0) - - # Else, default numpy_less operator - return numpy_less(x, y) - - def numpy_less_or_equal( x: numpy.ndarray, y: numpy.ndarray, + *, + lsbs_to_remove: Optional[int] = None, ) -> Tuple[numpy.ndarray]: """Compute less or equal in numpy according to ONNX spec. @@ -1113,11 +1079,16 @@ def numpy_less_or_equal( Args: x (numpy.ndarray): Input tensor y (numpy.ndarray): Input tensor + lsbs_to_remove (Optional[int]): The number of the least significant bits to remove Returns: Tuple[numpy.ndarray]: Output tensor """ + if lsbs_to_remove is not None and lsbs_to_remove > 0: + return rounded_comparison(y, x, lsbs_to_remove, operation=lambda x: x >= 0) + + # Else, default numpy_less_or_equal operator return (numpy.less_equal(x, y),) @@ -1140,31 +1111,6 @@ def numpy_less_or_equal_float( return cast_to_float(numpy_less_or_equal(x, y)) -def numpy_rounded_less_or_equal( - x: numpy.ndarray, - y: numpy.ndarray, - lsbs_to_remove: int, -) -> Tuple[numpy.ndarray]: - """Compute rounded less or equal according to ONNX spec. - - See https://github.com/onnx/onnx/blob/main/docs/Changelog.md#LessOrEqual-12 - - Args: - x (numpy.ndarray): Input tensor - y (numpy.ndarray): Input tensor - lsbs_to_remove (int): The number of the least significant bits to remove - - Returns: - Tuple[numpy.ndarray]: Output tensor - """ - - if lsbs_to_remove > 0: - return rounded_comparison(y, x, lsbs_to_remove, operation=lambda x: x >= 0) - - # Else, default numpy_less_or_equal operator - return numpy_less_or_equal(x, y) - - def numpy_identity( x: numpy.ndarray, ) -> Tuple[numpy.ndarray]: diff --git a/src/concrete/ml/sklearn/base.py b/src/concrete/ml/sklearn/base.py index 6df902098..83f7a5282 100644 --- a/src/concrete/ml/sklearn/base.py +++ b/src/concrete/ml/sklearn/base.py @@ -1321,6 +1321,7 @@ def fit(self, X: Data, y: Target, **fit_parameters): framework=self.framework, output_n_bits=self.n_bits, ) + self._is_fitted = True return self diff --git a/src/concrete/ml/sklearn/tree_to_numpy.py b/src/concrete/ml/sklearn/tree_to_numpy.py index 3c891e60e..4ad730c73 100644 --- a/src/concrete/ml/sklearn/tree_to_numpy.py +++ b/src/concrete/ml/sklearn/tree_to_numpy.py @@ -1,7 +1,7 @@ """Implements the conversion of a tree model to a numpy function.""" import math import warnings -from typing import Callable, List, Tuple +from typing import Callable, List, Optional, Tuple import numpy import onnx @@ -381,6 +381,8 @@ def tree_to_numpy( # mypy assert output_n_bits is not None + lsbs_to_remove: Optional[List[int]] = None + assert_true( framework in ["xgboost", "sklearn"], f"framework={framework} is not supported. It must be either 'xgboost' or 'sklearn'", @@ -394,7 +396,6 @@ def tree_to_numpy( # First LSB refers to Less or LessOrEqual comparisons # Second LSB refers to Equal comparison lsbs_to_remove = compute_lsb_to_remove_for_trees(onnx_model, q_x) - onnx_model = replace_operator_with_rounded_version(onnx_model, lsbs_to_remove) # Get the expected number of ONNX outputs in the sklearn model. expected_number_of_outputs = 1 if is_regressor_or_partial_regressor(model) else 2 @@ -410,9 +411,7 @@ def tree_to_numpy( # Get the numpy inference for the quantized tree (_tree_inference). # Use check_model = False here since we have custom onnx operator that won't be recognised. - _tree_inference, onnx_model = get_equivalent_numpy_forward_from_onnx( - onnx_model, check_model=False - ) + _tree_inference, onnx_model = get_equivalent_numpy_forward_from_onnx(onnx_model, lsbs_to_remove) return (_tree_inference, [q_y.quantizer], onnx_model) diff --git a/tests/sklearn/test_dump_onnx.py b/tests/sklearn/test_dump_onnx.py index 19c53c572..454518d5b 100644 --- a/tests/sklearn/test_dump_onnx.py +++ b/tests/sklearn/test_dump_onnx.py @@ -214,11 +214,11 @@ def test_dump( %input_0[DOUBLE, symx10] ) { %/_operators.0/Gemm_output_0 = Gemm[alpha = 1, beta = 0, transB = 1](%_operators.0.weight_1, %input_0) - %/_operators.0/LessOrEqual_output_0 = RoundedLessOrEqual[lsbs_to_remove = 7](%/_operators.0/Gemm_output_0, %_operators.0.bias_1) + %/_operators.0/LessOrEqual_output_0 = LessOrEqual(%/_operators.0/Gemm_output_0, %_operators.0.bias_1) %/_operators.0/Reshape_output_0 = Reshape[allowzero = 0](%/_operators.0/LessOrEqual_output_0, %/_operators.0/Constant_output_0) %/_operators.0/MatMul_output_0 = MatMul(%_operators.0.weight_2, %/_operators.0/Reshape_output_0) %/_operators.0/Reshape_1_output_0 = Reshape[allowzero = 0](%/_operators.0/MatMul_output_0, %/_operators.0/Constant_1_output_0) - %/_operators.0/Equal_output_0 = RoundedEqual[lsbs_to_remove = 5](%_operators.0.bias_2, %/_operators.0/Reshape_1_output_0) + %/_operators.0/Equal_output_0 = Equal(%_operators.0.bias_2, %/_operators.0/Reshape_1_output_0) %/_operators.0/Reshape_2_output_0 = Reshape[allowzero = 0](%/_operators.0/Equal_output_0, %/_operators.0/Constant_2_output_0) %/_operators.0/MatMul_1_output_0 = MatMul(%_operators.0.weight_3, %/_operators.0/Reshape_2_output_0) %/_operators.0/Reshape_3_output_0 = Reshape[allowzero = 0](%/_operators.0/MatMul_1_output_0, %/_operators.0/Constant_3_output_0) @@ -286,11 +286,11 @@ def test_dump( %input_0[DOUBLE, symx10] ) { %/_operators.0/Gemm_output_0 = Gemm[alpha = 1, beta = 0, transB = 1](%_operators.0.weight_1, %input_0) - %/_operators.0/LessOrEqual_output_0 = RoundedLessOrEqual[lsbs_to_remove = 7](%/_operators.0/Gemm_output_0, %_operators.0.bias_1) + %/_operators.0/LessOrEqual_output_0 = LessOrEqual(%/_operators.0/Gemm_output_0, %_operators.0.bias_1) %/_operators.0/Reshape_output_0 = Reshape[allowzero = 0](%/_operators.0/LessOrEqual_output_0, %/_operators.0/Constant_output_0) %/_operators.0/MatMul_output_0 = MatMul(%_operators.0.weight_2, %/_operators.0/Reshape_output_0) %/_operators.0/Reshape_1_output_0 = Reshape[allowzero = 0](%/_operators.0/MatMul_output_0, %/_operators.0/Constant_1_output_0) - %/_operators.0/Equal_output_0 = RoundedEqual[lsbs_to_remove = 5](%_operators.0.bias_2, %/_operators.0/Reshape_1_output_0) + %/_operators.0/Equal_output_0 = Equal(%_operators.0.bias_2, %/_operators.0/Reshape_1_output_0) %/_operators.0/Reshape_2_output_0 = Reshape[allowzero = 0](%/_operators.0/Equal_output_0, %/_operators.0/Constant_2_output_0) %/_operators.0/MatMul_1_output_0 = MatMul(%_operators.0.weight_3, %/_operators.0/Reshape_2_output_0) %/_operators.0/Reshape_3_output_0 = Reshape[allowzero = 0](%/_operators.0/MatMul_1_output_0, %/_operators.0/Constant_3_output_0) @@ -328,11 +328,11 @@ def test_dump( %input_0[DOUBLE, symx10] ) { %/_operators.0/Gemm_output_0 = Gemm[alpha = 1, beta = 0, transB = 1](%_operators.0.weight_1, %input_0) - %/_operators.0/Less_output_0 = RoundedLess[lsbs_to_remove = 7](%/_operators.0/Gemm_output_0, %_operators.0.bias_1) + %/_operators.0/Less_output_0 = Less(%/_operators.0/Gemm_output_0, %_operators.0.bias_1) %/_operators.0/Reshape_output_0 = Reshape[allowzero = 0](%/_operators.0/Less_output_0, %/_operators.0/Constant_output_0) %/_operators.0/MatMul_output_0 = MatMul(%_operators.0.weight_2, %/_operators.0/Reshape_output_0) %/_operators.0/Reshape_1_output_0 = Reshape[allowzero = 0](%/_operators.0/MatMul_output_0, %/_operators.0/Constant_1_output_0) - %/_operators.0/Equal_output_0 = RoundedEqual[lsbs_to_remove = 0](%_operators.0.bias_2, %/_operators.0/Reshape_1_output_0) + %/_operators.0/Equal_output_0 = Equal(%_operators.0.bias_2, %/_operators.0/Reshape_1_output_0) %/_operators.0/Reshape_2_output_0 = Reshape[allowzero = 0](%/_operators.0/Equal_output_0, %/_operators.0/Constant_2_output_0) %/_operators.0/MatMul_1_output_0 = MatMul(%_operators.0.weight_3, %/_operators.0/Reshape_2_output_0) %/_operators.0/Reshape_3_output_0 = Reshape[allowzero = 0](%/_operators.0/MatMul_1_output_0, %/_operators.0/Constant_3_output_0) @@ -376,11 +376,11 @@ def test_dump( %/_operators.0/Constant_4_output_0[INT64, 3] ) { %/_operators.0/Gemm_output_0 = Gemm[alpha = 1, beta = 0, transB = 1](%_operators.0.weight_1, %input_0) - %/_operators.0/Less_output_0 = RoundedLess[lsbs_to_remove = 7](%/_operators.0/Gemm_output_0, %_operators.0.bias_1) + %/_operators.0/Less_output_0 = Less(%/_operators.0/Gemm_output_0, %_operators.0.bias_1) %/_operators.0/Reshape_output_0 = Reshape[allowzero = 0](%/_operators.0/Less_output_0, %/_operators.0/Constant_output_0) %/_operators.0/MatMul_output_0 = MatMul(%_operators.0.weight_2, %/_operators.0/Reshape_output_0) %/_operators.0/Reshape_1_output_0 = Reshape[allowzero = 0](%/_operators.0/MatMul_output_0, %/_operators.0/Constant_1_output_0) - %/_operators.0/Equal_output_0 = RoundedEqual[lsbs_to_remove = 0](%_operators.0.bias_2, %/_operators.0/Reshape_1_output_0) + %/_operators.0/Equal_output_0 = Equal(%_operators.0.bias_2, %/_operators.0/Reshape_1_output_0) %/_operators.0/Reshape_2_output_0 = Reshape[allowzero = 0](%/_operators.0/Equal_output_0, %/_operators.0/Constant_2_output_0) %/_operators.0/MatMul_1_output_0 = MatMul(%_operators.0.weight_3, %/_operators.0/Reshape_2_output_0) %/_operators.0/Reshape_3_output_0 = Reshape[allowzero = 0](%/_operators.0/MatMul_1_output_0, %/_operators.0/Constant_3_output_0) From aac906fd64153d23ce808a0d7d2e95a4bb3ee731 Mon Sep 17 00:00:00 2001 From: kcelia Date: Fri, 1 Dec 2023 16:39:14 +0100 Subject: [PATCH 14/73] chore: remove replace_operator_with_rounded_version function --- src/concrete/ml/sklearn/base.py | 9 ++-- src/concrete/ml/sklearn/tree_to_numpy.py | 66 ------------------------ tests/sklearn/test_sklearn_models.py | 4 +- 3 files changed, 7 insertions(+), 72 deletions(-) diff --git a/src/concrete/ml/sklearn/base.py b/src/concrete/ml/sklearn/base.py index 83f7a5282..91030059c 100644 --- a/src/concrete/ml/sklearn/base.py +++ b/src/concrete/ml/sklearn/base.py @@ -1331,11 +1331,12 @@ def disable_rounding(self): self._use_rounding = False + warnings.simplefilter("always") warnings.warn( - "Using tree models without the rounding function is deprecated. " - "Consider setting 'use_rounding' to True for accelerated execution " - "of FHE calculations and key generation.", - category=UserWarning, + "Using Concrete tree-based models without the `rounding feature` s deprecated. " + "Consider setting '_use_rounding' to `True` for improved speed in FHE computation and " + "key generation.", + category=DeprecationWarning, stacklevel=2, ) diff --git a/src/concrete/ml/sklearn/tree_to_numpy.py b/src/concrete/ml/sklearn/tree_to_numpy.py index 4ad730c73..59eee0ffd 100644 --- a/src/concrete/ml/sklearn/tree_to_numpy.py +++ b/src/concrete/ml/sklearn/tree_to_numpy.py @@ -248,70 +248,6 @@ def tree_onnx_graph_preprocessing( remove_node_types(onnx_model, op_types_to_remove=["Cast"]) -def replace_operator_with_rounded_version(onnx_model, lsbs_to_remove): - """Replace comparisons with rounded comparisons. - - Args: - onnx_model (onnx.ModelProto): The ONNX model. - lsbs_to_remove (List[int]): A list of two integers specifying the number of LSBs to remove. - - Returns: - onnx.ModelProto: The modified ONNX model. - """ - - assert_true(isinstance(lsbs_to_remove, list), "lsbs_to_remove must be a list.") - assert_true(len(lsbs_to_remove) == 2, "lsbs_to_remove must have exactly two values.") - - # Mapping of original operators to their rounded counterparts - operator_mapping = { - "Less": "RoundedLess", - "LessOrEqual": "RoundedLessOrEqual", - "Equal": "RoundedEqual", - } - - # Track if the required operators have been replaced - comparison_replaced = False - equal_replaced = False - - new_nodes = [] - - for node in onnx_model.graph.node: - # The first comparison in the tree involves the operators `<` and `<=` - # Assign the first value of LBS in `lsbs_to_remove` to the relevant nodes - if not comparison_replaced and node.op_type in operator_mapping and node.op_type != "Equal": - # Use the first value in lsbs_to_remove for the comparison operator - lsbs = lsbs_to_remove[0] - comparison_replaced = True - # The second comparison in the tree involves only the operator `==` - # Assign the second value of LBS in `lsbs_to_remove` to the relevant node - elif not equal_replaced and node.op_type == "Equal": - lsbs = lsbs_to_remove[1] - equal_replaced = True - else: - new_nodes.append(node) - continue - - # Create a new node with the corresponding rounded operator - rounded_node = onnx.helper.make_node( - operator_mapping[node.op_type], - inputs=node.input, - outputs=node.output, - lsbs_to_remove=lsbs, - ) - new_nodes.append(rounded_node) - - # Ensure that both a comparison and an equal operator were replaced - assert_true( - comparison_replaced and equal_replaced, "Required operators not found in the model." - ) - - # Replace the graph's node list with the new list - onnx_model.graph.ClearField("node") - onnx_model.graph.node.extend(new_nodes) - - return onnx_model - - def tree_values_preprocessing( onnx_model: onnx.ModelProto, framework: str, @@ -409,8 +345,6 @@ def tree_to_numpy( # but also rounding the threshold such that they are now integers q_y = tree_values_preprocessing(onnx_model, framework, output_n_bits) - # Get the numpy inference for the quantized tree (_tree_inference). - # Use check_model = False here since we have custom onnx operator that won't be recognised. _tree_inference, onnx_model = get_equivalent_numpy_forward_from_onnx(onnx_model, lsbs_to_remove) return (_tree_inference, [q_y.quantizer], onnx_model) diff --git a/tests/sklearn/test_sklearn_models.py b/tests/sklearn/test_sklearn_models.py index e1f68ca6e..9aec8d47d 100644 --- a/tests/sklearn/test_sklearn_models.py +++ b/tests/sklearn/test_sklearn_models.py @@ -1172,7 +1172,7 @@ def check_rounding_consistency( # Fit and compile without rounding with warnings.catch_warnings(): - warnings.simplefilter("ignore", category=UserWarning) + warnings.simplefilter("ignore", category=DeprecationWarning) model.disable_rounding() fit_and_compile(model, x, y) @@ -1816,7 +1816,7 @@ def test_linear_models_have_no_tlu( # Test only tree-based models @pytest.mark.parametrize("model_class, parameters", get_sklearn_tree_models_and_datasets()) -@pytest.mark.parametrize("n_bits", [2, 3, 4, 5]) +@pytest.mark.parametrize("n_bits", [2, 3, 4, 5, 6]) def test_rounding_consistency( model_class, parameters, From 992f23015578e7e0bfb765d920b96f2485f9dedc Mon Sep 17 00:00:00 2001 From: kcelia Date: Mon, 4 Dec 2023 09:37:56 +0100 Subject: [PATCH 15/73] chore: update --- src/concrete/ml/sklearn/tree_to_numpy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/concrete/ml/sklearn/tree_to_numpy.py b/src/concrete/ml/sklearn/tree_to_numpy.py index 59eee0ffd..1625d22e3 100644 --- a/src/concrete/ml/sklearn/tree_to_numpy.py +++ b/src/concrete/ml/sklearn/tree_to_numpy.py @@ -393,7 +393,7 @@ def get_lsbs_to_remove(array: numpy.ndarray) -> int: """ initial_bitwidth = get_bitwidth(array) - lsbs_to_remove = initial_bitwidth + lsbs_to_remove = initial_bitwidth + 1 while lsbs_to_remove > 0: half = 1 << (lsbs_to_remove - 1) From 0243dc75fcc9863bf892a1ee6ac96369bc552e31 Mon Sep 17 00:00:00 2001 From: kcelia Date: Mon, 4 Dec 2023 10:15:02 +0100 Subject: [PATCH 16/73] chore: add comments --- tests/sklearn/test_sklearn_models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/sklearn/test_sklearn_models.py b/tests/sklearn/test_sklearn_models.py index 9aec8d47d..62662d3f0 100644 --- a/tests/sklearn/test_sklearn_models.py +++ b/tests/sklearn/test_sklearn_models.py @@ -424,6 +424,7 @@ def check_serialization_dump_load(model, x, use_dump_method): serialized_model_dict["serialized_value"].pop(attribute, None) re_serialized_model_dict["serialized_value"].pop(attribute, None) + # Check if the graphs are similar assert serialized_model_dict == re_serialized_model_dict # Check that the predictions made by both model are identical @@ -476,6 +477,7 @@ def check_serialization_dumps_loads(model, x, use_dump_method): serialized_model_dict["serialized_value"].pop(attribute, None) re_serialized_model_dict["serialized_value"].pop(attribute, None) + # Check if the graphs are similar assert serialized_model_dict == re_serialized_model_dict # Check that the predictions made by both model are identical From dc08fb009ae670a563d2775a6f653d0b1f874e9c Mon Sep 17 00:00:00 2001 From: kcelia Date: Mon, 4 Dec 2023 10:49:08 +0100 Subject: [PATCH 17/73] chore: fix divergence in graphs after serialization --- src/concrete/ml/sklearn/base.py | 44 ++++++++++++++++-------- src/concrete/ml/sklearn/rf.py | 8 +++-- src/concrete/ml/sklearn/tree.py | 8 +++-- src/concrete/ml/sklearn/tree_to_numpy.py | 7 ++-- src/concrete/ml/sklearn/xgb.py | 8 +++-- tests/sklearn/test_sklearn_models.py | 2 +- 6 files changed, 53 insertions(+), 24 deletions(-) diff --git a/src/concrete/ml/sklearn/base.py b/src/concrete/ml/sklearn/base.py index 91030059c..e857d7f9d 100644 --- a/src/concrete/ml/sklearn/base.py +++ b/src/concrete/ml/sklearn/base.py @@ -1288,6 +1288,36 @@ def __init__(self, n_bits: int): BaseEstimator.__init__(self) + @property + def use_rounding(self) -> bool: + """The rounding property. + + Returns: + bool: Whether to enable or disable rounding + """ + return self._use_rounding # pragma: no cover + + @use_rounding.setter + def use_rounding(self, value: bool) -> None: + """Set the rounding feature. + + Args: + value (bool): Whether to enable or disable rounding + """ + assert isinstance(value, bool) + + self._use_rounding = value + + if not value: + warnings.simplefilter("always") + warnings.warn( + "Using Concrete tree-based models without the `rounding feature` s deprecated. " + "Consider setting '_use_rounding' to `True` for improved speed in FHE computation " + "and key generation.", + category=DeprecationWarning, + stacklevel=2, + ) + def fit(self, X: Data, y: Target, **fit_parameters): # Reset for double fit self._is_fitted = False @@ -1326,20 +1356,6 @@ def fit(self, X: Data, y: Target, **fit_parameters): return self - def disable_rounding(self): - """Disable the rounding feature.""" - - self._use_rounding = False - - warnings.simplefilter("always") - warnings.warn( - "Using Concrete tree-based models without the `rounding feature` s deprecated. " - "Consider setting '_use_rounding' to `True` for improved speed in FHE computation and " - "key generation.", - category=DeprecationWarning, - stacklevel=2, - ) - def quantize_input(self, X: numpy.ndarray) -> numpy.ndarray: self.check_model_is_fitted() diff --git a/src/concrete/ml/sklearn/rf.py b/src/concrete/ml/sklearn/rf.py index 00685a047..72ed77a01 100644 --- a/src/concrete/ml/sklearn/rf.py +++ b/src/concrete/ml/sklearn/rf.py @@ -118,7 +118,9 @@ def load_dict(cls, metadata: Dict): obj._is_compiled = metadata["_is_compiled"] obj.input_quantizers = metadata["input_quantizers"] obj.framework = metadata["framework"] - obj._tree_inference, obj.output_quantizers, obj.onnx_model_ = tree_to_numpy( + obj.onnx_model_ = metadata["onnx_model_"] + obj.output_quantizers = metadata["output_quantizers"] + obj._tree_inference, _, _ = tree_to_numpy( obj.sklearn_model, numpy.zeros((len(obj.input_quantizers),))[None, ...], framework=obj.framework, @@ -251,7 +253,9 @@ def load_dict(cls, metadata: Dict): obj._is_compiled = metadata["_is_compiled"] obj.input_quantizers = metadata["input_quantizers"] obj.framework = metadata["framework"] - obj._tree_inference, obj.output_quantizers, obj.onnx_model_ = tree_to_numpy( + obj.onnx_model_ = metadata["onnx_model_"] + obj.output_quantizers = metadata["output_quantizers"] + obj._tree_inference, _, _ = tree_to_numpy( obj.sklearn_model, numpy.zeros((len(obj.input_quantizers),))[None, ...], framework=obj.framework, diff --git a/src/concrete/ml/sklearn/tree.py b/src/concrete/ml/sklearn/tree.py index b81558b77..9b6b6c3f0 100644 --- a/src/concrete/ml/sklearn/tree.py +++ b/src/concrete/ml/sklearn/tree.py @@ -113,7 +113,9 @@ def load_dict(cls, metadata: Dict): obj._is_compiled = metadata["_is_compiled"] obj.input_quantizers = metadata["input_quantizers"] obj.framework = metadata["framework"] - obj._tree_inference, obj.output_quantizers, obj.onnx_model_ = tree_to_numpy( + obj.onnx_model_ = metadata["onnx_model_"] + obj.output_quantizers = metadata["output_quantizers"] + obj._tree_inference, _, _ = tree_to_numpy( obj.sklearn_model, numpy.zeros((len(obj.input_quantizers),))[None, ...], framework=obj.framework, @@ -234,7 +236,9 @@ def load_dict(cls, metadata: Dict): obj._is_compiled = metadata["_is_compiled"] obj.input_quantizers = metadata["input_quantizers"] obj.framework = metadata["framework"] - obj._tree_inference, obj.output_quantizers, obj.onnx_model_ = tree_to_numpy( + obj.onnx_model_ = metadata["onnx_model_"] + obj.output_quantizers = metadata["output_quantizers"] + obj._tree_inference, _, _ = tree_to_numpy( obj.sklearn_model, numpy.zeros((len(obj.input_quantizers),))[None, ...], framework=obj.framework, diff --git a/src/concrete/ml/sklearn/tree_to_numpy.py b/src/concrete/ml/sklearn/tree_to_numpy.py index 1625d22e3..87650c2ad 100644 --- a/src/concrete/ml/sklearn/tree_to_numpy.py +++ b/src/concrete/ml/sklearn/tree_to_numpy.py @@ -355,7 +355,7 @@ def tree_to_numpy( def compute_lsb_to_remove_for_trees(onnx_model: onnx.ModelProto, q_x: numpy.ndarray) -> List[int]: """Compute the LSB to remove for the comparison operators in the trees. - Referring to this paper: https://scnakandala.github.io/papers/TR_2020_Hummingbird.pdf, there are + Referring to this paper: https://arxiv.org/pdf/2010.04804.pdf, there are 2 levels of comparison for trees, one at the level of X.A < B and a second at the level of I.C == D. @@ -393,7 +393,8 @@ def get_lsbs_to_remove(array: numpy.ndarray) -> int: """ initial_bitwidth = get_bitwidth(array) - lsbs_to_remove = initial_bitwidth + 1 + # The subtraction operation increases precision by 1 or 2 bits + lsbs_to_remove = initial_bitwidth + 2 while lsbs_to_remove > 0: half = 1 << (lsbs_to_remove - 1) @@ -450,7 +451,7 @@ def get_lsbs_to_remove(array: numpy.ndarray) -> int: lsbs_to_remove_stage_1 = get_lsbs_to_remove(stage_1) # The matrix I, as referenced in this paper: - # https://scnakandala.github.io/papers/TR_2020_Hummingbird.pdf, results from the condition: + # https://arxiv.org/pdf/2010.04804.pdf, results from the condition: # X.A < B and consists exclusively of binary elements, 1 and 0. # Given this assumption, we randomly generate it. matrix_q = numpy.random.randint(0, 2, size=(stage_1.shape)) diff --git a/src/concrete/ml/sklearn/xgb.py b/src/concrete/ml/sklearn/xgb.py index c3d23b46d..b707409a8 100644 --- a/src/concrete/ml/sklearn/xgb.py +++ b/src/concrete/ml/sklearn/xgb.py @@ -172,7 +172,9 @@ def load_dict(cls, metadata: Dict): obj._is_compiled = metadata["_is_compiled"] obj.input_quantizers = metadata["input_quantizers"] obj.framework = metadata["framework"] - obj._tree_inference, obj.output_quantizers, obj.onnx_model_ = tree_to_numpy( + obj.onnx_model_ = metadata["onnx_model_"] + obj.output_quantizers = metadata["output_quantizers"] + obj._tree_inference, _, _ = tree_to_numpy( obj.sklearn_model, numpy.zeros((len(obj.input_quantizers),))[None, ...], framework=obj.framework, @@ -399,7 +401,9 @@ def load_dict(cls, metadata: Dict): obj._is_compiled = metadata["_is_compiled"] obj.input_quantizers = metadata["input_quantizers"] obj.framework = metadata["framework"] - obj._tree_inference, obj.output_quantizers, obj.onnx_model_ = tree_to_numpy( + obj.onnx_model_ = metadata["onnx_model_"] + obj.output_quantizers = metadata["output_quantizers"] + obj._tree_inference, _, _ = tree_to_numpy( obj.sklearn_model, numpy.zeros((len(obj.input_quantizers),))[None, ...], framework=obj.framework, diff --git a/tests/sklearn/test_sklearn_models.py b/tests/sklearn/test_sklearn_models.py index 62662d3f0..adafc8d25 100644 --- a/tests/sklearn/test_sklearn_models.py +++ b/tests/sklearn/test_sklearn_models.py @@ -1175,7 +1175,7 @@ def check_rounding_consistency( with warnings.catch_warnings(): warnings.simplefilter("ignore", category=DeprecationWarning) - model.disable_rounding() + model.use_rounding = False fit_and_compile(model, x, y) From 78f3185a63490216a7e4415ccbb28ffdfd97b064 Mon Sep 17 00:00:00 2001 From: kcelia Date: Tue, 5 Dec 2023 10:48:39 +0100 Subject: [PATCH 18/73] chore: update --- src/concrete/ml/sklearn/rf.py | 14 +++++-- src/concrete/ml/sklearn/tree.py | 14 +++++-- src/concrete/ml/sklearn/xgb.py | 14 +++++-- tests/sklearn/test_sklearn_models.py | 60 ++++++++++++++++++---------- 4 files changed, 70 insertions(+), 32 deletions(-) diff --git a/src/concrete/ml/sklearn/rf.py b/src/concrete/ml/sklearn/rf.py index 72ed77a01..5c87c021f 100644 --- a/src/concrete/ml/sklearn/rf.py +++ b/src/concrete/ml/sklearn/rf.py @@ -84,6 +84,7 @@ def dump_dict(self) -> Dict[str, Any]: metadata["onnx_model_"] = self.onnx_model_ metadata["framework"] = self.framework metadata["post_processing_params"] = self.post_processing_params + metadata["_use_rounding"] = self._use_rounding # Scikit-Learn metadata["n_estimators"] = self.n_estimators @@ -120,12 +121,14 @@ def load_dict(cls, metadata: Dict): obj.framework = metadata["framework"] obj.onnx_model_ = metadata["onnx_model_"] obj.output_quantizers = metadata["output_quantizers"] - obj._tree_inference, _, _ = tree_to_numpy( + obj._use_rounding = metadata["_use_rounding"] + obj._tree_inference = tree_to_numpy( obj.sklearn_model, numpy.zeros((len(obj.input_quantizers),))[None, ...], + use_rounding=obj._use_rounding, framework=obj.framework, output_n_bits=obj.n_bits, - ) + )[0] obj.post_processing_params = metadata["post_processing_params"] # Scikit-Learn @@ -219,6 +222,7 @@ def dump_dict(self) -> Dict[str, Any]: metadata["onnx_model_"] = self.onnx_model_ metadata["framework"] = self.framework metadata["post_processing_params"] = self.post_processing_params + metadata["_use_rounding"] = self._use_rounding # Scikit-Learn metadata["n_estimators"] = self.n_estimators @@ -255,12 +259,14 @@ def load_dict(cls, metadata: Dict): obj.framework = metadata["framework"] obj.onnx_model_ = metadata["onnx_model_"] obj.output_quantizers = metadata["output_quantizers"] - obj._tree_inference, _, _ = tree_to_numpy( + obj._use_rounding = metadata["_use_rounding"] + obj._tree_inference = tree_to_numpy( obj.sklearn_model, numpy.zeros((len(obj.input_quantizers),))[None, ...], + use_rounding=obj._use_rounding, framework=obj.framework, output_n_bits=obj.n_bits, - ) + )[0] obj.post_processing_params = metadata["post_processing_params"] # Scikit-Learn diff --git a/src/concrete/ml/sklearn/tree.py b/src/concrete/ml/sklearn/tree.py index 9b6b6c3f0..15f5de749 100644 --- a/src/concrete/ml/sklearn/tree.py +++ b/src/concrete/ml/sklearn/tree.py @@ -84,6 +84,7 @@ def dump_dict(self) -> Dict[str, Any]: metadata["onnx_model_"] = self.onnx_model_ metadata["framework"] = self.framework metadata["post_processing_params"] = self.post_processing_params + metadata["_use_rounding"] = self._use_rounding # Scikit-Learn metadata["criterion"] = self.criterion @@ -115,12 +116,14 @@ def load_dict(cls, metadata: Dict): obj.framework = metadata["framework"] obj.onnx_model_ = metadata["onnx_model_"] obj.output_quantizers = metadata["output_quantizers"] - obj._tree_inference, _, _ = tree_to_numpy( + obj._use_rounding = metadata["_use_rounding"] + obj._tree_inference = tree_to_numpy( obj.sklearn_model, numpy.zeros((len(obj.input_quantizers),))[None, ...], + use_rounding=obj._use_rounding, framework=obj.framework, output_n_bits=obj.n_bits, - ) + )[0] obj.post_processing_params = metadata["post_processing_params"] # Scikit-Learn @@ -208,6 +211,7 @@ def dump_dict(self) -> Dict[str, Any]: metadata["onnx_model_"] = self.onnx_model_ metadata["framework"] = self.framework metadata["post_processing_params"] = self.post_processing_params + metadata["_use_rounding"] = self._use_rounding # Scikit-Learn metadata["criterion"] = self.criterion @@ -238,12 +242,14 @@ def load_dict(cls, metadata: Dict): obj.framework = metadata["framework"] obj.onnx_model_ = metadata["onnx_model_"] obj.output_quantizers = metadata["output_quantizers"] - obj._tree_inference, _, _ = tree_to_numpy( + obj._use_rounding = metadata["_use_rounding"] + obj._tree_inference = tree_to_numpy( obj.sklearn_model, numpy.zeros((len(obj.input_quantizers),))[None, ...], + use_rounding=obj._use_rounding, framework=obj.framework, output_n_bits=obj.n_bits, - ) + )[0] obj.post_processing_params = metadata["post_processing_params"] # Scikit-Learn diff --git a/src/concrete/ml/sklearn/xgb.py b/src/concrete/ml/sklearn/xgb.py index b707409a8..eab922e43 100644 --- a/src/concrete/ml/sklearn/xgb.py +++ b/src/concrete/ml/sklearn/xgb.py @@ -125,6 +125,7 @@ def dump_dict(self) -> Dict[str, Any]: metadata["onnx_model_"] = self.onnx_model_ metadata["framework"] = self.framework metadata["post_processing_params"] = self.post_processing_params + metadata["_use_rounding"] = self._use_rounding # XGBoost metadata["max_depth"] = self.max_depth @@ -174,12 +175,14 @@ def load_dict(cls, metadata: Dict): obj.framework = metadata["framework"] obj.onnx_model_ = metadata["onnx_model_"] obj.output_quantizers = metadata["output_quantizers"] - obj._tree_inference, _, _ = tree_to_numpy( + obj._use_rounding = metadata["_use_rounding"] + obj._tree_inference = tree_to_numpy( obj.sklearn_model, numpy.zeros((len(obj.input_quantizers),))[None, ...], + use_rounding=obj._use_rounding, framework=obj.framework, output_n_bits=obj.n_bits, - ) + )[0] obj.post_processing_params = metadata["post_processing_params"] # XGBoost @@ -354,6 +357,7 @@ def dump_dict(self) -> Dict[str, Any]: metadata["onnx_model_"] = self.onnx_model_ metadata["framework"] = self.framework metadata["post_processing_params"] = self.post_processing_params + metadata["_use_rounding"] = self._use_rounding # XGBoost metadata["max_depth"] = self.max_depth @@ -403,12 +407,14 @@ def load_dict(cls, metadata: Dict): obj.framework = metadata["framework"] obj.onnx_model_ = metadata["onnx_model_"] obj.output_quantizers = metadata["output_quantizers"] - obj._tree_inference, _, _ = tree_to_numpy( + obj._use_rounding = metadata["_use_rounding"] + obj._tree_inference = tree_to_numpy( obj.sklearn_model, numpy.zeros((len(obj.input_quantizers),))[None, ...], + use_rounding=obj._use_rounding, framework=obj.framework, output_n_bits=obj.n_bits, - ) + )[0] obj.post_processing_params = metadata["post_processing_params"] # XGBoost diff --git a/tests/sklearn/test_sklearn_models.py b/tests/sklearn/test_sklearn_models.py index adafc8d25..b7d9ea552 100644 --- a/tests/sklearn/test_sklearn_models.py +++ b/tests/sklearn/test_sklearn_models.py @@ -23,7 +23,6 @@ import copy import json -import random import tempfile # pylint: disable=too-many-lines, too-many-arguments @@ -163,6 +162,17 @@ def fit_and_compile(model, x, y): model.compile(x) +def get_random_samples(x, n_sample): + """Selects `n_sample` random elements from a 2D NumPy array.""" + + # Sanity checks + assert 0 < n_sample < x.shape[0] + assert len(x.shape) == 2 + + random_rows_indices = numpy.random.choice(x.shape[0], size=n_sample, replace=False) + return x[random_rows_indices] + + def check_correctness_with_sklearn( model_class, x, @@ -1159,17 +1169,26 @@ def check_rounding_consistency( y, predict_method, metric, + is_weekly_option, ): """Test that Concrete ML witout and with rounding are 'equivalent'.""" - random_int = random.randint(0, x.shape[0] - 1) + # Run the test with more samples during weekly CIs + if is_weekly_option: + fhe_samples = 5 + else: + fhe_samples = 1 + + # Check that separated inference steps (encrypt, run, decrypt, post_processing, ...) are + # equivalent to built-in methods (predict, predict_proba, ...) + fhe_test = get_random_samples(x, fhe_samples) # Fit and compile with rounding enabled fit_and_compile(model, x, y) rounded_predict_quantized = predict_method(x, fhe="disable") rounded_predict_simulate = predict_method(x, fhe="simulate") - rounded_predict_fhe = predict_method(x[random_int, None], fhe="execute") + rounded_predict_fhe = predict_method(fhe_test, fhe="execute") # Fit and compile without rounding @@ -1181,7 +1200,7 @@ def check_rounding_consistency( not_rounded_predict_quantized = predict_method(x, fhe="disable") not_rounded_predict_simulate = predict_method(x, fhe="simulate") - not_rounded_predict_fhe = predict_method(x[random_int, None], fhe="execute") + not_rounded_predict_fhe = predict_method(fhe_test, fhe="execute") metric(rounded_predict_quantized, not_rounded_predict_quantized) metric(rounded_predict_simulate, not_rounded_predict_simulate) @@ -1540,7 +1559,8 @@ def test_predict_correctness( print(f"Check prediction correctness for {fhe_samples} samples.") # Check prediction correctness between quantized clear and FHE simulation or execution - check_is_good_execution_for_cml_vs_circuit(x[:fhe_samples], model=model, simulate=simulate) + fhe_test = get_random_samples(x, fhe_samples) + check_is_good_execution_for_cml_vs_circuit(fhe_test, model=model, simulate=simulate) @pytest.mark.parametrize("model_class, parameters", MODELS_AND_DATASETS) @@ -1602,7 +1622,8 @@ def test_separated_inference( # Check that separated inference steps (encrypt, run, decrypt, post_processing, ...) are # equivalent to built-in methods (predict, predict_proba, ...) - check_separated_inference(model, fhe_circuit, x[:fhe_samples], check_float_array_equal) + fhe_test = get_random_samples(x, fhe_samples) + check_separated_inference(model, fhe_circuit, fhe_test, check_float_array_equal) @pytest.mark.parametrize("model_class, parameters", UNIQUE_MODELS_AND_DATASETS) @@ -1839,19 +1860,18 @@ def test_rounding_consistency( # Check `predict_proba` for classifiers if is_classifier_or_partial_classifier(model): - check_rounding_consistency( - model, - x, - y, - predict_method=model.predict_proba, - metric=check_r2_score, - ) + predict_method = model.predict_proba + metric = check_r2_score else: # Check `predict` for regressors - check_rounding_consistency( - model, - x, - y, - predict_method=model.predict, - metric=check_accuracy, - ) + predict_method = model.predict + metric = check_accuracy + + check_rounding_consistency( + model, + x, + y, + predict_method, + metric, + is_weekly_option, + ) From 4f9859b645e378eaa6e4261de59ecded8bb44ae4 Mon Sep 17 00:00:00 2001 From: kcelia Date: Tue, 5 Dec 2023 13:09:35 +0100 Subject: [PATCH 19/73] chore: coverage --- src/concrete/ml/sklearn/tree_to_numpy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/concrete/ml/sklearn/tree_to_numpy.py b/src/concrete/ml/sklearn/tree_to_numpy.py index 87650c2ad..f307209bc 100644 --- a/src/concrete/ml/sklearn/tree_to_numpy.py +++ b/src/concrete/ml/sklearn/tree_to_numpy.py @@ -399,9 +399,9 @@ def get_lsbs_to_remove(array: numpy.ndarray) -> int: while lsbs_to_remove > 0: half = 1 << (lsbs_to_remove - 1) if get_bitwidth(array - half) <= initial_bitwidth: - lsbs_to_remove -= 1 + lsbs_to_remove -= 1 # pragma: no cover else: - break + break # pragma: no cover return lsbs_to_remove From 251e91d688036c5c200ffddf2049c1215c8ac25d Mon Sep 17 00:00:00 2001 From: kcelia Date: Tue, 5 Dec 2023 14:35:36 +0100 Subject: [PATCH 20/73] chore: update lsbs 2 --- src/concrete/ml/onnx/convert.py | 20 +++--- src/concrete/ml/onnx/onnx_utils.py | 14 +++-- src/concrete/ml/onnx/ops_impl.py | 9 ++- src/concrete/ml/pytest/utils.py | 23 +++++++ src/concrete/ml/sklearn/base.py | 8 +-- src/concrete/ml/sklearn/tree_to_numpy.py | 79 ++++++++++++++---------- tests/sklearn/test_sklearn_models.py | 22 ++----- 7 files changed, 105 insertions(+), 70 deletions(-) diff --git a/src/concrete/ml/onnx/convert.py b/src/concrete/ml/onnx/convert.py index bc19f2f52..629a3ac74 100644 --- a/src/concrete/ml/onnx/convert.py +++ b/src/concrete/ml/onnx/convert.py @@ -2,7 +2,7 @@ import tempfile from pathlib import Path -from typing import Callable, List, Tuple, Union +from typing import Callable, Optional, Tuple, Union import numpy import onnx @@ -145,7 +145,7 @@ def get_equivalent_numpy_forward_from_torch( output_onnx_file_path.unlink() equivalent_numpy_forward, equivalent_onnx_model = get_equivalent_numpy_forward_from_onnx( - equivalent_onnx_model, check_model=True + equivalent_onnx_model ) with output_onnx_file_path.open("wb") as file: file.write(equivalent_onnx_model.SerializeToString()) @@ -158,8 +158,8 @@ def get_equivalent_numpy_forward_from_torch( def get_equivalent_numpy_forward_from_onnx( onnx_model: onnx.ModelProto, - lsbs_to_remove: List[int] = None, check_model: bool = True, + lsbs_to_remove: Optional[Tuple[int, int]] = None, ) -> Tuple[Callable[..., Tuple[numpy.ndarray, ...]], onnx.ModelProto]: """Get the numpy equivalent forward of the provided ONNX model. @@ -168,10 +168,11 @@ def get_equivalent_numpy_forward_from_onnx( forward. check_model (bool): set to True to run the onnx checker on the model. Defaults to True. - lsbs_to_remove (List[int]): Contains the values of the least significant bits to - remove during tree traversal. The first value pertains to the first comparison (either - "less" or "less_or_equal"), and the second value relates to the "Equal" comparison - operation. Default value set to None, when the rounding feature is not used. + lsbs_to_remove (Optional[Tuple[int, int]]): Contains the values of the least significant + bits to remove during tree traversal only. The first value pertains to the first + comparison (either "less" or "less_or_equal"), and the second value relates to the + "Equal" comparison operation. Default value set to None, when the rounding feature + is not used. Raises: ValueError: Raised if there is an unsupported ONNX operator required to convert the torch @@ -181,6 +182,9 @@ def get_equivalent_numpy_forward_from_onnx( Callable[..., Tuple[numpy.ndarray, ...]]: The function that will execute the equivalent numpy function. """ + + # All onnx models should be checked, "check_model" parameter must be removed + # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4157 if check_model: checker.check_model(onnx_model) @@ -195,11 +199,13 @@ def get_equivalent_numpy_forward_from_onnx( "eliminate_unused_initializer", ] equivalent_onnx_model = onnxoptimizer.optimize(onnx_model, onnx_passes) + checker.check_model(equivalent_onnx_model) # Custom optimization # ONNX optimizer does not optimize Mat-Mult + Bias pattern into GEMM if the input isn't a matrix # We manually do the optimization for this case equivalent_onnx_model = fuse_matmul_bias_to_gemm(equivalent_onnx_model) + checker.check_model(equivalent_onnx_model) # Check supported operators required_onnx_operators = set(get_op_type(node) for node in equivalent_onnx_model.graph.node) diff --git a/src/concrete/ml/onnx/onnx_utils.py b/src/concrete/ml/onnx/onnx_utils.py index 1bcb40c4f..09965121c 100644 --- a/src/concrete/ml/onnx/onnx_utils.py +++ b/src/concrete/ml/onnx/onnx_utils.py @@ -213,7 +213,7 @@ # Original file: # https://github.com/google/jax/blob/f6d329b2d9b5f83c6a59e5739aa1ca8d4d1ffa1c/examples/onnx2xla.py -from typing import Any, Callable, Dict, List, Optional, Tuple +from typing import Any, Callable, Dict, Optional, Tuple import numpy import onnx @@ -443,17 +443,17 @@ def get_op_type(node): def execute_onnx_with_numpy( graph: onnx.GraphProto, - lsbs_to_remove: Optional[List[int]], + lsbs_to_remove: Optional[Tuple[int, int]], *inputs: numpy.ndarray, ) -> Tuple[numpy.ndarray, ...]: """Execute the provided ONNX graph on the given inputs. Args: graph (onnx.GraphProto): The ONNX graph to execute. - lsbs_to_remove (Optional[List[int]]): Contains the values of the least significant bits to - remove during tree traversal. The first value pertains to the first comparison (either - "less" or "less_or_equal"), and the second value relates to the "Equal" comparison - operation. Default value set to None, when the rounding feature is not used. + lsbs_to_remove (Optional[Tuple[int, int]]): Contains the values of the least significant + bits to remove during tree traversal. The first value pertains to the first comparison + (either "less" or "less_or_equal"), and the second value relates to the "Equal" + comparison operation. Default value set to None, when the rounding feature is not used. *inputs: The inputs of the graph. Returns: @@ -470,6 +470,8 @@ def execute_onnx_with_numpy( curr_inputs = (node_results[input_name] for input_name in node.input) attributes = {attribute.name: get_attribute(attribute) for attribute in node.attribute} + # For trees, the first LSB refers to `Less` or `LessOrEqual` comparisons and the second + # LSB refers to `Equal` comparison if lsbs_to_remove is not None and node.op_type in SUPPORTED_ROUNDED_OPERATIONS: attributes["lsbs_to_remove"] = ( lsbs_to_remove[0] if node.op_type != "Equal" else lsbs_to_remove[1] diff --git a/src/concrete/ml/onnx/ops_impl.py b/src/concrete/ml/onnx/ops_impl.py index 1e9fdcbd4..62368786d 100644 --- a/src/concrete/ml/onnx/ops_impl.py +++ b/src/concrete/ml/onnx/ops_impl.py @@ -897,7 +897,7 @@ def numpy_equal( Args: x (numpy.ndarray): Input tensor y (numpy.ndarray): Input tensor - lsbs_to_remove (Optional[int]): The number of the least significant bits to remove. + lsbs_to_remove (Optional[int]): Number of the least significant bits to remove. Returns: Tuple[numpy.ndarray]: Output tensor @@ -905,6 +905,7 @@ def numpy_equal( # In the case of trees, x == y <=> x <= y or x < y - 1, because y is the max sum. if lsbs_to_remove is not None and lsbs_to_remove > 0: + print("equal", lsbs_to_remove) return rounded_comparison(y, x, lsbs_to_remove, operation=lambda x: x >= 0) # Else, default numpy_equal operator @@ -1034,13 +1035,14 @@ def numpy_less( Args: x (numpy.ndarray): Input tensor y (numpy.ndarray): Input tensor - lsbs_to_remove (Optional[int]): The number of the least significant bits to remove + lsbs_to_remove (Optional[int]): Number of the least significant bits to remove Returns: Tuple[numpy.ndarray]: Output tensor """ if lsbs_to_remove is not None and lsbs_to_remove > 0: + print("less", lsbs_to_remove) return rounded_comparison(x, y, lsbs_to_remove, operation=lambda x: x < 0) # Else, default numpy_less operator @@ -1079,13 +1081,14 @@ def numpy_less_or_equal( Args: x (numpy.ndarray): Input tensor y (numpy.ndarray): Input tensor - lsbs_to_remove (Optional[int]): The number of the least significant bits to remove + lsbs_to_remove (Optional[int]): Number of the least significant bits to remove Returns: Tuple[numpy.ndarray]: Output tensor """ if lsbs_to_remove is not None and lsbs_to_remove > 0: + print("equal or less", lsbs_to_remove) return rounded_comparison(y, x, lsbs_to_remove, operation=lambda x: x >= 0) # Else, default numpy_less_or_equal operator diff --git a/src/concrete/ml/pytest/utils.py b/src/concrete/ml/pytest/utils.py index 2c20d2f4b..53153b9a1 100644 --- a/src/concrete/ml/pytest/utils.py +++ b/src/concrete/ml/pytest/utils.py @@ -661,3 +661,26 @@ def check_serialization( "Loaded object (from file) is not equal to the initial one, using equal method " f"{equal_method}." ) + + +def get_random_samples(x: numpy.ndarray, n_sample: int) -> numpy.ndarray: + """Select `n_sample` random elements from a 2D NumPy array. + + Args: + x (numpy.ndarray): The 2D NumPy array from which random rows will be selected. + n_sample (int): The number of rows to randomly select. + + Returns: + numpy.ndarray: A new 2D NumPy array containing the randomly selected rows. + + Raises: + AssertionError: If `n_sample` is not within the range (0, x.shape[0]) or + if `x` is not a 2D array. + """ + + # Sanity checks + assert 0 < n_sample < x.shape[0] + assert len(x.shape) == 2 + + random_rows_indices = numpy.random.choice(x.shape[0], size=n_sample, replace=False) + return x[random_rows_indices] diff --git a/src/concrete/ml/sklearn/base.py b/src/concrete/ml/sklearn/base.py index e857d7f9d..d82c04b56 100644 --- a/src/concrete/ml/sklearn/base.py +++ b/src/concrete/ml/sklearn/base.py @@ -1283,7 +1283,7 @@ def __init__(self, n_bits: int): #: The model's inference function. Is None if the model is not fitted. self._tree_inference: Optional[Callable] = None - #: Set to `True` to enable the rounding feature, or `False` to disable it. + #: Boolean that indicates whether the rounding feature is enabled or not. self._use_rounding: bool = True BaseEstimator.__init__(self) @@ -1311,8 +1311,8 @@ def use_rounding(self, value: bool) -> None: if not value: warnings.simplefilter("always") warnings.warn( - "Using Concrete tree-based models without the `rounding feature` s deprecated. " - "Consider setting '_use_rounding' to `True` for improved speed in FHE computation " + "Using Concrete tree-based models without the `rounding feature` is deprecated. " + "Consider setting 'use_rounding' to `True` for making the FHE inference faster " "and key generation.", category=DeprecationWarning, stacklevel=2, @@ -1345,7 +1345,7 @@ def fit(self, X: Data, y: Target, **fit_parameters): # Convert the tree inference with Numpy operators self._tree_inference, self.output_quantizers, self.onnx_model_ = tree_to_numpy( - self.sklearn_model, # type: ignore[arg-type] + self.sklearn_model, q_X, use_rounding=self._use_rounding, framework=self.framework, diff --git a/src/concrete/ml/sklearn/tree_to_numpy.py b/src/concrete/ml/sklearn/tree_to_numpy.py index f307209bc..a528dd17c 100644 --- a/src/concrete/ml/sklearn/tree_to_numpy.py +++ b/src/concrete/ml/sklearn/tree_to_numpy.py @@ -28,6 +28,9 @@ # pylint: disable=too-many-branches +# Most significant bits to retain when applying rounding to the tree +MSB_TO_KEEP_FOR_TREES = 4 + def get_onnx_model(model: Callable, x: numpy.ndarray, framework: str) -> onnx.ModelProto: """Create ONNX model with Hummingbird convert method. @@ -294,17 +297,17 @@ def tree_values_preprocessing( # pylint: disable=too-many-locals def tree_to_numpy( model: Callable, - q_x: numpy.ndarray, + x: numpy.ndarray, framework: str, - use_rounding: bool = False, + use_rounding: Optional[bool] = False, output_n_bits: int = MAX_BITWIDTH_BACKWARD_COMPATIBLE, ) -> Tuple[Callable, List[UniformQuantizer], onnx.ModelProto]: """Convert the tree inference to a numpy functions using Hummingbird. Args: model (Callable): The tree model to convert. - q_x (numpy.ndarray): The quantized input data. - use_rounding (bool): Use rounding feature or not. + x (numpy.ndarray): The input data. + use_rounding (Optional[bool]): Use rounding feature or not. framework (str): The framework from which the ONNX model is generated. (options: 'xgboost', 'sklearn') output_n_bits (int): The number of bits of the output. Default to 8. @@ -317,7 +320,7 @@ def tree_to_numpy( # mypy assert output_n_bits is not None - lsbs_to_remove: Optional[List[int]] = None + lsbs_to_remove: Optional[Tuple[int, int]] = None assert_true( framework in ["xgboost", "sklearn"], @@ -325,13 +328,16 @@ def tree_to_numpy( ) # Execute with 1 example for efficiency in large data scenarios to prevent slowdown - onnx_model = get_onnx_model(model, q_x[:1], framework) + onnx_model = get_onnx_model(model, x[:1], framework) - # compute LSB to remove in stage 1 and stage 2 + # Compute for tree-based models the LSB to remove in stage 1 and stage 2 if use_rounding: # First LSB refers to Less or LessOrEqual comparisons # Second LSB refers to Equal comparison - lsbs_to_remove = compute_lsb_to_remove_for_trees(onnx_model, q_x) + lsbs_to_remove = _compute_lsb_to_remove_for_trees(onnx_model, x) + + # mypy + assert len(lsbs_to_remove) == 2 # Get the expected number of ONNX outputs in the sklearn model. expected_number_of_outputs = 1 if is_regressor_or_partial_regressor(model) else 2 @@ -345,14 +351,18 @@ def tree_to_numpy( # but also rounding the threshold such that they are now integers q_y = tree_values_preprocessing(onnx_model, framework, output_n_bits) - _tree_inference, onnx_model = get_equivalent_numpy_forward_from_onnx(onnx_model, lsbs_to_remove) + _tree_inference, onnx_model = get_equivalent_numpy_forward_from_onnx( + onnx_model, lsbs_to_remove=lsbs_to_remove + ) return (_tree_inference, [q_y.quantizer], onnx_model) # Remove this function once the truncate feature is released # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4143 -def compute_lsb_to_remove_for_trees(onnx_model: onnx.ModelProto, q_x: numpy.ndarray) -> List[int]: +def _compute_lsb_to_remove_for_trees( + onnx_model: onnx.ModelProto, q_x: numpy.ndarray +) -> Tuple[int, int]: """Compute the LSB to remove for the comparison operators in the trees. Referring to this paper: https://arxiv.org/pdf/2010.04804.pdf, there are @@ -364,7 +374,7 @@ def compute_lsb_to_remove_for_trees(onnx_model: onnx.ModelProto, q_x: numpy.ndar q_x (numpy.ndarray): The quantized inputs Returns: - List: the number of LSB to remove for level 1 and level 2 + Tuple[int, int]: the number of LSB to remove for level 1 and level 2 """ def get_bitwidth(array: numpy.ndarray) -> int: @@ -378,6 +388,7 @@ def get_bitwidth(array: numpy.ndarray) -> int: """ max_val = numpy.max(numpy.abs(array)) + # + 1 is added to include the sign bit bitwidth = math.ceil(math.log2(max_val + 1)) + 1 return bitwidth @@ -386,24 +397,27 @@ def get_lsbs_to_remove(array: numpy.ndarray) -> int: """Update the number of LSBs to remove based on overflow detection. Args: - array (umpy.ndarray): The array for which the bitwidth needs to be checked. + array (numpy.ndarray): The array for which the bitwidth needs to be checked. Returns: int: The updated LSB to remove. """ initial_bitwidth = get_bitwidth(array) - # The subtraction operation increases precision by 1 or 2 bits - lsbs_to_remove = initial_bitwidth + 2 - while lsbs_to_remove > 0: - half = 1 << (lsbs_to_remove - 1) - if get_bitwidth(array - half) <= initial_bitwidth: - lsbs_to_remove -= 1 # pragma: no cover - else: - break # pragma: no cover + if initial_bitwidth - MSB_TO_KEEP_FOR_TREES > 0: + lsbs_to_remove = initial_bitwidth + + while lsbs_to_remove > 0: + half = 1 << (lsbs_to_remove - 1) - return lsbs_to_remove + # The subtraction operation may increase or decrease the precision by 1 or 2 bits + if get_bitwidth(array - half) <= initial_bitwidth: + lsbs_to_remove -= 1 # pragma: no cover + else: + return lsbs_to_remove + + return 0 quant_params = { onnx_init.name: numpy_helper.to_array(onnx_init) @@ -419,11 +433,13 @@ def get_lsbs_to_remove(array: numpy.ndarray) -> int: # shape: (nodes, features) or (trees * nodes, features) mat_1 = quant_params[key_mat_1] + # shape: (nodes, 1) or (trees * nodes, 1) bias_1 = quant_params[key_bias_1] # shape: (trees, leaves, nodes) mat_2 = quant_params[key_mat_2] + # shape: (leaves, 1) or (trees * leaves, 1) bias_2 = quant_params[key_bias_2] @@ -439,28 +455,25 @@ def get_lsbs_to_remove(array: numpy.ndarray) -> int: # If operator is `<`, np.less(x, y) is equivalent to: # round_bit_pattern((x - y) - half, lsbs_to_remove=r) < 0. - # Therfore, stage_1 = (q_x @ mat_1.transpose(0, 2, 1)) - bias_1 + # Therefore, stage_1 = (q_x @ mat_1.transpose(0, 2, 1)) - bias_1 if "Less" in required_onnx_operators: stage_1 = (q_x @ mat_1.transpose(0, 2, 1)) - bias_1 + matrix_q = stage_1 < 0 + + # Else, if operator is `<=`, np.less_equal(x, y) is equivalent to: + # round_bit_pattern((y - x) - half, lsbs_to_remove=r) >= 0. + # Therefore, stage_1 = bias_1 - (q_x @ mat_1.transpose(0, 2, 1)) elif "LessOrEqual" in required_onnx_operators: - # If operator is `<=`, np.less_equal(x, y) is equivalent to: - # round_bit_pattern((y - x) - half, lsbs_to_remove=r) >= 0. - # Therfore, stage_1 = bias_1 - (q_x @ mat_1.transpose(0, 2, 1)) stage_1 = bias_1 - (q_x @ mat_1.transpose(0, 2, 1)) + matrix_q = stage_1 >= 0 lsbs_to_remove_stage_1 = get_lsbs_to_remove(stage_1) - # The matrix I, as referenced in this paper: - # https://arxiv.org/pdf/2010.04804.pdf, results from the condition: - # X.A < B and consists exclusively of binary elements, 1 and 0. - # Given this assumption, we randomly generate it. - matrix_q = numpy.random.randint(0, 2, size=(stage_1.shape)) - # If operator is `==`, np.equal(x, y) is equivalent to: # round_bit_pattern((x - y) - half, lsbs_to_remove=r) >= 0. - # Therfore, stage_2 = bias_1 - (q_x @ mat_2.transpose(0, 2, 1)) + # Therefore, stage_2 = bias_1 - (q_x @ mat_2.transpose(0, 2, 1)) stage_2 = ((bias_2 - matrix_q @ mat_2.transpose(0, 2, 1))).sum(axis=0) lsbs_to_remove_stage_2 = get_lsbs_to_remove(stage_2) - return [lsbs_to_remove_stage_1, lsbs_to_remove_stage_2] + return (lsbs_to_remove_stage_1, lsbs_to_remove_stage_2) diff --git a/tests/sklearn/test_sklearn_models.py b/tests/sklearn/test_sklearn_models.py index b7d9ea552..7b79a4142 100644 --- a/tests/sklearn/test_sklearn_models.py +++ b/tests/sklearn/test_sklearn_models.py @@ -54,6 +54,7 @@ from concrete.ml.pytest.utils import ( MODELS_AND_DATASETS, UNIQUE_MODELS_AND_DATASETS, + get_random_samples, get_sklearn_all_models_and_datasets, get_sklearn_linear_models_and_datasets, get_sklearn_neighbors_models_and_datasets, @@ -162,17 +163,6 @@ def fit_and_compile(model, x, y): model.compile(x) -def get_random_samples(x, n_sample): - """Selects `n_sample` random elements from a 2D NumPy array.""" - - # Sanity checks - assert 0 < n_sample < x.shape[0] - assert len(x.shape) == 2 - - random_rows_indices = numpy.random.choice(x.shape[0], size=n_sample, replace=False) - return x[random_rows_indices] - - def check_correctness_with_sklearn( model_class, x, @@ -434,7 +424,7 @@ def check_serialization_dump_load(model, x, use_dump_method): serialized_model_dict["serialized_value"].pop(attribute, None) re_serialized_model_dict["serialized_value"].pop(attribute, None) - # Check if the graphs are similar + # Check if the serialized models are identical assert serialized_model_dict == re_serialized_model_dict # Check that the predictions made by both model are identical @@ -487,7 +477,7 @@ def check_serialization_dumps_loads(model, x, use_dump_method): serialized_model_dict["serialized_value"].pop(attribute, None) re_serialized_model_dict["serialized_value"].pop(attribute, None) - # Check if the graphs are similar + # Check if the serialized models are identical assert serialized_model_dict == re_serialized_model_dict # Check that the predictions made by both model are identical @@ -1171,7 +1161,7 @@ def check_rounding_consistency( metric, is_weekly_option, ): - """Test that Concrete ML witout and with rounding are 'equivalent'.""" + """Test that Concrete ML without and with rounding are 'equivalent'.""" # Run the test with more samples during weekly CIs if is_weekly_option: @@ -1179,8 +1169,6 @@ def check_rounding_consistency( else: fhe_samples = 1 - # Check that separated inference steps (encrypt, run, decrypt, post_processing, ...) are - # equivalent to built-in methods (predict, predict_proba, ...) fhe_test = get_random_samples(x, fhe_samples) # Fit and compile with rounding enabled @@ -1850,7 +1838,7 @@ def test_rounding_consistency( is_weekly_option, verbose=True, ): - """Test that Concrete ML witout and with rounding are 'equivalent'.""" + """Test that Concrete ML without and with rounding are 'equivalent'.""" if verbose: print("Run check_rounding_consistency") From 4d68c957ea8be8ab475e3ee167b63044feb84487 Mon Sep 17 00:00:00 2001 From: kcelia Date: Wed, 6 Dec 2023 14:51:31 +0100 Subject: [PATCH 21/73] chore: update --- src/concrete/ml/onnx/convert.py | 29 ++++++++++++++------ src/concrete/ml/onnx/onnx_utils.py | 19 +++++++------ src/concrete/ml/onnx/ops_impl.py | 32 +++++++++++----------- src/concrete/ml/sklearn/tree_to_numpy.py | 35 ++++++++++++------------ 4 files changed, 65 insertions(+), 50 deletions(-) diff --git a/src/concrete/ml/onnx/convert.py b/src/concrete/ml/onnx/convert.py index 629a3ac74..93a34da2d 100644 --- a/src/concrete/ml/onnx/convert.py +++ b/src/concrete/ml/onnx/convert.py @@ -1,6 +1,7 @@ """ONNX conversion related code.""" import tempfile +import warnings from pathlib import Path from typing import Callable, Optional, Tuple, Union @@ -159,7 +160,7 @@ def get_equivalent_numpy_forward_from_torch( def get_equivalent_numpy_forward_from_onnx( onnx_model: onnx.ModelProto, check_model: bool = True, - lsbs_to_remove: Optional[Tuple[int, int]] = None, + lsbs_to_remove_for_trees: Optional[Tuple[int, int]] = None, ) -> Tuple[Callable[..., Tuple[numpy.ndarray, ...]], onnx.ModelProto]: """Get the numpy equivalent forward of the provided ONNX model. @@ -168,11 +169,11 @@ def get_equivalent_numpy_forward_from_onnx( forward. check_model (bool): set to True to run the onnx checker on the model. Defaults to True. - lsbs_to_remove (Optional[Tuple[int, int]]): Contains the values of the least significant - bits to remove during tree traversal only. The first value pertains to the first - comparison (either "less" or "less_or_equal"), and the second value relates to the - "Equal" comparison operation. Default value set to None, when the rounding feature - is not used. + lsbs_to_remove_for_trees (Optional[Tuple[int, int]]): This parameter is exclusively used for + optimizing tree-based models. It contains the values of the least significant bits to + remove during the tree traversal, where the first value refers to the first comparison + (either "less" or "less_or_equal"), while the second value refers to the "Equal" + comparison operation. Default to None, as it is not applicable to other types of models. Raises: ValueError: Raised if there is an unsupported ONNX operator required to convert the torch @@ -185,8 +186,16 @@ def get_equivalent_numpy_forward_from_onnx( # All onnx models should be checked, "check_model" parameter must be removed # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4157 - if check_model: - checker.check_model(onnx_model) + if not check_model: + warnings.simplefilter("always") + warnings.warn( + "`check_model` parameter should always be set to True, to ensure proper onnx model " + "verification and avoid bypassing essential onnx model validation checks.", + category=UserWarning, + stacklevel=2, + ) + + checker.check_model(onnx_model) # Optimize ONNX graph # List of all currently supported onnx optimizer passes @@ -219,5 +228,7 @@ def get_equivalent_numpy_forward_from_onnx( # Return lambda of numpy equivalent of onnx execution return ( - lambda *args: execute_onnx_with_numpy(equivalent_onnx_model.graph, lsbs_to_remove, *args) + lambda *args: execute_onnx_with_numpy( + equivalent_onnx_model.graph, lsbs_to_remove_for_trees, *args + ) ), equivalent_onnx_model diff --git a/src/concrete/ml/onnx/onnx_utils.py b/src/concrete/ml/onnx/onnx_utils.py index 09965121c..b30305c26 100644 --- a/src/concrete/ml/onnx/onnx_utils.py +++ b/src/concrete/ml/onnx/onnx_utils.py @@ -443,17 +443,18 @@ def get_op_type(node): def execute_onnx_with_numpy( graph: onnx.GraphProto, - lsbs_to_remove: Optional[Tuple[int, int]], + lsbs_to_remove_for_trees: Optional[Tuple[int, int]], *inputs: numpy.ndarray, ) -> Tuple[numpy.ndarray, ...]: """Execute the provided ONNX graph on the given inputs. Args: graph (onnx.GraphProto): The ONNX graph to execute. - lsbs_to_remove (Optional[Tuple[int, int]]): Contains the values of the least significant - bits to remove during tree traversal. The first value pertains to the first comparison - (either "less" or "less_or_equal"), and the second value relates to the "Equal" - comparison operation. Default value set to None, when the rounding feature is not used. + lsbs_to_remove_for_trees (Optional[Tuple[int, int]]): This parameter is exclusively used for + optimizing tree-based models. It contains the values of the least significant bits to + remove during the tree traversal, where the first value refers to the first comparison + (either "less" or "less_or_equal"), while the second value refers to the "Equal" + comparison operation. Default to None, as it is not applicable to other types of models. *inputs: The inputs of the graph. Returns: @@ -472,9 +473,11 @@ def execute_onnx_with_numpy( # For trees, the first LSB refers to `Less` or `LessOrEqual` comparisons and the second # LSB refers to `Equal` comparison - if lsbs_to_remove is not None and node.op_type in SUPPORTED_ROUNDED_OPERATIONS: - attributes["lsbs_to_remove"] = ( - lsbs_to_remove[0] if node.op_type != "Equal" else lsbs_to_remove[1] + if lsbs_to_remove_for_trees is not None and node.op_type in SUPPORTED_ROUNDED_OPERATIONS: + attributes["lsbs_to_remove_for_trees"] = ( + lsbs_to_remove_for_trees[0] + if node.op_type != "Equal" + else lsbs_to_remove_for_trees[1] ) outputs = ONNX_OPS_TO_NUMPY_IMPL_BOOL[node.op_type](*curr_inputs, **attributes) diff --git a/src/concrete/ml/onnx/ops_impl.py b/src/concrete/ml/onnx/ops_impl.py index 62368786d..bdc6e3ce4 100644 --- a/src/concrete/ml/onnx/ops_impl.py +++ b/src/concrete/ml/onnx/ops_impl.py @@ -888,7 +888,7 @@ def numpy_equal( x: numpy.ndarray, y: numpy.ndarray, *, - lsbs_to_remove: Optional[int] = None, + lsbs_to_remove_for_trees: Optional[int] = None, ) -> Tuple[numpy.ndarray]: """Compute equal in numpy according to ONNX spec. @@ -897,16 +897,16 @@ def numpy_equal( Args: x (numpy.ndarray): Input tensor y (numpy.ndarray): Input tensor - lsbs_to_remove (Optional[int]): Number of the least significant bits to remove. + lsbs_to_remove_for_trees (Optional[int]): Number of the least significant bits to remove + for tree-based models only. Returns: Tuple[numpy.ndarray]: Output tensor """ - # In the case of trees, x == y <=> x <= y or x < y - 1, because y is the max sum. - if lsbs_to_remove is not None and lsbs_to_remove > 0: - print("equal", lsbs_to_remove) - return rounded_comparison(y, x, lsbs_to_remove, operation=lambda x: x >= 0) + # For tree-based models, x == y <=> x <= y or x < y - 1, because y is the max sum. + if lsbs_to_remove_for_trees is not None and lsbs_to_remove_for_trees > 0: + return rounded_comparison(y, x, lsbs_to_remove_for_trees, operation=lambda x: x >= 0) # Else, default numpy_equal operator return (numpy.equal(x, y),) @@ -1026,7 +1026,7 @@ def numpy_less( x: numpy.ndarray, y: numpy.ndarray, *, - lsbs_to_remove: Optional[int] = None, + lsbs_to_remove_for_trees: Optional[int] = None, ) -> Tuple[numpy.ndarray]: """Compute less in numpy according to ONNX spec. @@ -1035,15 +1035,15 @@ def numpy_less( Args: x (numpy.ndarray): Input tensor y (numpy.ndarray): Input tensor - lsbs_to_remove (Optional[int]): Number of the least significant bits to remove + lsbs_to_remove_for_trees (Optional[int]): Number of the least significant bits to remove + for tree-based models only. Returns: Tuple[numpy.ndarray]: Output tensor """ - if lsbs_to_remove is not None and lsbs_to_remove > 0: - print("less", lsbs_to_remove) - return rounded_comparison(x, y, lsbs_to_remove, operation=lambda x: x < 0) + if lsbs_to_remove_for_trees is not None and lsbs_to_remove_for_trees > 0: + return rounded_comparison(x, y, lsbs_to_remove_for_trees, operation=lambda x: x < 0) # Else, default numpy_less operator return (numpy.less(x, y),) @@ -1072,7 +1072,7 @@ def numpy_less_or_equal( x: numpy.ndarray, y: numpy.ndarray, *, - lsbs_to_remove: Optional[int] = None, + lsbs_to_remove_for_trees: Optional[int] = None, ) -> Tuple[numpy.ndarray]: """Compute less or equal in numpy according to ONNX spec. @@ -1081,15 +1081,15 @@ def numpy_less_or_equal( Args: x (numpy.ndarray): Input tensor y (numpy.ndarray): Input tensor - lsbs_to_remove (Optional[int]): Number of the least significant bits to remove + lsbs_to_remove_for_trees (Optional[int]): Number of the least significant bits to remove + for tree-based models only. Returns: Tuple[numpy.ndarray]: Output tensor """ - if lsbs_to_remove is not None and lsbs_to_remove > 0: - print("equal or less", lsbs_to_remove) - return rounded_comparison(y, x, lsbs_to_remove, operation=lambda x: x >= 0) + if lsbs_to_remove_for_trees is not None and lsbs_to_remove_for_trees > 0: + return rounded_comparison(y, x, lsbs_to_remove_for_trees, operation=lambda x: x >= 0) # Else, default numpy_less_or_equal operator return (numpy.less_equal(x, y),) diff --git a/src/concrete/ml/sklearn/tree_to_numpy.py b/src/concrete/ml/sklearn/tree_to_numpy.py index a528dd17c..ac5d96425 100644 --- a/src/concrete/ml/sklearn/tree_to_numpy.py +++ b/src/concrete/ml/sklearn/tree_to_numpy.py @@ -307,7 +307,8 @@ def tree_to_numpy( Args: model (Callable): The tree model to convert. x (numpy.ndarray): The input data. - use_rounding (Optional[bool]): Use rounding feature or not. + use_rounding (Optional[bool]): This parameter is exclusively used to tree-based models. + It determines whether the rounding feature is enabled or disabled. framework (str): The framework from which the ONNX model is generated. (options: 'xgboost', 'sklearn') output_n_bits (int): The number of bits of the output. Default to 8. @@ -320,7 +321,7 @@ def tree_to_numpy( # mypy assert output_n_bits is not None - lsbs_to_remove: Optional[Tuple[int, int]] = None + lsbs_to_remove_for_trees: Optional[Tuple[int, int]] = None assert_true( framework in ["xgboost", "sklearn"], @@ -334,10 +335,10 @@ def tree_to_numpy( if use_rounding: # First LSB refers to Less or LessOrEqual comparisons # Second LSB refers to Equal comparison - lsbs_to_remove = _compute_lsb_to_remove_for_trees(onnx_model, x) + lsbs_to_remove_for_trees = _compute_lsb_to_remove_for_trees(onnx_model, x) # mypy - assert len(lsbs_to_remove) == 2 + assert len(lsbs_to_remove_for_trees) == 2 # Get the expected number of ONNX outputs in the sklearn model. expected_number_of_outputs = 1 if is_regressor_or_partial_regressor(model) else 2 @@ -352,7 +353,7 @@ def tree_to_numpy( q_y = tree_values_preprocessing(onnx_model, framework, output_n_bits) _tree_inference, onnx_model = get_equivalent_numpy_forward_from_onnx( - onnx_model, lsbs_to_remove=lsbs_to_remove + onnx_model, lsbs_to_remove_for_trees=lsbs_to_remove_for_trees ) return (_tree_inference, [q_y.quantizer], onnx_model) @@ -393,7 +394,7 @@ def get_bitwidth(array: numpy.ndarray) -> int: bitwidth = math.ceil(math.log2(max_val + 1)) + 1 return bitwidth - def get_lsbs_to_remove(array: numpy.ndarray) -> int: + def get_lsbs_to_remove_for_trees(array: numpy.ndarray) -> int: """Update the number of LSBs to remove based on overflow detection. Args: @@ -406,16 +407,16 @@ def get_lsbs_to_remove(array: numpy.ndarray) -> int: initial_bitwidth = get_bitwidth(array) if initial_bitwidth - MSB_TO_KEEP_FOR_TREES > 0: - lsbs_to_remove = initial_bitwidth + lsbs_to_remove_for_trees = initial_bitwidth - while lsbs_to_remove > 0: - half = 1 << (lsbs_to_remove - 1) + while lsbs_to_remove_for_trees > 0: + half = 1 << (lsbs_to_remove_for_trees - 1) # The subtraction operation may increase or decrease the precision by 1 or 2 bits if get_bitwidth(array - half) <= initial_bitwidth: - lsbs_to_remove -= 1 # pragma: no cover + lsbs_to_remove_for_trees -= 1 else: - return lsbs_to_remove + return lsbs_to_remove_for_trees return 0 @@ -454,26 +455,26 @@ def get_lsbs_to_remove(array: numpy.ndarray) -> int: required_onnx_operators = set(get_op_type(node) for node in onnx_model.graph.node) # If operator is `<`, np.less(x, y) is equivalent to: - # round_bit_pattern((x - y) - half, lsbs_to_remove=r) < 0. + # round_bit_pattern((x - y) - half, lsbs_to_remove_for_trees=r) < 0. # Therefore, stage_1 = (q_x @ mat_1.transpose(0, 2, 1)) - bias_1 if "Less" in required_onnx_operators: stage_1 = (q_x @ mat_1.transpose(0, 2, 1)) - bias_1 matrix_q = stage_1 < 0 # Else, if operator is `<=`, np.less_equal(x, y) is equivalent to: - # round_bit_pattern((y - x) - half, lsbs_to_remove=r) >= 0. + # round_bit_pattern((y - x) - half, lsbs_to_remove_for_trees=r) >= 0. # Therefore, stage_1 = bias_1 - (q_x @ mat_1.transpose(0, 2, 1)) elif "LessOrEqual" in required_onnx_operators: stage_1 = bias_1 - (q_x @ mat_1.transpose(0, 2, 1)) matrix_q = stage_1 >= 0 - lsbs_to_remove_stage_1 = get_lsbs_to_remove(stage_1) + lsbs_to_remove_for_trees_stage_1 = get_lsbs_to_remove_for_trees(stage_1) # If operator is `==`, np.equal(x, y) is equivalent to: - # round_bit_pattern((x - y) - half, lsbs_to_remove=r) >= 0. + # round_bit_pattern((x - y) - half, lsbs_to_remove_for_trees=r) >= 0. # Therefore, stage_2 = bias_1 - (q_x @ mat_2.transpose(0, 2, 1)) stage_2 = ((bias_2 - matrix_q @ mat_2.transpose(0, 2, 1))).sum(axis=0) - lsbs_to_remove_stage_2 = get_lsbs_to_remove(stage_2) + lsbs_to_remove_for_trees_stage_2 = get_lsbs_to_remove_for_trees(stage_2) - return (lsbs_to_remove_stage_1, lsbs_to_remove_stage_2) + return (lsbs_to_remove_for_trees_stage_1, lsbs_to_remove_for_trees_stage_2) From d186ec8a4f988d3d63699c8fbea81d19ca1bcb1f Mon Sep 17 00:00:00 2001 From: kcelia Date: Wed, 6 Dec 2023 19:54:43 +0100 Subject: [PATCH 22/73] chore: update --- src/concrete/ml/onnx/convert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/concrete/ml/onnx/convert.py b/src/concrete/ml/onnx/convert.py index 93a34da2d..3a56dee5f 100644 --- a/src/concrete/ml/onnx/convert.py +++ b/src/concrete/ml/onnx/convert.py @@ -186,7 +186,7 @@ def get_equivalent_numpy_forward_from_onnx( # All onnx models should be checked, "check_model" parameter must be removed # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4157 - if not check_model: + if not check_model: # pragma: no cover warnings.simplefilter("always") warnings.warn( "`check_model` parameter should always be set to True, to ensure proper onnx model " From c1fcf5fe87fa411df7e3cf2e4f3e7ec42b049fa1 Mon Sep 17 00:00:00 2001 From: kcelia Date: Thu, 7 Dec 2023 00:14:54 +0100 Subject: [PATCH 23/73] chore: update with new version --- src/concrete/ml/onnx/convert.py | 76 ++++++++++++++++++----- src/concrete/ml/onnx/onnx_utils.py | 72 ++++++++++++++++++---- src/concrete/ml/onnx/ops_impl.py | 78 +++++++++++++++++++++--- src/concrete/ml/sklearn/base.py | 6 +- src/concrete/ml/sklearn/tree_to_numpy.py | 38 +++++++++--- tests/sklearn/test_sklearn_models.py | 25 ++++++-- 6 files changed, 243 insertions(+), 52 deletions(-) diff --git a/src/concrete/ml/onnx/convert.py b/src/concrete/ml/onnx/convert.py index 3a56dee5f..9f8b6bda1 100644 --- a/src/concrete/ml/onnx/convert.py +++ b/src/concrete/ml/onnx/convert.py @@ -11,7 +11,12 @@ import torch from onnx import checker, helper -from .onnx_utils import IMPLEMENTED_ONNX_OPS, execute_onnx_with_numpy, get_op_type +from .onnx_utils import ( + IMPLEMENTED_ONNX_OPS, + execute_onnx_with_numpy, + execute_onnx_with_numpy_trees, + get_op_type, +) OPSET_VERSION_FOR_ONNX_EXPORT = 14 @@ -157,11 +162,7 @@ def get_equivalent_numpy_forward_from_torch( ) -def get_equivalent_numpy_forward_from_onnx( - onnx_model: onnx.ModelProto, - check_model: bool = True, - lsbs_to_remove_for_trees: Optional[Tuple[int, int]] = None, -) -> Tuple[Callable[..., Tuple[numpy.ndarray, ...]], onnx.ModelProto]: +def preprocess_onnx_model(onnx_model: onnx.ModelProto, check_model: bool) -> onnx.ModelProto: """Get the numpy equivalent forward of the provided ONNX model. Args: @@ -169,19 +170,13 @@ def get_equivalent_numpy_forward_from_onnx( forward. check_model (bool): set to True to run the onnx checker on the model. Defaults to True. - lsbs_to_remove_for_trees (Optional[Tuple[int, int]]): This parameter is exclusively used for - optimizing tree-based models. It contains the values of the least significant bits to - remove during the tree traversal, where the first value refers to the first comparison - (either "less" or "less_or_equal"), while the second value refers to the "Equal" - comparison operation. Default to None, as it is not applicable to other types of models. Raises: ValueError: Raised if there is an unsupported ONNX operator required to convert the torch model to numpy. Returns: - Callable[..., Tuple[numpy.ndarray, ...]]: The function that will execute - the equivalent numpy function. + onnx.ModelProto: The preprocessed ONNX model. """ # All onnx models should be checked, "check_model" parameter must be removed @@ -226,9 +221,62 @@ def get_equivalent_numpy_forward_from_onnx( f"Available ONNX operators: {', '.join(sorted(IMPLEMENTED_ONNX_OPS))}" ) + return equivalent_onnx_model + + +def get_equivalent_numpy_forward_from_onnx( + onnx_model: onnx.ModelProto, + check_model: bool = True, +) -> Tuple[Callable[..., Tuple[numpy.ndarray, ...]], onnx.ModelProto]: + """Get the numpy equivalent forward of the provided ONNX model. + + Args: + onnx_model (onnx.ModelProto): the ONNX model for which to get the equivalent numpy + forward. + check_model (bool): set to True to run the onnx checker on the model. + Defaults to True. + + Returns: + Callable[..., Tuple[numpy.ndarray, ...]]: The function that will execute + the equivalent numpy function. + """ + + equivalent_onnx_model = preprocess_onnx_model(onnx_model, check_model) + + # Return lambda of numpy equivalent of onnx execution + return ( + lambda *args: execute_onnx_with_numpy(equivalent_onnx_model.graph, *args) + ), equivalent_onnx_model + + +def get_equivalent_numpy_forward_from_onnx_tree( + onnx_model: onnx.ModelProto, + check_model: bool = True, + lsbs_to_remove_for_trees: Optional[Tuple[int, int]] = None, +) -> Tuple[Callable[..., Tuple[numpy.ndarray, ...]], onnx.ModelProto]: + """Get the numpy equivalent forward of the provided ONNX model for tree-based models only. + + Args: + onnx_model (onnx.ModelProto): the ONNX model for which to get the equivalent numpy + forward. + check_model (bool): set to True to run the onnx checker on the model. + Defaults to True. + lsbs_to_remove_for_trees (Optional[Tuple[int, int]]): This parameter is exclusively used for + optimizing tree-based models. It contains the values of the least significant bits to + remove during the tree traversal, where the first value refers to the first comparison + (either "less" or "less_or_equal"), while the second value refers to the "Equal" + comparison operation. Default to None, as it is not applicable to other types of models. + + Returns: + Callable[..., Tuple[numpy.ndarray, ...]]: The function that will execute + the equivalent numpy function. + """ + + equivalent_onnx_model = preprocess_onnx_model(onnx_model, check_model) + # Return lambda of numpy equivalent of onnx execution return ( - lambda *args: execute_onnx_with_numpy( + lambda *args: execute_onnx_with_numpy_trees( equivalent_onnx_model.graph, lsbs_to_remove_for_trees, *args ) ), equivalent_onnx_model diff --git a/src/concrete/ml/onnx/onnx_utils.py b/src/concrete/ml/onnx/onnx_utils.py index b30305c26..efb8cd87f 100644 --- a/src/concrete/ml/onnx/onnx_utils.py +++ b/src/concrete/ml/onnx/onnx_utils.py @@ -295,6 +295,9 @@ numpy_transpose, numpy_unsqueeze, numpy_where, + rounded_numpy_equal_for_trees, + rounded_numpy_less_for_trees, + rounded_numpy_less_or_equal_for_trees, ) ATTR_TYPES = dict(onnx.AttributeProto.AttributeType.items()) @@ -404,9 +407,13 @@ "Less": numpy_less, "LessOrEqual": numpy_less_or_equal, } - # All numpy operators used for tree-based models that support auto rounding -SUPPORTED_ROUNDED_OPERATIONS = ["Less", "LessOrEqual", "Equal"] +ONNX_COMPARISON_OPS_TO_ROUNDED_TREES_NUMPY_IMPL_BOOL = { + "Less": rounded_numpy_less_for_trees, + "Equal": rounded_numpy_equal_for_trees, + "LessOrEqual": rounded_numpy_less_or_equal_for_trees, +} + # All numpy operators used in QuantizedOps ONNX_OPS_TO_NUMPY_IMPL.update(ONNX_COMPARISON_OPS_TO_NUMPY_IMPL_FLOAT) @@ -443,23 +450,58 @@ def get_op_type(node): def execute_onnx_with_numpy( graph: onnx.GraphProto, - lsbs_to_remove_for_trees: Optional[Tuple[int, int]], *inputs: numpy.ndarray, ) -> Tuple[numpy.ndarray, ...]: """Execute the provided ONNX graph on the given inputs. + Args: + graph (onnx.GraphProto): The ONNX graph to execute. + *inputs: The inputs of the graph. + + Returns: + Tuple[numpy.ndarray]: The result of the graph's execution. + """ + node_results: Dict[str, numpy.ndarray] = dict( + {graph_input.name: input_value for graph_input, input_value in zip(graph.input, inputs)}, + **{ + initializer.name: numpy_helper.to_array(initializer) + for initializer in graph.initializer + }, + ) + for node in graph.node: + curr_inputs = (node_results[input_name] for input_name in node.input) + attributes = {attribute.name: get_attribute(attribute) for attribute in node.attribute} + outputs = ONNX_OPS_TO_NUMPY_IMPL_BOOL[node.op_type](*curr_inputs, **attributes) + node_results.update(zip(node.output, outputs)) + + return tuple(node_results[output.name] for output in graph.output) + + +def execute_onnx_with_numpy_trees( + graph: onnx.GraphProto, + lsbs_to_remove_for_trees: Optional[Tuple[int, int]], + *inputs: numpy.ndarray, +) -> Tuple[numpy.ndarray, ...]: + """Execute the provided ONNX graph on the given inputs for tree-based models only. + Args: graph (onnx.GraphProto): The ONNX graph to execute. lsbs_to_remove_for_trees (Optional[Tuple[int, int]]): This parameter is exclusively used for optimizing tree-based models. It contains the values of the least significant bits to remove during the tree traversal, where the first value refers to the first comparison (either "less" or "less_or_equal"), while the second value refers to the "Equal" - comparison operation. Default to None, as it is not applicable to other types of models. + comparison operation. + Default to None. *inputs: The inputs of the graph. Returns: Tuple[numpy.ndarray]: The result of the graph's execution. """ + + # If no tree-based optimization is specified, return standard execution + if lsbs_to_remove_for_trees is None: + return execute_onnx_with_numpy(graph, *inputs) + node_results: Dict[str, numpy.ndarray] = dict( {graph_input.name: input_value for graph_input, input_value in zip(graph.input, inputs)}, **{ @@ -467,20 +509,24 @@ def execute_onnx_with_numpy( for initializer in graph.initializer }, ) + for node in graph.node: curr_inputs = (node_results[input_name] for input_name in node.input) attributes = {attribute.name: get_attribute(attribute) for attribute in node.attribute} - # For trees, the first LSB refers to `Less` or `LessOrEqual` comparisons and the second - # LSB refers to `Equal` comparison - if lsbs_to_remove_for_trees is not None and node.op_type in SUPPORTED_ROUNDED_OPERATIONS: - attributes["lsbs_to_remove_for_trees"] = ( - lsbs_to_remove_for_trees[0] - if node.op_type != "Equal" - else lsbs_to_remove_for_trees[1] - ) + if node.op_type in ONNX_COMPARISON_OPS_TO_ROUNDED_TREES_NUMPY_IMPL_BOOL: - outputs = ONNX_OPS_TO_NUMPY_IMPL_BOOL[node.op_type](*curr_inputs, **attributes) + # The first LSB refers to `Less` or `LessOrEqual` comparisons + # The second LSB refers to `Equal` comparison + stage = 0 if node.op_type != "Equal" else 1 + attributes["lsbs_to_remove_for_trees"] = lsbs_to_remove_for_trees[stage] + + # Use rounded numpy operation to relevant comparison nodes + op_type = ONNX_COMPARISON_OPS_TO_ROUNDED_TREES_NUMPY_IMPL_BOOL[node.op_type] + else: + op_type = ONNX_OPS_TO_NUMPY_IMPL_BOOL[node.op_type] # type: ignore[assignment] + + outputs = op_type(*curr_inputs, **attributes) node_results.update(zip(node.output, outputs)) return tuple(node_results[output.name] for output in graph.output) diff --git a/src/concrete/ml/onnx/ops_impl.py b/src/concrete/ml/onnx/ops_impl.py index bdc6e3ce4..a3b8f1707 100644 --- a/src/concrete/ml/onnx/ops_impl.py +++ b/src/concrete/ml/onnx/ops_impl.py @@ -887,10 +887,28 @@ def numpy_exp( def numpy_equal( x: numpy.ndarray, y: numpy.ndarray, +) -> Tuple[numpy.ndarray]: + """Compute equal in numpy according to ONNX spec. + + See https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Equal-11 + + Args: + x (numpy.ndarray): Input tensor + y (numpy.ndarray): Input tensor + + Returns: + Tuple[numpy.ndarray]: Output tensor + """ + return (numpy.equal(x, y),) + + +def rounded_numpy_equal_for_trees( + x: numpy.ndarray, + y: numpy.ndarray, *, lsbs_to_remove_for_trees: Optional[int] = None, ) -> Tuple[numpy.ndarray]: - """Compute equal in numpy according to ONNX spec. + """Compute rounded equal in numpy according to ONNX spec for tree-based models only. See https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Equal-11 @@ -904,9 +922,15 @@ def numpy_equal( Tuple[numpy.ndarray]: Output tensor """ - # For tree-based models, x == y <=> x <= y or x < y - 1, because y is the max sum. + # For tree-based models in the second stage, x == y is equivalent to x <= y or x < y - 1 + # Because y is the max sum, see this paper: https://arxiv.org/pdf/2010.04804.pdf + # The approach x <= y, is equivalent to: x - y <= 0 or y - x >= 0 + # We take y - x >= 0, because with `rounding_bit_pattern` feature, it gives accurate outputs + # compared to x - y <= 0. if lsbs_to_remove_for_trees is not None and lsbs_to_remove_for_trees > 0: - return rounded_comparison(y, x, lsbs_to_remove_for_trees, operation=lambda x: x >= 0) + return rounded_comparison( + y, x, lsbs_to_remove_for_trees, operation=lambda x: x >= 0 + ) # pragma: no cover # Else, default numpy_equal operator return (numpy.equal(x, y),) @@ -1025,10 +1049,28 @@ def numpy_greater_or_equal_float( def numpy_less( x: numpy.ndarray, y: numpy.ndarray, +) -> Tuple[numpy.ndarray]: + """Compute less in numpy according to ONNX spec. + + See https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Less-13 + + Args: + x (numpy.ndarray): Input tensor + y (numpy.ndarray): Input tensor + + Returns: + Tuple[numpy.ndarray]: Output tensor + """ + return (numpy.less(x, y),) + + +def rounded_numpy_less_for_trees( + x: numpy.ndarray, + y: numpy.ndarray, *, lsbs_to_remove_for_trees: Optional[int] = None, ) -> Tuple[numpy.ndarray]: - """Compute less in numpy according to ONNX spec. + """Compute rounded less in numpy according to ONNX spec for tree-based models only. See https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Less-13 @@ -1042,11 +1084,12 @@ def numpy_less( Tuple[numpy.ndarray]: Output tensor """ + # x < y is equivalent to (x - y < 0) or (y - x > 0) if lsbs_to_remove_for_trees is not None and lsbs_to_remove_for_trees > 0: return rounded_comparison(x, y, lsbs_to_remove_for_trees, operation=lambda x: x < 0) # Else, default numpy_less operator - return (numpy.less(x, y),) + return numpy_less(x, y) def numpy_less_float( @@ -1071,10 +1114,29 @@ def numpy_less_float( def numpy_less_or_equal( x: numpy.ndarray, y: numpy.ndarray, +) -> Tuple[numpy.ndarray]: + """Compute less or equal in numpy according to ONNX spec. + + See https://github.com/onnx/onnx/blob/main/docs/Changelog.md#LessOrEqual-12 + + Args: + x (numpy.ndarray): Input tensor + y (numpy.ndarray): Input tensor + + Returns: + Tuple[numpy.ndarray]: Output tensor + """ + + return (numpy.less_equal(x, y),) + + +def rounded_numpy_less_or_equal_for_trees( + x: numpy.ndarray, + y: numpy.ndarray, *, lsbs_to_remove_for_trees: Optional[int] = None, ) -> Tuple[numpy.ndarray]: - """Compute less or equal in numpy according to ONNX spec. + """Compute rounded less or equal in numpy according to ONNX spec for tree-based models only. See https://github.com/onnx/onnx/blob/main/docs/Changelog.md#LessOrEqual-12 @@ -1088,11 +1150,13 @@ def numpy_less_or_equal( Tuple[numpy.ndarray]: Output tensor """ + # x <= y is equivalent to (x - y <= 0) or (y - x >= 0) + # `rounding_bit_pattern` gives accurate results with (y-x <= 0) approach compred to (x-y <= 0) if lsbs_to_remove_for_trees is not None and lsbs_to_remove_for_trees > 0: return rounded_comparison(y, x, lsbs_to_remove_for_trees, operation=lambda x: x >= 0) # Else, default numpy_less_or_equal operator - return (numpy.less_equal(x, y),) + return numpy_less_or_equal(x, y) def numpy_less_or_equal_float( diff --git a/src/concrete/ml/sklearn/base.py b/src/concrete/ml/sklearn/base.py index d82c04b56..84f2884c4 100644 --- a/src/concrete/ml/sklearn/base.py +++ b/src/concrete/ml/sklearn/base.py @@ -1289,7 +1289,7 @@ def __init__(self, n_bits: int): BaseEstimator.__init__(self) @property - def use_rounding(self) -> bool: + def use_optimized_execution(self) -> bool: """The rounding property. Returns: @@ -1297,8 +1297,8 @@ def use_rounding(self) -> bool: """ return self._use_rounding # pragma: no cover - @use_rounding.setter - def use_rounding(self, value: bool) -> None: + @use_optimized_execution.setter + def use_optimized_execution(self, value: bool) -> None: """Set the rounding feature. Args: diff --git a/src/concrete/ml/sklearn/tree_to_numpy.py b/src/concrete/ml/sklearn/tree_to_numpy.py index ac5d96425..4d3c87511 100644 --- a/src/concrete/ml/sklearn/tree_to_numpy.py +++ b/src/concrete/ml/sklearn/tree_to_numpy.py @@ -13,7 +13,10 @@ get_onnx_opset_version, is_regressor_or_partial_regressor, ) -from ..onnx.convert import OPSET_VERSION_FOR_ONNX_EXPORT, get_equivalent_numpy_forward_from_onnx +from ..onnx.convert import ( + OPSET_VERSION_FOR_ONNX_EXPORT, + get_equivalent_numpy_forward_from_onnx_tree, +) from ..onnx.onnx_model_manipulations import clean_graph_at_node_op_type, remove_node_types from ..onnx.onnx_utils import get_op_type from ..quantization import QuantizedArray @@ -27,9 +30,13 @@ from hummingbird.ml import convert as hb_convert # noqa: E402 # pylint: disable=too-many-branches +# pylint: enable=wrong-import-position,wrong-import-order # Most significant bits to retain when applying rounding to the tree -MSB_TO_KEEP_FOR_TREES = 4 +MSB_TO_KEEP_FOR_TREES = 1 + +# Minimum circuit size to apply rounding +MIN_ROUNDING_THRESHOLD = 4 def get_onnx_model(model: Callable, x: numpy.ndarray, framework: str) -> onnx.ModelProto: @@ -352,7 +359,7 @@ def tree_to_numpy( # but also rounding the threshold such that they are now integers q_y = tree_values_preprocessing(onnx_model, framework, output_n_bits) - _tree_inference, onnx_model = get_equivalent_numpy_forward_from_onnx( + _tree_inference, onnx_model = get_equivalent_numpy_forward_from_onnx_tree( onnx_model, lsbs_to_remove_for_trees=lsbs_to_remove_for_trees ) @@ -397,6 +404,8 @@ def get_bitwidth(array: numpy.ndarray) -> int: def get_lsbs_to_remove_for_trees(array: numpy.ndarray) -> int: """Update the number of LSBs to remove based on overflow detection. + this function works only for MSB = 1 + Args: array (numpy.ndarray): The array for which the bitwidth needs to be checked. @@ -406,17 +415,26 @@ def get_lsbs_to_remove_for_trees(array: numpy.ndarray) -> int: initial_bitwidth = get_bitwidth(array) - if initial_bitwidth - MSB_TO_KEEP_FOR_TREES > 0: + # No need to compute if the bitwidth doesn't satisfy the threshold + if initial_bitwidth - MIN_ROUNDING_THRESHOLD > 0: lsbs_to_remove_for_trees = initial_bitwidth + half = 1 << (lsbs_to_remove_for_trees - 1) + # The subtraction operation may increase or decrease the precision by 1 or 2 bits + # In the following, we handle the case where the precision decreases while lsbs_to_remove_for_trees > 0: - half = 1 << (lsbs_to_remove_for_trees - 1) + new_bitwidth = get_bitwidth(array - half) - # The subtraction operation may increase or decrease the precision by 1 or 2 bits - if get_bitwidth(array - half) <= initial_bitwidth: - lsbs_to_remove_for_trees -= 1 + # Readjust the LSB + if initial_bitwidth - new_bitwidth > 0: + lsbs_to_remove_for_trees -= 1 # pragma: no cover + half = 1 << (lsbs_to_remove_for_trees - 1) # pragma: no cover else: - return lsbs_to_remove_for_trees + break + + assert MSB_TO_KEEP_FOR_TREES == new_bitwidth - lsbs_to_remove_for_trees + + return lsbs_to_remove_for_trees return 0 @@ -473,7 +491,7 @@ def get_lsbs_to_remove_for_trees(array: numpy.ndarray) -> int: # If operator is `==`, np.equal(x, y) is equivalent to: # round_bit_pattern((x - y) - half, lsbs_to_remove_for_trees=r) >= 0. # Therefore, stage_2 = bias_1 - (q_x @ mat_2.transpose(0, 2, 1)) - stage_2 = ((bias_2 - matrix_q @ mat_2.transpose(0, 2, 1))).sum(axis=0) + stage_2 = ((bias_2 - matrix_q @ mat_2.transpose(0, 2, 1))).max(axis=0) lsbs_to_remove_for_trees_stage_2 = get_lsbs_to_remove_for_trees(stage_2) diff --git a/tests/sklearn/test_sklearn_models.py b/tests/sklearn/test_sklearn_models.py index 7b79a4142..e7b4906eb 100644 --- a/tests/sklearn/test_sklearn_models.py +++ b/tests/sklearn/test_sklearn_models.py @@ -160,7 +160,9 @@ def fit_and_compile(model, x, y): warnings.simplefilter("ignore", category=ConvergenceWarning) model.fit(x, y) - model.compile(x) + circuit = model.compile(x) + + return circuit def check_correctness_with_sklearn( @@ -437,6 +439,10 @@ def check_serialization_dump_load(model, x, use_dump_method): y_pred_loaded_sklearn_model = loaded_model.sklearn_model.predict(x) assert numpy.array_equal(y_pred_sklearn_model, y_pred_loaded_sklearn_model) + # Check if the graphs are identical + loaded_model.compile(x) + assert (model.fhe_circuit.graph.format()) == loaded_model.fhe_circuit.graph.format() + def check_serialization_dumps_loads(model, x, use_dump_method): """Check that a model can be serialized two times using dumps/loads.""" @@ -490,6 +496,10 @@ def check_serialization_dumps_loads(model, x, use_dump_method): y_pred_loaded_sklearn_model = loaded_model.sklearn_model.predict(x) assert numpy.array_equal(y_pred_sklearn_model, y_pred_loaded_sklearn_model) + # Check if the graphs are identical + loaded_model.compile(x) + assert (model.fhe_circuit.graph.format()) == loaded_model.fhe_circuit.graph.format() + def check_offset(model_class, n_bits, x, y): """Check offset.""" @@ -1155,6 +1165,7 @@ def check_load_fitted_sklearn_linear_models(model_class, n_bits, x, y, check_flo def check_rounding_consistency( model, + n_bits, x, y, predict_method, @@ -1172,7 +1183,7 @@ def check_rounding_consistency( fhe_test = get_random_samples(x, fhe_samples) # Fit and compile with rounding enabled - fit_and_compile(model, x, y) + circuit_with_rounding = fit_and_compile(model, x, y) rounded_predict_quantized = predict_method(x, fhe="disable") rounded_predict_simulate = predict_method(x, fhe="simulate") @@ -1182,9 +1193,9 @@ def check_rounding_consistency( with warnings.catch_warnings(): warnings.simplefilter("ignore", category=DeprecationWarning) - model.use_rounding = False + model.use_optimized_execution = False - fit_and_compile(model, x, y) + _ = fit_and_compile(model, x, y) not_rounded_predict_quantized = predict_method(x, fhe="disable") not_rounded_predict_simulate = predict_method(x, fhe="simulate") @@ -1194,6 +1205,9 @@ def check_rounding_consistency( metric(rounded_predict_simulate, not_rounded_predict_simulate) metric(rounded_predict_fhe, not_rounded_predict_fhe) + # Check that the maximum bitwidth of the cuircuit with rounding is at most n_bits + 2 + assert circuit_with_rounding.graph.maximum_integer_bit_width() <= n_bits + 2 + # Neural network models are skipped for this test # The `fit_benchmark` function of QNNs returns a QAT model and a FP32 model that is similar @@ -1827,7 +1841,7 @@ def test_linear_models_have_no_tlu( # Test only tree-based models @pytest.mark.parametrize("model_class, parameters", get_sklearn_tree_models_and_datasets()) -@pytest.mark.parametrize("n_bits", [2, 3, 4, 5, 6]) +@pytest.mark.parametrize("n_bits", [5, 6]) def test_rounding_consistency( model_class, parameters, @@ -1857,6 +1871,7 @@ def test_rounding_consistency( check_rounding_consistency( model, + n_bits, x, y, predict_method, From 012ddb656cc3c4de448bc2b618579c7404b08898 Mon Sep 17 00:00:00 2001 From: kcelia Date: Thu, 7 Dec 2023 22:52:02 +0100 Subject: [PATCH 24/73] chore: update lsb computation --- src/concrete/ml/sklearn/tree_to_numpy.py | 30 +++++++----------------- tests/sklearn/test_sklearn_models.py | 2 +- 2 files changed, 10 insertions(+), 22 deletions(-) diff --git a/src/concrete/ml/sklearn/tree_to_numpy.py b/src/concrete/ml/sklearn/tree_to_numpy.py index 4d3c87511..932662c0b 100644 --- a/src/concrete/ml/sklearn/tree_to_numpy.py +++ b/src/concrete/ml/sklearn/tree_to_numpy.py @@ -35,8 +35,8 @@ # Most significant bits to retain when applying rounding to the tree MSB_TO_KEEP_FOR_TREES = 1 -# Minimum circuit size to apply rounding -MIN_ROUNDING_THRESHOLD = 4 +# Minimum bitwidth to apply rounding +MIN_CIRCUIT_THRESHOLD_FOR_TREES = 4 def get_onnx_model(model: Callable, x: numpy.ndarray, framework: str) -> onnx.ModelProto: @@ -413,30 +413,18 @@ def get_lsbs_to_remove_for_trees(array: numpy.ndarray) -> int: int: The updated LSB to remove. """ - initial_bitwidth = get_bitwidth(array) + lsbs_to_remove_for_trees: int = 0 - # No need to compute if the bitwidth doesn't satisfy the threshold - if initial_bitwidth - MIN_ROUNDING_THRESHOLD > 0: - lsbs_to_remove_for_trees = initial_bitwidth - half = 1 << (lsbs_to_remove_for_trees - 1) + prev_bitwidth = get_bitwidth(array) - # The subtraction operation may increase or decrease the precision by 1 or 2 bits - # In the following, we handle the case where the precision decreases - while lsbs_to_remove_for_trees > 0: - new_bitwidth = get_bitwidth(array - half) + if prev_bitwidth > MIN_CIRCUIT_THRESHOLD_FOR_TREES: - # Readjust the LSB - if initial_bitwidth - new_bitwidth > 0: - lsbs_to_remove_for_trees -= 1 # pragma: no cover - half = 1 << (lsbs_to_remove_for_trees - 1) # pragma: no cover - else: - break + if prev_bitwidth - MSB_TO_KEEP_FOR_TREES > 0: - assert MSB_TO_KEEP_FOR_TREES == new_bitwidth - lsbs_to_remove_for_trees + msb = MSB_TO_KEEP_FOR_TREES if MSB_TO_KEEP_FOR_TREES > 1 else 0 + lsbs_to_remove_for_trees = prev_bitwidth - msb - return lsbs_to_remove_for_trees - - return 0 + return lsbs_to_remove_for_trees quant_params = { onnx_init.name: numpy_helper.to_array(onnx_init) diff --git a/tests/sklearn/test_sklearn_models.py b/tests/sklearn/test_sklearn_models.py index e7b4906eb..6a581d721 100644 --- a/tests/sklearn/test_sklearn_models.py +++ b/tests/sklearn/test_sklearn_models.py @@ -1841,7 +1841,7 @@ def test_linear_models_have_no_tlu( # Test only tree-based models @pytest.mark.parametrize("model_class, parameters", get_sklearn_tree_models_and_datasets()) -@pytest.mark.parametrize("n_bits", [5, 6]) +@pytest.mark.parametrize("n_bits", [3, 5, 6]) def test_rounding_consistency( model_class, parameters, From 399ccc7a6c2de3f7997e8e7fc8c6ac403df6829c Mon Sep 17 00:00:00 2001 From: kcelia Date: Fri, 8 Dec 2023 09:06:24 +0100 Subject: [PATCH 25/73] chore: fix assert c_r1 < c_r0 + 2 --- tests/sklearn/test_sklearn_models.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/sklearn/test_sklearn_models.py b/tests/sklearn/test_sklearn_models.py index 6a581d721..9f5b8904e 100644 --- a/tests/sklearn/test_sklearn_models.py +++ b/tests/sklearn/test_sklearn_models.py @@ -1165,7 +1165,6 @@ def check_load_fitted_sklearn_linear_models(model_class, n_bits, x, y, check_flo def check_rounding_consistency( model, - n_bits, x, y, predict_method, @@ -1195,7 +1194,7 @@ def check_rounding_consistency( warnings.simplefilter("ignore", category=DeprecationWarning) model.use_optimized_execution = False - _ = fit_and_compile(model, x, y) + circuit_without_rounding = fit_and_compile(model, x, y) not_rounded_predict_quantized = predict_method(x, fhe="disable") not_rounded_predict_simulate = predict_method(x, fhe="simulate") @@ -1206,7 +1205,11 @@ def check_rounding_consistency( metric(rounded_predict_fhe, not_rounded_predict_fhe) # Check that the maximum bitwidth of the cuircuit with rounding is at most n_bits + 2 - assert circuit_with_rounding.graph.maximum_integer_bit_width() <= n_bits + 2 + max_bitwitdth_with_rounding = circuit_with_rounding.graph.maximum_integer_bit_width() + max_bitwitdth_without_rounding = circuit_without_rounding.graph.maximum_integer_bit_width() + + print(max_bitwitdth_with_rounding, max_bitwitdth_without_rounding) + assert max_bitwitdth_with_rounding <= max_bitwitdth_without_rounding + 2 # Neural network models are skipped for this test @@ -1871,7 +1874,6 @@ def test_rounding_consistency( check_rounding_consistency( model, - n_bits, x, y, predict_method, From 6956996688eba6f05ec87bd58b9efd55492a6a20 Mon Sep 17 00:00:00 2001 From: kcelia Date: Fri, 8 Dec 2023 09:29:42 +0100 Subject: [PATCH 26/73] chore: update serialized test --- tests/sklearn/test_sklearn_models.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/sklearn/test_sklearn_models.py b/tests/sklearn/test_sklearn_models.py index 9f5b8904e..e7f44c9f6 100644 --- a/tests/sklearn/test_sklearn_models.py +++ b/tests/sklearn/test_sklearn_models.py @@ -439,9 +439,8 @@ def check_serialization_dump_load(model, x, use_dump_method): y_pred_loaded_sklearn_model = loaded_model.sklearn_model.predict(x) assert numpy.array_equal(y_pred_sklearn_model, y_pred_loaded_sklearn_model) - # Check if the graphs are identical - loaded_model.compile(x) - assert (model.fhe_circuit.graph.format()) == loaded_model.fhe_circuit.graph.format() + # Add a test to check that graphs before and after the serialization are identical + # FIME: https://github.com/zama-ai/concrete-internal/issues/546 def check_serialization_dumps_loads(model, x, use_dump_method): @@ -496,9 +495,8 @@ def check_serialization_dumps_loads(model, x, use_dump_method): y_pred_loaded_sklearn_model = loaded_model.sklearn_model.predict(x) assert numpy.array_equal(y_pred_sklearn_model, y_pred_loaded_sklearn_model) - # Check if the graphs are identical - loaded_model.compile(x) - assert (model.fhe_circuit.graph.format()) == loaded_model.fhe_circuit.graph.format() + # Add a test to check that graphs before and after the serialization are identical + # FIME: https://github.com/zama-ai/concrete-internal/issues/546 def check_offset(model_class, n_bits, x, y): From a9b2b069570babe7217e70a7aadb575e7f43dd41 Mon Sep 17 00:00:00 2001 From: kcelia Date: Fri, 8 Dec 2023 09:41:13 +0100 Subject: [PATCH 27/73] chore: version for Jordan --- src/concrete/ml/onnx/onnx_impl_utils.py | 4 +++- src/concrete/ml/sklearn/tree_to_numpy.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/concrete/ml/onnx/onnx_impl_utils.py b/src/concrete/ml/onnx/onnx_impl_utils.py index 081406fe1..7e992c27c 100644 --- a/src/concrete/ml/onnx/onnx_impl_utils.py +++ b/src/concrete/ml/onnx/onnx_impl_utils.py @@ -268,6 +268,8 @@ def rounded_comparison( half = 1 << (lsbs_to_remove - 1) # To determine if 'x' 'operation' 'y' (operation being <, >, >=, <=), we evaluate 'x - y' - rounded_subtraction = round_bit_pattern((x - y) - half, lsbs_to_remove=lsbs_to_remove) + rounded_subtraction = round_bit_pattern( + (x - y) - half, lsbs_to_remove=lsbs_to_remove, overflow_protection=False + ) return (operation(rounded_subtraction),) diff --git a/src/concrete/ml/sklearn/tree_to_numpy.py b/src/concrete/ml/sklearn/tree_to_numpy.py index 932662c0b..a729b34b4 100644 --- a/src/concrete/ml/sklearn/tree_to_numpy.py +++ b/src/concrete/ml/sklearn/tree_to_numpy.py @@ -33,7 +33,7 @@ # pylint: enable=wrong-import-position,wrong-import-order # Most significant bits to retain when applying rounding to the tree -MSB_TO_KEEP_FOR_TREES = 1 +MSB_TO_KEEP_FOR_TREES = 4 # Minimum bitwidth to apply rounding MIN_CIRCUIT_THRESHOLD_FOR_TREES = 4 @@ -343,7 +343,7 @@ def tree_to_numpy( # First LSB refers to Less or LessOrEqual comparisons # Second LSB refers to Equal comparison lsbs_to_remove_for_trees = _compute_lsb_to_remove_for_trees(onnx_model, x) - + print("LSB computed", lsbs_to_remove_for_trees) # mypy assert len(lsbs_to_remove_for_trees) == 2 From ffa45772a70d63ad34460ae177e8cd361c8df3da Mon Sep 17 00:00:00 2001 From: kcelia Date: Fri, 8 Dec 2023 09:43:35 +0100 Subject: [PATCH 28/73] chore: version with MSB=1 --- src/concrete/ml/onnx/onnx_impl_utils.py | 4 +--- src/concrete/ml/sklearn/tree_to_numpy.py | 4 ++-- tests/sklearn/test_sklearn_models.py | 3 +-- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/concrete/ml/onnx/onnx_impl_utils.py b/src/concrete/ml/onnx/onnx_impl_utils.py index 7e992c27c..081406fe1 100644 --- a/src/concrete/ml/onnx/onnx_impl_utils.py +++ b/src/concrete/ml/onnx/onnx_impl_utils.py @@ -268,8 +268,6 @@ def rounded_comparison( half = 1 << (lsbs_to_remove - 1) # To determine if 'x' 'operation' 'y' (operation being <, >, >=, <=), we evaluate 'x - y' - rounded_subtraction = round_bit_pattern( - (x - y) - half, lsbs_to_remove=lsbs_to_remove, overflow_protection=False - ) + rounded_subtraction = round_bit_pattern((x - y) - half, lsbs_to_remove=lsbs_to_remove) return (operation(rounded_subtraction),) diff --git a/src/concrete/ml/sklearn/tree_to_numpy.py b/src/concrete/ml/sklearn/tree_to_numpy.py index a729b34b4..932662c0b 100644 --- a/src/concrete/ml/sklearn/tree_to_numpy.py +++ b/src/concrete/ml/sklearn/tree_to_numpy.py @@ -33,7 +33,7 @@ # pylint: enable=wrong-import-position,wrong-import-order # Most significant bits to retain when applying rounding to the tree -MSB_TO_KEEP_FOR_TREES = 4 +MSB_TO_KEEP_FOR_TREES = 1 # Minimum bitwidth to apply rounding MIN_CIRCUIT_THRESHOLD_FOR_TREES = 4 @@ -343,7 +343,7 @@ def tree_to_numpy( # First LSB refers to Less or LessOrEqual comparisons # Second LSB refers to Equal comparison lsbs_to_remove_for_trees = _compute_lsb_to_remove_for_trees(onnx_model, x) - print("LSB computed", lsbs_to_remove_for_trees) + # mypy assert len(lsbs_to_remove_for_trees) == 2 diff --git a/tests/sklearn/test_sklearn_models.py b/tests/sklearn/test_sklearn_models.py index e7f44c9f6..5f52a0afc 100644 --- a/tests/sklearn/test_sklearn_models.py +++ b/tests/sklearn/test_sklearn_models.py @@ -1206,7 +1206,6 @@ def check_rounding_consistency( max_bitwitdth_with_rounding = circuit_with_rounding.graph.maximum_integer_bit_width() max_bitwitdth_without_rounding = circuit_without_rounding.graph.maximum_integer_bit_width() - print(max_bitwitdth_with_rounding, max_bitwitdth_without_rounding) assert max_bitwitdth_with_rounding <= max_bitwitdth_without_rounding + 2 @@ -1842,7 +1841,7 @@ def test_linear_models_have_no_tlu( # Test only tree-based models @pytest.mark.parametrize("model_class, parameters", get_sklearn_tree_models_and_datasets()) -@pytest.mark.parametrize("n_bits", [3, 5, 6]) +@pytest.mark.parametrize("n_bits", [2, 6]) def test_rounding_consistency( model_class, parameters, From 9031531a0254fda1bbf6cccad1ba6030eeef4ee4 Mon Sep 17 00:00:00 2001 From: kcelia Date: Mon, 11 Dec 2023 09:56:57 +0100 Subject: [PATCH 29/73] chore: update --- src/concrete/ml/sklearn/tree_to_numpy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/concrete/ml/sklearn/tree_to_numpy.py b/src/concrete/ml/sklearn/tree_to_numpy.py index 932662c0b..15940e0ce 100644 --- a/src/concrete/ml/sklearn/tree_to_numpy.py +++ b/src/concrete/ml/sklearn/tree_to_numpy.py @@ -306,7 +306,7 @@ def tree_to_numpy( model: Callable, x: numpy.ndarray, framework: str, - use_rounding: Optional[bool] = False, + use_rounding: bool = True, output_n_bits: int = MAX_BITWIDTH_BACKWARD_COMPATIBLE, ) -> Tuple[Callable, List[UniformQuantizer], onnx.ModelProto]: """Convert the tree inference to a numpy functions using Hummingbird. @@ -314,7 +314,7 @@ def tree_to_numpy( Args: model (Callable): The tree model to convert. x (numpy.ndarray): The input data. - use_rounding (Optional[bool]): This parameter is exclusively used to tree-based models. + use_rounding (bool): This parameter is exclusively used to tree-based models. It determines whether the rounding feature is enabled or disabled. framework (str): The framework from which the ONNX model is generated. (options: 'xgboost', 'sklearn') From 71b56b2ec6c7bf5e5ec7c71de36e54782523e4a7 Mon Sep 17 00:00:00 2001 From: kcelia Date: Mon, 11 Dec 2023 13:23:03 +0100 Subject: [PATCH 30/73] chore: remove rounding in serialization --- src/concrete/ml/sklearn/rf.py | 6 ------ src/concrete/ml/sklearn/tree.py | 6 ------ src/concrete/ml/sklearn/xgb.py | 6 ------ 3 files changed, 18 deletions(-) diff --git a/src/concrete/ml/sklearn/rf.py b/src/concrete/ml/sklearn/rf.py index 5c87c021f..e5f756664 100644 --- a/src/concrete/ml/sklearn/rf.py +++ b/src/concrete/ml/sklearn/rf.py @@ -84,7 +84,6 @@ def dump_dict(self) -> Dict[str, Any]: metadata["onnx_model_"] = self.onnx_model_ metadata["framework"] = self.framework metadata["post_processing_params"] = self.post_processing_params - metadata["_use_rounding"] = self._use_rounding # Scikit-Learn metadata["n_estimators"] = self.n_estimators @@ -121,11 +120,9 @@ def load_dict(cls, metadata: Dict): obj.framework = metadata["framework"] obj.onnx_model_ = metadata["onnx_model_"] obj.output_quantizers = metadata["output_quantizers"] - obj._use_rounding = metadata["_use_rounding"] obj._tree_inference = tree_to_numpy( obj.sklearn_model, numpy.zeros((len(obj.input_quantizers),))[None, ...], - use_rounding=obj._use_rounding, framework=obj.framework, output_n_bits=obj.n_bits, )[0] @@ -222,7 +219,6 @@ def dump_dict(self) -> Dict[str, Any]: metadata["onnx_model_"] = self.onnx_model_ metadata["framework"] = self.framework metadata["post_processing_params"] = self.post_processing_params - metadata["_use_rounding"] = self._use_rounding # Scikit-Learn metadata["n_estimators"] = self.n_estimators @@ -259,11 +255,9 @@ def load_dict(cls, metadata: Dict): obj.framework = metadata["framework"] obj.onnx_model_ = metadata["onnx_model_"] obj.output_quantizers = metadata["output_quantizers"] - obj._use_rounding = metadata["_use_rounding"] obj._tree_inference = tree_to_numpy( obj.sklearn_model, numpy.zeros((len(obj.input_quantizers),))[None, ...], - use_rounding=obj._use_rounding, framework=obj.framework, output_n_bits=obj.n_bits, )[0] diff --git a/src/concrete/ml/sklearn/tree.py b/src/concrete/ml/sklearn/tree.py index 15f5de749..1ea972cfd 100644 --- a/src/concrete/ml/sklearn/tree.py +++ b/src/concrete/ml/sklearn/tree.py @@ -84,7 +84,6 @@ def dump_dict(self) -> Dict[str, Any]: metadata["onnx_model_"] = self.onnx_model_ metadata["framework"] = self.framework metadata["post_processing_params"] = self.post_processing_params - metadata["_use_rounding"] = self._use_rounding # Scikit-Learn metadata["criterion"] = self.criterion @@ -116,11 +115,9 @@ def load_dict(cls, metadata: Dict): obj.framework = metadata["framework"] obj.onnx_model_ = metadata["onnx_model_"] obj.output_quantizers = metadata["output_quantizers"] - obj._use_rounding = metadata["_use_rounding"] obj._tree_inference = tree_to_numpy( obj.sklearn_model, numpy.zeros((len(obj.input_quantizers),))[None, ...], - use_rounding=obj._use_rounding, framework=obj.framework, output_n_bits=obj.n_bits, )[0] @@ -211,7 +208,6 @@ def dump_dict(self) -> Dict[str, Any]: metadata["onnx_model_"] = self.onnx_model_ metadata["framework"] = self.framework metadata["post_processing_params"] = self.post_processing_params - metadata["_use_rounding"] = self._use_rounding # Scikit-Learn metadata["criterion"] = self.criterion @@ -242,11 +238,9 @@ def load_dict(cls, metadata: Dict): obj.framework = metadata["framework"] obj.onnx_model_ = metadata["onnx_model_"] obj.output_quantizers = metadata["output_quantizers"] - obj._use_rounding = metadata["_use_rounding"] obj._tree_inference = tree_to_numpy( obj.sklearn_model, numpy.zeros((len(obj.input_quantizers),))[None, ...], - use_rounding=obj._use_rounding, framework=obj.framework, output_n_bits=obj.n_bits, )[0] diff --git a/src/concrete/ml/sklearn/xgb.py b/src/concrete/ml/sklearn/xgb.py index eab922e43..a10b1400b 100644 --- a/src/concrete/ml/sklearn/xgb.py +++ b/src/concrete/ml/sklearn/xgb.py @@ -125,7 +125,6 @@ def dump_dict(self) -> Dict[str, Any]: metadata["onnx_model_"] = self.onnx_model_ metadata["framework"] = self.framework metadata["post_processing_params"] = self.post_processing_params - metadata["_use_rounding"] = self._use_rounding # XGBoost metadata["max_depth"] = self.max_depth @@ -175,11 +174,9 @@ def load_dict(cls, metadata: Dict): obj.framework = metadata["framework"] obj.onnx_model_ = metadata["onnx_model_"] obj.output_quantizers = metadata["output_quantizers"] - obj._use_rounding = metadata["_use_rounding"] obj._tree_inference = tree_to_numpy( obj.sklearn_model, numpy.zeros((len(obj.input_quantizers),))[None, ...], - use_rounding=obj._use_rounding, framework=obj.framework, output_n_bits=obj.n_bits, )[0] @@ -357,7 +354,6 @@ def dump_dict(self) -> Dict[str, Any]: metadata["onnx_model_"] = self.onnx_model_ metadata["framework"] = self.framework metadata["post_processing_params"] = self.post_processing_params - metadata["_use_rounding"] = self._use_rounding # XGBoost metadata["max_depth"] = self.max_depth @@ -407,11 +403,9 @@ def load_dict(cls, metadata: Dict): obj.framework = metadata["framework"] obj.onnx_model_ = metadata["onnx_model_"] obj.output_quantizers = metadata["output_quantizers"] - obj._use_rounding = metadata["_use_rounding"] obj._tree_inference = tree_to_numpy( obj.sklearn_model, numpy.zeros((len(obj.input_quantizers),))[None, ...], - use_rounding=obj._use_rounding, framework=obj.framework, output_n_bits=obj.n_bits, )[0] From ae5147034e705dee02f08bbc80fccb7c6857dff6 Mon Sep 17 00:00:00 2001 From: kcelia Date: Mon, 11 Dec 2023 13:23:19 +0100 Subject: [PATCH 31/73] chore: update v3 --- src/concrete/ml/onnx/convert.py | 4 ++-- src/concrete/ml/onnx/onnx_utils.py | 4 +++- src/concrete/ml/onnx/ops_impl.py | 15 +++++++++------ tests/sklearn/test_sklearn_models.py | 2 +- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/concrete/ml/onnx/convert.py b/src/concrete/ml/onnx/convert.py index 9f8b6bda1..96b1f1eee 100644 --- a/src/concrete/ml/onnx/convert.py +++ b/src/concrete/ml/onnx/convert.py @@ -268,8 +268,8 @@ def get_equivalent_numpy_forward_from_onnx_tree( comparison operation. Default to None, as it is not applicable to other types of models. Returns: - Callable[..., Tuple[numpy.ndarray, ...]]: The function that will execute - the equivalent numpy function. + Tuple[Callable[..., Tuple[numpy.ndarray, ...]], onnx.ModelProto]: The function that will + execute the equivalent numpy function. """ equivalent_onnx_model = preprocess_onnx_model(onnx_model, check_model) diff --git a/src/concrete/ml/onnx/onnx_utils.py b/src/concrete/ml/onnx/onnx_utils.py index efb8cd87f..a8b7bfd71 100644 --- a/src/concrete/ml/onnx/onnx_utils.py +++ b/src/concrete/ml/onnx/onnx_utils.py @@ -498,6 +498,8 @@ def execute_onnx_with_numpy_trees( Tuple[numpy.ndarray]: The result of the graph's execution. """ + op_type: Callable[..., Tuple[numpy.ndarray[Any, Any], ...]] + # If no tree-based optimization is specified, return standard execution if lsbs_to_remove_for_trees is None: return execute_onnx_with_numpy(graph, *inputs) @@ -524,7 +526,7 @@ def execute_onnx_with_numpy_trees( # Use rounded numpy operation to relevant comparison nodes op_type = ONNX_COMPARISON_OPS_TO_ROUNDED_TREES_NUMPY_IMPL_BOOL[node.op_type] else: - op_type = ONNX_OPS_TO_NUMPY_IMPL_BOOL[node.op_type] # type: ignore[assignment] + op_type = ONNX_OPS_TO_NUMPY_IMPL_BOOL[node.op_type] outputs = op_type(*curr_inputs, **attributes) diff --git a/src/concrete/ml/onnx/ops_impl.py b/src/concrete/ml/onnx/ops_impl.py index a3b8f1707..74bc90927 100644 --- a/src/concrete/ml/onnx/ops_impl.py +++ b/src/concrete/ml/onnx/ops_impl.py @@ -924,9 +924,10 @@ def rounded_numpy_equal_for_trees( # For tree-based models in the second stage, x == y is equivalent to x <= y or x < y - 1 # Because y is the max sum, see this paper: https://arxiv.org/pdf/2010.04804.pdf - # The approach x <= y, is equivalent to: x - y <= 0 or y - x >= 0 - # We take y - x >= 0, because with `rounding_bit_pattern` feature, it gives accurate outputs - # compared to x - y <= 0. + # The approach x <= y, is equivalent to: + # x - y <= 0 => round_bit_pattern(x - y + half) <= 0 or + # y - x >= 0 => round_bit_pattern(y - x - half) >= 0 + # `rounding_bit_pattern` rounds to the closer if lsbs_to_remove_for_trees is not None and lsbs_to_remove_for_trees > 0: return rounded_comparison( y, x, lsbs_to_remove_for_trees, operation=lambda x: x >= 0 @@ -1084,7 +1085,8 @@ def rounded_numpy_less_for_trees( Tuple[numpy.ndarray]: Output tensor """ - # x < y is equivalent to (x - y < 0) or (y - x > 0) + # numpy.less(x, y) is equivalent to : + # x - y <= 0 => round_bit_pattern(x - y - half) < 0 if lsbs_to_remove_for_trees is not None and lsbs_to_remove_for_trees > 0: return rounded_comparison(x, y, lsbs_to_remove_for_trees, operation=lambda x: x < 0) @@ -1150,8 +1152,9 @@ def rounded_numpy_less_or_equal_for_trees( Tuple[numpy.ndarray]: Output tensor """ - # x <= y is equivalent to (x - y <= 0) or (y - x >= 0) - # `rounding_bit_pattern` gives accurate results with (y-x <= 0) approach compred to (x-y <= 0) + # numpy.less_equal(x, y) <= y is equivalent to : + # x - y <= 0 => round_bit_pattern(x - y + half) <= 0 or + # y - x >= 0 => round_bit_pattern(y - x - half) >= 0 if lsbs_to_remove_for_trees is not None and lsbs_to_remove_for_trees > 0: return rounded_comparison(y, x, lsbs_to_remove_for_trees, operation=lambda x: x >= 0) diff --git a/tests/sklearn/test_sklearn_models.py b/tests/sklearn/test_sklearn_models.py index 5f52a0afc..53c8da9e5 100644 --- a/tests/sklearn/test_sklearn_models.py +++ b/tests/sklearn/test_sklearn_models.py @@ -1841,7 +1841,7 @@ def test_linear_models_have_no_tlu( # Test only tree-based models @pytest.mark.parametrize("model_class, parameters", get_sklearn_tree_models_and_datasets()) -@pytest.mark.parametrize("n_bits", [2, 6]) +@pytest.mark.parametrize("n_bits", [2, 5]) def test_rounding_consistency( model_class, parameters, From 0ad84a6e5ae81d5fc5cb194b41cb45b010567a24 Mon Sep 17 00:00:00 2001 From: kcelia Date: Mon, 11 Dec 2023 15:47:54 +0100 Subject: [PATCH 32/73] chore: update v4 --- src/concrete/ml/onnx/onnx_impl_utils.py | 2 +- src/concrete/ml/onnx/ops_impl.py | 23 ++++--- src/concrete/ml/sklearn/base.py | 54 ++++++--------- tests/sklearn/test_sklearn_models.py | 87 ++++++++++++++++++------- 4 files changed, 96 insertions(+), 70 deletions(-) diff --git a/src/concrete/ml/onnx/onnx_impl_utils.py b/src/concrete/ml/onnx/onnx_impl_utils.py index 081406fe1..1a554e96b 100644 --- a/src/concrete/ml/onnx/onnx_impl_utils.py +++ b/src/concrete/ml/onnx/onnx_impl_utils.py @@ -5,8 +5,8 @@ import numpy from concrete.fhe import conv as cnp_conv from concrete.fhe import ones as cnp_ones -from concrete.fhe.tracing import Tracer from concrete.fhe import round_bit_pattern +from concrete.fhe.tracing import Tracer from ..common.debugging import assert_true diff --git a/src/concrete/ml/onnx/ops_impl.py b/src/concrete/ml/onnx/ops_impl.py index 74bc90927..22d0d7658 100644 --- a/src/concrete/ml/onnx/ops_impl.py +++ b/src/concrete/ml/onnx/ops_impl.py @@ -922,16 +922,16 @@ def rounded_numpy_equal_for_trees( Tuple[numpy.ndarray]: Output tensor """ - # For tree-based models in the second stage, x == y is equivalent to x <= y or x < y - 1 + # For tree-based models in the second stage, x == y is equivalent to x <= y # Because y is the max sum, see this paper: https://arxiv.org/pdf/2010.04804.pdf # The approach x <= y, is equivalent to: - # x - y <= 0 => round_bit_pattern(x - y + half) <= 0 or - # y - x >= 0 => round_bit_pattern(y - x - half) >= 0 - # `rounding_bit_pattern` rounds to the closer + # option 1: x - y <= 0 => round_bit_pattern(x - y + half) <= 0 or + # option 2: y - x >= 0 => round_bit_pattern(y - x - half) >= 0 + + # Option 2 is selected because it adheres to the established pattern in `rounded_comparison` + # which does: (a - b) - half. if lsbs_to_remove_for_trees is not None and lsbs_to_remove_for_trees > 0: - return rounded_comparison( - y, x, lsbs_to_remove_for_trees, operation=lambda x: x >= 0 - ) # pragma: no cover + return rounded_comparison(y, x, lsbs_to_remove_for_trees, operation=lambda x: x >= 0) # Else, default numpy_equal operator return (numpy.equal(x, y),) @@ -1153,8 +1153,11 @@ def rounded_numpy_less_or_equal_for_trees( """ # numpy.less_equal(x, y) <= y is equivalent to : - # x - y <= 0 => round_bit_pattern(x - y + half) <= 0 or - # y - x >= 0 => round_bit_pattern(y - x - half) >= 0 + # option 1: x - y <= 0 => round_bit_pattern(x - y + half) <= 0 or + # option 2: y - x >= 0 => round_bit_pattern(y - x - half) >= 0 + + # Option 2 is selected because it adheres to the established pattern in `rounded_comparison` + # which does: (a - b) - half. if lsbs_to_remove_for_trees is not None and lsbs_to_remove_for_trees > 0: return rounded_comparison(y, x, lsbs_to_remove_for_trees, operation=lambda x: x >= 0) @@ -1522,7 +1525,7 @@ def numpy_batchnorm( training_mode (int): if the model was exported in training mode this is set to 1, else 0 Returns: - numpy.ndarray: Normalized tensor + numpy.ndarray: Normalized tenso """ assert_true( diff --git a/src/concrete/ml/sklearn/base.py b/src/concrete/ml/sklearn/base.py index 84f2884c4..63f05a64a 100644 --- a/src/concrete/ml/sklearn/base.py +++ b/src/concrete/ml/sklearn/base.py @@ -2,6 +2,7 @@ from __future__ import annotations import copy +import os import tempfile # Disable pylint as some names like X and q_X are used, following scikit-Learn's standard. The file @@ -90,8 +91,14 @@ # Define QNN's attribute that will be auto-generated when fitting QNN_AUTO_KWARGS = ["module__n_outputs", "module__input_dim"] +# Enable rounding feature for all tree-based models by default +# Note: This setting is fixed and cannot be altered by users +# However, for internal testing purposes, we retain the capability to disable this feature +os.environ["TREES_USE_ROUNDING"] = "1" # pylint: disable=too-many-public-methods + + class BaseEstimator: """Base class for all estimators in Concrete ML. @@ -1283,41 +1290,8 @@ def __init__(self, n_bits: int): #: The model's inference function. Is None if the model is not fitted. self._tree_inference: Optional[Callable] = None - #: Boolean that indicates whether the rounding feature is enabled or not. - self._use_rounding: bool = True - BaseEstimator.__init__(self) - @property - def use_optimized_execution(self) -> bool: - """The rounding property. - - Returns: - bool: Whether to enable or disable rounding - """ - return self._use_rounding # pragma: no cover - - @use_optimized_execution.setter - def use_optimized_execution(self, value: bool) -> None: - """Set the rounding feature. - - Args: - value (bool): Whether to enable or disable rounding - """ - assert isinstance(value, bool) - - self._use_rounding = value - - if not value: - warnings.simplefilter("always") - warnings.warn( - "Using Concrete tree-based models without the `rounding feature` is deprecated. " - "Consider setting 'use_rounding' to `True` for making the FHE inference faster " - "and key generation.", - category=DeprecationWarning, - stacklevel=2, - ) - def fit(self, X: Data, y: Target, **fit_parameters): # Reset for double fit self._is_fitted = False @@ -1344,10 +1318,22 @@ def fit(self, X: Data, y: Target, **fit_parameters): assert self.sklearn_model is not None, self._sklearn_model_is_not_fitted_error_message() # Convert the tree inference with Numpy operators + enable_rounding = os.environ.get("TREES_USE_ROUNDING", "1") == "1" + + if not enable_rounding: + warnings.simplefilter("always") + warnings.warn( + "Using Concrete tree-based models without the `rounding feature` is deprecated. " + "Consider setting 'use_rounding' to `True` for making the FHE inference faster " + "and key generation.", + category=DeprecationWarning, + stacklevel=2, + ) + self._tree_inference, self.output_quantizers, self.onnx_model_ = tree_to_numpy( self.sklearn_model, q_X, - use_rounding=self._use_rounding, + use_rounding=enable_rounding, framework=self.framework, output_n_bits=self.n_bits, ) diff --git a/tests/sklearn/test_sklearn_models.py b/tests/sklearn/test_sklearn_models.py index 53c8da9e5..029ecd8e2 100644 --- a/tests/sklearn/test_sklearn_models.py +++ b/tests/sklearn/test_sklearn_models.py @@ -23,6 +23,7 @@ import copy import json +import os import tempfile # pylint: disable=too-many-lines, too-many-arguments @@ -103,6 +104,9 @@ # the CRT. N_BITS_THRESHOLD_FOR_CRT_FHE_CIRCUITS = 9 +# Current maximum n_bits value for tree-based models +N_BITS_MAXIMUM_THRESHOLD_FOR_TREE_BASED_MODELS = 9 + def get_dataset(model_class, parameters, n_bits, load_data, is_weekly_option): """Prepare the the (x, y) data-set.""" @@ -440,7 +444,7 @@ def check_serialization_dump_load(model, x, use_dump_method): assert numpy.array_equal(y_pred_sklearn_model, y_pred_loaded_sklearn_model) # Add a test to check that graphs before and after the serialization are identical - # FIME: https://github.com/zama-ai/concrete-internal/issues/546 + # FIME: https://github.com/zama-ai/concrete-ml-internal/issues/4175 def check_serialization_dumps_loads(model, x, use_dump_method): @@ -496,7 +500,7 @@ def check_serialization_dumps_loads(model, x, use_dump_method): assert numpy.array_equal(y_pred_sklearn_model, y_pred_loaded_sklearn_model) # Add a test to check that graphs before and after the serialization are identical - # FIME: https://github.com/zama-ai/concrete-internal/issues/546 + # FIME: https://github.com/zama-ai/concrete-ml-internal/issues/4175 def check_offset(model_class, n_bits, x, y): @@ -1163,6 +1167,7 @@ def check_load_fitted_sklearn_linear_models(model_class, n_bits, x, y, check_flo def check_rounding_consistency( model, + n_bits, x, y, predict_method, @@ -1173,40 +1178,61 @@ def check_rounding_consistency( # Run the test with more samples during weekly CIs if is_weekly_option: - fhe_samples = 5 - else: - fhe_samples = 1 + fhe_test = get_random_samples(x, n_sample=5) - fhe_test = get_random_samples(x, fhe_samples) + # Check that rounding is enabled + rounding_enabled = os.getenv("TREES_USE_ROUNDING") == "1" + assert rounding_enabled # Fit and compile with rounding enabled circuit_with_rounding = fit_and_compile(model, x, y) rounded_predict_quantized = predict_method(x, fhe="disable") rounded_predict_simulate = predict_method(x, fhe="simulate") - rounded_predict_fhe = predict_method(fhe_test, fhe="execute") - # Fit and compile without rounding + # Compute the FHE predictions only during weekly CIs + if is_weekly_option: + rounded_predict_fhe = predict_method(fhe_test, fhe="execute") - with warnings.catch_warnings(): - warnings.simplefilter("ignore", category=DeprecationWarning) - model.use_optimized_execution = False + # For models that has less than 9 bits of quantization, we repeat the same experiment, but this + # time with rounding disabled + if n_bits <= N_BITS_MAXIMUM_THRESHOLD_FOR_TREE_BASED_MODELS: + with pytest.MonkeyPatch.context() as mp_context: - circuit_without_rounding = fit_and_compile(model, x, y) + # Disable rounding + mp_context.setenv("TREES_USE_ROUNDING", "0") - not_rounded_predict_quantized = predict_method(x, fhe="disable") - not_rounded_predict_simulate = predict_method(x, fhe="simulate") - not_rounded_predict_fhe = predict_method(fhe_test, fhe="execute") + # Check that rounding is disabled + rounding_disabled = os.environ.get("TREES_USE_ROUNDING") == "0" + assert rounding_disabled - metric(rounded_predict_quantized, not_rounded_predict_quantized) - metric(rounded_predict_simulate, not_rounded_predict_simulate) - metric(rounded_predict_fhe, not_rounded_predict_fhe) + with pytest.warns( + DeprecationWarning, + match=( + "Using Concrete tree-based models without the `rounding feature` is " + "deprecated.*" + ), + ): - # Check that the maximum bitwidth of the cuircuit with rounding is at most n_bits + 2 - max_bitwitdth_with_rounding = circuit_with_rounding.graph.maximum_integer_bit_width() - max_bitwitdth_without_rounding = circuit_without_rounding.graph.maximum_integer_bit_width() + # Fit and compile without rounding + circuit_without_rounding = fit_and_compile(model, x, y) - assert max_bitwitdth_with_rounding <= max_bitwitdth_without_rounding + 2 + not_rounded_predict_quantized = predict_method(x, fhe="disable") + not_rounded_predict_simulate = predict_method(x, fhe="simulate") + + metric(rounded_predict_quantized, not_rounded_predict_quantized) + metric(rounded_predict_simulate, not_rounded_predict_simulate) + + # Compute the FHE predictions only during weekly CIs + if is_weekly_option: + not_rounded_predict_fhe = predict_method(fhe_test, fhe="execute") + metric(rounded_predict_fhe, not_rounded_predict_fhe) + + # Check that the maximum bit-width of the circuit with rounding is at most n_bits + 2 + max_bitwitdth_with_rounding = circuit_with_rounding.graph.maximum_integer_bit_width() + max_bitwitdth_without_rounding = circuit_without_rounding.graph.maximum_integer_bit_width() + + assert max_bitwitdth_with_rounding <= max_bitwitdth_without_rounding + 2 # Neural network models are skipped for this test @@ -1839,10 +1865,9 @@ def test_linear_models_have_no_tlu( check_circuit_has_no_tlu(fhe_circuit) -# Test only tree-based models @pytest.mark.parametrize("model_class, parameters", get_sklearn_tree_models_and_datasets()) -@pytest.mark.parametrize("n_bits", [2, 5]) -def test_rounding_consistency( +@pytest.mark.parametrize("n_bits", [2, 5, 11]) +def test_rounding_consistency_for_regular_models( model_class, parameters, n_bits, @@ -1858,6 +1883,17 @@ def test_rounding_consistency( print("Run check_rounding_consistency") model = instantiate_model_generic(model_class, n_bits=n_bits) + + # According to https://arxiv.org/pdf/2010.04804.pdf, there are two levels of comparison for + # trees: one at the level of X.A < B, and another at the level of I.C == D. + # To trigger the rounding in the second level, we need datasets with larger sizes and higher + # quantization precision are required. + # However, the test will be executed exclusively with rounding enabled, as tree-based models + # without rounding cannot be compiled. + if n_bits > 9: + parameters["n_samples"] = 1000 + parameters["n_features"] = 10 + x, y = get_dataset(model_class, parameters, n_bits, load_data, is_weekly_option) # Check `predict_proba` for classifiers @@ -1871,6 +1907,7 @@ def test_rounding_consistency( check_rounding_consistency( model, + n_bits, x, y, predict_method, From 586d7dbf69b7aafe1dc7366f0f184eff51e7ebd4 Mon Sep 17 00:00:00 2001 From: kcelia Date: Wed, 13 Dec 2023 15:53:34 +0100 Subject: [PATCH 33/73] chore: remove the assert that checks that : the maximum_bit (of the circuit without rounding) = maximum_bitwidth (of the circuit with rounding) Remove the test with rounding for n_bits=11 --- src/concrete/ml/onnx/ops_impl.py | 6 ++- tests/sklearn/test_sklearn_models.py | 68 ++++++++++------------------ 2 files changed, 29 insertions(+), 45 deletions(-) diff --git a/src/concrete/ml/onnx/ops_impl.py b/src/concrete/ml/onnx/ops_impl.py index 22d0d7658..281da62eb 100644 --- a/src/concrete/ml/onnx/ops_impl.py +++ b/src/concrete/ml/onnx/ops_impl.py @@ -902,6 +902,8 @@ def numpy_equal( return (numpy.equal(x, y),) +# Remove `# pragma: no cover` once the following issue will be resolved +# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4179 def rounded_numpy_equal_for_trees( x: numpy.ndarray, y: numpy.ndarray, @@ -931,7 +933,9 @@ def rounded_numpy_equal_for_trees( # Option 2 is selected because it adheres to the established pattern in `rounded_comparison` # which does: (a - b) - half. if lsbs_to_remove_for_trees is not None and lsbs_to_remove_for_trees > 0: - return rounded_comparison(y, x, lsbs_to_remove_for_trees, operation=lambda x: x >= 0) + return rounded_comparison( + y, x, lsbs_to_remove_for_trees, operation=lambda x: x >= 0 + ) # pragma: no cover # Else, default numpy_equal operator return (numpy.equal(x, y),) diff --git a/tests/sklearn/test_sklearn_models.py b/tests/sklearn/test_sklearn_models.py index 029ecd8e2..c3540ca94 100644 --- a/tests/sklearn/test_sklearn_models.py +++ b/tests/sklearn/test_sklearn_models.py @@ -104,9 +104,6 @@ # the CRT. N_BITS_THRESHOLD_FOR_CRT_FHE_CIRCUITS = 9 -# Current maximum n_bits value for tree-based models -N_BITS_MAXIMUM_THRESHOLD_FOR_TREE_BASED_MODELS = 9 - def get_dataset(model_class, parameters, n_bits, load_data, is_weekly_option): """Prepare the the (x, y) data-set.""" @@ -164,9 +161,7 @@ def fit_and_compile(model, x, y): warnings.simplefilter("ignore", category=ConvergenceWarning) model.fit(x, y) - circuit = model.compile(x) - - return circuit + model.compile(x) def check_correctness_with_sklearn( @@ -1167,7 +1162,6 @@ def check_load_fitted_sklearn_linear_models(model_class, n_bits, x, y, check_flo def check_rounding_consistency( model, - n_bits, x, y, predict_method, @@ -1185,7 +1179,7 @@ def check_rounding_consistency( assert rounding_enabled # Fit and compile with rounding enabled - circuit_with_rounding = fit_and_compile(model, x, y) + fit_and_compile(model, x, y) rounded_predict_quantized = predict_method(x, fhe="disable") rounded_predict_simulate = predict_method(x, fhe="simulate") @@ -1194,31 +1188,27 @@ def check_rounding_consistency( if is_weekly_option: rounded_predict_fhe = predict_method(fhe_test, fhe="execute") - # For models that has less than 9 bits of quantization, we repeat the same experiment, but this - # time with rounding disabled - if n_bits <= N_BITS_MAXIMUM_THRESHOLD_FOR_TREE_BASED_MODELS: - with pytest.MonkeyPatch.context() as mp_context: + with pytest.MonkeyPatch.context() as mp_context: - # Disable rounding - mp_context.setenv("TREES_USE_ROUNDING", "0") + # Disable rounding + mp_context.setenv("TREES_USE_ROUNDING", "0") - # Check that rounding is disabled - rounding_disabled = os.environ.get("TREES_USE_ROUNDING") == "0" - assert rounding_disabled + # Check that rounding is disabled + rounding_disabled = os.environ.get("TREES_USE_ROUNDING") == "0" + assert rounding_disabled - with pytest.warns( - DeprecationWarning, - match=( - "Using Concrete tree-based models without the `rounding feature` is " - "deprecated.*" - ), - ): + with pytest.warns( + DeprecationWarning, + match=( + "Using Concrete tree-based models without the `rounding feature` is " "deprecated.*" + ), + ): - # Fit and compile without rounding - circuit_without_rounding = fit_and_compile(model, x, y) + # Fit and compile without rounding + fit_and_compile(model, x, y) - not_rounded_predict_quantized = predict_method(x, fhe="disable") - not_rounded_predict_simulate = predict_method(x, fhe="simulate") + not_rounded_predict_quantized = predict_method(x, fhe="disable") + not_rounded_predict_simulate = predict_method(x, fhe="simulate") metric(rounded_predict_quantized, not_rounded_predict_quantized) metric(rounded_predict_simulate, not_rounded_predict_simulate) @@ -1228,11 +1218,9 @@ def check_rounding_consistency( not_rounded_predict_fhe = predict_method(fhe_test, fhe="execute") metric(rounded_predict_fhe, not_rounded_predict_fhe) - # Check that the maximum bit-width of the circuit with rounding is at most n_bits + 2 - max_bitwitdth_with_rounding = circuit_with_rounding.graph.maximum_integer_bit_width() - max_bitwitdth_without_rounding = circuit_without_rounding.graph.maximum_integer_bit_width() - - assert max_bitwitdth_with_rounding <= max_bitwitdth_without_rounding + 2 + # Check that the maximum bit-width of the circuit with rounding is at most: + # maximum bit-width (of the circuit without rounding) + 2 + # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4178 # Neural network models are skipped for this test @@ -1865,6 +1853,9 @@ def test_linear_models_have_no_tlu( check_circuit_has_no_tlu(fhe_circuit) +# This test does not check rounding at level 2 +# Additional tests for this purpose should be added in future updates +# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4179 @pytest.mark.parametrize("model_class, parameters", get_sklearn_tree_models_and_datasets()) @pytest.mark.parametrize("n_bits", [2, 5, 11]) def test_rounding_consistency_for_regular_models( @@ -1884,16 +1875,6 @@ def test_rounding_consistency_for_regular_models( model = instantiate_model_generic(model_class, n_bits=n_bits) - # According to https://arxiv.org/pdf/2010.04804.pdf, there are two levels of comparison for - # trees: one at the level of X.A < B, and another at the level of I.C == D. - # To trigger the rounding in the second level, we need datasets with larger sizes and higher - # quantization precision are required. - # However, the test will be executed exclusively with rounding enabled, as tree-based models - # without rounding cannot be compiled. - if n_bits > 9: - parameters["n_samples"] = 1000 - parameters["n_features"] = 10 - x, y = get_dataset(model_class, parameters, n_bits, load_data, is_weekly_option) # Check `predict_proba` for classifiers @@ -1907,7 +1888,6 @@ def test_rounding_consistency_for_regular_models( check_rounding_consistency( model, - n_bits, x, y, predict_method, From 094df3ebea0a843a8c66dcb271d383c2d218bec6 Mon Sep 17 00:00:00 2001 From: kcelia Date: Wed, 13 Dec 2023 17:59:19 +0100 Subject: [PATCH 34/73] chore: first test with truncate --- src/concrete/ml/onnx/convert.py | 6 +- src/concrete/ml/onnx/onnx_impl_utils.py | 13 +-- src/concrete/ml/onnx/onnx_utils.py | 12 +- src/concrete/ml/onnx/ops_impl.py | 40 +++---- src/concrete/ml/sklearn/base.py | 26 ++++- src/concrete/ml/sklearn/tree_to_numpy.py | 137 +---------------------- 6 files changed, 55 insertions(+), 179 deletions(-) diff --git a/src/concrete/ml/onnx/convert.py b/src/concrete/ml/onnx/convert.py index 96b1f1eee..6fdf5c497 100644 --- a/src/concrete/ml/onnx/convert.py +++ b/src/concrete/ml/onnx/convert.py @@ -252,7 +252,7 @@ def get_equivalent_numpy_forward_from_onnx( def get_equivalent_numpy_forward_from_onnx_tree( onnx_model: onnx.ModelProto, check_model: bool = True, - lsbs_to_remove_for_trees: Optional[Tuple[int, int]] = None, + auto_truncate = None, ) -> Tuple[Callable[..., Tuple[numpy.ndarray, ...]], onnx.ModelProto]: """Get the numpy equivalent forward of the provided ONNX model for tree-based models only. @@ -261,7 +261,7 @@ def get_equivalent_numpy_forward_from_onnx_tree( forward. check_model (bool): set to True to run the onnx checker on the model. Defaults to True. - lsbs_to_remove_for_trees (Optional[Tuple[int, int]]): This parameter is exclusively used for + auto_truncate: This parameter is exclusively used for optimizing tree-based models. It contains the values of the least significant bits to remove during the tree traversal, where the first value refers to the first comparison (either "less" or "less_or_equal"), while the second value refers to the "Equal" @@ -277,6 +277,6 @@ def get_equivalent_numpy_forward_from_onnx_tree( # Return lambda of numpy equivalent of onnx execution return ( lambda *args: execute_onnx_with_numpy_trees( - equivalent_onnx_model.graph, lsbs_to_remove_for_trees, *args + equivalent_onnx_model.graph, auto_truncate, *args ) ), equivalent_onnx_model diff --git a/src/concrete/ml/onnx/onnx_impl_utils.py b/src/concrete/ml/onnx/onnx_impl_utils.py index 1a554e96b..9946241fa 100644 --- a/src/concrete/ml/onnx/onnx_impl_utils.py +++ b/src/concrete/ml/onnx/onnx_impl_utils.py @@ -5,7 +5,7 @@ import numpy from concrete.fhe import conv as cnp_conv from concrete.fhe import ones as cnp_ones -from concrete.fhe import round_bit_pattern +from concrete.fhe import truncate_bit_pattern from concrete.fhe.tracing import Tracer from ..common.debugging import assert_true @@ -238,7 +238,7 @@ def onnx_avgpool_compute_norm_const( # - Adjust the typing # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4143 def rounded_comparison( - x: numpy.ndarray, y: numpy.ndarray, lsbs_to_remove: int, operation: ComparisonOperationType + x: numpy.ndarray, y: numpy.ndarray, auto_truncate, operation: ComparisonOperationType ) -> Tuple[bool]: """Comparison operation using `round_bit_pattern` function. @@ -260,14 +260,7 @@ def rounded_comparison( Tuple[bool]: If x and y satisfy the comparison operator. """ - assert isinstance(lsbs_to_remove, int) - - # Workaround: in this context, `round_bit_pattern` is used as a truncate operation. - # Consequently, we subtract a term, called `half` that will subsequently be re-added during the - # `round_bit_pattern` process. - half = 1 << (lsbs_to_remove - 1) - # To determine if 'x' 'operation' 'y' (operation being <, >, >=, <=), we evaluate 'x - y' - rounded_subtraction = round_bit_pattern((x - y) - half, lsbs_to_remove=lsbs_to_remove) + rounded_subtraction = truncate_bit_pattern(x - y, lsbs_to_remove=auto_truncate) return (operation(rounded_subtraction),) diff --git a/src/concrete/ml/onnx/onnx_utils.py b/src/concrete/ml/onnx/onnx_utils.py index a8b7bfd71..f566fb5ab 100644 --- a/src/concrete/ml/onnx/onnx_utils.py +++ b/src/concrete/ml/onnx/onnx_utils.py @@ -479,14 +479,14 @@ def execute_onnx_with_numpy( def execute_onnx_with_numpy_trees( graph: onnx.GraphProto, - lsbs_to_remove_for_trees: Optional[Tuple[int, int]], + auto_truncate, *inputs: numpy.ndarray, ) -> Tuple[numpy.ndarray, ...]: """Execute the provided ONNX graph on the given inputs for tree-based models only. Args: graph (onnx.GraphProto): The ONNX graph to execute. - lsbs_to_remove_for_trees (Optional[Tuple[int, int]]): This parameter is exclusively used for + auto_truncate: This parameter is exclusively used for optimizing tree-based models. It contains the values of the least significant bits to remove during the tree traversal, where the first value refers to the first comparison (either "less" or "less_or_equal"), while the second value refers to the "Equal" @@ -501,7 +501,7 @@ def execute_onnx_with_numpy_trees( op_type: Callable[..., Tuple[numpy.ndarray[Any, Any], ...]] # If no tree-based optimization is specified, return standard execution - if lsbs_to_remove_for_trees is None: + if auto_truncate is None: return execute_onnx_with_numpy(graph, *inputs) node_results: Dict[str, numpy.ndarray] = dict( @@ -517,11 +517,7 @@ def execute_onnx_with_numpy_trees( attributes = {attribute.name: get_attribute(attribute) for attribute in node.attribute} if node.op_type in ONNX_COMPARISON_OPS_TO_ROUNDED_TREES_NUMPY_IMPL_BOOL: - - # The first LSB refers to `Less` or `LessOrEqual` comparisons - # The second LSB refers to `Equal` comparison - stage = 0 if node.op_type != "Equal" else 1 - attributes["lsbs_to_remove_for_trees"] = lsbs_to_remove_for_trees[stage] + attributes["auto_truncate"] = auto_truncate # Use rounded numpy operation to relevant comparison nodes op_type = ONNX_COMPARISON_OPS_TO_ROUNDED_TREES_NUMPY_IMPL_BOOL[node.op_type] diff --git a/src/concrete/ml/onnx/ops_impl.py b/src/concrete/ml/onnx/ops_impl.py index 281da62eb..89c81d4e2 100644 --- a/src/concrete/ml/onnx/ops_impl.py +++ b/src/concrete/ml/onnx/ops_impl.py @@ -908,7 +908,7 @@ def rounded_numpy_equal_for_trees( x: numpy.ndarray, y: numpy.ndarray, *, - lsbs_to_remove_for_trees: Optional[int] = None, + auto_truncate = None, ) -> Tuple[numpy.ndarray]: """Compute rounded equal in numpy according to ONNX spec for tree-based models only. @@ -917,7 +917,7 @@ def rounded_numpy_equal_for_trees( Args: x (numpy.ndarray): Input tensor y (numpy.ndarray): Input tensor - lsbs_to_remove_for_trees (Optional[int]): Number of the least significant bits to remove + auto_truncate: Number of the least significant bits to remove for tree-based models only. Returns: @@ -932,10 +932,10 @@ def rounded_numpy_equal_for_trees( # Option 2 is selected because it adheres to the established pattern in `rounded_comparison` # which does: (a - b) - half. - if lsbs_to_remove_for_trees is not None and lsbs_to_remove_for_trees > 0: - return rounded_comparison( - y, x, lsbs_to_remove_for_trees, operation=lambda x: x >= 0 - ) # pragma: no cover + # if auto_truncate is not None: + # return rounded_comparison( + # y, x, auto_truncate, operation=lambda x: x >= 0 + # ) # pragma: no cover # Else, default numpy_equal operator return (numpy.equal(x, y),) @@ -1073,7 +1073,7 @@ def rounded_numpy_less_for_trees( x: numpy.ndarray, y: numpy.ndarray, *, - lsbs_to_remove_for_trees: Optional[int] = None, + auto_truncate, ) -> Tuple[numpy.ndarray]: """Compute rounded less in numpy according to ONNX spec for tree-based models only. @@ -1082,7 +1082,7 @@ def rounded_numpy_less_for_trees( Args: x (numpy.ndarray): Input tensor y (numpy.ndarray): Input tensor - lsbs_to_remove_for_trees (Optional[int]): Number of the least significant bits to remove + auto_truncate: Number of the least significant bits to remove for tree-based models only. Returns: @@ -1091,8 +1091,9 @@ def rounded_numpy_less_for_trees( # numpy.less(x, y) is equivalent to : # x - y <= 0 => round_bit_pattern(x - y - half) < 0 - if lsbs_to_remove_for_trees is not None and lsbs_to_remove_for_trees > 0: - return rounded_comparison(x, y, lsbs_to_remove_for_trees, operation=lambda x: x < 0) + if auto_truncate is not None: + #print("Use truncate for <") + return rounded_comparison(x, y, auto_truncate, operation=lambda x: x < 0) # Else, default numpy_less operator return numpy_less(x, y) @@ -1140,7 +1141,7 @@ def rounded_numpy_less_or_equal_for_trees( x: numpy.ndarray, y: numpy.ndarray, *, - lsbs_to_remove_for_trees: Optional[int] = None, + auto_truncate = None, ) -> Tuple[numpy.ndarray]: """Compute rounded less or equal in numpy according to ONNX spec for tree-based models only. @@ -1149,21 +1150,22 @@ def rounded_numpy_less_or_equal_for_trees( Args: x (numpy.ndarray): Input tensor y (numpy.ndarray): Input tensor - lsbs_to_remove_for_trees (Optional[int]): Number of the least significant bits to remove + auto_truncate: Number of the least significant bits to remove for tree-based models only. Returns: Tuple[numpy.ndarray]: Output tensor """ - # numpy.less_equal(x, y) <= y is equivalent to : - # option 1: x - y <= 0 => round_bit_pattern(x - y + half) <= 0 or - # option 2: y - x >= 0 => round_bit_pattern(y - x - half) >= 0 + # numpy.less_equal(x, y) <= 0 is equivalent to : + # np.less_equal(x, y), truncate_bit_pattern((y - x), lsbs_to_remove=r) >= 0 + # option 1: x - y <= 0 => round_bit_pattern(x - y) <= 0 + # gives bad results for : 0 < x - y <= 2**lsbs_to_remove because truncate_bit_pattern(x - y, lsb) = 0 + # option 2: y - x >= 0 => round_bit_pattern(y - x) >= 0 - # Option 2 is selected because it adheres to the established pattern in `rounded_comparison` - # which does: (a - b) - half. - if lsbs_to_remove_for_trees is not None and lsbs_to_remove_for_trees > 0: - return rounded_comparison(y, x, lsbs_to_remove_for_trees, operation=lambda x: x >= 0) + if auto_truncate is not None: + #print("Use truncate for <=") + return rounded_comparison(y, x, auto_truncate, operation=lambda x: x >= 0) # Else, default numpy_less_or_equal operator return numpy_less_or_equal(x, y) diff --git a/src/concrete/ml/sklearn/base.py b/src/concrete/ml/sklearn/base.py index 63f05a64a..7edec4b1a 100644 --- a/src/concrete/ml/sklearn/base.py +++ b/src/concrete/ml/sklearn/base.py @@ -91,14 +91,16 @@ # Define QNN's attribute that will be auto-generated when fitting QNN_AUTO_KWARGS = ["module__n_outputs", "module__input_dim"] -# Enable rounding feature for all tree-based models by default +# Most significant bits to retain when applying rounding to the tree +MSB_TO_KEEP_FOR_TREES = 1 + +# Enable truncate feature for all tree-based models by default # Note: This setting is fixed and cannot be altered by users # However, for internal testing purposes, we retain the capability to disable this feature os.environ["TREES_USE_ROUNDING"] = "1" # pylint: disable=too-many-public-methods - class BaseEstimator: """Base class for all estimators in Concrete ML. @@ -521,6 +523,7 @@ def compile( Returns: Circuit: The compiled Circuit. """ + print("compilation stage 2") # Reset for double compile self._is_compiled = False @@ -1290,6 +1293,9 @@ def __init__(self, n_bits: int): #: The model's inference function. Is None if the model is not fitted. self._tree_inference: Optional[Callable] = None + #: Determines the LSB to remove given a `target_msbs` + self.auto_truncate = cnp.AutoTruncator(target_msbs=MSB_TO_KEEP_FOR_TREES) + BaseEstimator.__init__(self) def fit(self, X: Data, y: Target, **fit_parameters): @@ -1318,9 +1324,9 @@ def fit(self, X: Data, y: Target, **fit_parameters): assert self.sklearn_model is not None, self._sklearn_model_is_not_fitted_error_message() # Convert the tree inference with Numpy operators - enable_rounding = os.environ.get("TREES_USE_ROUNDING", "1") == "1" + enable_truncate = os.environ.get("TREES_USE_ROUNDING", "1") == "1" - if not enable_rounding: + if not enable_truncate: warnings.simplefilter("always") warnings.warn( "Using Concrete tree-based models without the `rounding feature` is deprecated. " @@ -1329,15 +1335,24 @@ def fit(self, X: Data, y: Target, **fit_parameters): category=DeprecationWarning, stacklevel=2, ) + self.auto_truncate = None + print(f"{self.auto_truncate=}") + self._tree_inference, self.output_quantizers, self.onnx_model_ = tree_to_numpy( self.sklearn_model, q_X, - use_rounding=enable_rounding, + auto_truncate=self.auto_truncate, framework=self.framework, output_n_bits=self.n_bits, ) + # Adjust the truncate + inputset = numpy.array(list(_get_inputset_generator(q_X))).astype(int) + self.auto_truncate.adjust(self._tree_inference, inputset) + + self._tree_inference(q_X.astype("int")) + self._is_fitted = True return self @@ -1377,6 +1392,7 @@ def _get_module_to_compile(self) -> Union[Compiler, QuantizedModule]: return compiler def compile(self, *args, **kwargs) -> Circuit: + print("Compilation base.py") BaseEstimator.compile(self, *args, **kwargs) # Check that the graph only has a single output diff --git a/src/concrete/ml/sklearn/tree_to_numpy.py b/src/concrete/ml/sklearn/tree_to_numpy.py index 15940e0ce..4c23f5365 100644 --- a/src/concrete/ml/sklearn/tree_to_numpy.py +++ b/src/concrete/ml/sklearn/tree_to_numpy.py @@ -306,7 +306,7 @@ def tree_to_numpy( model: Callable, x: numpy.ndarray, framework: str, - use_rounding: bool = True, + auto_truncate = None, output_n_bits: int = MAX_BITWIDTH_BACKWARD_COMPATIBLE, ) -> Tuple[Callable, List[UniformQuantizer], onnx.ModelProto]: """Convert the tree inference to a numpy functions using Hummingbird. @@ -314,7 +314,7 @@ def tree_to_numpy( Args: model (Callable): The tree model to convert. x (numpy.ndarray): The input data. - use_rounding (bool): This parameter is exclusively used to tree-based models. + auto_truncate: This parameter is exclusively used to tree-based models. It determines whether the rounding feature is enabled or disabled. framework (str): The framework from which the ONNX model is generated. (options: 'xgboost', 'sklearn') @@ -328,8 +328,6 @@ def tree_to_numpy( # mypy assert output_n_bits is not None - lsbs_to_remove_for_trees: Optional[Tuple[int, int]] = None - assert_true( framework in ["xgboost", "sklearn"], f"framework={framework} is not supported. It must be either 'xgboost' or 'sklearn'", @@ -338,15 +336,6 @@ def tree_to_numpy( # Execute with 1 example for efficiency in large data scenarios to prevent slowdown onnx_model = get_onnx_model(model, x[:1], framework) - # Compute for tree-based models the LSB to remove in stage 1 and stage 2 - if use_rounding: - # First LSB refers to Less or LessOrEqual comparisons - # Second LSB refers to Equal comparison - lsbs_to_remove_for_trees = _compute_lsb_to_remove_for_trees(onnx_model, x) - - # mypy - assert len(lsbs_to_remove_for_trees) == 2 - # Get the expected number of ONNX outputs in the sklearn model. expected_number_of_outputs = 1 if is_regressor_or_partial_regressor(model) else 2 @@ -360,127 +349,7 @@ def tree_to_numpy( q_y = tree_values_preprocessing(onnx_model, framework, output_n_bits) _tree_inference, onnx_model = get_equivalent_numpy_forward_from_onnx_tree( - onnx_model, lsbs_to_remove_for_trees=lsbs_to_remove_for_trees + onnx_model, auto_truncate=auto_truncate ) return (_tree_inference, [q_y.quantizer], onnx_model) - - -# Remove this function once the truncate feature is released -# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4143 -def _compute_lsb_to_remove_for_trees( - onnx_model: onnx.ModelProto, q_x: numpy.ndarray -) -> Tuple[int, int]: - """Compute the LSB to remove for the comparison operators in the trees. - - Referring to this paper: https://arxiv.org/pdf/2010.04804.pdf, there are - 2 levels of comparison for trees, one at the level of X.A < B and a second at - the level of I.C == D. - - Args: - onnx_model (onnx.ModelProto): The model to clean - q_x (numpy.ndarray): The quantized inputs - - Returns: - Tuple[int, int]: the number of LSB to remove for level 1 and level 2 - """ - - def get_bitwidth(array: numpy.ndarray) -> int: - """Compute the bitwidth required to represent the largest value in `array`. - - Args: - array (umpy.ndarray): The array for which the bitwidth needs to be checked. - - Returns: - int: The required bits to represent the array. - """ - - max_val = numpy.max(numpy.abs(array)) - - # + 1 is added to include the sign bit - bitwidth = math.ceil(math.log2(max_val + 1)) + 1 - return bitwidth - - def get_lsbs_to_remove_for_trees(array: numpy.ndarray) -> int: - """Update the number of LSBs to remove based on overflow detection. - - this function works only for MSB = 1 - - Args: - array (numpy.ndarray): The array for which the bitwidth needs to be checked. - - Returns: - int: The updated LSB to remove. - """ - - lsbs_to_remove_for_trees: int = 0 - - prev_bitwidth = get_bitwidth(array) - - if prev_bitwidth > MIN_CIRCUIT_THRESHOLD_FOR_TREES: - - if prev_bitwidth - MSB_TO_KEEP_FOR_TREES > 0: - - msb = MSB_TO_KEEP_FOR_TREES if MSB_TO_KEEP_FOR_TREES > 1 else 0 - lsbs_to_remove_for_trees = prev_bitwidth - msb - - return lsbs_to_remove_for_trees - - quant_params = { - onnx_init.name: numpy_helper.to_array(onnx_init) - for onnx_init in onnx_model.graph.initializer - if "weight" in onnx_init.name or "bias" in onnx_init.name - } - - key_mat_1 = [key for key in quant_params.keys() if "_1" in key and "weight" in key][0] - key_bias_1 = [key for key in quant_params.keys() if "_1" in key and "bias" in key][0] - - key_mat_2 = [key for key in quant_params.keys() if "_2" in key and "weight" in key][0] - key_bias_2 = [key for key in quant_params.keys() if "_2" in key and "bias" in key][0] - - # shape: (nodes, features) or (trees * nodes, features) - mat_1 = quant_params[key_mat_1] - - # shape: (nodes, 1) or (trees * nodes, 1) - bias_1 = quant_params[key_bias_1] - - # shape: (trees, leaves, nodes) - mat_2 = quant_params[key_mat_2] - - # shape: (leaves, 1) or (trees * leaves, 1) - bias_2 = quant_params[key_bias_2] - - n_features = mat_1.shape[1] - n_nodes = mat_2.shape[2] - n_leaves = mat_2.shape[1] - - mat_1 = mat_1.reshape(-1, n_nodes, n_features) - bias_1 = bias_1.reshape(-1, 1, n_nodes) - bias_2 = bias_2.reshape(-1, 1, n_leaves) - - required_onnx_operators = set(get_op_type(node) for node in onnx_model.graph.node) - - # If operator is `<`, np.less(x, y) is equivalent to: - # round_bit_pattern((x - y) - half, lsbs_to_remove_for_trees=r) < 0. - # Therefore, stage_1 = (q_x @ mat_1.transpose(0, 2, 1)) - bias_1 - if "Less" in required_onnx_operators: - stage_1 = (q_x @ mat_1.transpose(0, 2, 1)) - bias_1 - matrix_q = stage_1 < 0 - - # Else, if operator is `<=`, np.less_equal(x, y) is equivalent to: - # round_bit_pattern((y - x) - half, lsbs_to_remove_for_trees=r) >= 0. - # Therefore, stage_1 = bias_1 - (q_x @ mat_1.transpose(0, 2, 1)) - elif "LessOrEqual" in required_onnx_operators: - stage_1 = bias_1 - (q_x @ mat_1.transpose(0, 2, 1)) - matrix_q = stage_1 >= 0 - - lsbs_to_remove_for_trees_stage_1 = get_lsbs_to_remove_for_trees(stage_1) - - # If operator is `==`, np.equal(x, y) is equivalent to: - # round_bit_pattern((x - y) - half, lsbs_to_remove_for_trees=r) >= 0. - # Therefore, stage_2 = bias_1 - (q_x @ mat_2.transpose(0, 2, 1)) - stage_2 = ((bias_2 - matrix_q @ mat_2.transpose(0, 2, 1))).max(axis=0) - - lsbs_to_remove_for_trees_stage_2 = get_lsbs_to_remove_for_trees(stage_2) - - return (lsbs_to_remove_for_trees_stage_1, lsbs_to_remove_for_trees_stage_2) From 51dd5f6c3687a744446b68ddd461e74faaa1c6b6 Mon Sep 17 00:00:00 2001 From: kcelia Date: Fri, 15 Dec 2023 07:56:50 +0100 Subject: [PATCH 35/73] chore: first test with truncate --- src/concrete/ml/onnx/ops_impl.py | 8 ++++---- src/concrete/ml/sklearn/base.py | 14 +++++++------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/concrete/ml/onnx/ops_impl.py b/src/concrete/ml/onnx/ops_impl.py index 89c81d4e2..7138c55b8 100644 --- a/src/concrete/ml/onnx/ops_impl.py +++ b/src/concrete/ml/onnx/ops_impl.py @@ -932,10 +932,10 @@ def rounded_numpy_equal_for_trees( # Option 2 is selected because it adheres to the established pattern in `rounded_comparison` # which does: (a - b) - half. - # if auto_truncate is not None: - # return rounded_comparison( - # y, x, auto_truncate, operation=lambda x: x >= 0 - # ) # pragma: no cover + if auto_truncate is not None: + return rounded_comparison( + y, x, auto_truncate, operation=lambda x: x >= 0 + ) # pragma: no cover # Else, default numpy_equal operator return (numpy.equal(x, y),) diff --git a/src/concrete/ml/sklearn/base.py b/src/concrete/ml/sklearn/base.py index 7edec4b1a..95ee6642b 100644 --- a/src/concrete/ml/sklearn/base.py +++ b/src/concrete/ml/sklearn/base.py @@ -1293,9 +1293,6 @@ def __init__(self, n_bits: int): #: The model's inference function. Is None if the model is not fitted. self._tree_inference: Optional[Callable] = None - #: Determines the LSB to remove given a `target_msbs` - self.auto_truncate = cnp.AutoTruncator(target_msbs=MSB_TO_KEEP_FOR_TREES) - BaseEstimator.__init__(self) def fit(self, X: Data, y: Target, **fit_parameters): @@ -1304,6 +1301,9 @@ def fit(self, X: Data, y: Target, **fit_parameters): self.input_quantizers = [] self.output_quantizers = [] + #: Determines the LSB to remove given a `target_msbs` + self.auto_truncate = cnp.AutoTruncator(target_msbs=MSB_TO_KEEP_FOR_TREES) + X, y = check_X_y_and_assert_multi_output(X, y) q_X = numpy.zeros_like(X) @@ -1348,10 +1348,10 @@ def fit(self, X: Data, y: Target, **fit_parameters): ) # Adjust the truncate - inputset = numpy.array(list(_get_inputset_generator(q_X))).astype(int) - self.auto_truncate.adjust(self._tree_inference, inputset) - - self._tree_inference(q_X.astype("int")) + if enable_truncate: + inputset = numpy.array(list(_get_inputset_generator(q_X))).astype(int) + self.auto_truncate.adjust(self._tree_inference, inputset) + self._tree_inference(q_X.astype("int")) self._is_fitted = True From e284bd2ffa24c82bcd4619247bce623c4457b950 Mon Sep 17 00:00:00 2001 From: jfrery Date: Mon, 18 Dec 2023 16:49:54 +0100 Subject: [PATCH 36/73] chore: run ensemble model aggregation in FHE closes https://github.com/zama-ai/concrete-ml-internal/issues/451 --- src/concrete/ml/sklearn/base.py | 9 --------- src/concrete/ml/sklearn/tree_to_numpy.py | 12 ++++++------ tests/sklearn/test_dump_onnx.py | 13 +++++++++---- 3 files changed, 15 insertions(+), 19 deletions(-) diff --git a/src/concrete/ml/sklearn/base.py b/src/concrete/ml/sklearn/base.py index d1275c130..f63f63902 100644 --- a/src/concrete/ml/sklearn/base.py +++ b/src/concrete/ml/sklearn/base.py @@ -1408,15 +1408,6 @@ def predict(self, X: Data, fhe: Union[FheMode, str] = FheMode.DISABLE) -> numpy. y_pred = self.post_processing(y_pred) return y_pred - def post_processing(self, y_preds: numpy.ndarray) -> numpy.ndarray: - # Sum all tree outputs - # Remove the sum once we handle multi-precision circuits - # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/451 - y_preds = numpy.sum(y_preds, axis=-1) - - assert_true(y_preds.ndim == 2, "y_preds should be a 2D array") - return y_preds - class BaseTreeRegressorMixin(BaseTreeEstimatorMixin, sklearn.base.RegressorMixin, ABC): """Mixin class for tree-based regressors. diff --git a/src/concrete/ml/sklearn/tree_to_numpy.py b/src/concrete/ml/sklearn/tree_to_numpy.py index 15940e0ce..42c241fe7 100644 --- a/src/concrete/ml/sklearn/tree_to_numpy.py +++ b/src/concrete/ml/sklearn/tree_to_numpy.py @@ -17,7 +17,7 @@ OPSET_VERSION_FOR_ONNX_EXPORT, get_equivalent_numpy_forward_from_onnx_tree, ) -from ..onnx.onnx_model_manipulations import clean_graph_at_node_op_type, remove_node_types +from ..onnx.onnx_model_manipulations import clean_graph_after_node_op_type, remove_node_types from ..onnx.onnx_utils import get_op_type from ..quantization import QuantizedArray from ..quantization.quantizers import UniformQuantizer @@ -141,12 +141,12 @@ def add_transpose_after_last_node(onnx_model: onnx.ModelProto): # Get the output node output_node = onnx_model.graph.output[0] - # Create the node with perm attribute equal to (2, 1, 0) + # Create the node with perm attribute equal to (1, 0) transpose_node = onnx.helper.make_node( "Transpose", inputs=[output_node.name], outputs=["transposed_output"], - perm=[2, 1, 0], + perm=[1, 0], ) onnx_model.graph.node.append(transpose_node) @@ -237,9 +237,9 @@ def tree_onnx_graph_preprocessing( if len(onnx_model.graph.output) == 1: assert_add_node_and_constant_in_xgboost_regressor_graph(onnx_model) - # Cut the graph at the ReduceSum node as large sum are not yet supported. - # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/451 - clean_graph_at_node_op_type(onnx_model, "ReduceSum") + # Cut the graph after the ReduceSum node to remove + # argmax, sigmoid, softmax from the graph. + clean_graph_after_node_op_type(onnx_model, "ReduceSum") if framework == "xgboost": # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/2778 diff --git a/tests/sklearn/test_dump_onnx.py b/tests/sklearn/test_dump_onnx.py index ecfafc879..637a25ac8 100644 --- a/tests/sklearn/test_dump_onnx.py +++ b/tests/sklearn/test_dump_onnx.py @@ -222,7 +222,8 @@ def test_dump( %/_operators.0/Reshape_2_output_0 = Reshape[allowzero = 0](%/_operators.0/Equal_output_0, %/_operators.0/Constant_2_output_0) %/_operators.0/MatMul_1_output_0 = MatMul(%_operators.0.weight_3, %/_operators.0/Reshape_2_output_0) %/_operators.0/Reshape_3_output_0 = Reshape[allowzero = 0](%/_operators.0/MatMul_1_output_0, %/_operators.0/Constant_3_output_0) - %transposed_output = Transpose[perm = [2, 1, 0]](%/_operators.0/Reshape_3_output_0) + %/_operators.0/ReduceSum_output_0 = ReduceSum[keepdims = 0](%/_operators.0/Reshape_3_output_0, %onnx::ReduceSum_22) + %transposed_output = Transpose[perm = [1, 0]](%/_operators.0/ReduceSum_output_0) return %transposed_output }""", "RandomForestClassifier": """graph torch_jit ( @@ -294,7 +295,8 @@ def test_dump( %/_operators.0/Reshape_2_output_0 = Reshape[allowzero = 0](%/_operators.0/Equal_output_0, %/_operators.0/Constant_2_output_0) %/_operators.0/MatMul_1_output_0 = MatMul(%_operators.0.weight_3, %/_operators.0/Reshape_2_output_0) %/_operators.0/Reshape_3_output_0 = Reshape[allowzero = 0](%/_operators.0/MatMul_1_output_0, %/_operators.0/Constant_3_output_0) - %transposed_output = Transpose[perm = [2, 1, 0]](%/_operators.0/Reshape_3_output_0) + %/_operators.0/ReduceSum_output_0 = ReduceSum[keepdims = 0](%/_operators.0/Reshape_3_output_0, %onnx::ReduceSum_22) + %transposed_output = Transpose[perm = [1, 0]](%/_operators.0/ReduceSum_output_0) return %transposed_output }""", "GammaRegressor": """graph torch_jit ( @@ -339,7 +341,8 @@ def test_dump( %/_operators.0/Squeeze_output_0 = Squeeze(%/_operators.0/Reshape_3_output_0, %axes_squeeze) %/_operators.0/Transpose_output_0 = Transpose[perm = [1, 0]](%/_operators.0/Squeeze_output_0) %/_operators.0/Reshape_4_output_0 = Reshape[allowzero = 0](%/_operators.0/Transpose_output_0, %/_operators.0/Constant_4_output_0) - return %/_operators.0/Reshape_4_output_0 + %/_operators.0/ReduceSum_output_0 = ReduceSum[keepdims = 0](%/_operators.0/Reshape_4_output_0, %onnx::ReduceSum_26) + return %/_operators.0/ReduceSum_output_0 }""", "RandomForestRegressor": """graph torch_jit ( %input_0[DOUBLE, symx10] @@ -374,6 +377,7 @@ def test_dump( %/_operators.0/Constant_2_output_0[INT64, 3] %/_operators.0/Constant_3_output_0[INT64, 3] %/_operators.0/Constant_4_output_0[INT64, 3] + %onnx::ReduceSum_27[INT64, 1] ) { %/_operators.0/Gemm_output_0 = Gemm[alpha = 1, beta = 0, transB = 1](%_operators.0.weight_1, %input_0) %/_operators.0/Less_output_0 = Less(%/_operators.0/Gemm_output_0, %_operators.0.bias_1) @@ -387,7 +391,8 @@ def test_dump( %/_operators.0/Squeeze_output_0 = Squeeze(%/_operators.0/Reshape_3_output_0, %axes_squeeze) %/_operators.0/Transpose_output_0 = Transpose[perm = [1, 0]](%/_operators.0/Squeeze_output_0) %/_operators.0/Reshape_4_output_0 = Reshape[allowzero = 0](%/_operators.0/Transpose_output_0, %/_operators.0/Constant_4_output_0) - return %/_operators.0/Reshape_4_output_0 + %/_operators.0/ReduceSum_output_0 = ReduceSum[keepdims = 0](%/_operators.0/Reshape_4_output_0, %onnx::ReduceSum_27) + return %/_operators.0/ReduceSum_output_0 }""", "LinearRegression": """graph torch_jit ( %input_0[DOUBLE, symx10] From 8c1e99a0dd0ace387c94cc59f31184ad2459c6b9 Mon Sep 17 00:00:00 2001 From: jfrery Date: Mon, 18 Dec 2023 14:35:50 +0100 Subject: [PATCH 37/73] chore: refresh notebooks --- .../ExperimentPrivacyTreePaper.ipynb | 42 +++++++++---------- .../advanced_examples/KNearestNeighbors.ipynb | 3 ++ 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/docs/advanced_examples/ExperimentPrivacyTreePaper.ipynb b/docs/advanced_examples/ExperimentPrivacyTreePaper.ipynb index 388454993..23ed43935 100644 --- a/docs/advanced_examples/ExperimentPrivacyTreePaper.ipynb +++ b/docs/advanced_examples/ExperimentPrivacyTreePaper.ipynb @@ -494,7 +494,7 @@ " 87.4\\% ± 1.2\\%\n", " 82.4\\% ± 1.8\\%\n", " -\n", - " 0.003\n", + " 0.004\n", " -\n", " \n", " \n", @@ -518,7 +518,7 @@ " \n", " FHE-RF\n", " 90.9\\% ± 1.1\\%\n", - " 87.5\\% ± 1.5\\%\n", + " 87.5\\% ± 1.6\\%\n", " 84.6\\% ± 1.7\\%\n", " 750.000\n", " 1.623\n", @@ -554,7 +554,7 @@ " \n", " \n", " FHE-XGB\n", - " 97.0\\% ± 2.4\\%\n", + " 96.8\\% ± 2.5\\%\n", " -\n", " -\n", " 900.000\n", @@ -792,9 +792,9 @@ " \n", " \n", " FHE-RF\n", - " 96.9\\% ± 1.2\\%\n", + " 96.8\\% ± 1.3\\%\n", " 95.4\\% ± 1.8\\%\n", - " 93.6\\% ± 2.2\\%\n", + " 93.5\\% ± 2.3\\%\n", " 700.000\n", " 1.477\n", " 576x\n", @@ -805,7 +805,7 @@ " 93.9\\% ± 1.5\\%\n", " 91.4\\% ± 2.3\\%\n", " -\n", - " 0.003\n", + " 0.002\n", " -\n", " \n", " \n", @@ -818,11 +818,11 @@ " FP32-DT 90.3\\% ± 1.0\\% 87.4\\% ± 1.2\\% \n", " FHE-XGB 94.5\\% ± 0.8\\% 92.9\\% ± 1.1\\% \n", " FP32-XGB 95.0\\% ± 0.7\\% 93.6\\% ± 0.9\\% \n", - " FHE-RF 90.9\\% ± 1.1\\% 87.5\\% ± 1.5\\% \n", + " FHE-RF 90.9\\% ± 1.1\\% 87.5\\% ± 1.6\\% \n", " FP32-RF 91.8\\% ± 1.1\\% 89.0\\% ± 1.4\\% \n", "wine (#features: 13) FHE-DT 90.8\\% ± 5.2\\% - \n", " FP32-DT 90.5\\% ± 5.0\\% - \n", - " FHE-XGB 97.0\\% ± 2.4\\% - \n", + " FHE-XGB 96.8\\% ± 2.5\\% - \n", " FP32-XGB 96.2\\% ± 2.9\\% - \n", " FHE-RF 98.5\\% ± 1.4\\% - \n", " FP32-RF 98.1\\% ± 2.0\\% - \n", @@ -848,7 +848,7 @@ " FP32-DT 97.2\\% ± 0.7\\% 96.1\\% ± 0.9\\% \n", " FHE-XGB 100.0\\% ± 0.0\\% 100.0\\% ± 0.0\\% \n", " FP32-XGB 100.0\\% ± 0.0\\% 100.0\\% ± 0.0\\% \n", - " FHE-RF 96.9\\% ± 1.2\\% 95.4\\% ± 1.8\\% \n", + " FHE-RF 96.8\\% ± 1.3\\% 95.4\\% ± 1.8\\% \n", " FP32-RF 95.9\\% ± 1.1\\% 93.9\\% ± 1.5\\% \n", "\n", " AP nodes Time (s) \\\n", @@ -1610,19 +1610,19 @@ "metadata": {}, "outputs": [ { - "name": "stderr", + "name": "stdout", "output_type": "stream", "text": [ - "The PostScript backend does not support transparency; partially transparent artists will be rendered opaque.\n" + "ap relative: [0.49626943 0.70187731 0.82640876 0.89067066 0.98315255 1.02264581\n", + " 1.02436888 1.01090038 1.01268386], f1_relative: [0.06488922 0.65490682 0.87590196 0.90861806 0.97920588 1.00604989\n", + " 1.00914511 1.00274636 1.00389957]\n" ] }, { - "name": "stdout", + "name": "stderr", "output_type": "stream", "text": [ - "ap relative: [0.49626943 0.70187731 0.82640876 0.89067066 0.98315255 1.02264581\n", - " 1.02436888 1.01090038 1.01268386], f1_relative: [0.06488922 0.65490682 0.87590196 0.90861806 0.97920588 1.00604989\n", - " 1.00914511 1.00274636 1.00389957]\n" + "The PostScript backend does not support transparency; partially transparent artists will be rendered opaque.\n" ] }, { @@ -1646,9 +1646,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "ap relative: [0.43556747 0.69054787 0.8789863 0.94180188 0.97097036 0.99094624\n", - " 0.99348364 0.99626825 0.99932372], f1_relative: [0. 0.65970362 0.91412713 0.95762445 0.97789164 0.99281277\n", - " 0.99447789 0.99697611 0.99969255]\n" + "ap relative: [0.43556747 0.69054787 0.8789863 0.94213852 0.97097036 0.99083622\n", + " 0.99365961 0.99626825 0.99920411], f1_relative: [0. 0.65970362 0.91412713 0.95780357 0.97789164 0.99271147\n", + " 0.99456864 0.99697611 0.99959059]\n" ] }, { @@ -1672,9 +1672,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "ap relative: [0.45810941 0.66176353 0.85701522 0.93668402 0.96541385 0.98353791\n", - " 0.99091316 0.99133601 0.99740638], f1_relative: [0. 0.57332946 0.87035559 0.9402579 0.96505021 0.983713\n", - " 0.99082334 0.99224022 0.99758998]\n" + "ap relative: [0.45810941 0.65828111 0.85617664 0.93660034 0.96541385 0.98342004\n", + " 0.99091316 0.9911998 0.99740638], f1_relative: [0. 0.56676488 0.86901886 0.93986022 0.96505021 0.98359134\n", + " 0.99082334 0.99211045 0.99758998]\n" ] }, { diff --git a/docs/advanced_examples/KNearestNeighbors.ipynb b/docs/advanced_examples/KNearestNeighbors.ipynb index d7ae1b8c1..9b4b7ed7e 100644 --- a/docs/advanced_examples/KNearestNeighbors.ipynb +++ b/docs/advanced_examples/KNearestNeighbors.ipynb @@ -287,6 +287,9 @@ "data": { "text/html": [ "\n", "\n", " \n", From 70f1775fac8e217e5c53863feab0e0939f2fce4e Mon Sep 17 00:00:00 2001 From: kcelia Date: Mon, 25 Dec 2023 20:28:47 +0100 Subject: [PATCH 38/73] chore: update celia --- src/concrete/ml/quantization/post_training.py | 3 ++- src/concrete/ml/sklearn/base.py | 3 ++- src/concrete/ml/sklearn/tree_to_numpy.py | 9 +++++++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/concrete/ml/quantization/post_training.py b/src/concrete/ml/quantization/post_training.py index 9389ab05f..c22ac8b85 100644 --- a/src/concrete/ml/quantization/post_training.py +++ b/src/concrete/ml/quantization/post_training.py @@ -50,7 +50,7 @@ def get_n_bits_dict(n_bits: Union[int, Dict[str, int]]) -> Dict[str, int]: or ( isinstance(n_bits, Dict) and set(n_bits.keys()).issubset( - {"model_inputs", "op_weights", "model_outputs", "op_inputs"} + {"model_inputs", "op_weights", "model_outputs", "op_inputs", "leaves"} ) and {"op_weights", "op_inputs"}.issubset(set(n_bits.keys())) ), @@ -69,6 +69,7 @@ def get_n_bits_dict(n_bits: Union[int, Dict[str, int]]) -> Dict[str, int]: "op_weights": n_bits, "op_inputs": n_bits, "model_outputs": max(DEFAULT_MODEL_BITS, n_bits), + "leaves": n_bits, } # If model_inputs or model_outputs are not given, we consider a default value diff --git a/src/concrete/ml/sklearn/base.py b/src/concrete/ml/sklearn/base.py index f63f63902..587968ff8 100644 --- a/src/concrete/ml/sklearn/base.py +++ b/src/concrete/ml/sklearn/base.py @@ -1292,6 +1292,7 @@ def __init__(self, n_bits: int): #: The model's inference function. Is None if the model is not fitted. self._tree_inference: Optional[Callable] = None + BaseEstimator.__init__(self) def fit(self, X: Data, y: Target, **fit_parameters): @@ -1306,7 +1307,7 @@ def fit(self, X: Data, y: Target, **fit_parameters): # Quantization of each feature in X for i in range(X.shape[1]): - input_quantizer = QuantizedArray(n_bits=self.n_bits, values=X[:, i]).quantizer + input_quantizer = QuantizedArray(n_bits=self.n_bits["op_inputs"], values=X[:, i]).quantizer self.input_quantizers.append(input_quantizer) q_X[:, i] = input_quantizer.quant(X[:, i]) diff --git a/src/concrete/ml/sklearn/tree_to_numpy.py b/src/concrete/ml/sklearn/tree_to_numpy.py index 42c241fe7..5983819aa 100644 --- a/src/concrete/ml/sklearn/tree_to_numpy.py +++ b/src/concrete/ml/sklearn/tree_to_numpy.py @@ -261,7 +261,7 @@ def tree_onnx_graph_preprocessing( def tree_values_preprocessing( onnx_model: onnx.ModelProto, framework: str, - output_n_bits: int, + n_bits: int, ) -> QuantizedArray: """Pre-process tree values. @@ -277,18 +277,23 @@ def tree_values_preprocessing( # Modify ONNX graph to fit in FHE for i, initializer in enumerate(onnx_model.graph.initializer): + + # All constants in our tree should be integers. # Tree thresholds can be rounded up or down (depending on the tree implementation) # while the final probabilities/regression values must be quantized. # We extract the value stored in each initializer node into the init_tensor. init_tensor = numpy_helper.to_array(initializer) + #print(initializer.name, init_tensor.shape) if "weight_3" in initializer.name: + #print(init_tensor) # weight_3 is the prediction tensor, apply the required pre-processing - q_y = preprocess_tree_predictions(init_tensor, output_n_bits) + q_y = preprocess_tree_predictions(init_tensor, n_bits["leaves"]) # Get the preprocessed tree predictions to replace the current (non-quantized) # values in the onnx_model. init_tensor = q_y.qvalues + elif "bias_1" in initializer.name: if framework == "xgboost": # xgboost uses "<" (Less) operator thus we must round up. From 6262f12b9f7c73705561b749957562193a1cfea0 Mon Sep 17 00:00:00 2001 From: kcelia Date: Thu, 11 Jan 2024 11:35:40 +0100 Subject: [PATCH 39/73] chore: add op_input and op_leaves --- src/concrete/ml/quantization/__init__.py | 2 +- src/concrete/ml/quantization/post_training.py | 43 ++++++++++++++++++- src/concrete/ml/sklearn/base.py | 33 +++++++++++--- src/concrete/ml/sklearn/tree_to_numpy.py | 11 +++-- 4 files changed, 73 insertions(+), 16 deletions(-) diff --git a/src/concrete/ml/quantization/__init__.py b/src/concrete/ml/quantization/__init__.py index 845b5dc11..58f90f916 100644 --- a/src/concrete/ml/quantization/__init__.py +++ b/src/concrete/ml/quantization/__init__.py @@ -1,6 +1,6 @@ """Modules for quantization.""" from .base_quantized_op import QuantizedOp -from .post_training import PostTrainingAffineQuantization, PostTrainingQATImporter, get_n_bits_dict +from .post_training import PostTrainingAffineQuantization, PostTrainingQATImporter, get_n_bits_dict, get_n_bits_dict_trees from .quantized_module import QuantizedModule from .quantized_ops import ( QuantizedAbs, diff --git a/src/concrete/ml/quantization/post_training.py b/src/concrete/ml/quantization/post_training.py index c22ac8b85..ffa259fc2 100644 --- a/src/concrete/ml/quantization/post_training.py +++ b/src/concrete/ml/quantization/post_training.py @@ -24,6 +24,46 @@ from .quantizers import QuantizationOptions, QuantizedArray, UniformQuantizer +def get_n_bits_dict_trees(n_bits: Union[int, Dict[str, int]]) -> Dict[str, int]: + """Convert the n_bits parameter into a proper dictionary for tree based-models. + + Args: + n_bits (int, Dict[str, int]): number of bits for quantization, can be a single value or + a dictionary with the following keys : + - "op_inputs" (mandatory) + - "op_leaves" (optional) + TODO + + Returns: + n_bits_dict (Dict[str, int]): TODO + """ + + assert_true( + isinstance(n_bits, int) + or (isinstance(n_bits, Dict) and set(n_bits.keys()).issubset({"op_inputs", "op_leaves"})), + "Invalid n_bits, either pass an integer or a dictionary containing integer values for " + "the following keys:\n" + "- `op_inputs` and `op_leaves` (mandatory)", + ) + + # If a single integer is passed, we use a default value for the model's input and + # output bits + if isinstance(n_bits, int): + n_bits_dict = { + "op_inputs": n_bits, + "op_leaves": n_bits, + } + # If model_inputs or model_outputs are not given, we consider a default value + elif isinstance(n_bits, Dict): + n_bits_dict = { + "model_inputs": n_bits, + "model_outputs": n_bits, + } + + n_bits_dict.update(n_bits) + return n_bits_dict + + def get_n_bits_dict(n_bits: Union[int, Dict[str, int]]) -> Dict[str, int]: """Convert the n_bits parameter into a proper dictionary. @@ -50,7 +90,7 @@ def get_n_bits_dict(n_bits: Union[int, Dict[str, int]]) -> Dict[str, int]: or ( isinstance(n_bits, Dict) and set(n_bits.keys()).issubset( - {"model_inputs", "op_weights", "model_outputs", "op_inputs", "leaves"} + {"model_inputs", "op_weights", "model_outputs", "op_inputs"} ) and {"op_weights", "op_inputs"}.issubset(set(n_bits.keys())) ), @@ -69,7 +109,6 @@ def get_n_bits_dict(n_bits: Union[int, Dict[str, int]]) -> Dict[str, int]: "op_weights": n_bits, "op_inputs": n_bits, "model_outputs": max(DEFAULT_MODEL_BITS, n_bits), - "leaves": n_bits, } # If model_inputs or model_outputs are not given, we consider a default value diff --git a/src/concrete/ml/sklearn/base.py b/src/concrete/ml/sklearn/base.py index 587968ff8..1935f25f7 100644 --- a/src/concrete/ml/sklearn/base.py +++ b/src/concrete/ml/sklearn/base.py @@ -49,7 +49,12 @@ # The sigmoid and softmax functions are already defined in the ONNX module and thus are imported # here in order to avoid duplicating them. from ..onnx.ops_impl import numpy_sigmoid, numpy_softmax -from ..quantization import PostTrainingQATImporter, QuantizedArray, get_n_bits_dict +from ..quantization import ( + PostTrainingQATImporter, + QuantizedArray, + get_n_bits_dict, + get_n_bits_dict_trees, +) from ..quantization.quantized_module import QuantizedModule, _get_inputset_generator from ..quantization.quantizers import ( QuantizationOptions, @@ -98,6 +103,9 @@ # However, for internal testing purposes, we retain the capability to disable this feature os.environ["TREES_USE_ROUNDING"] = "1" +# TODO +os.environ["TREES_USE_FHE_SUM"] = "0" + # pylint: disable=too-many-public-methods @@ -1285,14 +1293,22 @@ def __init__(self, n_bits: int): """Initialize the TreeBasedEstimatorMixin. Args: - n_bits (int): The number of bits used for quantization. + n_bits (int, Dict[str, int]): Number of bits to quantize the model. If an int is passed + for n_bits, the value will be used for quantizing inputs and leaves. If a dict is + passed, then it should contain "op_inputs" and "op_leaves" as keys with + corresponding number of quantization bits so that: + - op_inputs : number of bits to quantize the input values + - op_leaves: number of bits to quantize the leaves + Default to 6. """ - self.n_bits: int = n_bits + self.n_bits: Union[int, Dict[str, int]] = n_bits + + # Convert the n_bits attribute into a proper dictionary + self.n_bits = get_n_bits_dict_trees(self.n_bits) #: The model's inference function. Is None if the model is not fitted. self._tree_inference: Optional[Callable] = None - BaseEstimator.__init__(self) def fit(self, X: Data, y: Target, **fit_parameters): @@ -1307,7 +1323,9 @@ def fit(self, X: Data, y: Target, **fit_parameters): # Quantization of each feature in X for i in range(X.shape[1]): - input_quantizer = QuantizedArray(n_bits=self.n_bits["op_inputs"], values=X[:, i]).quantizer + input_quantizer = QuantizedArray( + n_bits=self.n_bits["op_inputs"], values=X[:, i] + ).quantizer self.input_quantizers.append(input_quantizer) q_X[:, i] = input_quantizer.quant(X[:, i]) @@ -1320,7 +1338,7 @@ def fit(self, X: Data, y: Target, **fit_parameters): # Check that the underlying sklearn model has been set and fit assert self.sklearn_model is not None, self._sklearn_model_is_not_fitted_error_message() - # Convert the tree inference with Numpy operators + # Enable rounding feature enable_rounding = os.environ.get("TREES_USE_ROUNDING", "1") == "1" if not enable_rounding: @@ -1333,12 +1351,13 @@ def fit(self, X: Data, y: Target, **fit_parameters): stacklevel=2, ) + # Convert the tree inference with Numpy operators self._tree_inference, self.output_quantizers, self.onnx_model_ = tree_to_numpy( self.sklearn_model, q_X, use_rounding=enable_rounding, framework=self.framework, - output_n_bits=self.n_bits, + output_n_bits=self.n_bits["op_leaves"], ) self._is_fitted = True diff --git a/src/concrete/ml/sklearn/tree_to_numpy.py b/src/concrete/ml/sklearn/tree_to_numpy.py index 5983819aa..3d1025a1f 100644 --- a/src/concrete/ml/sklearn/tree_to_numpy.py +++ b/src/concrete/ml/sklearn/tree_to_numpy.py @@ -277,23 +277,22 @@ def tree_values_preprocessing( # Modify ONNX graph to fit in FHE for i, initializer in enumerate(onnx_model.graph.initializer): - - + # All constants in our tree should be integers. # Tree thresholds can be rounded up or down (depending on the tree implementation) # while the final probabilities/regression values must be quantized. # We extract the value stored in each initializer node into the init_tensor. init_tensor = numpy_helper.to_array(initializer) - #print(initializer.name, init_tensor.shape) + # print(initializer.name, init_tensor.shape) if "weight_3" in initializer.name: - #print(init_tensor) + # print(init_tensor) # weight_3 is the prediction tensor, apply the required pre-processing - q_y = preprocess_tree_predictions(init_tensor, n_bits["leaves"]) + q_y = preprocess_tree_predictions(init_tensor, n_bits) # Get the preprocessed tree predictions to replace the current (non-quantized) # values in the onnx_model. init_tensor = q_y.qvalues - + elif "bias_1" in initializer.name: if framework == "xgboost": # xgboost uses "<" (Less) operator thus we must round up. From 67e3572df80bea5a686251ee0b04cfed4709bada Mon Sep 17 00:00:00 2001 From: kcelia Date: Thu, 11 Jan 2024 15:10:25 +0100 Subject: [PATCH 40/73] chore: restore non fhe computation --- src/concrete/ml/quantization/__init__.py | 7 ++++++- src/concrete/ml/sklearn/base.py | 16 +++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/concrete/ml/quantization/__init__.py b/src/concrete/ml/quantization/__init__.py index 58f90f916..6669d76a7 100644 --- a/src/concrete/ml/quantization/__init__.py +++ b/src/concrete/ml/quantization/__init__.py @@ -1,6 +1,11 @@ """Modules for quantization.""" from .base_quantized_op import QuantizedOp -from .post_training import PostTrainingAffineQuantization, PostTrainingQATImporter, get_n_bits_dict, get_n_bits_dict_trees +from .post_training import ( + PostTrainingAffineQuantization, + PostTrainingQATImporter, + get_n_bits_dict, + get_n_bits_dict_trees, +) from .quantized_module import QuantizedModule from .quantized_ops import ( QuantizedAbs, diff --git a/src/concrete/ml/sklearn/base.py b/src/concrete/ml/sklearn/base.py index 1935f25f7..510a9d2db 100644 --- a/src/concrete/ml/sklearn/base.py +++ b/src/concrete/ml/sklearn/base.py @@ -104,7 +104,7 @@ os.environ["TREES_USE_ROUNDING"] = "1" # TODO -os.environ["TREES_USE_FHE_SUM"] = "0" +os.environ["TREES_USE_FHE_SUM"] = "1" # pylint: disable=too-many-public-methods @@ -1428,6 +1428,20 @@ def predict(self, X: Data, fhe: Union[FheMode, str] = FheMode.DISABLE) -> numpy. y_pred = self.post_processing(y_pred) return y_pred + def post_processing(self, y_preds: numpy.ndarray) -> numpy.ndarray: + # Sum all tree outputs + # Remove the sum once we handle multi-precision circuits + # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/451 + if os.getenv("TREES_USE_FHE_SUM") == "0": + print("post_processing: Non FHE SUM") + y_preds = numpy.sum(y_preds, axis=-1) + + assert_true(y_preds.ndim == 2, "y_preds should be a 2D array") + return y_preds + else: + print("post_processing: FHE SUM") + return super().post_processing(y_preds) + class BaseTreeRegressorMixin(BaseTreeEstimatorMixin, sklearn.base.RegressorMixin, ABC): """Mixin class for tree-based regressors. From 8e2e056a221c69a926c47b99b55d9d181d7fbb32 Mon Sep 17 00:00:00 2001 From: kcelia Date: Thu, 11 Jan 2024 15:10:59 +0100 Subject: [PATCH 41/73] chore: update dump test --- tests/sklearn/test_dump_onnx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/sklearn/test_dump_onnx.py b/tests/sklearn/test_dump_onnx.py index 637a25ac8..a6727139a 100644 --- a/tests/sklearn/test_dump_onnx.py +++ b/tests/sklearn/test_dump_onnx.py @@ -1,6 +1,6 @@ """Tests for the sklearn decision trees.""" - +import os import warnings from functools import partial From 3e2289a23da3dfb874b701da49d630364df61e50 Mon Sep 17 00:00:00 2001 From: kcelia Date: Thu, 11 Jan 2024 16:30:27 +0100 Subject: [PATCH 42/73] chore: update test dump --- tests/sklearn/test_dump_onnx.py | 54 ++++++++++++++++++++++++++------- 1 file changed, 43 insertions(+), 11 deletions(-) diff --git a/tests/sklearn/test_dump_onnx.py b/tests/sklearn/test_dump_onnx.py index a6727139a..a5fe6497a 100644 --- a/tests/sklearn/test_dump_onnx.py +++ b/tests/sklearn/test_dump_onnx.py @@ -222,9 +222,15 @@ def test_dump( %/_operators.0/Reshape_2_output_0 = Reshape[allowzero = 0](%/_operators.0/Equal_output_0, %/_operators.0/Constant_2_output_0) %/_operators.0/MatMul_1_output_0 = MatMul(%_operators.0.weight_3, %/_operators.0/Reshape_2_output_0) %/_operators.0/Reshape_3_output_0 = Reshape[allowzero = 0](%/_operators.0/MatMul_1_output_0, %/_operators.0/Constant_3_output_0) - %/_operators.0/ReduceSum_output_0 = ReduceSum[keepdims = 0](%/_operators.0/Reshape_3_output_0, %onnx::ReduceSum_22) + """ + + ( + """%/_operators.0/ReduceSum_output_0 = ReduceSum[keepdims = 0](%/_operators.0/Reshape_3_output_0, %onnx::ReduceSum_22) %transposed_output = Transpose[perm = [1, 0]](%/_operators.0/ReduceSum_output_0) - return %transposed_output + """ + if os.getenv("TREES_USE_FHE_SUM") == "1" + else "%transposed_output = Transpose[perm = [2, 1, 0]](%/_operators.0/Reshape_3_output_0)" + ) + + """return %transposed_output }""", "RandomForestClassifier": """graph torch_jit ( %input_0[DOUBLE, symx10] @@ -295,9 +301,15 @@ def test_dump( %/_operators.0/Reshape_2_output_0 = Reshape[allowzero = 0](%/_operators.0/Equal_output_0, %/_operators.0/Constant_2_output_0) %/_operators.0/MatMul_1_output_0 = MatMul(%_operators.0.weight_3, %/_operators.0/Reshape_2_output_0) %/_operators.0/Reshape_3_output_0 = Reshape[allowzero = 0](%/_operators.0/MatMul_1_output_0, %/_operators.0/Constant_3_output_0) - %/_operators.0/ReduceSum_output_0 = ReduceSum[keepdims = 0](%/_operators.0/Reshape_3_output_0, %onnx::ReduceSum_22) + """ + + ( + """%/_operators.0/ReduceSum_output_0 = ReduceSum[keepdims = 0](%/_operators.0/Reshape_3_output_0, %onnx::ReduceSum_22) %transposed_output = Transpose[perm = [1, 0]](%/_operators.0/ReduceSum_output_0) - return %transposed_output + """ + if os.getenv("TREES_USE_FHE_SUM") == "1" + else "" + ) + + """return %transposed_output }""", "GammaRegressor": """graph torch_jit ( %input_0[DOUBLE, symx10] @@ -341,9 +353,15 @@ def test_dump( %/_operators.0/Squeeze_output_0 = Squeeze(%/_operators.0/Reshape_3_output_0, %axes_squeeze) %/_operators.0/Transpose_output_0 = Transpose[perm = [1, 0]](%/_operators.0/Squeeze_output_0) %/_operators.0/Reshape_4_output_0 = Reshape[allowzero = 0](%/_operators.0/Transpose_output_0, %/_operators.0/Constant_4_output_0) - %/_operators.0/ReduceSum_output_0 = ReduceSum[keepdims = 0](%/_operators.0/Reshape_4_output_0, %onnx::ReduceSum_26) + """ + + ( + """%/_operators.0/ReduceSum_output_0 = ReduceSum[keepdims = 0](%/_operators.0/Reshape_4_output_0, %onnx::ReduceSum_26) return %/_operators.0/ReduceSum_output_0 -}""", +}""" + if os.getenv("TREES_USE_FHE_SUM") == "1" + else """return %/_operators.0/Reshape_4_output_0 + }""" + ), "RandomForestRegressor": """graph torch_jit ( %input_0[DOUBLE, symx10] ) { @@ -360,8 +378,15 @@ def test_dump( %/_operators.0/MatMul_1_output_0 = MatMul(%_operators.0.weight_3, %/_operators.0/Reshape_2_output_0) %/_operators.0/Constant_3_output_0 = Constant[value = ]() %/_operators.0/Reshape_3_output_0 = Reshape[allowzero = 0](%/_operators.0/MatMul_1_output_0, %/_operators.0/Constant_3_output_0) - %transposed_output = Transpose[perm = [2, 1, 0]](%/_operators.0/Reshape_3_output_0) - return %transposed_output + """ + + ( + """%/_operators.0/ReduceSum_output_0 = ReduceSum[keepdims = 0](%/_operators.0/Reshape_3_output_0, %onnx::ReduceSum_22) + %transposed_output = Transpose[perm = [1, 0]](%/_operators.0/ReduceSum_output_0) + """ + if os.getenv("TREES_USE_FHE_SUM") == "1" + else "%transposed_output = Transpose[perm = [2, 1, 0]](%/_operators.0/Reshape_3_output_0)" + ) + + """return %transposed_output }""", "XGBRegressor": """graph torch_jit ( %input_0[DOUBLE, symx10] @@ -377,7 +402,9 @@ def test_dump( %/_operators.0/Constant_2_output_0[INT64, 3] %/_operators.0/Constant_3_output_0[INT64, 3] %/_operators.0/Constant_4_output_0[INT64, 3] - %onnx::ReduceSum_27[INT64, 1] + """ + + ("%onnx::ReduceSum_27[INT64, 1]" if os.getenv("TREES_USE_FHE_SUM") == "1" else "") + + """ ) { %/_operators.0/Gemm_output_0 = Gemm[alpha = 1, beta = 0, transB = 1](%_operators.0.weight_1, %input_0) %/_operators.0/Less_output_0 = Less(%/_operators.0/Gemm_output_0, %_operators.0.bias_1) @@ -391,9 +418,14 @@ def test_dump( %/_operators.0/Squeeze_output_0 = Squeeze(%/_operators.0/Reshape_3_output_0, %axes_squeeze) %/_operators.0/Transpose_output_0 = Transpose[perm = [1, 0]](%/_operators.0/Squeeze_output_0) %/_operators.0/Reshape_4_output_0 = Reshape[allowzero = 0](%/_operators.0/Transpose_output_0, %/_operators.0/Constant_4_output_0) - %/_operators.0/ReduceSum_output_0 = ReduceSum[keepdims = 0](%/_operators.0/Reshape_4_output_0, %onnx::ReduceSum_27) + """ + + ( + """%/_operators.0/ReduceSum_output_0 = ReduceSum[keepdims = 0](%/_operators.0/Reshape_4_output_0, %onnx::ReduceSum_27) return %/_operators.0/ReduceSum_output_0 -}""", +}""" + if os.getenv("TREES_USE_FHE_SUM") == "1" + else "return %/_operators.0/Reshape_4_output_0" + ), "LinearRegression": """graph torch_jit ( %input_0[DOUBLE, symx10] ) initializers ( From a9c8385dee5f45e90ce77aab5c92d41d018795e6 Mon Sep 17 00:00:00 2001 From: kcelia Date: Fri, 12 Jan 2024 10:54:07 +0100 Subject: [PATCH 43/73] chore: fix pipeline test --- src/concrete/ml/quantization/post_training.py | 13 +++++++++++-- src/concrete/ml/sklearn/base.py | 10 +++++++--- tests/sklearn/test_sklearn_models.py | 5 +++++ 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/src/concrete/ml/quantization/post_training.py b/src/concrete/ml/quantization/post_training.py index ffa259fc2..3ba0168c7 100644 --- a/src/concrete/ml/quantization/post_training.py +++ b/src/concrete/ml/quantization/post_training.py @@ -53,14 +53,23 @@ def get_n_bits_dict_trees(n_bits: Union[int, Dict[str, int]]) -> Dict[str, int]: "op_inputs": n_bits, "op_leaves": n_bits, } + # If model_inputs or model_outputs are not given, we consider a default value elif isinstance(n_bits, Dict): n_bits_dict = { - "model_inputs": n_bits, - "model_outputs": n_bits, + "model_inputs": DEFAULT_MODEL_BITS, + "model_outputs": max(DEFAULT_MODEL_BITS, n_bits["op_inputs"]), # TODO } n_bits_dict.update(n_bits) + + assert_true( + n_bits_dict["op_inputs"] >= n_bits_dict["op_leaves"], + "Using fewer bits to represent the model_outputs than the op inputs is not " + f"recommended. Got op_leaves: {n_bits_dict['op_leaves']} and op_inputs: " + f"{n_bits_dict['op_inputs']}", + ) + return n_bits_dict diff --git a/src/concrete/ml/sklearn/base.py b/src/concrete/ml/sklearn/base.py index 510a9d2db..77e3dfca2 100644 --- a/src/concrete/ml/sklearn/base.py +++ b/src/concrete/ml/sklearn/base.py @@ -1303,9 +1303,6 @@ def __init__(self, n_bits: int): """ self.n_bits: Union[int, Dict[str, int]] = n_bits - # Convert the n_bits attribute into a proper dictionary - self.n_bits = get_n_bits_dict_trees(self.n_bits) - #: The model's inference function. Is None if the model is not fitted. self._tree_inference: Optional[Callable] = None @@ -1321,6 +1318,10 @@ def fit(self, X: Data, y: Target, **fit_parameters): q_X = numpy.zeros_like(X) + # Convert the n_bits attribute into a proper dictionary + self.n_bits = get_n_bits_dict_trees(self.n_bits) + print(f"{self.n_bits=}") + # Quantization of each feature in X for i in range(X.shape[1]): input_quantizer = QuantizedArray( @@ -1338,6 +1339,9 @@ def fit(self, X: Data, y: Target, **fit_parameters): # Check that the underlying sklearn model has been set and fit assert self.sklearn_model is not None, self._sklearn_model_is_not_fitted_error_message() + # Convert the n_bits attribute into a proper dictionary + self.n_bits = get_n_bits_dict_trees(self.n_bits) + # Enable rounding feature enable_rounding = os.environ.get("TREES_USE_ROUNDING", "1") == "1" diff --git a/tests/sklearn/test_sklearn_models.py b/tests/sklearn/test_sklearn_models.py index d3e68f731..aac20fb90 100644 --- a/tests/sklearn/test_sklearn_models.py +++ b/tests/sklearn/test_sklearn_models.py @@ -726,12 +726,16 @@ def check_pipeline(model_class, x, y): {key: value} for key, values in hyper_param_combinations.items() for value in values ] + print(f"{hyperparameters_list=}") + # Take one of the hyper_parameters randomly (testing everything would be too long) if len(hyperparameters_list) == 0: hyper_parameters = {} else: hyper_parameters = hyperparameters_list[numpy.random.randint(0, len(hyperparameters_list))] + print(f"{hyperparameters_list=}") + pipe_cv = Pipeline( [ ("pca", PCA(n_components=2, random_state=numpy.random.randint(0, 2**15))), @@ -748,6 +752,7 @@ def check_pipeline(model_class, x, y): } else: + print("ELSE") param_grid = { "model__n_bits": [2, 3], } From a43f04c11125f041389e4b09cc7d759a2ee0f553 Mon Sep 17 00:00:00 2001 From: kcelia Date: Fri, 12 Jan 2024 11:27:12 +0100 Subject: [PATCH 44/73] chore: fix rounding test by decreasing the n_bits value because no crypto params are found and fix serialization function --- src/concrete/ml/quantization/post_training.py | 8 +++----- src/concrete/ml/sklearn/rf.py | 4 ++-- src/concrete/ml/sklearn/tree.py | 4 ++-- src/concrete/ml/sklearn/xgb.py | 4 ++-- tests/sklearn/test_sklearn_models.py | 2 +- 5 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/concrete/ml/quantization/post_training.py b/src/concrete/ml/quantization/post_training.py index 3ba0168c7..70f198fa9 100644 --- a/src/concrete/ml/quantization/post_training.py +++ b/src/concrete/ml/quantization/post_training.py @@ -46,6 +46,8 @@ def get_n_bits_dict_trees(n_bits: Union[int, Dict[str, int]]) -> Dict[str, int]: "- `op_inputs` and `op_leaves` (mandatory)", ) + n_bits_dict = {} + # If a single integer is passed, we use a default value for the model's input and # output bits if isinstance(n_bits, int): @@ -55,12 +57,8 @@ def get_n_bits_dict_trees(n_bits: Union[int, Dict[str, int]]) -> Dict[str, int]: } # If model_inputs or model_outputs are not given, we consider a default value - elif isinstance(n_bits, Dict): - n_bits_dict = { - "model_inputs": DEFAULT_MODEL_BITS, - "model_outputs": max(DEFAULT_MODEL_BITS, n_bits["op_inputs"]), # TODO - } + elif isinstance(n_bits, Dict): n_bits_dict.update(n_bits) assert_true( diff --git a/src/concrete/ml/sklearn/rf.py b/src/concrete/ml/sklearn/rf.py index e5f756664..14aabf060 100644 --- a/src/concrete/ml/sklearn/rf.py +++ b/src/concrete/ml/sklearn/rf.py @@ -124,7 +124,7 @@ def load_dict(cls, metadata: Dict): obj.sklearn_model, numpy.zeros((len(obj.input_quantizers),))[None, ...], framework=obj.framework, - output_n_bits=obj.n_bits, + output_n_bits=obj.n_bits["op_leaves"], )[0] obj.post_processing_params = metadata["post_processing_params"] @@ -259,7 +259,7 @@ def load_dict(cls, metadata: Dict): obj.sklearn_model, numpy.zeros((len(obj.input_quantizers),))[None, ...], framework=obj.framework, - output_n_bits=obj.n_bits, + output_n_bits=obj.n_bits["op_leaves"], )[0] obj.post_processing_params = metadata["post_processing_params"] diff --git a/src/concrete/ml/sklearn/tree.py b/src/concrete/ml/sklearn/tree.py index 1ea972cfd..c870d2804 100644 --- a/src/concrete/ml/sklearn/tree.py +++ b/src/concrete/ml/sklearn/tree.py @@ -119,7 +119,7 @@ def load_dict(cls, metadata: Dict): obj.sklearn_model, numpy.zeros((len(obj.input_quantizers),))[None, ...], framework=obj.framework, - output_n_bits=obj.n_bits, + output_n_bits=obj.n_bits["op_leaves"], )[0] obj.post_processing_params = metadata["post_processing_params"] @@ -242,7 +242,7 @@ def load_dict(cls, metadata: Dict): obj.sklearn_model, numpy.zeros((len(obj.input_quantizers),))[None, ...], framework=obj.framework, - output_n_bits=obj.n_bits, + output_n_bits=obj.n_bits["op_leaves"], )[0] obj.post_processing_params = metadata["post_processing_params"] diff --git a/src/concrete/ml/sklearn/xgb.py b/src/concrete/ml/sklearn/xgb.py index a10b1400b..a2b121f4d 100644 --- a/src/concrete/ml/sklearn/xgb.py +++ b/src/concrete/ml/sklearn/xgb.py @@ -178,7 +178,7 @@ def load_dict(cls, metadata: Dict): obj.sklearn_model, numpy.zeros((len(obj.input_quantizers),))[None, ...], framework=obj.framework, - output_n_bits=obj.n_bits, + output_n_bits=obj.n_bits["op_leaves"], )[0] obj.post_processing_params = metadata["post_processing_params"] @@ -407,7 +407,7 @@ def load_dict(cls, metadata: Dict): obj.sklearn_model, numpy.zeros((len(obj.input_quantizers),))[None, ...], framework=obj.framework, - output_n_bits=obj.n_bits, + output_n_bits=obj.n_bits["op_leaves"], )[0] obj.post_processing_params = metadata["post_processing_params"] diff --git a/tests/sklearn/test_sklearn_models.py b/tests/sklearn/test_sklearn_models.py index aac20fb90..26fac9d03 100644 --- a/tests/sklearn/test_sklearn_models.py +++ b/tests/sklearn/test_sklearn_models.py @@ -1839,7 +1839,7 @@ def test_linear_models_have_no_tlu( # Additional tests for this purpose should be added in future updates # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4179 @pytest.mark.parametrize("model_class, parameters", get_sklearn_tree_models_and_datasets()) -@pytest.mark.parametrize("n_bits", [2, 5, 11]) +@pytest.mark.parametrize("n_bits", [2, 5, 10]) def test_rounding_consistency_for_regular_models( model_class, parameters, From ecf5c668ef37b684f8840ecdecaac64c4054f41b Mon Sep 17 00:00:00 2001 From: kcelia Date: Fri, 12 Jan 2024 13:09:43 +0100 Subject: [PATCH 45/73] chore: reduce n_bits in simulation test to 4 bits otherwise OOM --- tests/sklearn/test_sklearn_models.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/sklearn/test_sklearn_models.py b/tests/sklearn/test_sklearn_models.py index 26fac9d03..3427792b9 100644 --- a/tests/sklearn/test_sklearn_models.py +++ b/tests/sklearn/test_sklearn_models.py @@ -1663,7 +1663,11 @@ def test_p_error_simulation( The test checks that models compiled with a large p_error value predicts very different results with simulation or in FHE compared to the expected clear quantized ones. """ - n_bits = get_n_bits_non_correctness(model_class) + + if os.getenv('TREES_USE_FHE_SUM') == "1": + n_bits = 4 + else: + n_bits = get_n_bits_non_correctness(model_class) # Get data-set, initialize and fit the model model, x = preamble(model_class, parameters, n_bits, load_data, is_weekly_option) @@ -1672,7 +1676,7 @@ def test_p_error_simulation( is_linear_model = is_model_class_in_a_list(model_class, _get_sklearn_linear_models()) # Compile with a large p_error to be sure the result is random. - model.compile(x, **error_param) + c = model.compile(x, **error_param) def check_for_divergent_predictions(x, model, fhe, max_iterations=N_ALLOWED_FHE_RUN): """Detect divergence between simulated/FHE execution and clear run.""" @@ -1693,7 +1697,11 @@ def check_for_divergent_predictions(x, model, fhe, max_iterations=N_ALLOWED_FHE_ return True return False + print("Start simulation") + print(model) + simulation_diff_found = check_for_divergent_predictions(x, model, fhe="simulate") + print("execution") fhe_diff_found = check_for_divergent_predictions(x, model, fhe="execute") # Check for differences in predictions From 9007362d274be734cac3561857ead659a3bdaa2f Mon Sep 17 00:00:00 2001 From: kcelia Date: Fri, 12 Jan 2024 14:07:54 +0100 Subject: [PATCH 46/73] chore: add a test for fhe sum --- src/concrete/ml/sklearn/base.py | 14 ---- src/concrete/ml/sklearn/tree_to_numpy.py | 7 +- tests/sklearn/test_sklearn_models.py | 94 +++++++++++++++++++++++- 3 files changed, 98 insertions(+), 17 deletions(-) diff --git a/src/concrete/ml/sklearn/base.py b/src/concrete/ml/sklearn/base.py index 77e3dfca2..539cfd74e 100644 --- a/src/concrete/ml/sklearn/base.py +++ b/src/concrete/ml/sklearn/base.py @@ -1432,20 +1432,6 @@ def predict(self, X: Data, fhe: Union[FheMode, str] = FheMode.DISABLE) -> numpy. y_pred = self.post_processing(y_pred) return y_pred - def post_processing(self, y_preds: numpy.ndarray) -> numpy.ndarray: - # Sum all tree outputs - # Remove the sum once we handle multi-precision circuits - # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/451 - if os.getenv("TREES_USE_FHE_SUM") == "0": - print("post_processing: Non FHE SUM") - y_preds = numpy.sum(y_preds, axis=-1) - - assert_true(y_preds.ndim == 2, "y_preds should be a 2D array") - return y_preds - else: - print("post_processing: FHE SUM") - return super().post_processing(y_preds) - class BaseTreeRegressorMixin(BaseTreeEstimatorMixin, sklearn.base.RegressorMixin, ABC): """Mixin class for tree-based regressors. diff --git a/src/concrete/ml/sklearn/tree_to_numpy.py b/src/concrete/ml/sklearn/tree_to_numpy.py index 3d1025a1f..a748197b9 100644 --- a/src/concrete/ml/sklearn/tree_to_numpy.py +++ b/src/concrete/ml/sklearn/tree_to_numpy.py @@ -1,5 +1,6 @@ """Implements the conversion of a tree model to a numpy function.""" import math +import os import warnings from typing import Callable, List, Optional, Tuple @@ -141,12 +142,16 @@ def add_transpose_after_last_node(onnx_model: onnx.ModelProto): # Get the output node output_node = onnx_model.graph.output[0] + if os.getenv("TREES_USE_FHE_SUM") == "1": + perm = [1, 0] + else: + perm = [2, 1, 0] # Create the node with perm attribute equal to (1, 0) transpose_node = onnx.helper.make_node( "Transpose", inputs=[output_node.name], outputs=["transposed_output"], - perm=[1, 0], + perm=perm, ) onnx_model.graph.node.append(transpose_node) diff --git a/tests/sklearn/test_sklearn_models.py b/tests/sklearn/test_sklearn_models.py index 3427792b9..beeceed4e 100644 --- a/tests/sklearn/test_sklearn_models.py +++ b/tests/sklearn/test_sklearn_models.py @@ -1664,9 +1664,9 @@ def test_p_error_simulation( with simulation or in FHE compared to the expected clear quantized ones. """ - if os.getenv('TREES_USE_FHE_SUM') == "1": + if os.getenv("TREES_USE_FHE_SUM") == "1": n_bits = 4 - else: + else: n_bits = get_n_bits_non_correctness(model_class) # Get data-set, initialize and fit the model @@ -1884,3 +1884,93 @@ def test_rounding_consistency_for_regular_models( metric, is_weekly_option, ) + + +def check_fhe_sum_consistency( + x, + predict_method, + metric, + is_weekly_option, +): + """Test that Concrete ML without and with rounding are 'equivalent'.""" + + # Run the test with more samples during weekly CIs + if is_weekly_option: + fhe_test = get_random_samples(x, n_sample=5) + + # By default, FHE_SUM is disabled + fhe_sum_disabled = os.getenv("TREES_USE_FHE_SUM") == "1" + assert fhe_sum_disabled + + non_fhe_sume_predict_quantized = predict_method(x, fhe="disable") + non_fhe_sume_predict_simulate = predict_method(x, fhe="simulate") + + # Compute the FHE predictions only during weekly CIs + if is_weekly_option: + rounded_predict_fhe = predict_method(fhe_test, fhe="execute") + + print("ROUNGING ENABLED") + + with pytest.MonkeyPatch.context() as mp_context: + + # Enable FHE sum + mp_context.setenv("TREES_USE_FHE_SUM", "0") + + # Check that rounding is disabled + fhe_sum_enbled = os.environ.get("TREES_USE_FHE_SUM") == "0" + assert fhe_sum_enbled + + fhe_sum_predict_quantized = predict_method(x, fhe="disable") + fhe_sum_predict_simulate = predict_method(x, fhe="simulate") + + metric(non_fhe_sume_predict_quantized, fhe_sum_predict_quantized) + metric(non_fhe_sume_predict_simulate, fhe_sum_predict_simulate) + + # Compute the FHE predictions only during weekly CIs + if is_weekly_option: + not_rounded_predict_fhe = predict_method(fhe_test, fhe="execute") + metric(rounded_predict_fhe, not_rounded_predict_fhe) + + # Check that the maximum bit-width of the circuit with rounding is at most: + # maximum bit-width (of the circuit without rounding) + 2 + # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4178 + + +@pytest.mark.parametrize("model_class, parameters", get_sklearn_tree_models_and_datasets()) +@pytest.mark.parametrize("n_bits", [2, 5, 10]) +def test_fhe_sum_for_tree_based_models( + model_class, + parameters, + n_bits, + load_data, + check_r2_score, + check_accuracy, + is_weekly_option, + default_configuration, + verbose=True, +): + """Test that Concrete ML without and with rounding are 'equivalent'.""" + + if verbose: + print("Run check_rounding_consistency") + + model, x = preamble(model_class, parameters, n_bits, load_data, is_weekly_option) + + # Compile the model to make sure we consider all possible attributes during the serialization + model.compile(x, default_configuration) + + # Check `predict_proba` for classifiers + if is_classifier_or_partial_classifier(model): + predict_method = model.predict_proba + metric = check_r2_score + else: + # Check `predict` for regressors + predict_method = model.predict + metric = check_accuracy + + check_fhe_sum_consistency( + x, + predict_method, + metric, + is_weekly_option, + ) From 8fab929ad3c6b611f31474b9c2e370c06edc497d Mon Sep 17 00:00:00 2001 From: kcelia Date: Mon, 15 Jan 2024 11:17:33 +0100 Subject: [PATCH 47/73] chore: update --- src/concrete/ml/quantization/post_training.py | 12 ++- src/concrete/ml/sklearn/base.py | 16 +++- src/concrete/ml/sklearn/rf.py | 10 +- src/concrete/ml/sklearn/tree.py | 10 +- src/concrete/ml/sklearn/tree_to_numpy.py | 8 +- src/concrete/ml/sklearn/xgb.py | 8 +- tests/sklearn/test_sklearn_models.py | 96 +++++++++---------- 7 files changed, 84 insertions(+), 76 deletions(-) diff --git a/src/concrete/ml/quantization/post_training.py b/src/concrete/ml/quantization/post_training.py index 70f198fa9..22d1a3cb5 100644 --- a/src/concrete/ml/quantization/post_training.py +++ b/src/concrete/ml/quantization/post_training.py @@ -23,6 +23,8 @@ from .quantized_ops import QuantizedBrevitasQuant from .quantizers import QuantizationOptions, QuantizedArray, UniformQuantizer +# pylint: disable=too-many-lines + def get_n_bits_dict_trees(n_bits: Union[int, Dict[str, int]]) -> Dict[str, int]: """Convert the n_bits parameter into a proper dictionary for tree based-models. @@ -32,10 +34,12 @@ def get_n_bits_dict_trees(n_bits: Union[int, Dict[str, int]]) -> Dict[str, int]: a dictionary with the following keys : - "op_inputs" (mandatory) - "op_leaves" (optional) - TODO + When using a single integer for n_bits, its value is assigned to "op_inputs" and + "op_leaves" bits. Returns: - n_bits_dict (Dict[str, int]): TODO + n_bits_dict (Dict[str, int]): A dictionary properly representing the number of bits to use + for quantization. """ assert_true( @@ -46,7 +50,7 @@ def get_n_bits_dict_trees(n_bits: Union[int, Dict[str, int]]) -> Dict[str, int]: "- `op_inputs` and `op_leaves` (mandatory)", ) - n_bits_dict = {} + n_bits_dict: Dict = {} # If a single integer is passed, we use a default value for the model's input and # output bits @@ -56,8 +60,6 @@ def get_n_bits_dict_trees(n_bits: Union[int, Dict[str, int]]) -> Dict[str, int]: "op_leaves": n_bits, } - # If model_inputs or model_outputs are not given, we consider a default value - elif isinstance(n_bits, Dict): n_bits_dict.update(n_bits) diff --git a/src/concrete/ml/sklearn/base.py b/src/concrete/ml/sklearn/base.py index 539cfd74e..aacf301a2 100644 --- a/src/concrete/ml/sklearn/base.py +++ b/src/concrete/ml/sklearn/base.py @@ -103,8 +103,8 @@ # However, for internal testing purposes, we retain the capability to disable this feature os.environ["TREES_USE_ROUNDING"] = "1" -# TODO -os.environ["TREES_USE_FHE_SUM"] = "1" +# By default, the decision of the tree ensembles is made in clear +os.environ["TREES_USE_FHE_SUM"] = "0" # pylint: disable=too-many-public-methods @@ -1289,7 +1289,7 @@ def __init_subclass__(cls): _TREE_MODELS.add(cls) _ALL_SKLEARN_MODELS.add(cls) - def __init__(self, n_bits: int): + def __init__(self, n_bits: Union[int, Dict[str, int]]): """Initialize the TreeBasedEstimatorMixin. Args: @@ -1432,6 +1432,16 @@ def predict(self, X: Data, fhe: Union[FheMode, str] = FheMode.DISABLE) -> numpy. y_pred = self.post_processing(y_pred) return y_pred + # def post_processing(self, y_preds: numpy.ndarray) -> numpy.ndarray: + + + # # Sum all tree outputs + # # Remove the sum once we handle multi-precision circuits + # # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/451 + # y_preds = numpy.sum(y_preds, axis=-1) + + # assert_true(y_preds.ndim == 2, "y_preds should be a 2D array") + # return y_preds class BaseTreeRegressorMixin(BaseTreeEstimatorMixin, sklearn.base.RegressorMixin, ABC): """Mixin class for tree-based regressors. diff --git a/src/concrete/ml/sklearn/rf.py b/src/concrete/ml/sklearn/rf.py index 14aabf060..2ebca55b8 100644 --- a/src/concrete/ml/sklearn/rf.py +++ b/src/concrete/ml/sklearn/rf.py @@ -1,5 +1,5 @@ """Implement RandomForest models.""" -from typing import Any, Dict +from typing import Any, Dict, Union import numpy import sklearn.ensemble @@ -19,7 +19,7 @@ class RandomForestClassifier(BaseTreeClassifierMixin): # pylint: disable-next=too-many-arguments def __init__( self, - n_bits: int = 6, + n_bits: Union[int, Dict[str, int]] = 6, n_estimators=20, criterion="gini", max_depth=4, @@ -124,7 +124,7 @@ def load_dict(cls, metadata: Dict): obj.sklearn_model, numpy.zeros((len(obj.input_quantizers),))[None, ...], framework=obj.framework, - output_n_bits=obj.n_bits["op_leaves"], + output_n_bits=obj.n_bits["op_leaves"] if isinstance(obj.n_bits, Dict) else obj.n_bits, )[0] obj.post_processing_params = metadata["post_processing_params"] @@ -162,7 +162,7 @@ class RandomForestRegressor(BaseTreeRegressorMixin): # pylint: disable-next=too-many-arguments def __init__( self, - n_bits: int = 6, + n_bits: Union[int, Dict[str, int]] = 6, n_estimators=20, criterion="squared_error", max_depth=4, @@ -259,7 +259,7 @@ def load_dict(cls, metadata: Dict): obj.sklearn_model, numpy.zeros((len(obj.input_quantizers),))[None, ...], framework=obj.framework, - output_n_bits=obj.n_bits["op_leaves"], + output_n_bits=obj.n_bits["op_leaves"] if isinstance(obj.n_bits, Dict) else obj.n_bits, )[0] obj.post_processing_params = metadata["post_processing_params"] diff --git a/src/concrete/ml/sklearn/tree.py b/src/concrete/ml/sklearn/tree.py index c870d2804..5ba1f8cff 100644 --- a/src/concrete/ml/sklearn/tree.py +++ b/src/concrete/ml/sklearn/tree.py @@ -1,5 +1,5 @@ """Implement DecisionTree models.""" -from typing import Any, Dict +from typing import Any, Dict, Union import numpy import sklearn.tree @@ -31,7 +31,7 @@ def __init__( min_impurity_decrease=0.0, class_weight=None, ccp_alpha: float = 0.0, - n_bits: int = 6, + n_bits: Union[int, Dict[str, int]] = 6, ): """Initialize the DecisionTreeClassifier. @@ -119,7 +119,7 @@ def load_dict(cls, metadata: Dict): obj.sklearn_model, numpy.zeros((len(obj.input_quantizers),))[None, ...], framework=obj.framework, - output_n_bits=obj.n_bits["op_leaves"], + output_n_bits=obj.n_bits["op_leaves"] if isinstance(obj.n_bits, Dict) else obj.n_bits, )[0] obj.post_processing_params = metadata["post_processing_params"] @@ -162,7 +162,7 @@ def __init__( max_leaf_nodes=None, min_impurity_decrease=0.0, ccp_alpha=0.0, - n_bits: int = 6, + n_bits: Union[int, Dict[str, int]] = 6, ): """Initialize the DecisionTreeRegressor. @@ -242,7 +242,7 @@ def load_dict(cls, metadata: Dict): obj.sklearn_model, numpy.zeros((len(obj.input_quantizers),))[None, ...], framework=obj.framework, - output_n_bits=obj.n_bits["op_leaves"], + output_n_bits=obj.n_bits["op_leaves"] if isinstance(obj.n_bits, Dict) else obj.n_bits, )[0] obj.post_processing_params = metadata["post_processing_params"] diff --git a/src/concrete/ml/sklearn/tree_to_numpy.py b/src/concrete/ml/sklearn/tree_to_numpy.py index a748197b9..49b86705e 100644 --- a/src/concrete/ml/sklearn/tree_to_numpy.py +++ b/src/concrete/ml/sklearn/tree_to_numpy.py @@ -143,10 +143,12 @@ def add_transpose_after_last_node(onnx_model: onnx.ModelProto): output_node = onnx_model.graph.output[0] if os.getenv("TREES_USE_FHE_SUM") == "1": + # Create the node with perm attribute equal to (1, 0) perm = [1, 0] else: + # Create the node with perm attribute equal to (2, 1, 0) perm = [2, 1, 0] - # Create the node with perm attribute equal to (1, 0) + transpose_node = onnx.helper.make_node( "Transpose", inputs=[output_node.name], @@ -266,7 +268,7 @@ def tree_onnx_graph_preprocessing( def tree_values_preprocessing( onnx_model: onnx.ModelProto, framework: str, - n_bits: int, + output_n_bits: int, ) -> QuantizedArray: """Pre-process tree values. @@ -292,7 +294,7 @@ def tree_values_preprocessing( if "weight_3" in initializer.name: # print(init_tensor) # weight_3 is the prediction tensor, apply the required pre-processing - q_y = preprocess_tree_predictions(init_tensor, n_bits) + q_y = preprocess_tree_predictions(init_tensor, output_n_bits) # Get the preprocessed tree predictions to replace the current (non-quantized) # values in the onnx_model. diff --git a/src/concrete/ml/sklearn/xgb.py b/src/concrete/ml/sklearn/xgb.py index a2b121f4d..28722b706 100644 --- a/src/concrete/ml/sklearn/xgb.py +++ b/src/concrete/ml/sklearn/xgb.py @@ -27,7 +27,7 @@ class XGBClassifier(BaseTreeClassifierMixin): # pylint: disable=too-many-arguments,too-many-locals def __init__( self, - n_bits: int = 6, + n_bits: Union[int, Dict[str, int]] = 6, max_depth: Optional[int] = 3, learning_rate: Optional[float] = None, n_estimators: Optional[int] = 20, @@ -178,7 +178,7 @@ def load_dict(cls, metadata: Dict): obj.sklearn_model, numpy.zeros((len(obj.input_quantizers),))[None, ...], framework=obj.framework, - output_n_bits=obj.n_bits["op_leaves"], + output_n_bits=obj.n_bits["op_leaves"] if isinstance(obj.n_bits, Dict) else obj.n_bits, )[0] obj.post_processing_params = metadata["post_processing_params"] @@ -233,7 +233,7 @@ class XGBRegressor(BaseTreeRegressorMixin): # pylint: disable=too-many-arguments,too-many-locals def __init__( self, - n_bits: int = 6, + n_bits: Union[int, Dict[str, int]] = 6, max_depth: Optional[int] = 3, learning_rate: Optional[float] = None, n_estimators: Optional[int] = 20, @@ -407,7 +407,7 @@ def load_dict(cls, metadata: Dict): obj.sklearn_model, numpy.zeros((len(obj.input_quantizers),))[None, ...], framework=obj.framework, - output_n_bits=obj.n_bits["op_leaves"], + output_n_bits=obj.n_bits["op_leaves"] if isinstance(obj.n_bits, Dict) else obj.n_bits, )[0] obj.post_processing_params = metadata["post_processing_params"] diff --git a/tests/sklearn/test_sklearn_models.py b/tests/sklearn/test_sklearn_models.py index beeceed4e..1910d15b4 100644 --- a/tests/sklearn/test_sklearn_models.py +++ b/tests/sklearn/test_sklearn_models.py @@ -1205,6 +1205,50 @@ def check_rounding_consistency( # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4178 +def check_fhe_sum_consistency( + x, + predict_method, + metric, + is_weekly_option, +): + """Test that Concrete ML without and with rounding are 'equivalent'.""" + + # Run the test with more samples during weekly CIs + if is_weekly_option: + fhe_test = get_random_samples(x, n_sample=5) + + # By default, FHE_SUM is disabled + fhe_sum_disabled = os.getenv("TREES_USE_FHE_SUM") == "1" + assert fhe_sum_disabled + + non_fhe_sum_predict_quantized = predict_method(x, fhe="disable") + non_fhe_sum_predict_simulate = predict_method(x, fhe="simulate") + + # Compute the FHE predictions only during weekly CIs + if is_weekly_option: + rounded_predict_fhe = predict_method(fhe_test, fhe="execute") + + with pytest.MonkeyPatch.context() as mp_context: + + # Enable FHE sum + mp_context.setenv("TREES_USE_FHE_SUM", "0") + + # Check that rounding is disabled + fhe_sum_enbled = os.environ.get("TREES_USE_FHE_SUM") == "0" + assert fhe_sum_enbled + + fhe_sum_predict_quantized = predict_method(x, fhe="disable") + fhe_sum_predict_simulate = predict_method(x, fhe="simulate") + + metric(non_fhe_sum_predict_quantized, fhe_sum_predict_quantized) + metric(non_fhe_sum_predict_simulate, fhe_sum_predict_simulate) + + # Compute the FHE predictions only during weekly CIs + if is_weekly_option: + not_rounded_predict_fhe = predict_method(fhe_test, fhe="execute") + metric(rounded_predict_fhe, not_rounded_predict_fhe) + + # Neural network models are skipped for this test # The `fit_benchmark` function of QNNs returns a QAT model and a FP32 model that is similar # in structure but trained from scratch. Furthermore, the `n_bits` setting @@ -1676,7 +1720,7 @@ def test_p_error_simulation( is_linear_model = is_model_class_in_a_list(model_class, _get_sklearn_linear_models()) # Compile with a large p_error to be sure the result is random. - c = model.compile(x, **error_param) + model.compile(x, **error_param) def check_for_divergent_predictions(x, model, fhe, max_iterations=N_ALLOWED_FHE_RUN): """Detect divergence between simulated/FHE execution and clear run.""" @@ -1886,56 +1930,6 @@ def test_rounding_consistency_for_regular_models( ) -def check_fhe_sum_consistency( - x, - predict_method, - metric, - is_weekly_option, -): - """Test that Concrete ML without and with rounding are 'equivalent'.""" - - # Run the test with more samples during weekly CIs - if is_weekly_option: - fhe_test = get_random_samples(x, n_sample=5) - - # By default, FHE_SUM is disabled - fhe_sum_disabled = os.getenv("TREES_USE_FHE_SUM") == "1" - assert fhe_sum_disabled - - non_fhe_sume_predict_quantized = predict_method(x, fhe="disable") - non_fhe_sume_predict_simulate = predict_method(x, fhe="simulate") - - # Compute the FHE predictions only during weekly CIs - if is_weekly_option: - rounded_predict_fhe = predict_method(fhe_test, fhe="execute") - - print("ROUNGING ENABLED") - - with pytest.MonkeyPatch.context() as mp_context: - - # Enable FHE sum - mp_context.setenv("TREES_USE_FHE_SUM", "0") - - # Check that rounding is disabled - fhe_sum_enbled = os.environ.get("TREES_USE_FHE_SUM") == "0" - assert fhe_sum_enbled - - fhe_sum_predict_quantized = predict_method(x, fhe="disable") - fhe_sum_predict_simulate = predict_method(x, fhe="simulate") - - metric(non_fhe_sume_predict_quantized, fhe_sum_predict_quantized) - metric(non_fhe_sume_predict_simulate, fhe_sum_predict_simulate) - - # Compute the FHE predictions only during weekly CIs - if is_weekly_option: - not_rounded_predict_fhe = predict_method(fhe_test, fhe="execute") - metric(rounded_predict_fhe, not_rounded_predict_fhe) - - # Check that the maximum bit-width of the circuit with rounding is at most: - # maximum bit-width (of the circuit without rounding) + 2 - # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4178 - - @pytest.mark.parametrize("model_class, parameters", get_sklearn_tree_models_and_datasets()) @pytest.mark.parametrize("n_bits", [2, 5, 10]) def test_fhe_sum_for_tree_based_models( From ba26a5cc06bebe2114f7e85e7d030f9e40671b4a Mon Sep 17 00:00:00 2001 From: kcelia Date: Mon, 15 Jan 2024 15:21:54 +0100 Subject: [PATCH 48/73] chore: update --- src/concrete/ml/sklearn/base.py | 17 +++-- src/concrete/ml/sklearn/tree_to_numpy.py | 18 +++-- tests/sklearn/test_dump_onnx.py | 15 ++-- tests/sklearn/test_sklearn_models.py | 94 ++++++++++++------------ 4 files changed, 77 insertions(+), 67 deletions(-) diff --git a/src/concrete/ml/sklearn/base.py b/src/concrete/ml/sklearn/base.py index aacf301a2..179e5a5c4 100644 --- a/src/concrete/ml/sklearn/base.py +++ b/src/concrete/ml/sklearn/base.py @@ -1320,7 +1320,6 @@ def fit(self, X: Data, y: Target, **fit_parameters): # Convert the n_bits attribute into a proper dictionary self.n_bits = get_n_bits_dict_trees(self.n_bits) - print(f"{self.n_bits=}") # Quantization of each feature in X for i in range(X.shape[1]): @@ -1432,16 +1431,18 @@ def predict(self, X: Data, fhe: Union[FheMode, str] = FheMode.DISABLE) -> numpy. y_pred = self.post_processing(y_pred) return y_pred - # def post_processing(self, y_preds: numpy.ndarray) -> numpy.ndarray: + def post_processing(self, y_preds: numpy.ndarray) -> numpy.ndarray: + # Sum all tree outputs + # Remove the sum once we handle multi-precision circuits + # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/451 + if os.getenv("TREES_USE_FHE_SUM") == "0": + y_preds = numpy.sum(y_preds, axis=-1) + assert_true(y_preds.ndim == 2, "y_preds should be a 2D array") + return y_preds - # # Sum all tree outputs - # # Remove the sum once we handle multi-precision circuits - # # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/451 - # y_preds = numpy.sum(y_preds, axis=-1) + return super().post_processing(y_preds) - # assert_true(y_preds.ndim == 2, "y_preds should be a 2D array") - # return y_preds class BaseTreeRegressorMixin(BaseTreeEstimatorMixin, sklearn.base.RegressorMixin, ABC): """Mixin class for tree-based regressors. diff --git a/src/concrete/ml/sklearn/tree_to_numpy.py b/src/concrete/ml/sklearn/tree_to_numpy.py index 49b86705e..65a115759 100644 --- a/src/concrete/ml/sklearn/tree_to_numpy.py +++ b/src/concrete/ml/sklearn/tree_to_numpy.py @@ -18,7 +18,11 @@ OPSET_VERSION_FOR_ONNX_EXPORT, get_equivalent_numpy_forward_from_onnx_tree, ) -from ..onnx.onnx_model_manipulations import clean_graph_after_node_op_type, remove_node_types +from ..onnx.onnx_model_manipulations import ( + clean_graph_after_node_op_type, + clean_graph_at_node_op_type, + remove_node_types, +) from ..onnx.onnx_utils import get_op_type from ..quantization import QuantizedArray from ..quantization.quantizers import UniformQuantizer @@ -142,13 +146,14 @@ def add_transpose_after_last_node(onnx_model: onnx.ModelProto): # Get the output node output_node = onnx_model.graph.output[0] + # When using FHE sum for tree ensembles, create the node with perm attribute equal to (1, 0) if os.getenv("TREES_USE_FHE_SUM") == "1": - # Create the node with perm attribute equal to (1, 0) perm = [1, 0] + + # Otherwise, create the node with perm attribute equal to (2, 1, 0) else: - # Create the node with perm attribute equal to (2, 1, 0) perm = [2, 1, 0] - + transpose_node = onnx.helper.make_node( "Transpose", inputs=[output_node.name], @@ -246,7 +251,10 @@ def tree_onnx_graph_preprocessing( # Cut the graph after the ReduceSum node to remove # argmax, sigmoid, softmax from the graph. - clean_graph_after_node_op_type(onnx_model, "ReduceSum") + if os.getenv("TREES_USE_FHE_SUM") == "1": + clean_graph_after_node_op_type(onnx_model, "ReduceSum") + else: + clean_graph_at_node_op_type(onnx_model, "ReduceSum") if framework == "xgboost": # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/2778 diff --git a/tests/sklearn/test_dump_onnx.py b/tests/sklearn/test_dump_onnx.py index a5fe6497a..484e2c0d7 100644 --- a/tests/sklearn/test_dump_onnx.py +++ b/tests/sklearn/test_dump_onnx.py @@ -69,7 +69,6 @@ def check_onnx_file_dump(model_class, parameters, load_data, str_expected, defau str_model = onnx.helper.printable_graph(onnx_model.graph) print(f"{model_name}:") print(str_model) - # Test equality when it does not depend on seeds # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3266 if not is_model_class_in_a_list(model_class, _get_sklearn_tree_models(select="RandomForest")): @@ -228,7 +227,7 @@ def test_dump( %transposed_output = Transpose[perm = [1, 0]](%/_operators.0/ReduceSum_output_0) """ if os.getenv("TREES_USE_FHE_SUM") == "1" - else "%transposed_output = Transpose[perm = [2, 1, 0]](%/_operators.0/Reshape_3_output_0)" + else "%transposed_output = Transpose[perm = [2, 1, 0]](%/_operators.0/Reshape_3_output_0)\n " ) + """return %transposed_output }""", @@ -307,7 +306,7 @@ def test_dump( %transposed_output = Transpose[perm = [1, 0]](%/_operators.0/ReduceSum_output_0) """ if os.getenv("TREES_USE_FHE_SUM") == "1" - else "" + else "%transposed_output = Transpose[perm = [2, 1, 0]](%/_operators.0/Reshape_3_output_0)\n " ) + """return %transposed_output }""", @@ -359,8 +358,7 @@ def test_dump( return %/_operators.0/ReduceSum_output_0 }""" if os.getenv("TREES_USE_FHE_SUM") == "1" - else """return %/_operators.0/Reshape_4_output_0 - }""" + else "return %/_operators.0/Reshape_4_output_0\n}" ), "RandomForestRegressor": """graph torch_jit ( %input_0[DOUBLE, symx10] @@ -401,9 +399,8 @@ def test_dump( %/_operators.0/Constant_1_output_0[INT64, 2] %/_operators.0/Constant_2_output_0[INT64, 3] %/_operators.0/Constant_3_output_0[INT64, 3] - %/_operators.0/Constant_4_output_0[INT64, 3] - """ - + ("%onnx::ReduceSum_27[INT64, 1]" if os.getenv("TREES_USE_FHE_SUM") == "1" else "") + %/_operators.0/Constant_4_output_0[INT64, 3]""" + + ("\n %onnx::ReduceSum_27[INT64, 1]" if os.getenv("TREES_USE_FHE_SUM") == "1" else "") + """ ) { %/_operators.0/Gemm_output_0 = Gemm[alpha = 1, beta = 0, transB = 1](%_operators.0.weight_1, %input_0) @@ -424,7 +421,7 @@ def test_dump( return %/_operators.0/ReduceSum_output_0 }""" if os.getenv("TREES_USE_FHE_SUM") == "1" - else "return %/_operators.0/Reshape_4_output_0" + else """return %/_operators.0/Reshape_4_output_0\n}""" ), "LinearRegression": """graph torch_jit ( %input_0[DOUBLE, symx10] diff --git a/tests/sklearn/test_sklearn_models.py b/tests/sklearn/test_sklearn_models.py index 1910d15b4..8fcc086dd 100644 --- a/tests/sklearn/test_sklearn_models.py +++ b/tests/sklearn/test_sklearn_models.py @@ -46,6 +46,7 @@ from concrete.ml.common.serialization.loaders import load, loads from concrete.ml.common.utils import ( USE_OLD_VL, + array_allclose_and_same_shape, get_model_class, get_model_name, is_classifier_or_partial_classifier, @@ -726,16 +727,12 @@ def check_pipeline(model_class, x, y): {key: value} for key, values in hyper_param_combinations.items() for value in values ] - print(f"{hyperparameters_list=}") - # Take one of the hyper_parameters randomly (testing everything would be too long) if len(hyperparameters_list) == 0: hyper_parameters = {} else: hyper_parameters = hyperparameters_list[numpy.random.randint(0, len(hyperparameters_list))] - print(f"{hyperparameters_list=}") - pipe_cv = Pipeline( [ ("pca", PCA(n_components=2, random_state=numpy.random.randint(0, 2**15))), @@ -752,7 +749,6 @@ def check_pipeline(model_class, x, y): } else: - print("ELSE") param_grid = { "model__n_bits": [2, 3], } @@ -1206,47 +1202,73 @@ def check_rounding_consistency( def check_fhe_sum_consistency( + model_class, x, - predict_method, - metric, + y, + n_bits, is_weekly_option, ): - """Test that Concrete ML without and with rounding are 'equivalent'.""" + """Test that Concrete ML without and with FHE sum are 'equivalent'.""" # Run the test with more samples during weekly CIs if is_weekly_option: fhe_test = get_random_samples(x, n_sample=5) - # By default, FHE_SUM is disabled - fhe_sum_disabled = os.getenv("TREES_USE_FHE_SUM") == "1" + # By default, the summation of tree ensemble outputs is done in clear + fhe_sum_disabled = os.getenv("TREES_USE_FHE_SUM") == "0" assert fhe_sum_disabled + model_ref = instantiate_model_generic(model_class, n_bits=n_bits) + fit_and_compile(model_ref, x, y) + + # Check `predict_proba` for classifiers and `predict` for regressors + predict_method = ( + model_ref.predict_proba + if is_classifier_or_partial_classifier(model_class) + else model_ref.predict + ) + non_fhe_sum_predict_quantized = predict_method(x, fhe="disable") non_fhe_sum_predict_simulate = predict_method(x, fhe="simulate") + # Sanity check + array_allclose_and_same_shape(non_fhe_sum_predict_quantized, non_fhe_sum_predict_simulate) + # Compute the FHE predictions only during weekly CIs if is_weekly_option: - rounded_predict_fhe = predict_method(fhe_test, fhe="execute") + non_fhe_sum_predict_fhe = predict_method(fhe_test, fhe="execute") with pytest.MonkeyPatch.context() as mp_context: - # Enable FHE sum - mp_context.setenv("TREES_USE_FHE_SUM", "0") + # Enable the FHE summation of tree ensemble outputs + mp_context.setenv("TREES_USE_FHE_SUM", "1") - # Check that rounding is disabled - fhe_sum_enbled = os.environ.get("TREES_USE_FHE_SUM") == "0" - assert fhe_sum_enbled + # Check that the summation of tree ensemble outputs is enabled + fhe_sum_enabled = os.environ.get("TREES_USE_FHE_SUM") == "1" + assert fhe_sum_enabled + + model = model_class(**model_ref.get_params()) + fit_and_compile(model, x, y) + + # Check `predict_proba` for classifiers and `predict` for regressors + predict_method = ( + model.predict_proba + if is_classifier_or_partial_classifier(model_class) + else model.predict + ) fhe_sum_predict_quantized = predict_method(x, fhe="disable") fhe_sum_predict_simulate = predict_method(x, fhe="simulate") - metric(non_fhe_sum_predict_quantized, fhe_sum_predict_quantized) - metric(non_fhe_sum_predict_simulate, fhe_sum_predict_simulate) + # Sanity check + array_allclose_and_same_shape(fhe_sum_predict_quantized, fhe_sum_predict_simulate) - # Compute the FHE predictions only during weekly CIs - if is_weekly_option: - not_rounded_predict_fhe = predict_method(fhe_test, fhe="execute") - metric(rounded_predict_fhe, not_rounded_predict_fhe) + # Check that we have the exact same predictions + array_allclose_and_same_shape(fhe_sum_predict_quantized, non_fhe_sum_predict_quantized) + array_allclose_and_same_shape(fhe_sum_predict_simulate, non_fhe_sum_predict_simulate) + if is_weekly_option: + fhe_sum_predict_fhe = predict_method(fhe_test, fhe="execute") + array_allclose_and_same_shape(fhe_sum_predict_fhe, non_fhe_sum_predict_fhe) # Neural network models are skipped for this test @@ -1741,11 +1763,7 @@ def check_for_divergent_predictions(x, model, fhe, max_iterations=N_ALLOWED_FHE_ return True return False - print("Start simulation") - print(model) - simulation_diff_found = check_for_divergent_predictions(x, model, fhe="simulate") - print("execution") fhe_diff_found = check_for_divergent_predictions(x, model, fhe="execute") # Check for differences in predictions @@ -1937,34 +1955,20 @@ def test_fhe_sum_for_tree_based_models( parameters, n_bits, load_data, - check_r2_score, - check_accuracy, is_weekly_option, - default_configuration, verbose=True, ): """Test that Concrete ML without and with rounding are 'equivalent'.""" if verbose: - print("Run check_rounding_consistency") - - model, x = preamble(model_class, parameters, n_bits, load_data, is_weekly_option) - - # Compile the model to make sure we consider all possible attributes during the serialization - model.compile(x, default_configuration) + print("Run check_fhe_sum_consistency") - # Check `predict_proba` for classifiers - if is_classifier_or_partial_classifier(model): - predict_method = model.predict_proba - metric = check_r2_score - else: - # Check `predict` for regressors - predict_method = model.predict - metric = check_accuracy + x, y = get_dataset(model_class, parameters, n_bits, load_data, is_weekly_option) check_fhe_sum_consistency( + model_class, x, - predict_method, - metric, + y, + n_bits, is_weekly_option, ) From 05256b212263a0d17ee0a612f03fe475454ed844 Mon Sep 17 00:00:00 2001 From: kcelia Date: Tue, 16 Jan 2024 12:43:04 +0100 Subject: [PATCH 49/73] chore: update --- src/concrete/ml/quantization/post_training.py | 11 +++++++---- src/concrete/ml/sklearn/base.py | 8 ++++---- tests/sklearn/test_sklearn_models.py | 2 +- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/concrete/ml/quantization/post_training.py b/src/concrete/ml/quantization/post_training.py index 22d1a3cb5..a8c453330 100644 --- a/src/concrete/ml/quantization/post_training.py +++ b/src/concrete/ml/quantization/post_training.py @@ -32,8 +32,10 @@ def get_n_bits_dict_trees(n_bits: Union[int, Dict[str, int]]) -> Dict[str, int]: Args: n_bits (int, Dict[str, int]): number of bits for quantization, can be a single value or a dictionary with the following keys : - - "op_inputs" (mandatory) - - "op_leaves" (optional) + - "op_inputs" (mandatory): number of bits to quantize the input values + - "op_leaves" (optional): number of bits to quantize the leaves, defaults to the value + of "op_inputs" if not specified. + When using a single integer for n_bits, its value is assigned to "op_inputs" and "op_leaves" bits. @@ -44,10 +46,11 @@ def get_n_bits_dict_trees(n_bits: Union[int, Dict[str, int]]) -> Dict[str, int]: assert_true( isinstance(n_bits, int) - or (isinstance(n_bits, Dict) and set(n_bits.keys()).issubset({"op_inputs", "op_leaves"})), + or (isinstance(n_bits, Dict) and not set(n_bits.keys()) - set(("op_leaves", "op_input"))), "Invalid n_bits, either pass an integer or a dictionary containing integer values for " "the following keys:\n" - "- `op_inputs` and `op_leaves` (mandatory)", + "- `op_inputs` (mandatory)\n" + "- `op_leaves` (optional)", ) n_bits_dict: Dict = {} diff --git a/src/concrete/ml/sklearn/base.py b/src/concrete/ml/sklearn/base.py index 179e5a5c4..792d37c98 100644 --- a/src/concrete/ml/sklearn/base.py +++ b/src/concrete/ml/sklearn/base.py @@ -101,10 +101,10 @@ # Enable rounding feature for all tree-based models by default # Note: This setting is fixed and cannot be altered by users # However, for internal testing purposes, we retain the capability to disable this feature -os.environ["TREES_USE_ROUNDING"] = "1" +os.environ["TREES_USE_ROUNDING"] = os.environ.get("TREES_USE_ROUNDING", "1") # By default, the decision of the tree ensembles is made in clear -os.environ["TREES_USE_FHE_SUM"] = "0" +os.environ["TREES_USE_FHE_SUM"] = os.environ.get("TREES_USE_FHE_SUM", "0") # pylint: disable=too-many-public-methods @@ -1297,8 +1297,8 @@ def __init__(self, n_bits: Union[int, Dict[str, int]]): for n_bits, the value will be used for quantizing inputs and leaves. If a dict is passed, then it should contain "op_inputs" and "op_leaves" as keys with corresponding number of quantization bits so that: - - op_inputs : number of bits to quantize the input values - - op_leaves: number of bits to quantize the leaves + - op_inputs (mandatory): number of bits to quantize the input values + - op_leaves (optional): number of bits to quantize the leaves Default to 6. """ self.n_bits: Union[int, Dict[str, int]] = n_bits diff --git a/tests/sklearn/test_sklearn_models.py b/tests/sklearn/test_sklearn_models.py index 8fcc086dd..d95024098 100644 --- a/tests/sklearn/test_sklearn_models.py +++ b/tests/sklearn/test_sklearn_models.py @@ -1958,7 +1958,7 @@ def test_fhe_sum_for_tree_based_models( is_weekly_option, verbose=True, ): - """Test that Concrete ML without and with rounding are 'equivalent'.""" + """Test that the tree ensembles' output are the same with and without the sum in FHE.""" if verbose: print("Run check_fhe_sum_consistency") From 03d498b0822fab1793aaa7f25731c36dffd90c87 Mon Sep 17 00:00:00 2001 From: kcelia Date: Tue, 16 Jan 2024 13:46:08 +0100 Subject: [PATCH 50/73] chore: remove useless prints --- src/concrete/ml/sklearn/tree_to_numpy.py | 2 -- tests/sklearn/test_sklearn_models.py | 6 +++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/concrete/ml/sklearn/tree_to_numpy.py b/src/concrete/ml/sklearn/tree_to_numpy.py index 65a115759..4536f6a9c 100644 --- a/src/concrete/ml/sklearn/tree_to_numpy.py +++ b/src/concrete/ml/sklearn/tree_to_numpy.py @@ -298,9 +298,7 @@ def tree_values_preprocessing( # while the final probabilities/regression values must be quantized. # We extract the value stored in each initializer node into the init_tensor. init_tensor = numpy_helper.to_array(initializer) - # print(initializer.name, init_tensor.shape) if "weight_3" in initializer.name: - # print(init_tensor) # weight_3 is the prediction tensor, apply the required pre-processing q_y = preprocess_tree_predictions(init_tensor, output_n_bits) diff --git a/tests/sklearn/test_sklearn_models.py b/tests/sklearn/test_sklearn_models.py index d95024098..c47c592f3 100644 --- a/tests/sklearn/test_sklearn_models.py +++ b/tests/sklearn/test_sklearn_models.py @@ -1201,7 +1201,7 @@ def check_rounding_consistency( # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4178 -def check_fhe_sum_consistency( +def check_fhe_sum_for_tree_based_models( model_class, x, y, @@ -1961,11 +1961,11 @@ def test_fhe_sum_for_tree_based_models( """Test that the tree ensembles' output are the same with and without the sum in FHE.""" if verbose: - print("Run check_fhe_sum_consistency") + print("Run check_fhe_sum_for_tree_based_models") x, y = get_dataset(model_class, parameters, n_bits, load_data, is_weekly_option) - check_fhe_sum_consistency( + check_fhe_sum_for_tree_based_models( model_class, x, y, From cf879d392b1b7308f810cd9764afcb3d00f12299 Mon Sep 17 00:00:00 2001 From: kcelia Date: Wed, 17 Jan 2024 10:29:59 +0100 Subject: [PATCH 51/73] chore: update get_n_bits_dict_trees --- src/concrete/ml/quantization/post_training.py | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/concrete/ml/quantization/post_training.py b/src/concrete/ml/quantization/post_training.py index a8c453330..e62c1a261 100644 --- a/src/concrete/ml/quantization/post_training.py +++ b/src/concrete/ml/quantization/post_training.py @@ -44,14 +44,13 @@ def get_n_bits_dict_trees(n_bits: Union[int, Dict[str, int]]) -> Dict[str, int]: for quantization. """ - assert_true( - isinstance(n_bits, int) - or (isinstance(n_bits, Dict) and not set(n_bits.keys()) - set(("op_leaves", "op_input"))), - "Invalid n_bits, either pass an integer or a dictionary containing integer values for " - "the following keys:\n" - "- `op_inputs` (mandatory)\n" - "- `op_leaves` (optional)", - ) + if not isinstance(n_bits, int) and not(isinstance(n_bits, Dict) and not set(n_bits.keys()) - set(("op_leaves", "op_inputs"))): + raise ValueError( + "Invalid n_bits, either pass an integer or a dictionary containing integer values for " + "the following keys:\n" + "- `op_inputs` (mandatory)\n" + "- `op_leaves` (optional)" + ) n_bits_dict: Dict = {} @@ -66,12 +65,13 @@ def get_n_bits_dict_trees(n_bits: Union[int, Dict[str, int]]) -> Dict[str, int]: elif isinstance(n_bits, Dict): n_bits_dict.update(n_bits) - assert_true( - n_bits_dict["op_inputs"] >= n_bits_dict["op_leaves"], - "Using fewer bits to represent the model_outputs than the op inputs is not " - f"recommended. Got op_leaves: {n_bits_dict['op_leaves']} and op_inputs: " - f"{n_bits_dict['op_inputs']}", - ) + if n_bits_dict["op_inputs"] < n_bits_dict["op_leaves"]: + + raise ValueError( + "Using fewer bits to represent the model_outputs than the op inputs is not " + f"recommended. Got op_leaves: {n_bits_dict['op_leaves']} and op_inputs: " + f"{n_bits_dict['op_inputs']}", + ) return n_bits_dict From ff1c6b14d85d8d8a1cd70cb529bf69290d91eef4 Mon Sep 17 00:00:00 2001 From: kcelia Date: Wed, 17 Jan 2024 15:18:17 +0100 Subject: [PATCH 52/73] chore: update add a test to check valid n_bits for trees make get_n_bits_trees and inspect_tree_n_bits private functions --- src/concrete/ml/quantization/__init__.py | 3 +- src/concrete/ml/quantization/post_training.py | 97 +++++++++++++------ src/concrete/ml/sklearn/base.py | 15 ++- tests/sklearn/test_sklearn_models.py | 34 +++++++ 4 files changed, 114 insertions(+), 35 deletions(-) diff --git a/src/concrete/ml/quantization/__init__.py b/src/concrete/ml/quantization/__init__.py index 6669d76a7..20fba6653 100644 --- a/src/concrete/ml/quantization/__init__.py +++ b/src/concrete/ml/quantization/__init__.py @@ -4,7 +4,8 @@ PostTrainingAffineQuantization, PostTrainingQATImporter, get_n_bits_dict, - get_n_bits_dict_trees, + _get_n_bits_dict_trees, + _inspect_tree_n_bits, ) from .quantized_module import QuantizedModule from .quantized_ops import ( diff --git a/src/concrete/ml/quantization/post_training.py b/src/concrete/ml/quantization/post_training.py index e62c1a261..89aad3aa0 100644 --- a/src/concrete/ml/quantization/post_training.py +++ b/src/concrete/ml/quantization/post_training.py @@ -26,15 +26,73 @@ # pylint: disable=too-many-lines -def get_n_bits_dict_trees(n_bits: Union[int, Dict[str, int]]) -> Dict[str, int]: +def _inspect_tree_n_bits(n_bits): + """Validate the 'n_bits' parameter for tree-based models. + + This function checks whether 'n_bits' is a valid integer or dictionary. + - If 'n_bits' is an integer, it must be a non-null positive, its value is assigned to + "op_inputs" and "op_leaves" bits + - If it is a dictionary, it should contain integer values for keys 'op_leaves' and 'op_inputs', + where 'op_leaves' should not exceed 'op_inputs'. + + The function raises a ValueError with a descriptive message if 'n_bits' does not meet + these criteria. + + Args: + n_bits (int, Dict[str, int]): number of bits for quantization, can be a single value or + a dictionary with the following keys : + - "op_inputs" (mandatory): number of bits to quantize the input values + - "op_leaves" (optional): number of bits to quantize the leaves, must be less than or + equal to `op_inputs`. defaults to the value of "op_inputs" if not specified. + + Raises: + ValueError: If 'n_bits' does not conform to the required format or value constraints. + """ + + detailed_message = ( + "Invalid `n_bits`, either pass a non-null positive integer or a dictionary containing " + "integer values for the following keys:\n" + "- `op_inputs` (mandatory): number of bits to quantize the input values\n" + "- `op_leaves` (optional): number of bits to quantize the leaves, must be less than or " + "equal to `op_inputs`. Defaults to the value of `op_inputs` if not specified." + "When using a single integer for n_bits, its value is assigned to `op_inputs` and " + "`op_leaves` bits.\n" + ) + + error_message = "" + + if isinstance(n_bits, int): + if n_bits <= 0: + error_message = "n_bits must be a non-null, positive integer" + elif isinstance(n_bits, dict): + if "op_inputs" not in n_bits.keys() or set(n_bits.keys()) - {"op_leaves", "op_inputs"}: + error_message = ( + "Invalid keys in `n_bits` dictionary. Only 'op_inputs' (mandatory) and " + "'op_leaves' (optional) are allowed" + ) + elif not all(isinstance(value, int) and value > 0 for value in n_bits.values()): + error_message = "All values in `n_bits` dictionary must be non-null, positive integers" + + elif n_bits.get("op_leaves", 0) > n_bits.get("op_inputs", 0): + error_message = "`op_leaves` must be less than or equal to `op_inputs`" + else: + error_message = "n_bits must be either an integer or a dictionary" + + if len(error_message) > 0: + raise ValueError( + f"{error_message}. Got `{type(n_bits)}` and `{n_bits}` value.\n{detailed_message}" + ) + + +def _get_n_bits_dict_trees(n_bits: Union[int, Dict[str, int]]) -> Dict[str, int]: """Convert the n_bits parameter into a proper dictionary for tree based-models. Args: n_bits (int, Dict[str, int]): number of bits for quantization, can be a single value or a dictionary with the following keys : - "op_inputs" (mandatory): number of bits to quantize the input values - - "op_leaves" (optional): number of bits to quantize the leaves, defaults to the value - of "op_inputs" if not specified. + - "op_leaves" (optional): number of bits to quantize the leaves, must be less than or + equal to `op_inputs`. defaults to the value of "op_inputs" if not specified. When using a single integer for n_bits, its value is assigned to "op_inputs" and "op_leaves" bits. @@ -44,36 +102,17 @@ def get_n_bits_dict_trees(n_bits: Union[int, Dict[str, int]]) -> Dict[str, int]: for quantization. """ - if not isinstance(n_bits, int) and not(isinstance(n_bits, Dict) and not set(n_bits.keys()) - set(("op_leaves", "op_inputs"))): - raise ValueError( - "Invalid n_bits, either pass an integer or a dictionary containing integer values for " - "the following keys:\n" - "- `op_inputs` (mandatory)\n" - "- `op_leaves` (optional)" - ) + _inspect_tree_n_bits(n_bits) - n_bits_dict: Dict = {} - - # If a single integer is passed, we use a default value for the model's input and - # output bits + # If a single integer is passed, we use a default value for the model's input and leaves if isinstance(n_bits, int): - n_bits_dict = { - "op_inputs": n_bits, - "op_leaves": n_bits, - } - - elif isinstance(n_bits, Dict): - n_bits_dict.update(n_bits) + return {"op_inputs": n_bits, "op_leaves": n_bits} - if n_bits_dict["op_inputs"] < n_bits_dict["op_leaves"]: + # Default `op_leaves` to `op_inputs` if not specified + if "op_leaves" not in n_bits: + n_bits["op_leaves"] = n_bits["op_inputs"] - raise ValueError( - "Using fewer bits to represent the model_outputs than the op inputs is not " - f"recommended. Got op_leaves: {n_bits_dict['op_leaves']} and op_inputs: " - f"{n_bits_dict['op_inputs']}", - ) - - return n_bits_dict + return n_bits def get_n_bits_dict(n_bits: Union[int, Dict[str, int]]) -> Dict[str, int]: diff --git a/src/concrete/ml/sklearn/base.py b/src/concrete/ml/sklearn/base.py index 792d37c98..62ff03cbe 100644 --- a/src/concrete/ml/sklearn/base.py +++ b/src/concrete/ml/sklearn/base.py @@ -53,7 +53,8 @@ PostTrainingQATImporter, QuantizedArray, get_n_bits_dict, - get_n_bits_dict_trees, + _get_n_bits_dict_trees, + _inspect_tree_n_bits, ) from ..quantization.quantized_module import QuantizedModule, _get_inputset_generator from ..quantization.quantizers import ( @@ -1301,6 +1302,10 @@ def __init__(self, n_bits: Union[int, Dict[str, int]]): - op_leaves (optional): number of bits to quantize the leaves Default to 6. """ + + # Check if 'n_bits' is a valid value + _inspect_tree_n_bits(n_bits) + self.n_bits: Union[int, Dict[str, int]] = n_bits #: The model's inference function. Is None if the model is not fitted. @@ -1319,7 +1324,7 @@ def fit(self, X: Data, y: Target, **fit_parameters): q_X = numpy.zeros_like(X) # Convert the n_bits attribute into a proper dictionary - self.n_bits = get_n_bits_dict_trees(self.n_bits) + self.n_bits = _get_n_bits_dict_trees(self.n_bits) # Quantization of each feature in X for i in range(X.shape[1]): @@ -1338,9 +1343,6 @@ def fit(self, X: Data, y: Target, **fit_parameters): # Check that the underlying sklearn model has been set and fit assert self.sklearn_model is not None, self._sklearn_model_is_not_fitted_error_message() - # Convert the n_bits attribute into a proper dictionary - self.n_bits = get_n_bits_dict_trees(self.n_bits) - # Enable rounding feature enable_rounding = os.environ.get("TREES_USE_ROUNDING", "1") == "1" @@ -1867,12 +1869,15 @@ def __init__(self, n_bits: int = 3): quantizing inputs and X_fit. Default to 3. """ self.n_bits: int = n_bits + # _q_fit_X: In distance metric algorithms, `_q_fit_X` stores the training set to compute # the similarity or distance measures. There is no `weights` attribute because there isn't # a training phase self._q_fit_X: numpy.ndarray + # _y: Labels of `_q_fit_X` self._y: numpy.ndarray + # _q_fit_X_quantizer: The quantizer to use for quantizing the model's training set self._q_fit_X_quantizer: Optional[UniformQuantizer] = None diff --git a/tests/sklearn/test_sklearn_models.py b/tests/sklearn/test_sklearn_models.py index c47c592f3..560edde27 100644 --- a/tests/sklearn/test_sklearn_models.py +++ b/tests/sklearn/test_sklearn_models.py @@ -1972,3 +1972,37 @@ def test_fhe_sum_for_tree_based_models( n_bits, is_weekly_option, ) + + +@pytest.mark.parametrize( + "n_bits, error_message", + [ + (0, "n_bits must be a non-null, positive integer"), + (-1, "n_bits must be a non-null, positive integer"), + # ( + # {"op_inputs": 4, "op_leaves": 2, "op_weights": 2}, + # "Invalid keys in `n_bits` dictionary. Only 'op_inputs' (mandatory) and 'op_leaves' " + # "(optional) are allowed", + # ), + ( + {"op_inputs": -2, "op_leaves": -5}, + "All values in `n_bits` dictionary must be non-null, positive integers", + ), + ({"op_inputs": 2, "op_leaves": 5}, "`op_leaves` must be less than or equal to `op_inputs`"), + (0.5, "n_bits must be either an integer or a dictionary"), + ], +) +@pytest.mark.parametrize("model_class", _get_sklearn_tree_models()) +def test_invalid_n_bits_setting(model_class, n_bits, error_message): + """Check if the model instantiation raises an exception with invalid 'n_bits' settings.""" + + with pytest.raises(ValueError, match=f"{error_message}. Got `{type(n_bits)}` and `{n_bits}`.*"): + instantiate_model_generic(model_class, n_bits=n_bits) + + +@pytest.mark.parametrize("n_bits", [5, {"op_inputs": 5}, {"op_inputs": 2, "op_leaves": 1}]) +@pytest.mark.parametrize("model_class", _get_sklearn_tree_models()) +def test_valid_n_bits_setting(model_class, n_bits): + """Check valid `n_bits' settings.""" + + instantiate_model_generic(model_class, n_bits=n_bits) From d4ca14081672bf171361f0f89fb87a723c6b94fc Mon Sep 17 00:00:00 2001 From: kcelia Date: Wed, 17 Jan 2024 15:41:36 +0100 Subject: [PATCH 53/73] chore: update comment --- src/concrete/ml/quantization/__init__.py | 2 +- src/concrete/ml/sklearn/base.py | 2 +- src/concrete/ml/sklearn/tree_to_numpy.py | 6 ++++++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/concrete/ml/quantization/__init__.py b/src/concrete/ml/quantization/__init__.py index 20fba6653..f9c94793e 100644 --- a/src/concrete/ml/quantization/__init__.py +++ b/src/concrete/ml/quantization/__init__.py @@ -3,9 +3,9 @@ from .post_training import ( PostTrainingAffineQuantization, PostTrainingQATImporter, - get_n_bits_dict, _get_n_bits_dict_trees, _inspect_tree_n_bits, + get_n_bits_dict, ) from .quantized_module import QuantizedModule from .quantized_ops import ( diff --git a/src/concrete/ml/sklearn/base.py b/src/concrete/ml/sklearn/base.py index 62ff03cbe..b951d2224 100644 --- a/src/concrete/ml/sklearn/base.py +++ b/src/concrete/ml/sklearn/base.py @@ -52,9 +52,9 @@ from ..quantization import ( PostTrainingQATImporter, QuantizedArray, - get_n_bits_dict, _get_n_bits_dict_trees, _inspect_tree_n_bits, + get_n_bits_dict, ) from ..quantization.quantized_module import QuantizedModule, _get_inputset_generator from ..quantization.quantizers import ( diff --git a/src/concrete/ml/sklearn/tree_to_numpy.py b/src/concrete/ml/sklearn/tree_to_numpy.py index 4536f6a9c..e9896f5e7 100644 --- a/src/concrete/ml/sklearn/tree_to_numpy.py +++ b/src/concrete/ml/sklearn/tree_to_numpy.py @@ -146,6 +146,12 @@ def add_transpose_after_last_node(onnx_model: onnx.ModelProto): # Get the output node output_node = onnx_model.graph.output[0] + # The state of the `TREES_USE_FHE_SUM` variable affects the structure of the model's ONNX graph. + # When the option is enabled, the graph is cut after the ReduceSum node. + # On the other hand, when it is disabled, the graph is cut at the ReduceSum node, + # which alters the output shape. + # Therefore, it is necessary to adjust this shape with the correct permutation. + # When using FHE sum for tree ensembles, create the node with perm attribute equal to (1, 0) if os.getenv("TREES_USE_FHE_SUM") == "1": perm = [1, 0] From f96333bc1dfbbb11e37da9275bdb2eb0ef576816 Mon Sep 17 00:00:00 2001 From: kcelia Date: Wed, 17 Jan 2024 16:49:22 +0100 Subject: [PATCH 54/73] chore: update simulated p_error test --- tests/sklearn/test_sklearn_models.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/sklearn/test_sklearn_models.py b/tests/sklearn/test_sklearn_models.py index 560edde27..1b6b2a58b 100644 --- a/tests/sklearn/test_sklearn_models.py +++ b/tests/sklearn/test_sklearn_models.py @@ -144,10 +144,16 @@ def preamble(model_class, parameters, n_bits, load_data, is_weekly_option): def get_n_bits_non_correctness(model_class): """Get the number of bits to use for non correctness related tests.""" + # KNN can only be compiled with small quantization bit numbers for now + # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3979 if get_model_name(model_class) == "KNeighborsClassifier": - # KNN can only be compiled with small quantization bit numbers for now - # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3979 n_bits = 2 + + # Adjust the quantization precision for tree-based model based on `TREES_USE_FHE_SUM` setting. + # When enabled, the circuit's bitwidth increases, potentially leading to Out-of-Memory issues. + # Therefore, the maximum quantization precision is 4 bits in this case. + elif model_class in _get_sklearn_tree_models() and os.environ.get("TREES_USE_FHE_SUM") == "1": + n_bits = min(min(N_BITS_REGULAR_BUILDS), 4) else: n_bits = min(N_BITS_REGULAR_BUILDS) @@ -1730,10 +1736,7 @@ def test_p_error_simulation( with simulation or in FHE compared to the expected clear quantized ones. """ - if os.getenv("TREES_USE_FHE_SUM") == "1": - n_bits = 4 - else: - n_bits = get_n_bits_non_correctness(model_class) + n_bits = get_n_bits_non_correctness(model_class) # Get data-set, initialize and fit the model model, x = preamble(model_class, parameters, n_bits, load_data, is_weekly_option) From cc4781f85b8505eb4469a432c62e16f88adebaac Mon Sep 17 00:00:00 2001 From: kcelia Date: Wed, 17 Jan 2024 17:59:12 +0100 Subject: [PATCH 55/73] chore: update coverage --- src/concrete/ml/quantization/post_training.py | 6 +++--- tests/sklearn/test_sklearn_models.py | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/concrete/ml/quantization/post_training.py b/src/concrete/ml/quantization/post_training.py index 89aad3aa0..34bfe8dee 100644 --- a/src/concrete/ml/quantization/post_training.py +++ b/src/concrete/ml/quantization/post_training.py @@ -67,8 +67,8 @@ def _inspect_tree_n_bits(n_bits): elif isinstance(n_bits, dict): if "op_inputs" not in n_bits.keys() or set(n_bits.keys()) - {"op_leaves", "op_inputs"}: error_message = ( - "Invalid keys in `n_bits` dictionary. Only 'op_inputs' (mandatory) and " - "'op_leaves' (optional) are allowed" + "Invalid keys in `n_bits` dictionary. Only 'op_inputs' (mandatory) and 'op_leaves' " + "(optional) are allowed" ) elif not all(isinstance(value, int) and value > 0 for value in n_bits.values()): error_message = "All values in `n_bits` dictionary must be non-null, positive integers" @@ -110,7 +110,7 @@ def _get_n_bits_dict_trees(n_bits: Union[int, Dict[str, int]]) -> Dict[str, int] # Default `op_leaves` to `op_inputs` if not specified if "op_leaves" not in n_bits: - n_bits["op_leaves"] = n_bits["op_inputs"] + n_bits["op_leaves"] = n_bits["op_inputs"] # pragma: no cover return n_bits diff --git a/tests/sklearn/test_sklearn_models.py b/tests/sklearn/test_sklearn_models.py index 1b6b2a58b..94f098d72 100644 --- a/tests/sklearn/test_sklearn_models.py +++ b/tests/sklearn/test_sklearn_models.py @@ -1982,11 +1982,11 @@ def test_fhe_sum_for_tree_based_models( [ (0, "n_bits must be a non-null, positive integer"), (-1, "n_bits must be a non-null, positive integer"), - # ( - # {"op_inputs": 4, "op_leaves": 2, "op_weights": 2}, - # "Invalid keys in `n_bits` dictionary. Only 'op_inputs' (mandatory) and 'op_leaves' " - # "(optional) are allowed", - # ), + ( + {"op_inputs": 4, "op_leaves": 2, "op_weights": 2}, + "Invalid keys in `n_bits` dictionary. Only 'op_inputs' \\(mandatory\\) and 'op_leaves' " + "\\(optional\\) are allowed", + ), ( {"op_inputs": -2, "op_leaves": -5}, "All values in `n_bits` dictionary must be non-null, positive integers", From a65200169b018ee9adcf15051c5a593c614eb3d2 Mon Sep 17 00:00:00 2001 From: kcelia Date: Thu, 18 Jan 2024 12:32:25 +0100 Subject: [PATCH 56/73] chore: update tests --- src/concrete/ml/quantization/post_training.py | 34 ++++++++++--------- tests/sklearn/test_sklearn_models.py | 32 +++++++++++++---- 2 files changed, 43 insertions(+), 23 deletions(-) diff --git a/src/concrete/ml/quantization/post_training.py b/src/concrete/ml/quantization/post_training.py index 34bfe8dee..621697f14 100644 --- a/src/concrete/ml/quantization/post_training.py +++ b/src/concrete/ml/quantization/post_training.py @@ -31,7 +31,7 @@ def _inspect_tree_n_bits(n_bits): This function checks whether 'n_bits' is a valid integer or dictionary. - If 'n_bits' is an integer, it must be a non-null positive, its value is assigned to - "op_inputs" and "op_leaves" bits + 'op_inputs' and 'op_leaves' bits - If it is a dictionary, it should contain integer values for keys 'op_leaves' and 'op_inputs', where 'op_leaves' should not exceed 'op_inputs'. @@ -43,20 +43,20 @@ def _inspect_tree_n_bits(n_bits): a dictionary with the following keys : - "op_inputs" (mandatory): number of bits to quantize the input values - "op_leaves" (optional): number of bits to quantize the leaves, must be less than or - equal to `op_inputs`. defaults to the value of "op_inputs" if not specified. + equal to 'op_inputs. defaults to the value of 'op_inputs if not specified. Raises: ValueError: If 'n_bits' does not conform to the required format or value constraints. """ detailed_message = ( - "Invalid `n_bits`, either pass a non-null positive integer or a dictionary containing " + "Invalid 'n_bits', either pass a non-null positive integer or a dictionary containing " "integer values for the following keys:\n" - "- `op_inputs` (mandatory): number of bits to quantize the input values\n" - "- `op_leaves` (optional): number of bits to quantize the leaves, must be less than or " - "equal to `op_inputs`. Defaults to the value of `op_inputs` if not specified." - "When using a single integer for n_bits, its value is assigned to `op_inputs` and " - "`op_leaves` bits.\n" + "- 'op_inputs' (mandatory): number of bits to quantize the input values\n" + "- 'op_leaves' (optional): number of bits to quantize the leaves, must be less than or " + "equal to 'op_inputs'. Defaults to the value of 'op_inputs' if not specified." + "When using a single integer for n_bits, its value is assigned to 'op_inputs' and " + "'op_leaves' bits.\n" ) error_message = "" @@ -65,22 +65,24 @@ def _inspect_tree_n_bits(n_bits): if n_bits <= 0: error_message = "n_bits must be a non-null, positive integer" elif isinstance(n_bits, dict): - if "op_inputs" not in n_bits.keys() or set(n_bits.keys()) - {"op_leaves", "op_inputs"}: + if "op_inputs" not in n_bits.keys(): + error_message = "Invalid keys in `n_bits` dictionary. The key 'op_inputs' is mandatory" + elif set(n_bits.keys()) - {"op_leaves", "op_inputs"}: error_message = ( - "Invalid keys in `n_bits` dictionary. Only 'op_inputs' (mandatory) and 'op_leaves' " + "Invalid keys in 'n_bits' dictionary. Only 'op_inputs' (mandatory) and 'op_leaves' " "(optional) are allowed" ) elif not all(isinstance(value, int) and value > 0 for value in n_bits.values()): - error_message = "All values in `n_bits` dictionary must be non-null, positive integers" + error_message = "All values in 'n_bits' dictionary must be non-null, positive integers" elif n_bits.get("op_leaves", 0) > n_bits.get("op_inputs", 0): - error_message = "`op_leaves` must be less than or equal to `op_inputs`" + error_message = "'op_leaves' must be less than or equal to 'op_inputs'" else: error_message = "n_bits must be either an integer or a dictionary" if len(error_message) > 0: raise ValueError( - f"{error_message}. Got `{type(n_bits)}` and `{n_bits}` value.\n{detailed_message}" + f"{error_message}. Got '{type(n_bits)}' and '{n_bits}' value.\n{detailed_message}" ) @@ -92,7 +94,7 @@ def _get_n_bits_dict_trees(n_bits: Union[int, Dict[str, int]]) -> Dict[str, int] a dictionary with the following keys : - "op_inputs" (mandatory): number of bits to quantize the input values - "op_leaves" (optional): number of bits to quantize the leaves, must be less than or - equal to `op_inputs`. defaults to the value of "op_inputs" if not specified. + equal to 'op_inputs'. defaults to the value of "op_inputs" if not specified. When using a single integer for n_bits, its value is assigned to "op_inputs" and "op_leaves" bits. @@ -108,9 +110,9 @@ def _get_n_bits_dict_trees(n_bits: Union[int, Dict[str, int]]) -> Dict[str, int] if isinstance(n_bits, int): return {"op_inputs": n_bits, "op_leaves": n_bits} - # Default `op_leaves` to `op_inputs` if not specified + # Default 'op_leaves' to 'op_inputs' if not specified if "op_leaves" not in n_bits: - n_bits["op_leaves"] = n_bits["op_inputs"] # pragma: no cover + n_bits["op_leaves"] = n_bits["op_inputs"] return n_bits diff --git a/tests/sklearn/test_sklearn_models.py b/tests/sklearn/test_sklearn_models.py index 94f098d72..1d6dbe406 100644 --- a/tests/sklearn/test_sklearn_models.py +++ b/tests/sklearn/test_sklearn_models.py @@ -1982,16 +1982,17 @@ def test_fhe_sum_for_tree_based_models( [ (0, "n_bits must be a non-null, positive integer"), (-1, "n_bits must be a non-null, positive integer"), + ({"op_leaves": 2}, "The key 'op_inputs' is mandatory"), ( {"op_inputs": 4, "op_leaves": 2, "op_weights": 2}, - "Invalid keys in `n_bits` dictionary. Only 'op_inputs' \\(mandatory\\) and 'op_leaves' " + "Invalid keys in 'n_bits' dictionary. Only 'op_inputs' \\(mandatory\\) and 'op_leaves' " "\\(optional\\) are allowed", ), ( {"op_inputs": -2, "op_leaves": -5}, - "All values in `n_bits` dictionary must be non-null, positive integers", + "All values in 'n_bits' dictionary must be non-null, positive integers", ), - ({"op_inputs": 2, "op_leaves": 5}, "`op_leaves` must be less than or equal to `op_inputs`"), + ({"op_inputs": 2, "op_leaves": 5}, "'op_leaves' must be less than or equal to 'op_inputs'"), (0.5, "n_bits must be either an integer or a dictionary"), ], ) @@ -1999,13 +2000,30 @@ def test_fhe_sum_for_tree_based_models( def test_invalid_n_bits_setting(model_class, n_bits, error_message): """Check if the model instantiation raises an exception with invalid 'n_bits' settings.""" - with pytest.raises(ValueError, match=f"{error_message}. Got `{type(n_bits)}` and `{n_bits}`.*"): + with pytest.raises(ValueError, match=f"{error_message}. Got '{type(n_bits)}' and '{n_bits}'.*"): instantiate_model_generic(model_class, n_bits=n_bits) @pytest.mark.parametrize("n_bits", [5, {"op_inputs": 5}, {"op_inputs": 2, "op_leaves": 1}]) -@pytest.mark.parametrize("model_class", _get_sklearn_tree_models()) -def test_valid_n_bits_setting(model_class, n_bits): +@pytest.mark.parametrize("model_class, parameters", get_sklearn_tree_models_and_datasets()) +def test_valid_n_bits_setting( + model_class, + n_bits, + parameters, + load_data, + is_weekly_option, + verbose=True, +): """Check valid `n_bits' settings.""" - instantiate_model_generic(model_class, n_bits=n_bits) + if verbose: + print("Run test_valid_n_bits_setting") + + x, y = get_dataset(model_class, parameters, n_bits, load_data, is_weekly_option) + + model = instantiate_model_generic(model_class, n_bits=n_bits) + + with warnings.catch_warnings(): + # Sometimes, we miss convergence, which is not a problem for our test + warnings.simplefilter("ignore", category=ConvergenceWarning) + model.fit(x, y) From 9839ce9623c4cca4776e72ca9d2e2477970bec43 Mon Sep 17 00:00:00 2001 From: kcelia Date: Thu, 18 Jan 2024 17:08:06 +0100 Subject: [PATCH 57/73] chore: update assert --- tests/sklearn/test_sklearn_models.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/tests/sklearn/test_sklearn_models.py b/tests/sklearn/test_sklearn_models.py index 1d6dbe406..1ec4d7c98 100644 --- a/tests/sklearn/test_sklearn_models.py +++ b/tests/sklearn/test_sklearn_models.py @@ -1159,8 +1159,7 @@ def check_rounding_consistency( fhe_test = get_random_samples(x, n_sample=5) # Check that rounding is enabled - rounding_enabled = os.getenv("TREES_USE_ROUNDING") == "1" - assert rounding_enabled + assert os.environ.get("TREES_USE_ROUNDING") == "1", "'TREES_USE_ROUNDING' is not enabled" # Fit and compile with rounding enabled fit_and_compile(model, x, y) @@ -1178,8 +1177,7 @@ def check_rounding_consistency( mp_context.setenv("TREES_USE_ROUNDING", "0") # Check that rounding is disabled - rounding_disabled = os.environ.get("TREES_USE_ROUNDING") == "0" - assert rounding_disabled + assert os.environ.get("TREES_USE_ROUNDING") == "0", "'TREES_USE_ROUNDING' is not disabled" with pytest.warns( DeprecationWarning, @@ -1221,8 +1219,7 @@ def check_fhe_sum_for_tree_based_models( fhe_test = get_random_samples(x, n_sample=5) # By default, the summation of tree ensemble outputs is done in clear - fhe_sum_disabled = os.getenv("TREES_USE_FHE_SUM") == "0" - assert fhe_sum_disabled + assert os.getenv("TREES_USE_FHE_SUM") == "0", "'TREES_USE_FHE_SUM' is not disabled" model_ref = instantiate_model_generic(model_class, n_bits=n_bits) fit_and_compile(model_ref, x, y) @@ -1250,8 +1247,7 @@ def check_fhe_sum_for_tree_based_models( mp_context.setenv("TREES_USE_FHE_SUM", "1") # Check that the summation of tree ensemble outputs is enabled - fhe_sum_enabled = os.environ.get("TREES_USE_FHE_SUM") == "1" - assert fhe_sum_enabled + assert os.getenv("TREES_USE_FHE_SUM") == "1", "'TREES_USE_FHE_SUM' is not enabled" model = model_class(**model_ref.get_params()) fit_and_compile(model, x, y) From 7cb13e08d064c3daef91ba1ae317477c87b50816 Mon Sep 17 00:00:00 2001 From: kcelia Date: Mon, 22 Jan 2024 12:15:56 +0100 Subject: [PATCH 58/73] chore: update comment --- src/concrete/ml/sklearn/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/concrete/ml/sklearn/base.py b/src/concrete/ml/sklearn/base.py index b951d2224..d732d664f 100644 --- a/src/concrete/ml/sklearn/base.py +++ b/src/concrete/ml/sklearn/base.py @@ -1303,9 +1303,10 @@ def __init__(self, n_bits: Union[int, Dict[str, int]]): Default to 6. """ - # Check if 'n_bits' is a valid value + # Check if 'n_bits' is a valid value. _inspect_tree_n_bits(n_bits) + #: The number of bits to quantize the model. self.n_bits: Union[int, Dict[str, int]] = n_bits #: The model's inference function. Is None if the model is not fitted. From 7d935754c3e397cd7a82d13bbdeb7fe7b42c2c2d Mon Sep 17 00:00:00 2001 From: kcelia Date: Mon, 22 Jan 2024 12:44:47 +0100 Subject: [PATCH 59/73] chore: update comment --- tests/sklearn/test_dump_onnx.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/sklearn/test_dump_onnx.py b/tests/sklearn/test_dump_onnx.py index 484e2c0d7..554b24d28 100644 --- a/tests/sklearn/test_dump_onnx.py +++ b/tests/sklearn/test_dump_onnx.py @@ -67,8 +67,9 @@ def check_onnx_file_dump(model_class, parameters, load_data, str_expected, defau del onnx_model.graph.initializer[0] str_model = onnx.helper.printable_graph(onnx_model.graph) - print(f"{model_name}:") - print(str_model) + print(f"\nCurrent {model_name=}:\n{str_model}") + print(f"\nExpected {model_name=}:\n{str_expected}") + # Test equality when it does not depend on seeds # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3266 if not is_model_class_in_a_list(model_class, _get_sklearn_tree_models(select="RandomForest")): From 70adfd5e11834ab8fff608493dd9345066a440a6 Mon Sep 17 00:00:00 2001 From: kcelia Date: Mon, 22 Jan 2024 13:14:17 +0100 Subject: [PATCH 60/73] chore: test dump in both cases (sum_fhe enabled and disabled) --- tests/sklearn/test_dump_onnx.py | 211 +++++++++++++++++--------------- 1 file changed, 112 insertions(+), 99 deletions(-) diff --git a/tests/sklearn/test_dump_onnx.py b/tests/sklearn/test_dump_onnx.py index 554b24d28..58925f6f0 100644 --- a/tests/sklearn/test_dump_onnx.py +++ b/tests/sklearn/test_dump_onnx.py @@ -20,107 +20,10 @@ # pylint: disable=line-too-long -def check_onnx_file_dump(model_class, parameters, load_data, str_expected, default_configuration): +def check_onnx_file_dump(model_class, parameters, load_data, default_configuration): """Fit the model and dump the corresponding ONNX.""" - # Get the data-set. The data generation is seeded in load_data. - x, y = load_data(model_class, **parameters) - - # Set the model - model = model_class() - - model_params = model.get_params() - if "random_state" in model_params: - model_params["random_state"] = numpy.random.randint(0, 2**15) - - model.set_params(**model_params) - - if get_model_name(model) == "KNeighborsClassifier": - # KNN can only be compiled with small quantization bit numbers for now - # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3979 - model.n_bits = 2 - - with warnings.catch_warnings(): - # Sometimes, we miss convergence, which is not a problem for our test - warnings.simplefilter("ignore", category=ConvergenceWarning) - - model.fit(x, y) - - with warnings.catch_warnings(): - # Use FHE simulation to not have issues with precision - model.compile(x, default_configuration) - - # Get ONNX model - onnx_model = model.onnx_model - - # Remove initializers, since they change from one seed to the other - model_name = get_model_name(model_class) - if model_name in [ - "DecisionTreeRegressor", - "DecisionTreeClassifier", - "RandomForestClassifier", - "RandomForestRegressor", - "XGBClassifier", - "KNeighborsClassifier", - ]: - while len(onnx_model.graph.initializer) > 0: - del onnx_model.graph.initializer[0] - - str_model = onnx.helper.printable_graph(onnx_model.graph) - print(f"\nCurrent {model_name=}:\n{str_model}") - print(f"\nExpected {model_name=}:\n{str_expected}") - - # Test equality when it does not depend on seeds - # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3266 - if not is_model_class_in_a_list(model_class, _get_sklearn_tree_models(select="RandomForest")): - # The expected graph is usually a string and we therefore directly test if it is equal to - # the retrieved graph's string. However, in some cases such as for TweedieRegressor models, - # this graph can slightly changed depending on some input's values. We then expected the - # string to match as least one of them expected strings (as a list) - if isinstance(str_expected, str): - assert str_model == str_expected - else: - assert str_model in str_expected - - -@pytest.mark.parametrize("model_class, parameters", UNIQUE_MODELS_AND_DATASETS) -def test_dump( - model_class, - parameters, - load_data, - default_configuration, -): - """Tests dump.""" - model_name = get_model_name(model_class) - - # Some models have been done with different n_classes which create different ONNX - if parameters.get("n_classes", 2) != 2 and model_name in ["LinearSVC", "LogisticRegression"]: - return - - if model_name == "NeuralNetClassifier": - model_class = partial( - NeuralNetClassifier, - module__n_layers=3, - module__power_of_two_scaling=False, - max_epochs=1, - verbose=0, - callbacks="disable", - ) - elif model_name == "NeuralNetRegressor": - model_class = partial( - NeuralNetRegressor, - module__n_layers=3, - module__n_w_bits=2, - module__n_a_bits=2, - module__n_accum_bits=7, # Stay with 7 bits for test exec time - module__n_hidden_neurons_multiplier=1, - module__power_of_two_scaling=False, - max_epochs=1, - verbose=0, - callbacks="disable", - ) - n_classes = parameters.get("n_classes", 2) # Ignore long lines here @@ -492,4 +395,114 @@ def test_dump( } str_expected = expected_strings.get(model_name, "") - check_onnx_file_dump(model_class, parameters, load_data, str_expected, default_configuration) + + # Get the data-set. The data generation is seeded in load_data. + x, y = load_data(model_class, **parameters) + + # Set the model + model = model_class() + + model_params = model.get_params() + if "random_state" in model_params: + model_params["random_state"] = numpy.random.randint(0, 2**15) + + model.set_params(**model_params) + + if model_name == "KNeighborsClassifier": + # KNN can only be compiled with small quantization bit numbers for now + # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3979 + model.n_bits = 2 + + with warnings.catch_warnings(): + # Sometimes, we miss convergence, which is not a problem for our test + warnings.simplefilter("ignore", category=ConvergenceWarning) + + model.fit(x, y) + + with warnings.catch_warnings(): + # Use FHE simulation to not have issues with precision + model.compile(x, default_configuration) + + # Get ONNX model + onnx_model = model.onnx_model + + # Remove initializers, since they change from one seed to the other + model_name = get_model_name(model_class) + if model_name in [ + "DecisionTreeRegressor", + "DecisionTreeClassifier", + "RandomForestClassifier", + "RandomForestRegressor", + "XGBClassifier", + "KNeighborsClassifier", + ]: + while len(onnx_model.graph.initializer) > 0: + del onnx_model.graph.initializer[0] + + str_model = onnx.helper.printable_graph(onnx_model.graph) + print(f"\nCurrent {model_name=}:\n{str_model}") + print(f"\nExpected {model_name=}:\n{str_expected}") + + # Test equality when it does not depend on seeds + # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3266 + if not is_model_class_in_a_list(model_class, _get_sklearn_tree_models(select="RandomForest")): + # The expected graph is usually a string and we therefore directly test if it is equal to + # the retrieved graph's string. However, in some cases such as for TweedieRegressor models, + # this graph can slightly changed depending on some input's values. We then expected the + # string to match as least one of them expected strings (as a list) + if isinstance(str_expected, str): + assert str_model == str_expected + else: + assert str_model in str_expected + + +@pytest.mark.parametrize("model_class, parameters", UNIQUE_MODELS_AND_DATASETS) +def test_dump( + model_class, + parameters, + load_data, + default_configuration, +): + """Tests dump.""" + + model_name = get_model_name(model_class) + + # Some models have been done with different n_classes which create different ONNX + if parameters.get("n_classes", 2) != 2 and model_name in ["LinearSVC", "LogisticRegression"]: + return + + if model_name == "NeuralNetClassifier": + model_class = partial( + NeuralNetClassifier, + module__n_layers=3, + module__power_of_two_scaling=False, + max_epochs=1, + verbose=0, + callbacks="disable", + ) + elif model_name == "NeuralNetRegressor": + model_class = partial( + NeuralNetRegressor, + module__n_layers=3, + module__n_w_bits=2, + module__n_a_bits=2, + module__n_accum_bits=7, # Stay with 7 bits for test exec time + module__n_hidden_neurons_multiplier=1, + module__power_of_two_scaling=False, + max_epochs=1, + verbose=0, + callbacks="disable", + ) + + # Check with 'TREES_USE_ROUNDING' disabled + assert os.environ.get("TREES_USE_FHE_SUM") == "0", "'TREES_USE_FHE_SUM' is not disabled" + check_onnx_file_dump(model_class, parameters, load_data, default_configuration) + + with pytest.MonkeyPatch.context() as mp_context: + + # Disable rounding + mp_context.setenv("TREES_USE_FHE_SUM", "1") + + # Check that rounding is disabled + assert os.environ.get("TREES_USE_FHE_SUM") == "1", "'TREES_USE_FHE_SUM' is enabled" + check_onnx_file_dump(model_class, parameters, load_data, default_configuration) From 783e7af01b7aca338d73fb805b677e41ea6ba4ca Mon Sep 17 00:00:00 2001 From: kcelia Date: Tue, 23 Jan 2024 00:43:32 +0100 Subject: [PATCH 61/73] chore: remove env var --- src/concrete/ml/sklearn/base.py | 40 +++++++++++++++++- src/concrete/ml/sklearn/tree_to_numpy.py | 32 ++++++++------ tests/sklearn/test_dump_onnx.py | 44 +++++++++---------- tests/sklearn/test_sklearn_models.py | 54 ++++++++---------------- 4 files changed, 95 insertions(+), 75 deletions(-) diff --git a/src/concrete/ml/sklearn/base.py b/src/concrete/ml/sklearn/base.py index d732d664f..dbd8ca4a8 100644 --- a/src/concrete/ml/sklearn/base.py +++ b/src/concrete/ml/sklearn/base.py @@ -105,7 +105,7 @@ os.environ["TREES_USE_ROUNDING"] = os.environ.get("TREES_USE_ROUNDING", "1") # By default, the decision of the tree ensembles is made in clear -os.environ["TREES_USE_FHE_SUM"] = os.environ.get("TREES_USE_FHE_SUM", "0") +TREES_USE_FHE_SUM = False # pylint: disable=too-many-public-methods @@ -1312,8 +1312,43 @@ def __init__(self, n_bits: Union[int, Dict[str, int]]): #: The model's inference function. Is None if the model is not fitted. self._tree_inference: Optional[Callable] = None + #: Wether to perform the sum of the output's tree ensembles in FHE or not. + self._use_fhe_sum = False + BaseEstimator.__init__(self) + @property + def use_fhe_sum(self) -> bool: + """Property getter for `use_fhe_sum`. + + Returns: + bool: The current setting of the `_use_fhe_sum` attribute. + """ + return self._use_fhe_sum + + @use_fhe_sum.setter + def use_fhe_sum(self, value) -> None: + """Property setter for `use_fhe_sum`. + + Args: + value (int): Whether to enable or disable the feature. + """ + + assert isinstance(value, bool), "Value must be a boolean type" + + if value is True: + warnings.simplefilter("always") + warnings.warn( + "Enabling `use_fhe_sum` computes the sum of the ouputs of tree ensembles in FHE.\n" + "This may slow down the computation and increase the maximum bitwidth.\n" + "To optimize performance, consider reducing the quantization leaf precision.\n" + "Additionally, the model must be refitted for these changes to take effect.", + category=UserWarning, + stacklevel=2, + ) + + self._use_fhe_sum = value + def fit(self, X: Data, y: Target, **fit_parameters): # Reset for double fit self._is_fitted = False @@ -1362,6 +1397,7 @@ def fit(self, X: Data, y: Target, **fit_parameters): self.sklearn_model, q_X, use_rounding=enable_rounding, + use_fhe_sum=self._use_fhe_sum, framework=self.framework, output_n_bits=self.n_bits["op_leaves"], ) @@ -1438,7 +1474,7 @@ def post_processing(self, y_preds: numpy.ndarray) -> numpy.ndarray: # Sum all tree outputs # Remove the sum once we handle multi-precision circuits # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/451 - if os.getenv("TREES_USE_FHE_SUM") == "0": + if not self._use_fhe_sum: y_preds = numpy.sum(y_preds, axis=-1) assert_true(y_preds.ndim == 2, "y_preds should be a 2D array") diff --git a/src/concrete/ml/sklearn/tree_to_numpy.py b/src/concrete/ml/sklearn/tree_to_numpy.py index e9896f5e7..c61ea9d1c 100644 --- a/src/concrete/ml/sklearn/tree_to_numpy.py +++ b/src/concrete/ml/sklearn/tree_to_numpy.py @@ -1,6 +1,5 @@ """Implements the conversion of a tree model to a numpy function.""" import math -import os import warnings from typing import Callable, List, Optional, Tuple @@ -137,28 +136,25 @@ def assert_add_node_and_constant_in_xgboost_regressor_graph(onnx_model: onnx.Mod ) -def add_transpose_after_last_node(onnx_model: onnx.ModelProto): +def add_transpose_after_last_node(onnx_model: onnx.ModelProto, use_fhe_sum: bool): """Add transpose after last node. Args: onnx_model (onnx.ModelProto): The ONNX model. + use_fhe_sum (bool): This parameter is exclusively used to tree-based models. + It determines whether the sum of the trees' outputs is computed in FHE. """ # Get the output node output_node = onnx_model.graph.output[0] - # The state of the `TREES_USE_FHE_SUM` variable affects the structure of the model's ONNX graph. + # The state of the 'use_fhe_sum' variable affects the structure of the model's ONNX graph. # When the option is enabled, the graph is cut after the ReduceSum node. - # On the other hand, when it is disabled, the graph is cut at the ReduceSum node, - # which alters the output shape. + # When it is disabled, the graph is cut at the ReduceSum node, which alters the output shape. # Therefore, it is necessary to adjust this shape with the correct permutation. # When using FHE sum for tree ensembles, create the node with perm attribute equal to (1, 0) - if os.getenv("TREES_USE_FHE_SUM") == "1": - perm = [1, 0] - # Otherwise, create the node with perm attribute equal to (2, 1, 0) - else: - perm = [2, 1, 0] + perm = [1, 0] if use_fhe_sum else [2, 1, 0] transpose_node = onnx.helper.make_node( "Transpose", @@ -222,7 +218,10 @@ def preprocess_tree_predictions( def tree_onnx_graph_preprocessing( - onnx_model: onnx.ModelProto, framework: str, expected_number_of_outputs: int + onnx_model: onnx.ModelProto, + framework: str, + expected_number_of_outputs: int, + use_fhe_sum: bool = False, ): """Apply pre-processing onto the ONNX graph. @@ -231,6 +230,8 @@ def tree_onnx_graph_preprocessing( framework (str): The framework from which the ONNX model is generated. (options: 'xgboost', 'sklearn') expected_number_of_outputs (int): The expected number of outputs in the ONNX model. + use_fhe_sum (bool): This parameter is exclusively used to tree-based models. + It determines whether the sum of the trees' outputs is computed in FHE. """ # Make sure the ONNX version returned by Hummingbird is OPSET_VERSION_FOR_ONNX_EXPORT onnx_version = get_onnx_opset_version(onnx_model) @@ -257,7 +258,7 @@ def tree_onnx_graph_preprocessing( # Cut the graph after the ReduceSum node to remove # argmax, sigmoid, softmax from the graph. - if os.getenv("TREES_USE_FHE_SUM") == "1": + if use_fhe_sum: clean_graph_after_node_op_type(onnx_model, "ReduceSum") else: clean_graph_at_node_op_type(onnx_model, "ReduceSum") @@ -273,7 +274,7 @@ def tree_onnx_graph_preprocessing( # sklearn models apply the reduce sum before the transpose. # To have equivalent output between xgboost in sklearn, # apply the transpose before returning the output. - add_transpose_after_last_node(onnx_model) + add_transpose_after_last_node(onnx_model, use_fhe_sum) # Cast nodes are not necessary so remove them. remove_node_types(onnx_model, op_types_to_remove=["Cast"]) @@ -330,6 +331,7 @@ def tree_to_numpy( x: numpy.ndarray, framework: str, use_rounding: bool = True, + use_fhe_sum: bool = False, output_n_bits: int = MAX_BITWIDTH_BACKWARD_COMPATIBLE, ) -> Tuple[Callable, List[UniformQuantizer], onnx.ModelProto]: """Convert the tree inference to a numpy functions using Hummingbird. @@ -339,6 +341,8 @@ def tree_to_numpy( x (numpy.ndarray): The input data. use_rounding (bool): This parameter is exclusively used to tree-based models. It determines whether the rounding feature is enabled or disabled. + use_fhe_sum (bool): This parameter is exclusively used to tree-based models. + It determines whether the sum of the trees' outputs is computed in FHE. framework (str): The framework from which the ONNX model is generated. (options: 'xgboost', 'sklearn') output_n_bits (int): The number of bits of the output. Default to 8. @@ -375,7 +379,7 @@ def tree_to_numpy( # ONNX graph pre-processing to make the model FHE friendly # i.e., delete irrelevant nodes and cut the graph before the final ensemble sum) - tree_onnx_graph_preprocessing(onnx_model, framework, expected_number_of_outputs) + tree_onnx_graph_preprocessing(onnx_model, framework, expected_number_of_outputs, use_fhe_sum) # Tree values pre-processing # i.e., mainly predictions quantization diff --git a/tests/sklearn/test_dump_onnx.py b/tests/sklearn/test_dump_onnx.py index 58925f6f0..fa398cd7e 100644 --- a/tests/sklearn/test_dump_onnx.py +++ b/tests/sklearn/test_dump_onnx.py @@ -1,6 +1,5 @@ """Tests for the sklearn decision trees.""" -import os import warnings from functools import partial @@ -20,12 +19,20 @@ # pylint: disable=line-too-long -def check_onnx_file_dump(model_class, parameters, load_data, default_configuration): +def check_onnx_file_dump(model_class, parameters, load_data, default_configuration, use_fhe_sum): """Fit the model and dump the corresponding ONNX.""" model_name = get_model_name(model_class) n_classes = parameters.get("n_classes", 2) + # Set the model + model = model_class() + + # Set `use_fhe_sum` + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=UserWarning) + model.use_fhe_sum = use_fhe_sum + # Ignore long lines here # ruff: noqa: E501 expected_strings = { @@ -130,7 +137,7 @@ def check_onnx_file_dump(model_class, parameters, load_data, default_configurati """%/_operators.0/ReduceSum_output_0 = ReduceSum[keepdims = 0](%/_operators.0/Reshape_3_output_0, %onnx::ReduceSum_22) %transposed_output = Transpose[perm = [1, 0]](%/_operators.0/ReduceSum_output_0) """ - if os.getenv("TREES_USE_FHE_SUM") == "1" + if use_fhe_sum else "%transposed_output = Transpose[perm = [2, 1, 0]](%/_operators.0/Reshape_3_output_0)\n " ) + """return %transposed_output @@ -209,7 +216,7 @@ def check_onnx_file_dump(model_class, parameters, load_data, default_configurati """%/_operators.0/ReduceSum_output_0 = ReduceSum[keepdims = 0](%/_operators.0/Reshape_3_output_0, %onnx::ReduceSum_22) %transposed_output = Transpose[perm = [1, 0]](%/_operators.0/ReduceSum_output_0) """ - if os.getenv("TREES_USE_FHE_SUM") == "1" + if use_fhe_sum is True else "%transposed_output = Transpose[perm = [2, 1, 0]](%/_operators.0/Reshape_3_output_0)\n " ) + """return %transposed_output @@ -261,7 +268,7 @@ def check_onnx_file_dump(model_class, parameters, load_data, default_configurati """%/_operators.0/ReduceSum_output_0 = ReduceSum[keepdims = 0](%/_operators.0/Reshape_4_output_0, %onnx::ReduceSum_26) return %/_operators.0/ReduceSum_output_0 }""" - if os.getenv("TREES_USE_FHE_SUM") == "1" + if use_fhe_sum is True else "return %/_operators.0/Reshape_4_output_0\n}" ), "RandomForestRegressor": """graph torch_jit ( @@ -285,7 +292,7 @@ def check_onnx_file_dump(model_class, parameters, load_data, default_configurati """%/_operators.0/ReduceSum_output_0 = ReduceSum[keepdims = 0](%/_operators.0/Reshape_3_output_0, %onnx::ReduceSum_22) %transposed_output = Transpose[perm = [1, 0]](%/_operators.0/ReduceSum_output_0) """ - if os.getenv("TREES_USE_FHE_SUM") == "1" + if use_fhe_sum is True else "%transposed_output = Transpose[perm = [2, 1, 0]](%/_operators.0/Reshape_3_output_0)" ) + """return %transposed_output @@ -304,7 +311,7 @@ def check_onnx_file_dump(model_class, parameters, load_data, default_configurati %/_operators.0/Constant_2_output_0[INT64, 3] %/_operators.0/Constant_3_output_0[INT64, 3] %/_operators.0/Constant_4_output_0[INT64, 3]""" - + ("\n %onnx::ReduceSum_27[INT64, 1]" if os.getenv("TREES_USE_FHE_SUM") == "1" else "") + + ("\n %onnx::ReduceSum_27[INT64, 1]" if use_fhe_sum is True else "") + """ ) { %/_operators.0/Gemm_output_0 = Gemm[alpha = 1, beta = 0, transB = 1](%_operators.0.weight_1, %input_0) @@ -324,7 +331,7 @@ def check_onnx_file_dump(model_class, parameters, load_data, default_configurati """%/_operators.0/ReduceSum_output_0 = ReduceSum[keepdims = 0](%/_operators.0/Reshape_4_output_0, %onnx::ReduceSum_27) return %/_operators.0/ReduceSum_output_0 }""" - if os.getenv("TREES_USE_FHE_SUM") == "1" + if use_fhe_sum is True else """return %/_operators.0/Reshape_4_output_0\n}""" ), "LinearRegression": """graph torch_jit ( @@ -399,9 +406,6 @@ def check_onnx_file_dump(model_class, parameters, load_data, default_configurati # Get the data-set. The data generation is seeded in load_data. x, y = load_data(model_class, **parameters) - # Set the model - model = model_class() - model_params = model.get_params() if "random_state" in model_params: model_params["random_state"] = numpy.random.randint(0, 2**15) @@ -494,15 +498,9 @@ def test_dump( callbacks="disable", ) - # Check with 'TREES_USE_ROUNDING' disabled - assert os.environ.get("TREES_USE_FHE_SUM") == "0", "'TREES_USE_FHE_SUM' is not disabled" - check_onnx_file_dump(model_class, parameters, load_data, default_configuration) - - with pytest.MonkeyPatch.context() as mp_context: - - # Disable rounding - mp_context.setenv("TREES_USE_FHE_SUM", "1") - - # Check that rounding is disabled - assert os.environ.get("TREES_USE_FHE_SUM") == "1", "'TREES_USE_FHE_SUM' is enabled" - check_onnx_file_dump(model_class, parameters, load_data, default_configuration) + check_onnx_file_dump( + model_class, parameters, load_data, default_configuration, use_fhe_sum=False + ) + check_onnx_file_dump( + model_class, parameters, load_data, default_configuration, use_fhe_sum=True + ) diff --git a/tests/sklearn/test_sklearn_models.py b/tests/sklearn/test_sklearn_models.py index 1ec4d7c98..50128e3ef 100644 --- a/tests/sklearn/test_sklearn_models.py +++ b/tests/sklearn/test_sklearn_models.py @@ -1206,10 +1206,10 @@ def check_rounding_consistency( def check_fhe_sum_for_tree_based_models( - model_class, + model, x, y, - n_bits, + predict_method, is_weekly_option, ): """Test that Concrete ML without and with FHE sum are 'equivalent'.""" @@ -1218,18 +1218,8 @@ def check_fhe_sum_for_tree_based_models( if is_weekly_option: fhe_test = get_random_samples(x, n_sample=5) - # By default, the summation of tree ensemble outputs is done in clear - assert os.getenv("TREES_USE_FHE_SUM") == "0", "'TREES_USE_FHE_SUM' is not disabled" - - model_ref = instantiate_model_generic(model_class, n_bits=n_bits) - fit_and_compile(model_ref, x, y) - - # Check `predict_proba` for classifiers and `predict` for regressors - predict_method = ( - model_ref.predict_proba - if is_classifier_or_partial_classifier(model_class) - else model_ref.predict - ) + assert not model.use_fhe_sum, "`use_fhe_sum` is disabled by default." + fit_and_compile(model, x, y) non_fhe_sum_predict_quantized = predict_method(x, fhe="disable") non_fhe_sum_predict_simulate = predict_method(x, fhe="simulate") @@ -1241,29 +1231,15 @@ def check_fhe_sum_for_tree_based_models( if is_weekly_option: non_fhe_sum_predict_fhe = predict_method(fhe_test, fhe="execute") - with pytest.MonkeyPatch.context() as mp_context: - - # Enable the FHE summation of tree ensemble outputs - mp_context.setenv("TREES_USE_FHE_SUM", "1") + model.use_fhe_sum = True - # Check that the summation of tree ensemble outputs is enabled - assert os.getenv("TREES_USE_FHE_SUM") == "1", "'TREES_USE_FHE_SUM' is not enabled" - - model = model_class(**model_ref.get_params()) - fit_and_compile(model, x, y) - - # Check `predict_proba` for classifiers and `predict` for regressors - predict_method = ( - model.predict_proba - if is_classifier_or_partial_classifier(model_class) - else model.predict - ) + fit_and_compile(model, x, y) - fhe_sum_predict_quantized = predict_method(x, fhe="disable") - fhe_sum_predict_simulate = predict_method(x, fhe="simulate") + fhe_sum_predict_quantized = predict_method(x, fhe="disable") + fhe_sum_predict_simulate = predict_method(x, fhe="simulate") - # Sanity check - array_allclose_and_same_shape(fhe_sum_predict_quantized, fhe_sum_predict_simulate) + # Sanity check + array_allclose_and_same_shape(fhe_sum_predict_quantized, fhe_sum_predict_simulate) # Check that we have the exact same predictions array_allclose_and_same_shape(fhe_sum_predict_quantized, non_fhe_sum_predict_quantized) @@ -1962,13 +1938,19 @@ def test_fhe_sum_for_tree_based_models( if verbose: print("Run check_fhe_sum_for_tree_based_models") + model = instantiate_model_generic(model_class, n_bits=n_bits) + x, y = get_dataset(model_class, parameters, n_bits, load_data, is_weekly_option) + predict_method = ( + model.predict_proba if is_classifier_or_partial_classifier(model) else model.predict + ) + check_fhe_sum_for_tree_based_models( - model_class, + model, x, y, - n_bits, + predict_method, is_weekly_option, ) From 39b2972a356d920b638a61ee09da50963afe7b57 Mon Sep 17 00:00:00 2001 From: kcelia Date: Tue, 23 Jan 2024 10:19:52 +0100 Subject: [PATCH 62/73] chore: restore knn notebook --- docs/advanced_examples/KNearestNeighbors.ipynb | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/advanced_examples/KNearestNeighbors.ipynb b/docs/advanced_examples/KNearestNeighbors.ipynb index 9b4b7ed7e..d7ae1b8c1 100644 --- a/docs/advanced_examples/KNearestNeighbors.ipynb +++ b/docs/advanced_examples/KNearestNeighbors.ipynb @@ -287,9 +287,6 @@ "data": { "text/html": [ "\n", "
\n", " \n", From 07b2f2a531d48b5124ccd11803d32c6e33c75e68 Mon Sep 17 00:00:00 2001 From: kcelia Date: Tue, 23 Jan 2024 10:22:26 +0100 Subject: [PATCH 63/73] chore: restore exp notebotebook --- .../ExperimentPrivacyTreePaper.ipynb | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/docs/advanced_examples/ExperimentPrivacyTreePaper.ipynb b/docs/advanced_examples/ExperimentPrivacyTreePaper.ipynb index 23ed43935..388454993 100644 --- a/docs/advanced_examples/ExperimentPrivacyTreePaper.ipynb +++ b/docs/advanced_examples/ExperimentPrivacyTreePaper.ipynb @@ -494,7 +494,7 @@ " \n", " \n", " \n", - " \n", + " \n", " \n", " \n", " \n", @@ -518,7 +518,7 @@ " \n", " \n", " \n", - " \n", + " \n", " \n", " \n", " \n", @@ -554,7 +554,7 @@ " \n", " \n", " \n", - " \n", + " \n", " \n", " \n", " \n", @@ -792,9 +792,9 @@ " \n", " \n", " \n", - " \n", + " \n", " \n", - " \n", + " \n", " \n", " \n", " \n", @@ -805,7 +805,7 @@ " \n", " \n", " \n", - " \n", + " \n", " \n", " \n", " \n", @@ -818,11 +818,11 @@ " FP32-DT 90.3\\% ± 1.0\\% 87.4\\% ± 1.2\\% \n", " FHE-XGB 94.5\\% ± 0.8\\% 92.9\\% ± 1.1\\% \n", " FP32-XGB 95.0\\% ± 0.7\\% 93.6\\% ± 0.9\\% \n", - " FHE-RF 90.9\\% ± 1.1\\% 87.5\\% ± 1.6\\% \n", + " FHE-RF 90.9\\% ± 1.1\\% 87.5\\% ± 1.5\\% \n", " FP32-RF 91.8\\% ± 1.1\\% 89.0\\% ± 1.4\\% \n", "wine (#features: 13) FHE-DT 90.8\\% ± 5.2\\% - \n", " FP32-DT 90.5\\% ± 5.0\\% - \n", - " FHE-XGB 96.8\\% ± 2.5\\% - \n", + " FHE-XGB 97.0\\% ± 2.4\\% - \n", " FP32-XGB 96.2\\% ± 2.9\\% - \n", " FHE-RF 98.5\\% ± 1.4\\% - \n", " FP32-RF 98.1\\% ± 2.0\\% - \n", @@ -848,7 +848,7 @@ " FP32-DT 97.2\\% ± 0.7\\% 96.1\\% ± 0.9\\% \n", " FHE-XGB 100.0\\% ± 0.0\\% 100.0\\% ± 0.0\\% \n", " FP32-XGB 100.0\\% ± 0.0\\% 100.0\\% ± 0.0\\% \n", - " FHE-RF 96.8\\% ± 1.3\\% 95.4\\% ± 1.8\\% \n", + " FHE-RF 96.9\\% ± 1.2\\% 95.4\\% ± 1.8\\% \n", " FP32-RF 95.9\\% ± 1.1\\% 93.9\\% ± 1.5\\% \n", "\n", " AP nodes Time (s) \\\n", @@ -1610,19 +1610,19 @@ "metadata": {}, "outputs": [ { - "name": "stdout", + "name": "stderr", "output_type": "stream", "text": [ - "ap relative: [0.49626943 0.70187731 0.82640876 0.89067066 0.98315255 1.02264581\n", - " 1.02436888 1.01090038 1.01268386], f1_relative: [0.06488922 0.65490682 0.87590196 0.90861806 0.97920588 1.00604989\n", - " 1.00914511 1.00274636 1.00389957]\n" + "The PostScript backend does not support transparency; partially transparent artists will be rendered opaque.\n" ] }, { - "name": "stderr", + "name": "stdout", "output_type": "stream", "text": [ - "The PostScript backend does not support transparency; partially transparent artists will be rendered opaque.\n" + "ap relative: [0.49626943 0.70187731 0.82640876 0.89067066 0.98315255 1.02264581\n", + " 1.02436888 1.01090038 1.01268386], f1_relative: [0.06488922 0.65490682 0.87590196 0.90861806 0.97920588 1.00604989\n", + " 1.00914511 1.00274636 1.00389957]\n" ] }, { @@ -1646,9 +1646,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "ap relative: [0.43556747 0.69054787 0.8789863 0.94213852 0.97097036 0.99083622\n", - " 0.99365961 0.99626825 0.99920411], f1_relative: [0. 0.65970362 0.91412713 0.95780357 0.97789164 0.99271147\n", - " 0.99456864 0.99697611 0.99959059]\n" + "ap relative: [0.43556747 0.69054787 0.8789863 0.94180188 0.97097036 0.99094624\n", + " 0.99348364 0.99626825 0.99932372], f1_relative: [0. 0.65970362 0.91412713 0.95762445 0.97789164 0.99281277\n", + " 0.99447789 0.99697611 0.99969255]\n" ] }, { @@ -1672,9 +1672,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "ap relative: [0.45810941 0.65828111 0.85617664 0.93660034 0.96541385 0.98342004\n", - " 0.99091316 0.9911998 0.99740638], f1_relative: [0. 0.56676488 0.86901886 0.93986022 0.96505021 0.98359134\n", - " 0.99082334 0.99211045 0.99758998]\n" + "ap relative: [0.45810941 0.66176353 0.85701522 0.93668402 0.96541385 0.98353791\n", + " 0.99091316 0.99133601 0.99740638], f1_relative: [0. 0.57332946 0.87035559 0.9402579 0.96505021 0.983713\n", + " 0.99082334 0.99224022 0.99758998]\n" ] }, { From 7fddeceb5661d737969f2b816d14e6ef2a691a2d Mon Sep 17 00:00:00 2001 From: kcelia Date: Tue, 23 Jan 2024 16:50:56 +0100 Subject: [PATCH 64/73] chore: update v1 --- src/concrete/ml/sklearn/base.py | 8 +++----- src/concrete/ml/sklearn/tree_to_numpy.py | 16 ++++++++-------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/src/concrete/ml/sklearn/base.py b/src/concrete/ml/sklearn/base.py index dbd8ca4a8..380ee1625 100644 --- a/src/concrete/ml/sklearn/base.py +++ b/src/concrete/ml/sklearn/base.py @@ -104,9 +104,6 @@ # However, for internal testing purposes, we retain the capability to disable this feature os.environ["TREES_USE_ROUNDING"] = os.environ.get("TREES_USE_ROUNDING", "1") -# By default, the decision of the tree ensembles is made in clear -TREES_USE_FHE_SUM = False - # pylint: disable=too-many-public-methods @@ -1313,6 +1310,7 @@ def __init__(self, n_bits: Union[int, Dict[str, int]]): self._tree_inference: Optional[Callable] = None #: Wether to perform the sum of the output's tree ensembles in FHE or not. + # By default, the decision of the tree ensembles is made in clear. self._use_fhe_sum = False BaseEstimator.__init__(self) @@ -1327,11 +1325,11 @@ def use_fhe_sum(self) -> bool: return self._use_fhe_sum @use_fhe_sum.setter - def use_fhe_sum(self, value) -> None: + def use_fhe_sum(self, value: bool) -> None: """Property setter for `use_fhe_sum`. Args: - value (int): Whether to enable or disable the feature. + value (bool): Whether to enable or disable the feature. """ assert isinstance(value, bool), "Value must be a boolean type" diff --git a/src/concrete/ml/sklearn/tree_to_numpy.py b/src/concrete/ml/sklearn/tree_to_numpy.py index c61ea9d1c..8f6beda61 100644 --- a/src/concrete/ml/sklearn/tree_to_numpy.py +++ b/src/concrete/ml/sklearn/tree_to_numpy.py @@ -141,8 +141,8 @@ def add_transpose_after_last_node(onnx_model: onnx.ModelProto, use_fhe_sum: bool Args: onnx_model (onnx.ModelProto): The ONNX model. - use_fhe_sum (bool): This parameter is exclusively used to tree-based models. - It determines whether the sum of the trees' outputs is computed in FHE. + use_fhe_sum (bool): Determines whether the sum of the trees' outputs is computed in FHE. + Default to False. """ # Get the output node output_node = onnx_model.graph.output[0] @@ -230,8 +230,8 @@ def tree_onnx_graph_preprocessing( framework (str): The framework from which the ONNX model is generated. (options: 'xgboost', 'sklearn') expected_number_of_outputs (int): The expected number of outputs in the ONNX model. - use_fhe_sum (bool): This parameter is exclusively used to tree-based models. - It determines whether the sum of the trees' outputs is computed in FHE. + use_fhe_sum (bool): Determines whether the sum of the trees' outputs is computed in FHE. + Default to False. """ # Make sure the ONNX version returned by Hummingbird is OPSET_VERSION_FOR_ONNX_EXPORT onnx_version = get_onnx_opset_version(onnx_model) @@ -339,10 +339,10 @@ def tree_to_numpy( Args: model (Callable): The tree model to convert. x (numpy.ndarray): The input data. - use_rounding (bool): This parameter is exclusively used to tree-based models. - It determines whether the rounding feature is enabled or disabled. - use_fhe_sum (bool): This parameter is exclusively used to tree-based models. - It determines whether the sum of the trees' outputs is computed in FHE. + use_rounding (bool): Determines whether the rounding feature is enabled or disabled. + Default to True. + use_fhe_sum (bool): Determines whether the sum of the trees' outputs is computed in FHE. + Default to False. framework (str): The framework from which the ONNX model is generated. (options: 'xgboost', 'sklearn') output_n_bits (int): The number of bits of the output. Default to 8. From 14d9dc0b0bca32982a99db216792ac8a794085ad Mon Sep 17 00:00:00 2001 From: kcelia Date: Tue, 23 Jan 2024 18:12:44 +0100 Subject: [PATCH 65/73] chore: update v2 --- src/concrete/ml/sklearn/base.py | 25 ++++++++++++------------ src/concrete/ml/sklearn/tree_to_numpy.py | 6 +++--- tests/sklearn/test_dump_onnx.py | 17 +++++++++------- tests/sklearn/test_sklearn_models.py | 17 ++++++++-------- 4 files changed, 34 insertions(+), 31 deletions(-) diff --git a/src/concrete/ml/sklearn/base.py b/src/concrete/ml/sklearn/base.py index 380ee1625..7308bd58f 100644 --- a/src/concrete/ml/sklearn/base.py +++ b/src/concrete/ml/sklearn/base.py @@ -1311,22 +1311,22 @@ def __init__(self, n_bits: Union[int, Dict[str, int]]): #: Wether to perform the sum of the output's tree ensembles in FHE or not. # By default, the decision of the tree ensembles is made in clear. - self._use_fhe_sum = False + self._fhe_ensembling = False BaseEstimator.__init__(self) @property - def use_fhe_sum(self) -> bool: - """Property getter for `use_fhe_sum`. + def fhe_ensembling(self) -> bool: + """Property getter for `_fhe_ensembling`. Returns: - bool: The current setting of the `_use_fhe_sum` attribute. + bool: The current setting of the `fhe_ensembling` attribute. """ - return self._use_fhe_sum + return self._fhe_ensembling - @use_fhe_sum.setter - def use_fhe_sum(self, value: bool) -> None: - """Property setter for `use_fhe_sum`. + @fhe_ensembling.setter + def fhe_ensembling(self, value: bool) -> None: + """Property setter for `fhe_ensembling`. Args: value (bool): Whether to enable or disable the feature. @@ -1335,9 +1335,10 @@ def use_fhe_sum(self, value: bool) -> None: assert isinstance(value, bool), "Value must be a boolean type" if value is True: + print("LAA") warnings.simplefilter("always") warnings.warn( - "Enabling `use_fhe_sum` computes the sum of the ouputs of tree ensembles in FHE.\n" + "Enabling `fhe_ensembling` computes the sum of the ouputs of tree ensembles in FHE.\n" "This may slow down the computation and increase the maximum bitwidth.\n" "To optimize performance, consider reducing the quantization leaf precision.\n" "Additionally, the model must be refitted for these changes to take effect.", @@ -1345,7 +1346,7 @@ def use_fhe_sum(self, value: bool) -> None: stacklevel=2, ) - self._use_fhe_sum = value + self._fhe_ensembling = value def fit(self, X: Data, y: Target, **fit_parameters): # Reset for double fit @@ -1395,7 +1396,7 @@ def fit(self, X: Data, y: Target, **fit_parameters): self.sklearn_model, q_X, use_rounding=enable_rounding, - use_fhe_sum=self._use_fhe_sum, + fhe_ensembling=self.fhe_ensembling, framework=self.framework, output_n_bits=self.n_bits["op_leaves"], ) @@ -1472,7 +1473,7 @@ def post_processing(self, y_preds: numpy.ndarray) -> numpy.ndarray: # Sum all tree outputs # Remove the sum once we handle multi-precision circuits # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/451 - if not self._use_fhe_sum: + if not self._fhe_ensembling: y_preds = numpy.sum(y_preds, axis=-1) assert_true(y_preds.ndim == 2, "y_preds should be a 2D array") diff --git a/src/concrete/ml/sklearn/tree_to_numpy.py b/src/concrete/ml/sklearn/tree_to_numpy.py index 8f6beda61..d3da342fe 100644 --- a/src/concrete/ml/sklearn/tree_to_numpy.py +++ b/src/concrete/ml/sklearn/tree_to_numpy.py @@ -331,7 +331,7 @@ def tree_to_numpy( x: numpy.ndarray, framework: str, use_rounding: bool = True, - use_fhe_sum: bool = False, + fhe_ensembling: bool = False, output_n_bits: int = MAX_BITWIDTH_BACKWARD_COMPATIBLE, ) -> Tuple[Callable, List[UniformQuantizer], onnx.ModelProto]: """Convert the tree inference to a numpy functions using Hummingbird. @@ -341,7 +341,7 @@ def tree_to_numpy( x (numpy.ndarray): The input data. use_rounding (bool): Determines whether the rounding feature is enabled or disabled. Default to True. - use_fhe_sum (bool): Determines whether the sum of the trees' outputs is computed in FHE. + fhe_ensembling (bool): Determines whether the sum of the trees' outputs is computed in FHE. Default to False. framework (str): The framework from which the ONNX model is generated. (options: 'xgboost', 'sklearn') @@ -379,7 +379,7 @@ def tree_to_numpy( # ONNX graph pre-processing to make the model FHE friendly # i.e., delete irrelevant nodes and cut the graph before the final ensemble sum) - tree_onnx_graph_preprocessing(onnx_model, framework, expected_number_of_outputs, use_fhe_sum) + tree_onnx_graph_preprocessing(onnx_model, framework, expected_number_of_outputs, fhe_ensembling) # Tree values pre-processing # i.e., mainly predictions quantization diff --git a/tests/sklearn/test_dump_onnx.py b/tests/sklearn/test_dump_onnx.py index fa398cd7e..4e7589319 100644 --- a/tests/sklearn/test_dump_onnx.py +++ b/tests/sklearn/test_dump_onnx.py @@ -19,7 +19,9 @@ # pylint: disable=line-too-long -def check_onnx_file_dump(model_class, parameters, load_data, default_configuration, use_fhe_sum): +def check_onnx_file_dump( + model_class, parameters, load_data, default_configuration, use_fhe_sum=False +): """Fit the model and dump the corresponding ONNX.""" model_name = get_model_name(model_class) @@ -498,9 +500,10 @@ def test_dump( callbacks="disable", ) - check_onnx_file_dump( - model_class, parameters, load_data, default_configuration, use_fhe_sum=False - ) - check_onnx_file_dump( - model_class, parameters, load_data, default_configuration, use_fhe_sum=True - ) + check_onnx_file_dump(model_class, parameters, load_data, default_configuration) + + # Additional tests exclusively dedicated for tree ensemble models. + if model_class in _get_sklearn_tree_models()[2:]: + check_onnx_file_dump( + model_class, parameters, load_data, default_configuration, use_fhe_sum=True + ) diff --git a/tests/sklearn/test_sklearn_models.py b/tests/sklearn/test_sklearn_models.py index 50128e3ef..446ad3d11 100644 --- a/tests/sklearn/test_sklearn_models.py +++ b/tests/sklearn/test_sklearn_models.py @@ -149,13 +149,7 @@ def get_n_bits_non_correctness(model_class): if get_model_name(model_class) == "KNeighborsClassifier": n_bits = 2 - # Adjust the quantization precision for tree-based model based on `TREES_USE_FHE_SUM` setting. - # When enabled, the circuit's bitwidth increases, potentially leading to Out-of-Memory issues. - # Therefore, the maximum quantization precision is 4 bits in this case. - elif model_class in _get_sklearn_tree_models() and os.environ.get("TREES_USE_FHE_SUM") == "1": - n_bits = min(min(N_BITS_REGULAR_BUILDS), 4) - else: - n_bits = min(N_BITS_REGULAR_BUILDS) + n_bits = min(N_BITS_REGULAR_BUILDS) return n_bits @@ -1218,7 +1212,7 @@ def check_fhe_sum_for_tree_based_models( if is_weekly_option: fhe_test = get_random_samples(x, n_sample=5) - assert not model.use_fhe_sum, "`use_fhe_sum` is disabled by default." + assert not model.fhe_ensembling, "`fhe_ensembling` is disabled by default." fit_and_compile(model, x, y) non_fhe_sum_predict_quantized = predict_method(x, fhe="disable") @@ -1231,7 +1225,8 @@ def check_fhe_sum_for_tree_based_models( if is_weekly_option: non_fhe_sum_predict_fhe = predict_method(fhe_test, fhe="execute") - model.use_fhe_sum = True + with pytest.warns(UserWarning, match="Enabling `fhe_ensembling` .*"): + model.fhe_ensembling = True fit_and_compile(model, x, y) @@ -1955,6 +1950,8 @@ def test_fhe_sum_for_tree_based_models( ) +# This test should be extended to all built-in models. +# FIXME: https://github.com/zama-ai/concrete-ml-internal#4234 @pytest.mark.parametrize( "n_bits, error_message", [ @@ -1982,6 +1979,8 @@ def test_invalid_n_bits_setting(model_class, n_bits, error_message): instantiate_model_generic(model_class, n_bits=n_bits) +# This test should be extended to all built-in models. +# FIXME: https://github.com/zama-ai/concrete-ml-internal#4234 @pytest.mark.parametrize("n_bits", [5, {"op_inputs": 5}, {"op_inputs": 2, "op_leaves": 1}]) @pytest.mark.parametrize("model_class, parameters", get_sklearn_tree_models_and_datasets()) def test_valid_n_bits_setting( From 324821652ec5cd8e2ba67478843da5afa12f09f3 Mon Sep 17 00:00:00 2001 From: kcelia Date: Tue, 23 Jan 2024 18:20:45 +0100 Subject: [PATCH 66/73] chore: update v3 --- src/concrete/ml/sklearn/tree_to_numpy.py | 16 ++++++++-------- tests/sklearn/test_sklearn_models.py | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/concrete/ml/sklearn/tree_to_numpy.py b/src/concrete/ml/sklearn/tree_to_numpy.py index d3da342fe..d2723e925 100644 --- a/src/concrete/ml/sklearn/tree_to_numpy.py +++ b/src/concrete/ml/sklearn/tree_to_numpy.py @@ -136,25 +136,25 @@ def assert_add_node_and_constant_in_xgboost_regressor_graph(onnx_model: onnx.Mod ) -def add_transpose_after_last_node(onnx_model: onnx.ModelProto, use_fhe_sum: bool): +def add_transpose_after_last_node(onnx_model: onnx.ModelProto, fhe_ensembling: bool): """Add transpose after last node. Args: onnx_model (onnx.ModelProto): The ONNX model. - use_fhe_sum (bool): Determines whether the sum of the trees' outputs is computed in FHE. + fhe_ensembling (bool): Determines whether the sum of the trees' outputs is computed in FHE. Default to False. """ # Get the output node output_node = onnx_model.graph.output[0] - # The state of the 'use_fhe_sum' variable affects the structure of the model's ONNX graph. + # The state of the 'fhe_ensembling' variable affects the structure of the model's ONNX graph. # When the option is enabled, the graph is cut after the ReduceSum node. # When it is disabled, the graph is cut at the ReduceSum node, which alters the output shape. # Therefore, it is necessary to adjust this shape with the correct permutation. # When using FHE sum for tree ensembles, create the node with perm attribute equal to (1, 0) # Otherwise, create the node with perm attribute equal to (2, 1, 0) - perm = [1, 0] if use_fhe_sum else [2, 1, 0] + perm = [1, 0] if fhe_ensembling else [2, 1, 0] transpose_node = onnx.helper.make_node( "Transpose", @@ -221,7 +221,7 @@ def tree_onnx_graph_preprocessing( onnx_model: onnx.ModelProto, framework: str, expected_number_of_outputs: int, - use_fhe_sum: bool = False, + fhe_ensembling: bool = False, ): """Apply pre-processing onto the ONNX graph. @@ -230,7 +230,7 @@ def tree_onnx_graph_preprocessing( framework (str): The framework from which the ONNX model is generated. (options: 'xgboost', 'sklearn') expected_number_of_outputs (int): The expected number of outputs in the ONNX model. - use_fhe_sum (bool): Determines whether the sum of the trees' outputs is computed in FHE. + fhe_ensembling (bool): Determines whether the sum of the trees' outputs is computed in FHE. Default to False. """ # Make sure the ONNX version returned by Hummingbird is OPSET_VERSION_FOR_ONNX_EXPORT @@ -258,7 +258,7 @@ def tree_onnx_graph_preprocessing( # Cut the graph after the ReduceSum node to remove # argmax, sigmoid, softmax from the graph. - if use_fhe_sum: + if fhe_ensembling: clean_graph_after_node_op_type(onnx_model, "ReduceSum") else: clean_graph_at_node_op_type(onnx_model, "ReduceSum") @@ -274,7 +274,7 @@ def tree_onnx_graph_preprocessing( # sklearn models apply the reduce sum before the transpose. # To have equivalent output between xgboost in sklearn, # apply the transpose before returning the output. - add_transpose_after_last_node(onnx_model, use_fhe_sum) + add_transpose_after_last_node(onnx_model, fhe_ensembling) # Cast nodes are not necessary so remove them. remove_node_types(onnx_model, op_types_to_remove=["Cast"]) diff --git a/tests/sklearn/test_sklearn_models.py b/tests/sklearn/test_sklearn_models.py index 446ad3d11..cd156f808 100644 --- a/tests/sklearn/test_sklearn_models.py +++ b/tests/sklearn/test_sklearn_models.py @@ -148,8 +148,8 @@ def get_n_bits_non_correctness(model_class): # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3979 if get_model_name(model_class) == "KNeighborsClassifier": n_bits = 2 - - n_bits = min(N_BITS_REGULAR_BUILDS) + else: + n_bits = min(N_BITS_REGULAR_BUILDS) return n_bits From ab45587539e5986bcbdd0cddc431161bffb72e92 Mon Sep 17 00:00:00 2001 From: kcelia Date: Tue, 23 Jan 2024 19:52:15 +0100 Subject: [PATCH 67/73] chore: update --- src/concrete/ml/sklearn/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/concrete/ml/sklearn/base.py b/src/concrete/ml/sklearn/base.py index 7308bd58f..b1ce3d55d 100644 --- a/src/concrete/ml/sklearn/base.py +++ b/src/concrete/ml/sklearn/base.py @@ -1338,8 +1338,8 @@ def fhe_ensembling(self, value: bool) -> None: print("LAA") warnings.simplefilter("always") warnings.warn( - "Enabling `fhe_ensembling` computes the sum of the ouputs of tree ensembles in FHE.\n" - "This may slow down the computation and increase the maximum bitwidth.\n" + "Enabling `fhe_ensembling` computes the sum of the ouputs of tree ensembles in " + "FHE.\nThis may slow down the computation and increase the maximum bitwidth.\n" "To optimize performance, consider reducing the quantization leaf precision.\n" "Additionally, the model must be refitted for these changes to take effect.", category=UserWarning, From 0ad0f6d1dad2014151cb839e6f18bb02878b7c44 Mon Sep 17 00:00:00 2001 From: kcelia Date: Wed, 24 Jan 2024 10:25:51 +0100 Subject: [PATCH 68/73] chore: update comments --- src/concrete/ml/quantization/post_training.py | 6 +++--- src/concrete/ml/sklearn/base.py | 1 - tests/sklearn/test_sklearn_models.py | 10 +++++----- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/concrete/ml/quantization/post_training.py b/src/concrete/ml/quantization/post_training.py index 621697f14..46bed0214 100644 --- a/src/concrete/ml/quantization/post_training.py +++ b/src/concrete/ml/quantization/post_training.py @@ -50,7 +50,7 @@ def _inspect_tree_n_bits(n_bits): """ detailed_message = ( - "Invalid 'n_bits', either pass a non-null positive integer or a dictionary containing " + "Invalid 'n_bits', either pass a strictly positive integer or a dictionary containing " "integer values for the following keys:\n" "- 'op_inputs' (mandatory): number of bits to quantize the input values\n" "- 'op_leaves' (optional): number of bits to quantize the leaves, must be less than or " @@ -63,7 +63,7 @@ def _inspect_tree_n_bits(n_bits): if isinstance(n_bits, int): if n_bits <= 0: - error_message = "n_bits must be a non-null, positive integer" + error_message = "n_bits must be a strictly positive integer" elif isinstance(n_bits, dict): if "op_inputs" not in n_bits.keys(): error_message = "Invalid keys in `n_bits` dictionary. The key 'op_inputs' is mandatory" @@ -73,7 +73,7 @@ def _inspect_tree_n_bits(n_bits): "(optional) are allowed" ) elif not all(isinstance(value, int) and value > 0 for value in n_bits.values()): - error_message = "All values in 'n_bits' dictionary must be non-null, positive integers" + error_message = "All values in 'n_bits' dictionary must be strictly positive integers" elif n_bits.get("op_leaves", 0) > n_bits.get("op_inputs", 0): error_message = "'op_leaves' must be less than or equal to 'op_inputs'" diff --git a/src/concrete/ml/sklearn/base.py b/src/concrete/ml/sklearn/base.py index b1ce3d55d..8b947b163 100644 --- a/src/concrete/ml/sklearn/base.py +++ b/src/concrete/ml/sklearn/base.py @@ -1335,7 +1335,6 @@ def fhe_ensembling(self, value: bool) -> None: assert isinstance(value, bool), "Value must be a boolean type" if value is True: - print("LAA") warnings.simplefilter("always") warnings.warn( "Enabling `fhe_ensembling` computes the sum of the ouputs of tree ensembles in " diff --git a/tests/sklearn/test_sklearn_models.py b/tests/sklearn/test_sklearn_models.py index cd156f808..6e5bd2a9e 100644 --- a/tests/sklearn/test_sklearn_models.py +++ b/tests/sklearn/test_sklearn_models.py @@ -1955,8 +1955,8 @@ def test_fhe_sum_for_tree_based_models( @pytest.mark.parametrize( "n_bits, error_message", [ - (0, "n_bits must be a non-null, positive integer"), - (-1, "n_bits must be a non-null, positive integer"), + (0, "n_bits must be a strictly positive integer"), + (-1, "n_bits must be a strictly positive integer"), ({"op_leaves": 2}, "The key 'op_inputs' is mandatory"), ( {"op_inputs": 4, "op_leaves": 2, "op_weights": 2}, @@ -1965,7 +1965,7 @@ def test_fhe_sum_for_tree_based_models( ), ( {"op_inputs": -2, "op_leaves": -5}, - "All values in 'n_bits' dictionary must be non-null, positive integers", + "All values in 'n_bits' dictionary must be strictly positive integers", ), ({"op_inputs": 2, "op_leaves": 5}, "'op_leaves' must be less than or equal to 'op_inputs'"), (0.5, "n_bits must be either an integer or a dictionary"), @@ -1973,7 +1973,7 @@ def test_fhe_sum_for_tree_based_models( ) @pytest.mark.parametrize("model_class", _get_sklearn_tree_models()) def test_invalid_n_bits_setting(model_class, n_bits, error_message): - """Check if the model instantiation raises an exception with invalid 'n_bits' settings.""" + """Check if the model instantiation raises an exception with invalid `n_bits` settings.""" with pytest.raises(ValueError, match=f"{error_message}. Got '{type(n_bits)}' and '{n_bits}'.*"): instantiate_model_generic(model_class, n_bits=n_bits) @@ -1991,7 +1991,7 @@ def test_valid_n_bits_setting( is_weekly_option, verbose=True, ): - """Check valid `n_bits' settings.""" + """Check valid `n_bits` settings.""" if verbose: print("Run test_valid_n_bits_setting") From 9b58948c5c91f78eff534c32ae6f48bce2460e2e Mon Sep 17 00:00:00 2001 From: kcelia Date: Wed, 24 Jan 2024 14:06:59 +0100 Subject: [PATCH 69/73] chore: update --- src/concrete/ml/sklearn/base.py | 37 +++------------------------- src/concrete/ml/sklearn/rf.py | 6 +++++ src/concrete/ml/sklearn/tree.py | 7 ++++++ src/concrete/ml/sklearn/xgb.py | 6 +++++ tests/sklearn/test_dump_onnx.py | 9 ++++--- tests/sklearn/test_sklearn_models.py | 7 +++--- 6 files changed, 31 insertions(+), 41 deletions(-) diff --git a/src/concrete/ml/sklearn/base.py b/src/concrete/ml/sklearn/base.py index 8b947b163..b42cf5518 100644 --- a/src/concrete/ml/sklearn/base.py +++ b/src/concrete/ml/sklearn/base.py @@ -1310,43 +1310,12 @@ def __init__(self, n_bits: Union[int, Dict[str, int]]): self._tree_inference: Optional[Callable] = None #: Wether to perform the sum of the output's tree ensembles in FHE or not. - # By default, the decision of the tree ensembles is made in clear. + # By default, the decision of the tree ensembles is made in clear (not in FHE). + # This attribute should not be modified by users. self._fhe_ensembling = False BaseEstimator.__init__(self) - @property - def fhe_ensembling(self) -> bool: - """Property getter for `_fhe_ensembling`. - - Returns: - bool: The current setting of the `fhe_ensembling` attribute. - """ - return self._fhe_ensembling - - @fhe_ensembling.setter - def fhe_ensembling(self, value: bool) -> None: - """Property setter for `fhe_ensembling`. - - Args: - value (bool): Whether to enable or disable the feature. - """ - - assert isinstance(value, bool), "Value must be a boolean type" - - if value is True: - warnings.simplefilter("always") - warnings.warn( - "Enabling `fhe_ensembling` computes the sum of the ouputs of tree ensembles in " - "FHE.\nThis may slow down the computation and increase the maximum bitwidth.\n" - "To optimize performance, consider reducing the quantization leaf precision.\n" - "Additionally, the model must be refitted for these changes to take effect.", - category=UserWarning, - stacklevel=2, - ) - - self._fhe_ensembling = value - def fit(self, X: Data, y: Target, **fit_parameters): # Reset for double fit self._is_fitted = False @@ -1395,7 +1364,7 @@ def fit(self, X: Data, y: Target, **fit_parameters): self.sklearn_model, q_X, use_rounding=enable_rounding, - fhe_ensembling=self.fhe_ensembling, + fhe_ensembling=self._fhe_ensembling, framework=self.framework, output_n_bits=self.n_bits["op_leaves"], ) diff --git a/src/concrete/ml/sklearn/rf.py b/src/concrete/ml/sklearn/rf.py index 2ebca55b8..f4521bf06 100644 --- a/src/concrete/ml/sklearn/rf.py +++ b/src/concrete/ml/sklearn/rf.py @@ -84,6 +84,7 @@ def dump_dict(self) -> Dict[str, Any]: metadata["onnx_model_"] = self.onnx_model_ metadata["framework"] = self.framework metadata["post_processing_params"] = self.post_processing_params + metadata["_fhe_ensembling"] = self._fhe_ensembling # Scikit-Learn metadata["n_estimators"] = self.n_estimators @@ -120,11 +121,13 @@ def load_dict(cls, metadata: Dict): obj.framework = metadata["framework"] obj.onnx_model_ = metadata["onnx_model_"] obj.output_quantizers = metadata["output_quantizers"] + obj._fhe_ensembling = metadata["_fhe_ensembling"] obj._tree_inference = tree_to_numpy( obj.sklearn_model, numpy.zeros((len(obj.input_quantizers),))[None, ...], framework=obj.framework, output_n_bits=obj.n_bits["op_leaves"] if isinstance(obj.n_bits, Dict) else obj.n_bits, + fhe_ensembling=obj._fhe_ensembling, )[0] obj.post_processing_params = metadata["post_processing_params"] @@ -219,6 +222,7 @@ def dump_dict(self) -> Dict[str, Any]: metadata["onnx_model_"] = self.onnx_model_ metadata["framework"] = self.framework metadata["post_processing_params"] = self.post_processing_params + metadata["_fhe_ensembling"] = self._fhe_ensembling # Scikit-Learn metadata["n_estimators"] = self.n_estimators @@ -255,11 +259,13 @@ def load_dict(cls, metadata: Dict): obj.framework = metadata["framework"] obj.onnx_model_ = metadata["onnx_model_"] obj.output_quantizers = metadata["output_quantizers"] + obj._fhe_ensembling = metadata["_fhe_ensembling"] obj._tree_inference = tree_to_numpy( obj.sklearn_model, numpy.zeros((len(obj.input_quantizers),))[None, ...], framework=obj.framework, output_n_bits=obj.n_bits["op_leaves"] if isinstance(obj.n_bits, Dict) else obj.n_bits, + fhe_ensembling=obj._fhe_ensembling, )[0] obj.post_processing_params = metadata["post_processing_params"] diff --git a/src/concrete/ml/sklearn/tree.py b/src/concrete/ml/sklearn/tree.py index 5ba1f8cff..b496d4e47 100644 --- a/src/concrete/ml/sklearn/tree.py +++ b/src/concrete/ml/sklearn/tree.py @@ -84,6 +84,7 @@ def dump_dict(self) -> Dict[str, Any]: metadata["onnx_model_"] = self.onnx_model_ metadata["framework"] = self.framework metadata["post_processing_params"] = self.post_processing_params + metadata["_fhe_ensembling"] = self._fhe_ensembling # Scikit-Learn metadata["criterion"] = self.criterion @@ -115,11 +116,13 @@ def load_dict(cls, metadata: Dict): obj.framework = metadata["framework"] obj.onnx_model_ = metadata["onnx_model_"] obj.output_quantizers = metadata["output_quantizers"] + obj._fhe_ensembling = metadata["_fhe_ensembling"] obj._tree_inference = tree_to_numpy( obj.sklearn_model, numpy.zeros((len(obj.input_quantizers),))[None, ...], framework=obj.framework, output_n_bits=obj.n_bits["op_leaves"] if isinstance(obj.n_bits, Dict) else obj.n_bits, + fhe_ensembling=obj._fhe_ensembling, )[0] obj.post_processing_params = metadata["post_processing_params"] @@ -208,6 +211,7 @@ def dump_dict(self) -> Dict[str, Any]: metadata["onnx_model_"] = self.onnx_model_ metadata["framework"] = self.framework metadata["post_processing_params"] = self.post_processing_params + metadata["_fhe_ensembling"] = self._fhe_ensembling # Scikit-Learn metadata["criterion"] = self.criterion @@ -233,16 +237,19 @@ def load_dict(cls, metadata: Dict): # Concrete-ML obj.sklearn_model = metadata["sklearn_model"] obj._is_fitted = metadata["_is_fitted"] + obj._fhe_ensembling = metadata["_fhe_ensembling"] obj._is_compiled = metadata["_is_compiled"] obj.input_quantizers = metadata["input_quantizers"] obj.framework = metadata["framework"] obj.onnx_model_ = metadata["onnx_model_"] obj.output_quantizers = metadata["output_quantizers"] + obj._fhe_ensembling = metadata["_fhe_ensembling"] obj._tree_inference = tree_to_numpy( obj.sklearn_model, numpy.zeros((len(obj.input_quantizers),))[None, ...], framework=obj.framework, output_n_bits=obj.n_bits["op_leaves"] if isinstance(obj.n_bits, Dict) else obj.n_bits, + fhe_ensembling=obj._fhe_ensembling, )[0] obj.post_processing_params = metadata["post_processing_params"] diff --git a/src/concrete/ml/sklearn/xgb.py b/src/concrete/ml/sklearn/xgb.py index 28722b706..8f3925fd7 100644 --- a/src/concrete/ml/sklearn/xgb.py +++ b/src/concrete/ml/sklearn/xgb.py @@ -125,6 +125,7 @@ def dump_dict(self) -> Dict[str, Any]: metadata["onnx_model_"] = self.onnx_model_ metadata["framework"] = self.framework metadata["post_processing_params"] = self.post_processing_params + metadata["_fhe_ensembling"] = self._fhe_ensembling # XGBoost metadata["max_depth"] = self.max_depth @@ -174,11 +175,13 @@ def load_dict(cls, metadata: Dict): obj.framework = metadata["framework"] obj.onnx_model_ = metadata["onnx_model_"] obj.output_quantizers = metadata["output_quantizers"] + obj._fhe_ensembling = metadata["_fhe_ensembling"] obj._tree_inference = tree_to_numpy( obj.sklearn_model, numpy.zeros((len(obj.input_quantizers),))[None, ...], framework=obj.framework, output_n_bits=obj.n_bits["op_leaves"] if isinstance(obj.n_bits, Dict) else obj.n_bits, + fhe_ensembling=obj._fhe_ensembling, )[0] obj.post_processing_params = metadata["post_processing_params"] @@ -354,6 +357,7 @@ def dump_dict(self) -> Dict[str, Any]: metadata["onnx_model_"] = self.onnx_model_ metadata["framework"] = self.framework metadata["post_processing_params"] = self.post_processing_params + metadata["_fhe_ensembling"] = self._fhe_ensembling # XGBoost metadata["max_depth"] = self.max_depth @@ -403,11 +407,13 @@ def load_dict(cls, metadata: Dict): obj.framework = metadata["framework"] obj.onnx_model_ = metadata["onnx_model_"] obj.output_quantizers = metadata["output_quantizers"] + obj._fhe_ensembling = metadata["_fhe_ensembling"] obj._tree_inference = tree_to_numpy( obj.sklearn_model, numpy.zeros((len(obj.input_quantizers),))[None, ...], framework=obj.framework, output_n_bits=obj.n_bits["op_leaves"] if isinstance(obj.n_bits, Dict) else obj.n_bits, + fhe_ensembling=obj._fhe_ensembling, )[0] obj.post_processing_params = metadata["post_processing_params"] diff --git a/tests/sklearn/test_dump_onnx.py b/tests/sklearn/test_dump_onnx.py index 4e7589319..7ff40b71e 100644 --- a/tests/sklearn/test_dump_onnx.py +++ b/tests/sklearn/test_dump_onnx.py @@ -30,10 +30,11 @@ def check_onnx_file_dump( # Set the model model = model_class() - # Set `use_fhe_sum` - with warnings.catch_warnings(): - warnings.simplefilter("ignore", category=UserWarning) - model.use_fhe_sum = use_fhe_sum + # Set `_fhe_ensembling` for tree based models only + if model_class in _get_sklearn_tree_models(): + + # pylint: disable=protected-access + model._fhe_ensembling = use_fhe_sum # Ignore long lines here # ruff: noqa: E501 diff --git a/tests/sklearn/test_sklearn_models.py b/tests/sklearn/test_sklearn_models.py index 6e5bd2a9e..02d773dfc 100644 --- a/tests/sklearn/test_sklearn_models.py +++ b/tests/sklearn/test_sklearn_models.py @@ -1212,7 +1212,8 @@ def check_fhe_sum_for_tree_based_models( if is_weekly_option: fhe_test = get_random_samples(x, n_sample=5) - assert not model.fhe_ensembling, "`fhe_ensembling` is disabled by default." + # pylint: disable=protected-access + assert not model._fhe_ensembling, "`_fhe_ensembling` is disabled by default." fit_and_compile(model, x, y) non_fhe_sum_predict_quantized = predict_method(x, fhe="disable") @@ -1225,8 +1226,8 @@ def check_fhe_sum_for_tree_based_models( if is_weekly_option: non_fhe_sum_predict_fhe = predict_method(fhe_test, fhe="execute") - with pytest.warns(UserWarning, match="Enabling `fhe_ensembling` .*"): - model.fhe_ensembling = True + # pylint: disable=protected-access + model._fhe_ensembling = True fit_and_compile(model, x, y) From 25bf8399f8945abda6f386fa90e70ac62de8fcb1 Mon Sep 17 00:00:00 2001 From: kcelia Date: Wed, 24 Jan 2024 15:13:43 +0100 Subject: [PATCH 70/73] chore: fix test dump --- tests/sklearn/test_dump_onnx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/sklearn/test_dump_onnx.py b/tests/sklearn/test_dump_onnx.py index 7ff40b71e..34e14d242 100644 --- a/tests/sklearn/test_dump_onnx.py +++ b/tests/sklearn/test_dump_onnx.py @@ -504,7 +504,7 @@ def test_dump( check_onnx_file_dump(model_class, parameters, load_data, default_configuration) # Additional tests exclusively dedicated for tree ensemble models. - if model_class in _get_sklearn_tree_models()[2:]: + if model_class in _get_sklearn_tree_models(): check_onnx_file_dump( model_class, parameters, load_data, default_configuration, use_fhe_sum=True ) From 5be7255c688afb73426464bc249b68f676ab4741 Mon Sep 17 00:00:00 2001 From: kcelia Date: Wed, 24 Jan 2024 20:07:11 +0100 Subject: [PATCH 71/73] chore: update comments --- src/concrete/ml/sklearn/tree_to_numpy.py | 2 +- tests/sklearn/test_sklearn_models.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/concrete/ml/sklearn/tree_to_numpy.py b/src/concrete/ml/sklearn/tree_to_numpy.py index d2723e925..b50944319 100644 --- a/src/concrete/ml/sklearn/tree_to_numpy.py +++ b/src/concrete/ml/sklearn/tree_to_numpy.py @@ -136,7 +136,7 @@ def assert_add_node_and_constant_in_xgboost_regressor_graph(onnx_model: onnx.Mod ) -def add_transpose_after_last_node(onnx_model: onnx.ModelProto, fhe_ensembling: bool): +def add_transpose_after_last_node(onnx_model: onnx.ModelProto, fhe_ensembling: bool = False): """Add transpose after last node. Args: diff --git a/tests/sklearn/test_sklearn_models.py b/tests/sklearn/test_sklearn_models.py index 02d773dfc..58423df78 100644 --- a/tests/sklearn/test_sklearn_models.py +++ b/tests/sklearn/test_sklearn_models.py @@ -1952,7 +1952,7 @@ def test_fhe_sum_for_tree_based_models( # This test should be extended to all built-in models. -# FIXME: https://github.com/zama-ai/concrete-ml-internal#4234 +# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4234 @pytest.mark.parametrize( "n_bits, error_message", [ @@ -1981,7 +1981,7 @@ def test_invalid_n_bits_setting(model_class, n_bits, error_message): # This test should be extended to all built-in models. -# FIXME: https://github.com/zama-ai/concrete-ml-internal#4234 +# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4234 @pytest.mark.parametrize("n_bits", [5, {"op_inputs": 5}, {"op_inputs": 2, "op_leaves": 1}]) @pytest.mark.parametrize("model_class, parameters", get_sklearn_tree_models_and_datasets()) def test_valid_n_bits_setting( From ee696ece4f624184e868334f9e27b90e1d77b77c Mon Sep 17 00:00:00 2001 From: kcelia Date: Thu, 25 Jan 2024 14:51:13 +0100 Subject: [PATCH 72/73] chore: remove comment --- src/concrete/ml/sklearn/base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/concrete/ml/sklearn/base.py b/src/concrete/ml/sklearn/base.py index b42cf5518..6ffdb8e2a 100644 --- a/src/concrete/ml/sklearn/base.py +++ b/src/concrete/ml/sklearn/base.py @@ -1303,7 +1303,6 @@ def __init__(self, n_bits: Union[int, Dict[str, int]]): # Check if 'n_bits' is a valid value. _inspect_tree_n_bits(n_bits) - #: The number of bits to quantize the model. self.n_bits: Union[int, Dict[str, int]] = n_bits #: The model's inference function. Is None if the model is not fitted. From 4e2fcda7d7c99a23af54389538f7bceb682950c7 Mon Sep 17 00:00:00 2001 From: kcelia Date: Mon, 29 Jan 2024 12:34:43 +0100 Subject: [PATCH 73/73] chore: update --- src/concrete/ml/onnx/convert.py | 6 ++-- src/concrete/ml/onnx/onnx_impl_utils.py | 27 ++++---------- src/concrete/ml/onnx/onnx_utils.py | 4 +-- src/concrete/ml/onnx/ops_impl.py | 24 +++++-------- src/concrete/ml/sklearn/base.py | 22 ++++++------ src/concrete/ml/sklearn/tree_to_numpy.py | 15 ++------ tests/sklearn/test_sklearn_models.py | 46 ++++++++++++------------ 7 files changed, 55 insertions(+), 89 deletions(-) diff --git a/src/concrete/ml/onnx/convert.py b/src/concrete/ml/onnx/convert.py index 54f7aad92..86ee5f0de 100644 --- a/src/concrete/ml/onnx/convert.py +++ b/src/concrete/ml/onnx/convert.py @@ -4,7 +4,7 @@ import tempfile import warnings from pathlib import Path -from typing import Callable, Optional, Tuple, Union +from typing import Callable, Tuple, Union import numpy import onnx @@ -255,7 +255,7 @@ def get_equivalent_numpy_forward_from_onnx( def get_equivalent_numpy_forward_from_onnx_tree( onnx_model: onnx.ModelProto, check_model: bool = True, - auto_truncate = None, + auto_truncate=None, ) -> Tuple[Callable[..., Tuple[numpy.ndarray, ...]], onnx.ModelProto]: """Get the numpy equivalent forward of the provided ONNX model for tree-based models only. @@ -264,7 +264,7 @@ def get_equivalent_numpy_forward_from_onnx_tree( forward. check_model (bool): set to True to run the onnx checker on the model. Defaults to True. - auto_truncate: This parameter is exclusively used for + auto_truncate (TODO): This parameter is exclusively used for optimizing tree-based models. It contains the values of the least significant bits to remove during the tree traversal, where the first value refers to the first comparison (either "less" or "less_or_equal"), while the second value refers to the "Equal" diff --git a/src/concrete/ml/onnx/onnx_impl_utils.py b/src/concrete/ml/onnx/onnx_impl_utils.py index 9a68052f0..156d845e8 100644 --- a/src/concrete/ml/onnx/onnx_impl_utils.py +++ b/src/concrete/ml/onnx/onnx_impl_utils.py @@ -3,10 +3,9 @@ from typing import Callable, Tuple, Union import numpy - -from concrete.fhe import truncate_bit_pattern, round_bit_pattern from concrete.fhe import conv as fhe_conv from concrete.fhe import ones as fhe_ones +from concrete.fhe import truncate_bit_pattern from concrete.fhe.tracing import Tracer from ..common.debugging import assert_true @@ -241,12 +240,12 @@ def onnx_avgpool_compute_norm_const( def rounded_comparison( x: numpy.ndarray, y: numpy.ndarray, auto_truncate, operation: ComparisonOperationType ) -> Tuple[bool]: - """Comparison operation using `round_bit_pattern` function. + """Comparison operation using `truncate_bit_pattern` function. - `round_bit_pattern` rounds the bit pattern of an integer to the closer + `truncate_bit_pattern` rounds the bit pattern of an integer to the closer It also checks for any potential overflow. If so, it readjusts the LSBs accordingly. - The parameter `lsbs_to_remove` in `round_bit_pattern` can either be an integer specifying the + The parameter `lsbs_to_remove` in `truncate_bit_pattern` can either be an integer specifying the number of LSBS to remove, or an `AutoRounder` object that determines the required number of LSBs based on the specified number of MSBs to retain. But in our case, we choose to compute the LSBs manually. @@ -254,8 +253,8 @@ def rounded_comparison( Args: x (numpy.ndarray): Input tensor y (numpy.ndarray): Input tensor - lsbs_to_remove (int): Number of the least significant bits to remove - operation (ComparisonOperationType): Comparison operation, which can `<`, `<=` and `==` + auto_truncate: TODO + operation: TODO Returns: Tuple[bool]: If x and y satisfy the comparison operator. @@ -265,17 +264,3 @@ def rounded_comparison( rounded_subtraction = truncate_bit_pattern(x - y, lsbs_to_remove=auto_truncate) return (operation(rounded_subtraction),) - - # # To determine if 'x' 'operation' 'y' (operation being <, >, >=, <=), we evaluate 'x - y' - # rounded_subtraction = truncate_bit_pattern(x - y, lsbs_to_remove=auto_truncate) - # assert isinstance(lsbs_to_remove, int) - - # # Workaround: in this context, `round_bit_pattern` is used as a truncate operation. - # # Consequently, we subtract a term, called `half` that will subsequently be re-added during the - # # `round_bit_pattern` process. - # half = 1 << (lsbs_to_remove - 1) - - # # To determine if 'x' 'operation' 'y' (operation being <, >, >=, <=), we evaluate 'x - y' - # rounded_subtraction = round_bit_pattern((x - y) - half, lsbs_to_remove=lsbs_to_remove) - - # return (operation(rounded_subtraction),) diff --git a/src/concrete/ml/onnx/onnx_utils.py b/src/concrete/ml/onnx/onnx_utils.py index da2ad80d3..1077e3f4b 100644 --- a/src/concrete/ml/onnx/onnx_utils.py +++ b/src/concrete/ml/onnx/onnx_utils.py @@ -213,7 +213,7 @@ # Original file: # https://github.com/google/jax/blob/f6d329b2d9b5f83c6a59e5739aa1ca8d4d1ffa1c/examples/onnx2xla.py -from typing import Any, Callable, Dict, Optional, Tuple +from typing import Any, Callable, Dict, Tuple import numpy import onnx @@ -413,7 +413,7 @@ } # All numpy operators used for tree-based models that support auto rounding ONNX_COMPARISON_OPS_TO_ROUNDED_TREES_NUMPY_IMPL_BOOL = { - "Less": rounded_numpy_less_for_trees, + "Less": rounded_numpy_less_for_trees, # type: ignore[dict-item] "Equal": rounded_numpy_equal_for_trees, "LessOrEqual": rounded_numpy_less_or_equal_for_trees, } diff --git a/src/concrete/ml/onnx/ops_impl.py b/src/concrete/ml/onnx/ops_impl.py index 661e13c98..7f2730dee 100644 --- a/src/concrete/ml/onnx/ops_impl.py +++ b/src/concrete/ml/onnx/ops_impl.py @@ -892,7 +892,7 @@ def rounded_numpy_equal_for_trees( x: numpy.ndarray, y: numpy.ndarray, *, - auto_truncate = None, + auto_truncate=None, ) -> Tuple[numpy.ndarray]: """Compute rounded equal in numpy according to ONNX spec for tree-based models only. @@ -925,7 +925,6 @@ def rounded_numpy_equal_for_trees( return (numpy.equal(x, y),) - def numpy_equal_float( x: numpy.ndarray, y: numpy.ndarray, @@ -1096,7 +1095,6 @@ def rounded_numpy_less_for_trees( # numpy.less(x, y) is equivalent to : # x - y <= 0 => round_bit_pattern(x - y - half) < 0 if auto_truncate is not None: - #print("Use truncate for <") return rounded_comparison(x, y, auto_truncate, operation=lambda x: x < 0) # Else, default numpy_less operator @@ -1145,7 +1143,7 @@ def rounded_numpy_less_or_equal_for_trees( x: numpy.ndarray, y: numpy.ndarray, *, - auto_truncate = None, + auto_truncate=None, ) -> Tuple[numpy.ndarray]: """Compute rounded less or equal in numpy according to ONNX spec for tree-based models only. @@ -1161,21 +1159,15 @@ def rounded_numpy_less_or_equal_for_trees( Tuple[numpy.ndarray]: Output tensor """ - # numpy.less_equal(x, y) <= 0 is equivalent to : - # np.less_equal(x, y), truncate_bit_pattern((y - x), lsbs_to_remove=r) >= 0 - # option 1: x - y <= 0 => round_bit_pattern(x - y) <= 0 - # gives bad results for : 0 < x - y <= 2**lsbs_to_remove because truncate_bit_pattern(x - y, lsb) = 0 - # option 2: y - x >= 0 => round_bit_pattern(y - x) >= 0 - - if auto_truncate is not None: - #print("Use truncate for <=") - return rounded_comparison(y, x, auto_truncate, operation=lambda x: x >= 0) # numpy.less_equal(x, y) <= y is equivalent to : - # option 1: x - y <= 0 => round_bit_pattern(x - y + half) <= 0 or - # option 2: y - x >= 0 => round_bit_pattern(y - x - half) >= 0 + # option 1: x - y <= 0 => truncate_bit_pattern(x - y + half) <= 0 or + # option 2: y - x >= 0 => truncate_bit_pattern(y - x - half) >= 0 # Option 2 is selected because it adheres to the established pattern in `rounded_comparison` - # which does: (a - b) - half. + # which does: (a - b). + + if auto_truncate is not None: + return rounded_comparison(y, x, auto_truncate, operation=lambda x: x >= 0) # Else, default numpy_less_or_equal operator return numpy_less_or_equal(x, y) diff --git a/src/concrete/ml/sklearn/base.py b/src/concrete/ml/sklearn/base.py index 46dcb6623..3b1275ac3 100644 --- a/src/concrete/ml/sklearn/base.py +++ b/src/concrete/ml/sklearn/base.py @@ -102,10 +102,10 @@ # Most significant bits to retain when applying rounding to the tree MSB_TO_KEEP_FOR_TREES = 1 -# Enable rounding feature for all tree-based models by default +# Enable truncate feature for all tree-based models by default # Note: This setting is fixed and cannot be altered by users # However, for internal testing purposes, we retain the capability to disable this feature -os.environ["TREES_USE_ROUNDING"] = os.environ.get("TREES_USE_ROUNDING", "1") +os.environ["TREES_USE_TRUNCATE"] = os.environ.get("TREES_USE_TRUNCATE", "1") # pylint: disable=too-many-public-methods @@ -1326,8 +1326,8 @@ def fit(self, X: Data, y: Target, **fit_parameters): self.output_quantizers = [] #: Determines the LSB to remove given a `target_msbs` - self.auto_truncate = cp.AutoTruncator(target_msbs=MSB_TO_KEEP_FOR_TREES) - + self._auto_truncate = cp.AutoTruncator(target_msbs=MSB_TO_KEEP_FOR_TREES) + X, y = check_X_y_and_assert_multi_output(X, y) q_X = numpy.zeros_like(X) @@ -1353,25 +1353,23 @@ def fit(self, X: Data, y: Target, **fit_parameters): assert self.sklearn_model is not None, self._sklearn_model_is_not_fitted_error_message() # Enable optimized computation - enable_truncate = os.environ.get("TREES_USE_ROUNDING", "1") == "1" + enable_truncate = os.environ.get("TREES_USE_TRUNCATE", "1") == "1" if not enable_truncate: warnings.simplefilter("always") warnings.warn( - "Using Concrete tree-based models without the `rounding feature` is deprecated. " - "Consider setting 'use_rounding' to `True` for making the FHE inference faster " + "Using Concrete tree-based models without the `truncate feature` is deprecated. " + "Consider setting 'use_truncate' to `True` for making the FHE inference faster " "and key generation.", category=DeprecationWarning, stacklevel=2, ) - self.auto_truncate = None + self._auto_truncate = None - print(f"{self.auto_truncate=}") - self._tree_inference, self.output_quantizers, self.onnx_model_ = tree_to_numpy( self.sklearn_model, q_X, - auto_truncate=self.auto_truncate, + auto_truncate=self._auto_truncate, fhe_ensembling=self._fhe_ensembling, framework=self.framework, output_n_bits=self.n_bits["op_leaves"], @@ -1380,7 +1378,7 @@ def fit(self, X: Data, y: Target, **fit_parameters): # Adjust the truncate if enable_truncate: inputset = numpy.array(list(_get_inputset_generator(q_X))).astype(int) - self.auto_truncate.adjust(self._tree_inference, inputset) + self._auto_truncate.adjust(self._tree_inference, inputset) self._tree_inference(q_X.astype("int")) self._is_fitted = True diff --git a/src/concrete/ml/sklearn/tree_to_numpy.py b/src/concrete/ml/sklearn/tree_to_numpy.py index 3e088d321..b3b72f88b 100644 --- a/src/concrete/ml/sklearn/tree_to_numpy.py +++ b/src/concrete/ml/sklearn/tree_to_numpy.py @@ -1,7 +1,7 @@ """Implements the conversion of a tree model to a numpy function.""" import math import warnings -from typing import Callable, List, Optional, Tuple +from typing import Callable, List, Tuple import numpy import onnx @@ -330,7 +330,7 @@ def tree_to_numpy( model: Callable, x: numpy.ndarray, framework: str, - auto_truncate: bool = None, + auto_truncate=None, fhe_ensembling: bool = False, output_n_bits: int = MAX_BITWIDTH_BACKWARD_COMPATIBLE, ) -> Tuple[Callable, List[UniformQuantizer], onnx.ModelProto]: @@ -339,7 +339,7 @@ def tree_to_numpy( Args: model (Callable): The tree model to convert. x (numpy.ndarray): The input data. - auto_truncate (bool): Determines whether the rounding feature is enabled or disabled. + auto_truncate (TODO): Determines whether the rounding feature is enabled or disabled. Default to True. fhe_ensembling (bool): Determines whether the sum of the trees' outputs is computed in FHE. Default to False. @@ -365,15 +365,6 @@ def tree_to_numpy( # Execute with 1 example for efficiency in large data scenarios to prevent slowdown onnx_model = get_onnx_model(model, x[:1], framework) - # # Compute for tree-based models the LSB to remove in stage 1 and stage 2 - # if use_rounding: - # # First LSB refers to Less or LessOrEqual comparisons - # # Second LSB refers to Equal comparison - # lsbs_to_remove_for_trees = _compute_lsb_to_remove_for_trees(onnx_model, x) - - # # mypy - # assert len(lsbs_to_remove_for_trees) == 2 - # Get the expected number of ONNX outputs in the sklearn model. expected_number_of_outputs = 1 if is_regressor_or_partial_regressor(model) else 2 diff --git a/tests/sklearn/test_sklearn_models.py b/tests/sklearn/test_sklearn_models.py index 58423df78..d77a67cf9 100644 --- a/tests/sklearn/test_sklearn_models.py +++ b/tests/sklearn/test_sklearn_models.py @@ -1138,7 +1138,7 @@ def check_load_fitted_sklearn_linear_models(model_class, n_bits, x, y, check_flo ) -def check_rounding_consistency( +def check_truncate_consistency( model, x, y, @@ -1146,53 +1146,53 @@ def check_rounding_consistency( metric, is_weekly_option, ): - """Test that Concrete ML without and with rounding are 'equivalent'.""" + """Test that Concrete ML without and with truncate are 'equivalent'.""" # Run the test with more samples during weekly CIs if is_weekly_option: fhe_test = get_random_samples(x, n_sample=5) - # Check that rounding is enabled - assert os.environ.get("TREES_USE_ROUNDING") == "1", "'TREES_USE_ROUNDING' is not enabled" + # Check that truncate is enabled + assert os.environ.get("TREES_USE_TRUNCATE") == "1", "'TREES_USE_TRUNCATE' is not enabled" - # Fit and compile with rounding enabled + # Fit and compile with truncate enabled fit_and_compile(model, x, y) - rounded_predict_quantized = predict_method(x, fhe="disable") - rounded_predict_simulate = predict_method(x, fhe="simulate") + truncate_predict_quantized = predict_method(x, fhe="disable") + truncate_predict_simulate = predict_method(x, fhe="simulate") # Compute the FHE predictions only during weekly CIs if is_weekly_option: - rounded_predict_fhe = predict_method(fhe_test, fhe="execute") + truncate_predict_fhe = predict_method(fhe_test, fhe="execute") with pytest.MonkeyPatch.context() as mp_context: - # Disable rounding - mp_context.setenv("TREES_USE_ROUNDING", "0") + # Disable truncate + mp_context.setenv("TREES_USE_TRUNCATE", "0") - # Check that rounding is disabled - assert os.environ.get("TREES_USE_ROUNDING") == "0", "'TREES_USE_ROUNDING' is not disabled" + # Check that truncate is disabled + assert os.environ.get("TREES_USE_TRUNCATE") == "0", "'TREES_USE_TRUNCATE' is not disabled" with pytest.warns( DeprecationWarning, match=( - "Using Concrete tree-based models without the `rounding feature` is " "deprecated.*" + "Using Concrete tree-based models without the `truncate feature` is " "deprecated.*" ), ): - # Fit and compile without rounding + # Fit and compile without truncate fit_and_compile(model, x, y) - not_rounded_predict_quantized = predict_method(x, fhe="disable") - not_rounded_predict_simulate = predict_method(x, fhe="simulate") + not_truncate_predict_quantized = predict_method(x, fhe="disable") + not_truncate_predict_simulate = predict_method(x, fhe="simulate") - metric(rounded_predict_quantized, not_rounded_predict_quantized) - metric(rounded_predict_simulate, not_rounded_predict_simulate) + metric(truncate_predict_quantized, not_truncate_predict_quantized) + metric(truncate_predict_simulate, not_truncate_predict_simulate) # Compute the FHE predictions only during weekly CIs if is_weekly_option: - not_rounded_predict_fhe = predict_method(fhe_test, fhe="execute") - metric(rounded_predict_fhe, not_rounded_predict_fhe) + not_truncate_predict_fhe = predict_method(fhe_test, fhe="execute") + metric(truncate_predict_fhe, not_truncate_predict_fhe) # Check that the maximum bit-width of the circuit with rounding is at most: # maximum bit-width (of the circuit without rounding) + 2 @@ -1881,7 +1881,7 @@ def test_linear_models_have_no_tlu( # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4179 @pytest.mark.parametrize("model_class, parameters", get_sklearn_tree_models_and_datasets()) @pytest.mark.parametrize("n_bits", [2, 5, 10]) -def test_rounding_consistency_for_regular_models( +def test_truncate_consistency_for_regular_models( model_class, parameters, n_bits, @@ -1894,7 +1894,7 @@ def test_rounding_consistency_for_regular_models( """Test that Concrete ML without and with rounding are 'equivalent'.""" if verbose: - print("Run check_rounding_consistency") + print("Run check_truncate_consistency") model = instantiate_model_generic(model_class, n_bits=n_bits) @@ -1909,7 +1909,7 @@ def test_rounding_consistency_for_regular_models( predict_method = model.predict metric = check_accuracy - check_rounding_consistency( + check_truncate_consistency( model, x, y,
87.4\\% ± 1.2\\%82.4\\% ± 1.8\\%-0.0040.003-
FHE-RF90.9\\% ± 1.1\\%87.5\\% ± 1.6\\%87.5\\% ± 1.5\\%84.6\\% ± 1.7\\%750.0001.623
FHE-XGB96.8\\% ± 2.5\\%97.0\\% ± 2.4\\%--900.000
FHE-RF96.8\\% ± 1.3\\%96.9\\% ± 1.2\\%95.4\\% ± 1.8\\%93.5\\% ± 2.3\\%93.6\\% ± 2.2\\%700.0001.477576x93.9\\% ± 1.5\\%91.4\\% ± 2.3\\%-0.0020.003-