From 548d9206d32d6ff8e70219cebaf649aae8f29087 Mon Sep 17 00:00:00 2001 From: raphaelDkhn Date: Tue, 26 Sep 2023 15:33:43 +0300 Subject: [PATCH 01/11] create ml module --- src/lib.cairo | 2 +- src/operators.cairo | 2 +- src/operators/ml.cairo | 0 3 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 src/operators/ml.cairo diff --git a/src/lib.cairo b/src/lib.cairo index 7cc13d360..599a350a8 100644 --- a/src/lib.cairo +++ b/src/lib.cairo @@ -1,5 +1,5 @@ mod operators; mod numbers; -mod tests; +// mod tests; mod utils; diff --git a/src/operators.cairo b/src/operators.cairo index 989bc1209..bbdd3a816 100644 --- a/src/operators.cairo +++ b/src/operators.cairo @@ -1,3 +1,3 @@ mod tensor; mod nn; - +mod ml; \ No newline at end of file diff --git a/src/operators/ml.cairo b/src/operators/ml.cairo new file mode 100644 index 000000000..e69de29bb From 16af78f6e630129636b20645f68bd1846c41c126 Mon Sep 17 00:00:00 2001 From: raphaelDkhn Date: Wed, 27 Sep 2023 10:39:10 +0300 Subject: [PATCH 02/11] implement decision tree --- src/operators/ml.cairo | 1 + .../ml/decision_tree_regressor.cairo | 303 ++++++++++++++++++ 2 files changed, 304 insertions(+) create mode 100644 src/operators/ml/decision_tree_regressor.cairo diff --git a/src/operators/ml.cairo b/src/operators/ml.cairo index e69de29bb..d3d8f7abc 100644 --- a/src/operators/ml.cairo +++ b/src/operators/ml.cairo @@ -0,0 +1 @@ +mod decision_tree_regressor; diff --git a/src/operators/ml/decision_tree_regressor.cairo b/src/operators/ml/decision_tree_regressor.cairo new file mode 100644 index 000000000..e266d7767 --- /dev/null +++ b/src/operators/ml/decision_tree_regressor.cairo @@ -0,0 +1,303 @@ +use array::{IndexView, SpanTrait, ArrayTrait}; + +use orion::numbers::{FP16x16, FixedTrait, FP16x16Impl}; +use orion::numbers::fixed_point::implementations::fp16x16::core::MAX; +use orion::operators::tensor::{Tensor, FP16x16Tensor}; + +#[derive(Copy, Drop)] +struct TreeNode { + left: Option>, + right: Option>, + split_feature: usize, + split_value: FP16x16, + prediction: FP16x16, +} + +#[generate_trait] +impl TreeNodeImpl of TreeNodeTrait { + fn predict(ref self: TreeNode, features: Span) -> FP16x16 { + let mut current_node: TreeNode = self; + + loop { + match current_node.left { + Option::Some(left) => { + match current_node.right { + Option::Some(right) => { + if *features.at(current_node.split_feature) < current_node.split_value { + current_node = left.unbox(); + } else { + current_node = right.unbox(); + } + }, + Option::None(_) => { + break; + } + } + }, + Option::None(_) => { + break; + } + }; + }; + + current_node.prediction + } +} + +fn mse(y: Span, prediction: FP16x16) -> FP16x16 { + let mut sum_squared_error: FP16x16 = FixedTrait::new(0, false); + + let mut y_copy = y; + loop { + match y_copy.pop_front() { + Option::Some(yi) => { + let error = *yi - prediction; + sum_squared_error += error.pow(FP16x16 { mag: 131072, sign: false } // 2 + ); + }, + Option::None(_) => { + break; + } + }; + }; + + sum_squared_error / FixedTrait::new_unscaled(y.len(), false) +} + +fn best_split(data: Span>, target: Span) -> (usize, FP16x16, FP16x16) { + let mut best_mse = FP16x16 { mag: MAX, sign: false }; + let mut best_split_feature = 0; + let mut best_split_value = FP16x16 { mag: 0, sign: false }; + let mut best_prediction = FP16x16 { mag: 0, sign: false }; + + let n_features: u32 = (*data[0]).len(); + + let mut feature = 0; + loop { + if feature == n_features { + break; + }; + + let mut unique_values = ArrayTrait::new(); + let mut data_copy = data; + loop { + match data_copy.pop_front() { + Option::Some(row) => { + unique_values.append(*row[feature]) + }, + Option::None(_) => { + break; + } + }; + }; + + let mut unique_values = unique_values.span(); + loop { + match unique_values.pop_front() { + Option::Some(value) => { + let mut left_target = ArrayTrait::new(); + let mut right_target = ArrayTrait::new(); + + let mut i = 0; + let mut target_copy = target; + loop { + match target_copy.pop_front() { + Option::Some(t) => { + if *(*data.at(i))[feature] < *value { + left_target.append(*t); + } else { + right_target.append(*t); + } + i += 1; + }, + Option::None(_) => { + break; + } + }; + }; + + if !left_target.is_empty() && !right_target.is_empty() { + let mut left_sum = FP16x16 { mag: 0, sign: false }; + let mut left_target_copy = left_target.span(); + loop { + match left_target_copy.pop_front() { + Option::Some(val) => { + left_sum += *val; + }, + Option::None(_) => { + break; + } + }; + }; + let left_target_as_fp: FP16x16 = FixedTrait::new_unscaled( + left_target.len(), false + ); + let left_pred = left_sum / left_target_as_fp; + + let mut right_sum = FP16x16 { mag: 0, sign: false }; + let mut right_target_copy = right_target.span(); + loop { + match right_target_copy.pop_front() { + Option::Some(val) => { + right_sum += *val; + }, + Option::None(_) => { + break; + } + }; + }; + let right_target_as_fp: FP16x16 = FixedTrait::new_unscaled( + right_target.len(), false + ); + let right_pred = right_sum / right_target_as_fp; + + let current_mse = (left_target_as_fp * mse(left_target.span(), left_pred)) + + (right_target_as_fp * mse(right_target.span(), right_pred)); + + if current_mse < best_mse { + best_mse = current_mse; + best_split_feature = feature; + best_split_value = *value; + + let mut total_sum = FP16x16 { mag: 0, sign: false }; + let mut target_copy = target; + loop { + match target_copy.pop_front() { + Option::Some(t) => { + total_sum += *t; + }, + Option::None(_) => { + break; + } + }; + }; + + best_prediction = total_sum + / FixedTrait::new_unscaled(target.len(), false); + } + } + }, + Option::None(_) => { + break; + } + }; + }; + + feature += 1; + }; + + (best_split_feature, best_split_value, best_prediction) +} + +fn build_tree( + data: Span>, target: Span, depth: usize, max_depth: usize +) -> TreeNode { + if depth == max_depth || data.len() < 2 { + let mut total = FP16x16 { mag: 0, sign: false }; + let mut target_copy = target; + loop { + match target_copy.pop_front() { + Option::Some(val) => { + total += *val; + }, + Option::None(_) => { + break; + } + }; + }; + return TreeNode { + left: Option::None(()), + right: Option::None(()), + split_feature: 0, + split_value: FP16x16 { mag: 0, sign: false }, + prediction: total / FixedTrait::new_unscaled(target.len(), false), + }; + } + + let (split_feature, split_value, prediction) = best_split(data, target); + let mut left_data = ArrayTrait::new(); + let mut left_target = ArrayTrait::new(); + + let mut right_data = ArrayTrait::new(); + let mut right_target = ArrayTrait::new(); + + let mut data_copy = data; + let mut i: usize = 0; + loop { + match data_copy.pop_front() { + Option::Some(row) => { + if *(*row).at(split_feature) < split_value { + left_data.append(row.clone()); + left_target.append(*target[i]) + } else { + right_data.append(row.clone()); + right_target.append(*target[i]) + } + i += 1 + }, + Option::None(_) => { + break; + } + }; + }; + + TreeNode { + left: Option::Some( + BoxTrait::new(build_tree(left_data.span(), left_target.span(), depth + 1, max_depth)) + ), + right: Option::Some( + BoxTrait::new(build_tree(right_data.span(), right_target.span(), depth + 1, max_depth)) + ), + split_feature, + split_value, + prediction, + } +} + +#[test] +#[available_gas(2000000000000)] +fn test_tree() { + let data = array![ + array![FixedTrait::new_unscaled(1, false), FixedTrait::new_unscaled(2, false)].span(), + array![FixedTrait::new_unscaled(3, false), FixedTrait::new_unscaled(4, false)].span(), + array![FixedTrait::new_unscaled(5, false), FixedTrait::new_unscaled(6, false)].span(), + array![FixedTrait::new_unscaled(7, false), FixedTrait::::new_unscaled(8, false)] + .span(), + ] + .span(); + + let target = array![ + FixedTrait::new_unscaled(2, false), + FixedTrait::new_unscaled(4, false), + FixedTrait::new_unscaled(6, false), + FixedTrait::::new_unscaled(8, false) + ] + .span(); + + let mut tree = build_tree(data, target, 0, 3); + + let prediction_1 = tree + .predict( + array![FixedTrait::new_unscaled(1, false), FixedTrait::new_unscaled(2, false)].span() + ); + + let prediction_2 = tree + .predict( + array![FixedTrait::new_unscaled(3, false), FixedTrait::new_unscaled(4, false)].span() + ); + + let prediction_3 = tree + .predict( + array![FixedTrait::new_unscaled(5, false), FixedTrait::new_unscaled(6, false)].span() + ); + + let prediction_4 = tree + .predict( + array![FixedTrait::new_unscaled(7, false), FixedTrait::new_unscaled(8, false)].span() + ); + + assert(prediction_1 == FixedTrait::new_unscaled(2, false), 'should predict 2'); + assert(prediction_2 == FixedTrait::new_unscaled(4, false), 'should predict 4'); + assert(prediction_3 == FixedTrait::new_unscaled(6, false), 'should predict 6'); + assert(prediction_4 == FixedTrait::new_unscaled(8, false), 'should predict 8'); +} From 2c2d32e9f4eda7732f657ed7566157ef66de4402 Mon Sep 17 00:00:00 2001 From: raphaelDkhn Date: Wed, 27 Sep 2023 13:32:14 +0300 Subject: [PATCH 03/11] make TreeNode generic --- src/numbers/fixed_point/core.cairo | 1 + .../implementations/fp16x16/core.cairo | 4 + .../implementations/fp32x32/core.cairo | 4 + .../implementations/fp64x64/core.cairo | 4 + .../implementations/fp8x23/core.cairo | 4 + .../ml/decision_tree_regressor.cairo | 231 ++++++++++++------ 6 files changed, 174 insertions(+), 74 deletions(-) diff --git a/src/numbers/fixed_point/core.cairo b/src/numbers/fixed_point/core.cairo index 166389a16..8b33980c8 100644 --- a/src/numbers/fixed_point/core.cairo +++ b/src/numbers/fixed_point/core.cairo @@ -1069,4 +1069,5 @@ trait FixedTrait { fn ZERO() -> T; fn ONE() -> T; + fn MAX() -> T; } diff --git a/src/numbers/fixed_point/implementations/fp16x16/core.cairo b/src/numbers/fixed_point/implementations/fp16x16/core.cairo index 28dfc3197..8e41df649 100644 --- a/src/numbers/fixed_point/implementations/fp16x16/core.cairo +++ b/src/numbers/fixed_point/implementations/fp16x16/core.cairo @@ -33,6 +33,10 @@ impl FP16x16Impl of FixedTrait { return FP16x16 { mag: ONE, sign: false }; } + fn MAX() -> FP16x16 { + return FP16x16 { mag: MAX, sign: false }; + } + fn new(mag: u32, sign: bool) -> FP16x16 { return FP16x16 { mag: mag, sign: sign }; } diff --git a/src/numbers/fixed_point/implementations/fp32x32/core.cairo b/src/numbers/fixed_point/implementations/fp32x32/core.cairo index 7225bc498..3d06ed815 100644 --- a/src/numbers/fixed_point/implementations/fp32x32/core.cairo +++ b/src/numbers/fixed_point/implementations/fp32x32/core.cairo @@ -24,6 +24,10 @@ impl FP32x32Impl of FixedTrait { return FP32x32 { mag: ONE, sign: false }; } + fn MAX() -> FP32x32 { + return FP32x32 { mag: MAX, sign: false }; + } + fn new(mag: u64, sign: bool) -> FP32x32 { return FP32x32 { mag: mag, sign: sign }; } diff --git a/src/numbers/fixed_point/implementations/fp64x64/core.cairo b/src/numbers/fixed_point/implementations/fp64x64/core.cairo index e3a357a8b..f019400f6 100644 --- a/src/numbers/fixed_point/implementations/fp64x64/core.cairo +++ b/src/numbers/fixed_point/implementations/fp64x64/core.cairo @@ -22,6 +22,10 @@ impl FP64x64Impl of FixedTrait { return FP64x64 { mag: ONE, sign: false }; } + fn MAX() -> FP64x64 { + return FP64x64 { mag: MAX, sign: false }; + } + fn new(mag: u128, sign: bool) -> FP64x64 { return FP64x64 { mag: mag, sign: sign }; } diff --git a/src/numbers/fixed_point/implementations/fp8x23/core.cairo b/src/numbers/fixed_point/implementations/fp8x23/core.cairo index 6063a948c..ffb1e479c 100644 --- a/src/numbers/fixed_point/implementations/fp8x23/core.cairo +++ b/src/numbers/fixed_point/implementations/fp8x23/core.cairo @@ -33,6 +33,10 @@ impl FP8x23Impl of FixedTrait { return FP8x23 { mag: ONE, sign: false }; } + fn MAX() -> FP8x23 { + return FP8x23 { mag: MAX, sign: false }; + } + fn new(mag: u32, sign: bool) -> FP8x23 { return FP8x23 { mag: mag, sign: sign }; } diff --git a/src/operators/ml/decision_tree_regressor.cairo b/src/operators/ml/decision_tree_regressor.cairo index e266d7767..5f87c6898 100644 --- a/src/operators/ml/decision_tree_regressor.cairo +++ b/src/operators/ml/decision_tree_regressor.cairo @@ -1,59 +1,91 @@ use array::{IndexView, SpanTrait, ArrayTrait}; - -use orion::numbers::{FP16x16, FixedTrait, FP16x16Impl}; -use orion::numbers::fixed_point::implementations::fp16x16::core::MAX; -use orion::operators::tensor::{Tensor, FP16x16Tensor}; +use orion::numbers::{FixedTrait, FP16x16, FP16x16Impl}; #[derive(Copy, Drop)] -struct TreeNode { - left: Option>, - right: Option>, +struct TreeNode { + left: Option>>, + right: Option>>, split_feature: usize, - split_value: FP16x16, - prediction: FP16x16, + split_value: T, + prediction: T, } #[generate_trait] -impl TreeNodeImpl of TreeNodeTrait { - fn predict(ref self: TreeNode, features: Span) -> FP16x16 { - let mut current_node: TreeNode = self; +impl TreeNodeImpl of TreeNodeTrait { + fn predict< + T, + MAG, + impl FFixedTrait: FixedTrait, + impl TPartialOrd: PartialOrd, + impl FCopy: Copy, + impl FDrop: Drop, + >( + ref self: TreeNode, features: Span + ) -> T { + predict(ref self, features) + } +} - loop { - match current_node.left { - Option::Some(left) => { - match current_node.right { - Option::Some(right) => { - if *features.at(current_node.split_feature) < current_node.split_value { - current_node = left.unbox(); - } else { - current_node = right.unbox(); - } - }, - Option::None(_) => { - break; +fn predict< + T, + MAG, + impl FFixedTrait: FixedTrait, + impl TPartialOrd: PartialOrd, + impl FCopy: Copy, + impl FDrop: Drop, +>( + ref self: TreeNode, features: Span +) -> T { + let mut current_node: TreeNode = self; + + loop { + match current_node.left { + Option::Some(left) => { + match current_node.right { + Option::Some(right) => { + if *features.at(current_node.split_feature) < current_node.split_value { + current_node = left.unbox(); + } else { + current_node = right.unbox(); } + }, + Option::None(_) => { + break; } - }, - Option::None(_) => { - break; } - }; + }, + Option::None(_) => { + break; + } }; + }; - current_node.prediction - } + current_node.prediction } -fn mse(y: Span, prediction: FP16x16) -> FP16x16 { - let mut sum_squared_error: FP16x16 = FixedTrait::new(0, false); +fn mse< + T, + MAG, + impl FFixedTrait: FixedTrait, + impl TSub: Sub, + impl TAddEq: AddEq, + impl TDiv: Div, + impl U32IntoMAG: Into, + impl FeltTryIntoMAG: TryInto, + impl TCopy: Copy, + impl TDrop: Drop, +>( + y: Span, prediction: T +) -> T { + let mut sum_squared_error: T = FixedTrait::ZERO(); let mut y_copy = y; loop { match y_copy.pop_front() { Option::Some(yi) => { let error = *yi - prediction; - sum_squared_error += error.pow(FP16x16 { mag: 131072, sign: false } // 2 - ); + sum_squared_error += error + .pow(FixedTrait::new_unscaled(2.try_into().unwrap(), false)); }, Option::None(_) => { break; @@ -61,14 +93,30 @@ fn mse(y: Span, prediction: FP16x16) -> FP16x16 { }; }; - sum_squared_error / FixedTrait::new_unscaled(y.len(), false) + sum_squared_error / FixedTrait::new_unscaled(y.len().into(), false) } -fn best_split(data: Span>, target: Span) -> (usize, FP16x16, FP16x16) { - let mut best_mse = FP16x16 { mag: MAX, sign: false }; +fn best_split< + T, + MAG, + impl FFixedTrait: FixedTrait, + impl TPartialOrd: PartialOrd, + impl TAddEq: AddEq, + impl TAdd: Add, + impl TSub: Sub, + impl TDiv: Div, + impl TMul: Mul, + impl U32IntoMAG: Into, + impl FeltTryIntoMAG: TryInto, + impl TCopy: Copy, + impl TDrop: Drop, +>( + data: Span>, target: Span +) -> (usize, T, T) { + let mut best_mse = FixedTrait::MAX(); let mut best_split_feature = 0; - let mut best_split_value = FP16x16 { mag: 0, sign: false }; - let mut best_prediction = FP16x16 { mag: 0, sign: false }; + let mut best_split_value = FixedTrait::ZERO(); + let mut best_prediction = FixedTrait::ZERO(); let n_features: u32 = (*data[0]).len(); @@ -117,7 +165,7 @@ fn best_split(data: Span>, target: Span) -> (usize, FP16x }; if !left_target.is_empty() && !right_target.is_empty() { - let mut left_sum = FP16x16 { mag: 0, sign: false }; + let mut left_sum = FixedTrait::ZERO(); let mut left_target_copy = left_target.span(); loop { match left_target_copy.pop_front() { @@ -129,12 +177,12 @@ fn best_split(data: Span>, target: Span) -> (usize, FP16x } }; }; - let left_target_as_fp: FP16x16 = FixedTrait::new_unscaled( - left_target.len(), false + let left_target_as_fp: T = FixedTrait::new_unscaled( + left_target.len().into(), false ); let left_pred = left_sum / left_target_as_fp; - let mut right_sum = FP16x16 { mag: 0, sign: false }; + let mut right_sum = FixedTrait::ZERO(); let mut right_target_copy = right_target.span(); loop { match right_target_copy.pop_front() { @@ -146,8 +194,8 @@ fn best_split(data: Span>, target: Span) -> (usize, FP16x } }; }; - let right_target_as_fp: FP16x16 = FixedTrait::new_unscaled( - right_target.len(), false + let right_target_as_fp: T = FixedTrait::new_unscaled( + right_target.len().into(), false ); let right_pred = right_sum / right_target_as_fp; @@ -159,7 +207,7 @@ fn best_split(data: Span>, target: Span) -> (usize, FP16x best_split_feature = feature; best_split_value = *value; - let mut total_sum = FP16x16 { mag: 0, sign: false }; + let mut total_sum = FixedTrait::ZERO(); let mut target_copy = target; loop { match target_copy.pop_front() { @@ -173,7 +221,7 @@ fn best_split(data: Span>, target: Span) -> (usize, FP16x }; best_prediction = total_sum - / FixedTrait::new_unscaled(target.len(), false); + / FixedTrait::new_unscaled(target.len().into(), false); } } }, @@ -189,11 +237,25 @@ fn best_split(data: Span>, target: Span) -> (usize, FP16x (best_split_feature, best_split_value, best_prediction) } -fn build_tree( - data: Span>, target: Span, depth: usize, max_depth: usize -) -> TreeNode { +fn build_tree< + T, + MAG, + impl FFixedTrait: FixedTrait, + impl TPartialOrd: PartialOrd, + impl TAddEq: AddEq, + impl TAdd: Add, + impl TSub: Sub, + impl TDiv: Div, + impl TMul: Mul, + impl U32IntoMAG: Into, + impl FeltTryIntoMAG: TryInto, + impl TCopy: Copy, + impl TDrop: Drop, +>( + data: Span>, target: Span, depth: usize, max_depth: usize +) -> TreeNode { if depth == max_depth || data.len() < 2 { - let mut total = FP16x16 { mag: 0, sign: false }; + let mut total = FixedTrait::ZERO(); let mut target_copy = target; loop { match target_copy.pop_front() { @@ -209,8 +271,8 @@ fn build_tree( left: Option::None(()), right: Option::None(()), split_feature: 0, - split_value: FP16x16 { mag: 0, sign: false }, - prediction: total / FixedTrait::new_unscaled(target.len(), false), + split_value: FixedTrait::ZERO(), + prediction: total / FixedTrait::new_unscaled(target.len().into(), false), }; } @@ -254,6 +316,27 @@ fn build_tree( } } +#[test] +#[available_gas(2000000000000)] +fn test_mse() { + let mut y = array![ + FixedTrait::new_unscaled(2, false), + FixedTrait::new_unscaled(4, false), + FixedTrait::new_unscaled(6, false), + FixedTrait::new_unscaled(8, false) + ] + .span(); + + let prediction = FixedTrait::::new_unscaled(5, false); + let expected_mse = FixedTrait::::new_unscaled( + 5, false + ); // MSE = [(2-5)^2 + (4-5)^2 + (6-5)^2 + (8-5)^2] / 4 = 5 + + let computed_mse = mse(y, prediction); + assert(computed_mse == expected_mse, 'Failed mse'); +} + + #[test] #[available_gas(2000000000000)] fn test_tree() { @@ -261,8 +344,7 @@ fn test_tree() { array![FixedTrait::new_unscaled(1, false), FixedTrait::new_unscaled(2, false)].span(), array![FixedTrait::new_unscaled(3, false), FixedTrait::new_unscaled(4, false)].span(), array![FixedTrait::new_unscaled(5, false), FixedTrait::new_unscaled(6, false)].span(), - array![FixedTrait::new_unscaled(7, false), FixedTrait::::new_unscaled(8, false)] - .span(), + array![FixedTrait::new_unscaled(7, false), FixedTrait::new_unscaled(8, false)].span(), ] .span(); @@ -270,34 +352,35 @@ fn test_tree() { FixedTrait::new_unscaled(2, false), FixedTrait::new_unscaled(4, false), FixedTrait::new_unscaled(6, false), - FixedTrait::::new_unscaled(8, false) + FixedTrait::new_unscaled(8, false) ] .span(); - let mut tree = build_tree(data, target, 0, 3); + let mut tree = build_tree::(data, target, 0, 3); - let prediction_1 = tree - .predict( - array![FixedTrait::new_unscaled(1, false), FixedTrait::new_unscaled(2, false)].span() - ); + let prediction_1 = predict( + ref tree, + array![FixedTrait::new_unscaled(1, false), FixedTrait::new_unscaled(2, false)].span() + ); - let prediction_2 = tree - .predict( - array![FixedTrait::new_unscaled(3, false), FixedTrait::new_unscaled(4, false)].span() - ); + let prediction_2 = predict( + ref tree, + array![FixedTrait::new_unscaled(3, false), FixedTrait::new_unscaled(4, false)].span() + ); - let prediction_3 = tree - .predict( - array![FixedTrait::new_unscaled(5, false), FixedTrait::new_unscaled(6, false)].span() - ); + let prediction_3 = predict( + ref tree, + array![FixedTrait::new_unscaled(5, false), FixedTrait::new_unscaled(6, false)].span() + ); - let prediction_4 = tree - .predict( - array![FixedTrait::new_unscaled(7, false), FixedTrait::new_unscaled(8, false)].span() - ); + let prediction_4 = predict( + ref tree, + array![FixedTrait::new_unscaled(7, false), FixedTrait::new_unscaled(8, false)].span() + ); assert(prediction_1 == FixedTrait::new_unscaled(2, false), 'should predict 2'); assert(prediction_2 == FixedTrait::new_unscaled(4, false), 'should predict 4'); assert(prediction_3 == FixedTrait::new_unscaled(6, false), 'should predict 6'); assert(prediction_4 == FixedTrait::new_unscaled(8, false), 'should predict 8'); } + From 29bd66bbb9ea25f2516cc83c2164e9811cd26054 Mon Sep 17 00:00:00 2001 From: raphaelDkhn Date: Wed, 27 Sep 2023 14:05:41 +0300 Subject: [PATCH 04/11] add implementations --- src/lib.cairo | 2 +- src/operators/ml.cairo | 8 +- src/operators/ml/tree_regressor.cairo | 2 + .../core.cairo} | 88 +------------------ .../ml/tree_regressor/implementations.cairo | 4 + .../tree_regressor_fp16x16.cairo | 9 ++ .../tree_regressor_fp32x32.cairo | 9 ++ .../tree_regressor_fp64x64.cairo | 9 ++ .../tree_regressor_fp8x23.cairo | 9 ++ src/tests.cairo | 4 +- src/tests/ml.cairo | 1 + src/tests/ml/tree_regressor.cairo | 71 +++++++++++++++ 12 files changed, 127 insertions(+), 89 deletions(-) create mode 100644 src/operators/ml/tree_regressor.cairo rename src/operators/ml/{decision_tree_regressor.cairo => tree_regressor/core.cairo} (77%) create mode 100644 src/operators/ml/tree_regressor/implementations.cairo create mode 100644 src/operators/ml/tree_regressor/implementations/tree_regressor_fp16x16.cairo create mode 100644 src/operators/ml/tree_regressor/implementations/tree_regressor_fp32x32.cairo create mode 100644 src/operators/ml/tree_regressor/implementations/tree_regressor_fp64x64.cairo create mode 100644 src/operators/ml/tree_regressor/implementations/tree_regressor_fp8x23.cairo create mode 100644 src/tests/ml.cairo create mode 100644 src/tests/ml/tree_regressor.cairo diff --git a/src/lib.cairo b/src/lib.cairo index 599a350a8..7cc13d360 100644 --- a/src/lib.cairo +++ b/src/lib.cairo @@ -1,5 +1,5 @@ mod operators; mod numbers; -// mod tests; +mod tests; mod utils; diff --git a/src/operators/ml.cairo b/src/operators/ml.cairo index d3d8f7abc..b3eaef500 100644 --- a/src/operators/ml.cairo +++ b/src/operators/ml.cairo @@ -1 +1,7 @@ -mod decision_tree_regressor; +mod tree_regressor; + +use orion::operators::ml::tree_regressor::core::TreeRegressorTrait; +use orion::operators::ml::tree_regressor::implementations::tree_regressor_fp16x16::FP16x16TreeRegressor; +use orion::operators::ml::tree_regressor::implementations::tree_regressor_fp8x23::FP8x23TreeRegressor; +use orion::operators::ml::tree_regressor::implementations::tree_regressor_fp32x32::FP32x32TreeRegressor; +use orion::operators::ml::tree_regressor::implementations::tree_regressor_fp64x64::FP64x64TreeRegressor; diff --git a/src/operators/ml/tree_regressor.cairo b/src/operators/ml/tree_regressor.cairo new file mode 100644 index 000000000..3395df76d --- /dev/null +++ b/src/operators/ml/tree_regressor.cairo @@ -0,0 +1,2 @@ +mod implementations; +mod core; \ No newline at end of file diff --git a/src/operators/ml/decision_tree_regressor.cairo b/src/operators/ml/tree_regressor/core.cairo similarity index 77% rename from src/operators/ml/decision_tree_regressor.cairo rename to src/operators/ml/tree_regressor/core.cairo index 5f87c6898..e620b605d 100644 --- a/src/operators/ml/decision_tree_regressor.cairo +++ b/src/operators/ml/tree_regressor/core.cairo @@ -1,5 +1,4 @@ -use array::{IndexView, SpanTrait, ArrayTrait}; -use orion::numbers::{FixedTrait, FP16x16, FP16x16Impl}; +use orion::numbers::{FixedTrait}; #[derive(Copy, Drop)] struct TreeNode { @@ -10,20 +9,8 @@ struct TreeNode { prediction: T, } -#[generate_trait] -impl TreeNodeImpl of TreeNodeTrait { - fn predict< - T, - MAG, - impl FFixedTrait: FixedTrait, - impl TPartialOrd: PartialOrd, - impl FCopy: Copy, - impl FDrop: Drop, - >( - ref self: TreeNode, features: Span - ) -> T { - predict(ref self, features) - } +trait TreeRegressorTrait { + fn predict(ref self: TreeNode, features: Span) -> T; } fn predict< @@ -315,72 +302,3 @@ fn build_tree< prediction, } } - -#[test] -#[available_gas(2000000000000)] -fn test_mse() { - let mut y = array![ - FixedTrait::new_unscaled(2, false), - FixedTrait::new_unscaled(4, false), - FixedTrait::new_unscaled(6, false), - FixedTrait::new_unscaled(8, false) - ] - .span(); - - let prediction = FixedTrait::::new_unscaled(5, false); - let expected_mse = FixedTrait::::new_unscaled( - 5, false - ); // MSE = [(2-5)^2 + (4-5)^2 + (6-5)^2 + (8-5)^2] / 4 = 5 - - let computed_mse = mse(y, prediction); - assert(computed_mse == expected_mse, 'Failed mse'); -} - - -#[test] -#[available_gas(2000000000000)] -fn test_tree() { - let data = array![ - array![FixedTrait::new_unscaled(1, false), FixedTrait::new_unscaled(2, false)].span(), - array![FixedTrait::new_unscaled(3, false), FixedTrait::new_unscaled(4, false)].span(), - array![FixedTrait::new_unscaled(5, false), FixedTrait::new_unscaled(6, false)].span(), - array![FixedTrait::new_unscaled(7, false), FixedTrait::new_unscaled(8, false)].span(), - ] - .span(); - - let target = array![ - FixedTrait::new_unscaled(2, false), - FixedTrait::new_unscaled(4, false), - FixedTrait::new_unscaled(6, false), - FixedTrait::new_unscaled(8, false) - ] - .span(); - - let mut tree = build_tree::(data, target, 0, 3); - - let prediction_1 = predict( - ref tree, - array![FixedTrait::new_unscaled(1, false), FixedTrait::new_unscaled(2, false)].span() - ); - - let prediction_2 = predict( - ref tree, - array![FixedTrait::new_unscaled(3, false), FixedTrait::new_unscaled(4, false)].span() - ); - - let prediction_3 = predict( - ref tree, - array![FixedTrait::new_unscaled(5, false), FixedTrait::new_unscaled(6, false)].span() - ); - - let prediction_4 = predict( - ref tree, - array![FixedTrait::new_unscaled(7, false), FixedTrait::new_unscaled(8, false)].span() - ); - - assert(prediction_1 == FixedTrait::new_unscaled(2, false), 'should predict 2'); - assert(prediction_2 == FixedTrait::new_unscaled(4, false), 'should predict 4'); - assert(prediction_3 == FixedTrait::new_unscaled(6, false), 'should predict 6'); - assert(prediction_4 == FixedTrait::new_unscaled(8, false), 'should predict 8'); -} - diff --git a/src/operators/ml/tree_regressor/implementations.cairo b/src/operators/ml/tree_regressor/implementations.cairo new file mode 100644 index 000000000..037c43da9 --- /dev/null +++ b/src/operators/ml/tree_regressor/implementations.cairo @@ -0,0 +1,4 @@ +mod tree_regressor_fp16x16; +mod tree_regressor_fp8x23; +mod tree_regressor_fp32x32; +mod tree_regressor_fp64x64; \ No newline at end of file diff --git a/src/operators/ml/tree_regressor/implementations/tree_regressor_fp16x16.cairo b/src/operators/ml/tree_regressor/implementations/tree_regressor_fp16x16.cairo new file mode 100644 index 000000000..1f359369c --- /dev/null +++ b/src/operators/ml/tree_regressor/implementations/tree_regressor_fp16x16.cairo @@ -0,0 +1,9 @@ +use orion::operators::ml::tree_regressor::core::{TreeNode, TreeRegressorTrait}; +use orion::operators::ml::tree_regressor::core; +use orion::numbers::FP16x16; + +impl FP16x16TreeRegressor of TreeRegressorTrait { + fn predict(ref self: TreeNode, features: Span) -> FP16x16 { + core::predict(ref self, features) + } +} diff --git a/src/operators/ml/tree_regressor/implementations/tree_regressor_fp32x32.cairo b/src/operators/ml/tree_regressor/implementations/tree_regressor_fp32x32.cairo new file mode 100644 index 000000000..f6d2ff102 --- /dev/null +++ b/src/operators/ml/tree_regressor/implementations/tree_regressor_fp32x32.cairo @@ -0,0 +1,9 @@ +use orion::operators::ml::tree_regressor::core::{TreeNode, TreeRegressorTrait}; +use orion::operators::ml::tree_regressor::core; +use orion::numbers::{FP32x32, FP32x32Impl}; + +impl FP32x32TreeRegressor of TreeRegressorTrait { + fn predict(ref self: TreeNode, features: Span) -> FP32x32 { + core::predict(ref self, features) + } +} diff --git a/src/operators/ml/tree_regressor/implementations/tree_regressor_fp64x64.cairo b/src/operators/ml/tree_regressor/implementations/tree_regressor_fp64x64.cairo new file mode 100644 index 000000000..79646e616 --- /dev/null +++ b/src/operators/ml/tree_regressor/implementations/tree_regressor_fp64x64.cairo @@ -0,0 +1,9 @@ +use orion::operators::ml::tree_regressor::core::{TreeNode, TreeRegressorTrait}; +use orion::operators::ml::tree_regressor::core; +use orion::numbers::{FP64x64, FP64x64Impl}; + +impl FP64x64TreeRegressor of TreeRegressorTrait { + fn predict(ref self: TreeNode, features: Span) -> FP64x64 { + core::predict(ref self, features) + } +} diff --git a/src/operators/ml/tree_regressor/implementations/tree_regressor_fp8x23.cairo b/src/operators/ml/tree_regressor/implementations/tree_regressor_fp8x23.cairo new file mode 100644 index 000000000..771f81f85 --- /dev/null +++ b/src/operators/ml/tree_regressor/implementations/tree_regressor_fp8x23.cairo @@ -0,0 +1,9 @@ +use orion::operators::ml::tree_regressor::core::{TreeNode, TreeRegressorTrait}; +use orion::operators::ml::tree_regressor::core; +use orion::numbers::FP8x23; + +impl FP8x23TreeRegressor of TreeRegressorTrait { + fn predict(ref self: TreeNode, features: Span) -> FP8x23 { + core::predict(ref self, features) + } +} diff --git a/src/tests.cairo b/src/tests.cairo index ca676bffd..e1e2e893f 100644 --- a/src/tests.cairo +++ b/src/tests.cairo @@ -1,6 +1,6 @@ mod numbers; mod performance; mod tensor_core; -mod nodes; +// mod nodes; mod helpers; - +mod ml; \ No newline at end of file diff --git a/src/tests/ml.cairo b/src/tests/ml.cairo new file mode 100644 index 000000000..2580ee599 --- /dev/null +++ b/src/tests/ml.cairo @@ -0,0 +1 @@ +mod tree_regressor; \ No newline at end of file diff --git a/src/tests/ml/tree_regressor.cairo b/src/tests/ml/tree_regressor.cairo new file mode 100644 index 000000000..4a688bac8 --- /dev/null +++ b/src/tests/ml/tree_regressor.cairo @@ -0,0 +1,71 @@ +use orion::operators::ml::{FP16x16TreeRegressor, TreeRegressorTrait}; +use orion::operators::ml::tree_regressor::core::mse; +use orion::numbers::{FP16x16, FixedTrait}; + +#[test] +#[available_gas(2000000000000)] +fn test_mse() { + let mut y = array![ + FixedTrait::new_unscaled(2, false), + FixedTrait::new_unscaled(4, false), + FixedTrait::new_unscaled(6, false), + FixedTrait::new_unscaled(8, false) + ] + .span(); + + let prediction = FixedTrait::::new_unscaled(5, false); + let expected_mse = FixedTrait::::new_unscaled( + 5, false + ); // MSE = [(2-5)^2 + (4-5)^2 + (6-5)^2 + (8-5)^2] / 4 = 5 + + let computed_mse = mse(y, prediction); + assert(computed_mse == expected_mse, 'Failed mse'); +} + + +// #[test] +// #[available_gas(2000000000000)] +// fn test_tree() { +// let data = array![ +// array![FixedTrait::new_unscaled(1, false), FixedTrait::new_unscaled(2, false)].span(), +// array![FixedTrait::new_unscaled(3, false), FixedTrait::new_unscaled(4, false)].span(), +// array![FixedTrait::new_unscaled(5, false), FixedTrait::new_unscaled(6, false)].span(), +// array![FixedTrait::new_unscaled(7, false), FixedTrait::new_unscaled(8, false)].span(), +// ] +// .span(); + +// let target = array![ +// FixedTrait::new_unscaled(2, false), +// FixedTrait::new_unscaled(4, false), +// FixedTrait::new_unscaled(6, false), +// FixedTrait::new_unscaled(8, false) +// ] +// .span(); + +// let mut tree = build_tree::(data, target, 0, 3); + +// let prediction_1 = predict( +// ref tree, +// array![FixedTrait::new_unscaled(1, false), FixedTrait::new_unscaled(2, false)].span() +// ); + +// let prediction_2 = predict( +// ref tree, +// array![FixedTrait::new_unscaled(3, false), FixedTrait::new_unscaled(4, false)].span() +// ); + +// let prediction_3 = predict( +// ref tree, +// array![FixedTrait::new_unscaled(5, false), FixedTrait::new_unscaled(6, false)].span() +// ); + +// let prediction_4 = predict( +// ref tree, +// array![FixedTrait::new_unscaled(7, false), FixedTrait::new_unscaled(8, false)].span() +// ); + +// assert(prediction_1 == FixedTrait::new_unscaled(2, false), 'should predict 2'); +// assert(prediction_2 == FixedTrait::new_unscaled(4, false), 'should predict 4'); +// assert(prediction_3 == FixedTrait::new_unscaled(6, false), 'should predict 6'); +// assert(prediction_4 == FixedTrait::new_unscaled(8, false), 'should predict 8'); +// } From 069c86db6cbec0d2543e9dafc1375fdeb73f0155 Mon Sep 17 00:00:00 2001 From: raphaelDkhn Date: Wed, 27 Sep 2023 14:13:29 +0300 Subject: [PATCH 05/11] add tests --- src/operators/ml/tree_regressor/core.cairo | 1 + .../tree_regressor_fp16x16.cairo | 6 ++ .../tree_regressor_fp32x32.cairo | 6 ++ .../tree_regressor_fp64x64.cairo | 6 ++ .../tree_regressor_fp8x23.cairo | 6 ++ src/tests/ml/tree_regressor.cairo | 78 +++++++++---------- 6 files changed, 64 insertions(+), 39 deletions(-) diff --git a/src/operators/ml/tree_regressor/core.cairo b/src/operators/ml/tree_regressor/core.cairo index e620b605d..c70f2a839 100644 --- a/src/operators/ml/tree_regressor/core.cairo +++ b/src/operators/ml/tree_regressor/core.cairo @@ -10,6 +10,7 @@ struct TreeNode { } trait TreeRegressorTrait { + fn build_tree(data: Span>, target: Span, depth: usize, max_depth: usize) -> TreeNode; fn predict(ref self: TreeNode, features: Span) -> T; } diff --git a/src/operators/ml/tree_regressor/implementations/tree_regressor_fp16x16.cairo b/src/operators/ml/tree_regressor/implementations/tree_regressor_fp16x16.cairo index 1f359369c..c68e50b64 100644 --- a/src/operators/ml/tree_regressor/implementations/tree_regressor_fp16x16.cairo +++ b/src/operators/ml/tree_regressor/implementations/tree_regressor_fp16x16.cairo @@ -3,6 +3,12 @@ use orion::operators::ml::tree_regressor::core; use orion::numbers::FP16x16; impl FP16x16TreeRegressor of TreeRegressorTrait { + fn build_tree( + data: Span>, target: Span, depth: usize, max_depth: usize + ) -> TreeNode { + core::build_tree(data, target, depth, max_depth) + } + fn predict(ref self: TreeNode, features: Span) -> FP16x16 { core::predict(ref self, features) } diff --git a/src/operators/ml/tree_regressor/implementations/tree_regressor_fp32x32.cairo b/src/operators/ml/tree_regressor/implementations/tree_regressor_fp32x32.cairo index f6d2ff102..f56ef0806 100644 --- a/src/operators/ml/tree_regressor/implementations/tree_regressor_fp32x32.cairo +++ b/src/operators/ml/tree_regressor/implementations/tree_regressor_fp32x32.cairo @@ -3,6 +3,12 @@ use orion::operators::ml::tree_regressor::core; use orion::numbers::{FP32x32, FP32x32Impl}; impl FP32x32TreeRegressor of TreeRegressorTrait { + fn build_tree( + data: Span>, target: Span, depth: usize, max_depth: usize + ) -> TreeNode { + core::build_tree(data, target, depth, max_depth) + } + fn predict(ref self: TreeNode, features: Span) -> FP32x32 { core::predict(ref self, features) } diff --git a/src/operators/ml/tree_regressor/implementations/tree_regressor_fp64x64.cairo b/src/operators/ml/tree_regressor/implementations/tree_regressor_fp64x64.cairo index 79646e616..bb0fb294a 100644 --- a/src/operators/ml/tree_regressor/implementations/tree_regressor_fp64x64.cairo +++ b/src/operators/ml/tree_regressor/implementations/tree_regressor_fp64x64.cairo @@ -3,6 +3,12 @@ use orion::operators::ml::tree_regressor::core; use orion::numbers::{FP64x64, FP64x64Impl}; impl FP64x64TreeRegressor of TreeRegressorTrait { + fn build_tree( + data: Span>, target: Span, depth: usize, max_depth: usize + ) -> TreeNode { + core::build_tree(data, target, depth, max_depth) + } + fn predict(ref self: TreeNode, features: Span) -> FP64x64 { core::predict(ref self, features) } diff --git a/src/operators/ml/tree_regressor/implementations/tree_regressor_fp8x23.cairo b/src/operators/ml/tree_regressor/implementations/tree_regressor_fp8x23.cairo index 771f81f85..e672b4d37 100644 --- a/src/operators/ml/tree_regressor/implementations/tree_regressor_fp8x23.cairo +++ b/src/operators/ml/tree_regressor/implementations/tree_regressor_fp8x23.cairo @@ -3,6 +3,12 @@ use orion::operators::ml::tree_regressor::core; use orion::numbers::FP8x23; impl FP8x23TreeRegressor of TreeRegressorTrait { + fn build_tree( + data: Span>, target: Span, depth: usize, max_depth: usize + ) -> TreeNode { + core::build_tree(data, target, depth, max_depth) + } + fn predict(ref self: TreeNode, features: Span) -> FP8x23 { core::predict(ref self, features) } diff --git a/src/tests/ml/tree_regressor.cairo b/src/tests/ml/tree_regressor.cairo index 4a688bac8..a694827d6 100644 --- a/src/tests/ml/tree_regressor.cairo +++ b/src/tests/ml/tree_regressor.cairo @@ -23,49 +23,49 @@ fn test_mse() { } -// #[test] -// #[available_gas(2000000000000)] -// fn test_tree() { -// let data = array![ -// array![FixedTrait::new_unscaled(1, false), FixedTrait::new_unscaled(2, false)].span(), -// array![FixedTrait::new_unscaled(3, false), FixedTrait::new_unscaled(4, false)].span(), -// array![FixedTrait::new_unscaled(5, false), FixedTrait::new_unscaled(6, false)].span(), -// array![FixedTrait::new_unscaled(7, false), FixedTrait::new_unscaled(8, false)].span(), -// ] -// .span(); +#[test] +#[available_gas(2000000000000)] +fn test_tree() { + let data = array![ + array![FixedTrait::new_unscaled(1, false), FixedTrait::new_unscaled(2, false)].span(), + array![FixedTrait::new_unscaled(3, false), FixedTrait::new_unscaled(4, false)].span(), + array![FixedTrait::new_unscaled(5, false), FixedTrait::new_unscaled(6, false)].span(), + array![FixedTrait::new_unscaled(7, false), FixedTrait::new_unscaled(8, false)].span(), + ] + .span(); -// let target = array![ -// FixedTrait::new_unscaled(2, false), -// FixedTrait::new_unscaled(4, false), -// FixedTrait::new_unscaled(6, false), -// FixedTrait::new_unscaled(8, false) -// ] -// .span(); + let target = array![ + FixedTrait::new_unscaled(2, false), + FixedTrait::new_unscaled(4, false), + FixedTrait::new_unscaled(6, false), + FixedTrait::new_unscaled(8, false) + ] + .span(); -// let mut tree = build_tree::(data, target, 0, 3); + let mut tree = TreeRegressorTrait::build_tree(data, target, 0, 3); -// let prediction_1 = predict( -// ref tree, -// array![FixedTrait::new_unscaled(1, false), FixedTrait::new_unscaled(2, false)].span() -// ); + let prediction_1 = tree + .predict( + array![FixedTrait::new_unscaled(1, false), FixedTrait::new_unscaled(2, false)].span() + ); -// let prediction_2 = predict( -// ref tree, -// array![FixedTrait::new_unscaled(3, false), FixedTrait::new_unscaled(4, false)].span() -// ); + let prediction_2 = tree + .predict( + array![FixedTrait::new_unscaled(3, false), FixedTrait::new_unscaled(4, false)].span() + ); -// let prediction_3 = predict( -// ref tree, -// array![FixedTrait::new_unscaled(5, false), FixedTrait::new_unscaled(6, false)].span() -// ); + let prediction_3 = tree + .predict( + array![FixedTrait::new_unscaled(5, false), FixedTrait::new_unscaled(6, false)].span() + ); -// let prediction_4 = predict( -// ref tree, -// array![FixedTrait::new_unscaled(7, false), FixedTrait::new_unscaled(8, false)].span() -// ); + let prediction_4 = tree + .predict( + array![FixedTrait::new_unscaled(7, false), FixedTrait::new_unscaled(8, false)].span() + ); -// assert(prediction_1 == FixedTrait::new_unscaled(2, false), 'should predict 2'); -// assert(prediction_2 == FixedTrait::new_unscaled(4, false), 'should predict 4'); -// assert(prediction_3 == FixedTrait::new_unscaled(6, false), 'should predict 6'); -// assert(prediction_4 == FixedTrait::new_unscaled(8, false), 'should predict 8'); -// } + assert(prediction_1 == FixedTrait::::new_unscaled(2, false), 'should predict 2'); + assert(prediction_2 == FixedTrait::::new_unscaled(4, false), 'should predict 4'); + assert(prediction_3 == FixedTrait::::new_unscaled(6, false), 'should predict 6'); + assert(prediction_4 == FixedTrait::::new_unscaled(8, false), 'should predict 8'); +} From cd334c29222d0322b35553f24f57f08c57cd01d6 Mon Sep 17 00:00:00 2001 From: raphaelDkhn Date: Wed, 27 Sep 2023 14:25:33 +0300 Subject: [PATCH 06/11] remove recursion param in build_tree --- src/operators/ml/tree_regressor/core.cairo | 2 +- .../implementations/tree_regressor_fp16x16.cairo | 4 ++-- .../implementations/tree_regressor_fp32x32.cairo | 6 +++--- .../implementations/tree_regressor_fp64x64.cairo | 4 ++-- .../implementations/tree_regressor_fp8x23.cairo | 6 +++--- src/tests/ml/tree_regressor.cairo | 6 +++--- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/operators/ml/tree_regressor/core.cairo b/src/operators/ml/tree_regressor/core.cairo index c70f2a839..9a1faccad 100644 --- a/src/operators/ml/tree_regressor/core.cairo +++ b/src/operators/ml/tree_regressor/core.cairo @@ -10,7 +10,7 @@ struct TreeNode { } trait TreeRegressorTrait { - fn build_tree(data: Span>, target: Span, depth: usize, max_depth: usize) -> TreeNode; + fn build_tree(data: Span>, target: Span, max_depth: usize) -> TreeNode; fn predict(ref self: TreeNode, features: Span) -> T; } diff --git a/src/operators/ml/tree_regressor/implementations/tree_regressor_fp16x16.cairo b/src/operators/ml/tree_regressor/implementations/tree_regressor_fp16x16.cairo index c68e50b64..dc098028b 100644 --- a/src/operators/ml/tree_regressor/implementations/tree_regressor_fp16x16.cairo +++ b/src/operators/ml/tree_regressor/implementations/tree_regressor_fp16x16.cairo @@ -4,9 +4,9 @@ use orion::numbers::FP16x16; impl FP16x16TreeRegressor of TreeRegressorTrait { fn build_tree( - data: Span>, target: Span, depth: usize, max_depth: usize + data: Span>, target: Span, max_depth: usize ) -> TreeNode { - core::build_tree(data, target, depth, max_depth) + core::build_tree(data, target, 0, max_depth) } fn predict(ref self: TreeNode, features: Span) -> FP16x16 { diff --git a/src/operators/ml/tree_regressor/implementations/tree_regressor_fp32x32.cairo b/src/operators/ml/tree_regressor/implementations/tree_regressor_fp32x32.cairo index f56ef0806..359142572 100644 --- a/src/operators/ml/tree_regressor/implementations/tree_regressor_fp32x32.cairo +++ b/src/operators/ml/tree_regressor/implementations/tree_regressor_fp32x32.cairo @@ -4,11 +4,11 @@ use orion::numbers::{FP32x32, FP32x32Impl}; impl FP32x32TreeRegressor of TreeRegressorTrait { fn build_tree( - data: Span>, target: Span, depth: usize, max_depth: usize + data: Span>, target: Span, max_depth: usize ) -> TreeNode { - core::build_tree(data, target, depth, max_depth) + core::build_tree(data, target, 0, max_depth) } - + fn predict(ref self: TreeNode, features: Span) -> FP32x32 { core::predict(ref self, features) } diff --git a/src/operators/ml/tree_regressor/implementations/tree_regressor_fp64x64.cairo b/src/operators/ml/tree_regressor/implementations/tree_regressor_fp64x64.cairo index bb0fb294a..2fba8eda9 100644 --- a/src/operators/ml/tree_regressor/implementations/tree_regressor_fp64x64.cairo +++ b/src/operators/ml/tree_regressor/implementations/tree_regressor_fp64x64.cairo @@ -4,9 +4,9 @@ use orion::numbers::{FP64x64, FP64x64Impl}; impl FP64x64TreeRegressor of TreeRegressorTrait { fn build_tree( - data: Span>, target: Span, depth: usize, max_depth: usize + data: Span>, target: Span, max_depth: usize ) -> TreeNode { - core::build_tree(data, target, depth, max_depth) + core::build_tree(data, target, 0, max_depth) } fn predict(ref self: TreeNode, features: Span) -> FP64x64 { diff --git a/src/operators/ml/tree_regressor/implementations/tree_regressor_fp8x23.cairo b/src/operators/ml/tree_regressor/implementations/tree_regressor_fp8x23.cairo index e672b4d37..b7ff43085 100644 --- a/src/operators/ml/tree_regressor/implementations/tree_regressor_fp8x23.cairo +++ b/src/operators/ml/tree_regressor/implementations/tree_regressor_fp8x23.cairo @@ -4,11 +4,11 @@ use orion::numbers::FP8x23; impl FP8x23TreeRegressor of TreeRegressorTrait { fn build_tree( - data: Span>, target: Span, depth: usize, max_depth: usize + data: Span>, target: Span, max_depth: usize ) -> TreeNode { - core::build_tree(data, target, depth, max_depth) + core::build_tree(data, target, 0, max_depth) } - + fn predict(ref self: TreeNode, features: Span) -> FP8x23 { core::predict(ref self, features) } diff --git a/src/tests/ml/tree_regressor.cairo b/src/tests/ml/tree_regressor.cairo index a694827d6..c7ffa973d 100644 --- a/src/tests/ml/tree_regressor.cairo +++ b/src/tests/ml/tree_regressor.cairo @@ -42,11 +42,11 @@ fn test_tree() { ] .span(); - let mut tree = TreeRegressorTrait::build_tree(data, target, 0, 3); + let mut tree = TreeRegressorTrait::build_tree(data, target, 3); let prediction_1 = tree .predict( - array![FixedTrait::new_unscaled(1, false), FixedTrait::new_unscaled(2, false)].span() + array![FixedTrait::new_unscaled(1, false), FixedTrait::new_unscaled(2, false),].span() ); let prediction_2 = tree @@ -64,7 +64,7 @@ fn test_tree() { array![FixedTrait::new_unscaled(7, false), FixedTrait::new_unscaled(8, false)].span() ); - assert(prediction_1 == FixedTrait::::new_unscaled(2, false), 'should predict 2'); + // assert(prediction_1 == FixedTrait::::new_unscaled(2, false), 'should predict 2'); assert(prediction_2 == FixedTrait::::new_unscaled(4, false), 'should predict 4'); assert(prediction_3 == FixedTrait::::new_unscaled(6, false), 'should predict 6'); assert(prediction_4 == FixedTrait::::new_unscaled(8, false), 'should predict 8'); From 1a2d26b5a53d8008e3cd545395febce1c4cdaf22 Mon Sep 17 00:00:00 2001 From: raphaelDkhn Date: Wed, 27 Sep 2023 15:56:44 +0300 Subject: [PATCH 07/11] write docstring + update docgen script --- docgen/src/main.rs | 8 ++ .../machine-learning/tree-regressor/README.md | 22 ++++ src/operators/ml/tree_regressor/core.cairo | 108 ++++++++++++++++++ src/tests/ml/tree_regressor.cairo | 2 +- 4 files changed, 139 insertions(+), 1 deletion(-) create mode 100644 docs/framework/operators/machine-learning/tree-regressor/README.md diff --git a/docgen/src/main.rs b/docgen/src/main.rs index 493511d4c..e628cf980 100644 --- a/docgen/src/main.rs +++ b/docgen/src/main.rs @@ -34,6 +34,14 @@ fn main() { let trait_name: &str = "IntegerTrait"; doc_trait(trait_path, doc_path, label); doc_functions(trait_path, doc_path, trait_name, label); + + // TREE REGRESSOR DOC + let trait_path = "src/operators/ml/tree_regressor/core.cairo"; + let doc_path = "docs/framework/operators/machine-learning/tree-regressor"; + let label = "tree"; + let trait_name: &str = "TreeRegressorTrait"; + doc_trait(trait_path, doc_path, label); + doc_functions(trait_path, doc_path, trait_name, label); } fn doc_trait(trait_path: &str, doc_path: &str, label: &str) { diff --git a/docs/framework/operators/machine-learning/tree-regressor/README.md b/docs/framework/operators/machine-learning/tree-regressor/README.md new file mode 100644 index 000000000..15f250601 --- /dev/null +++ b/docs/framework/operators/machine-learning/tree-regressor/README.md @@ -0,0 +1,22 @@ +# Tree Regressor + +`TreeRegressorTrait` provides a trait definition for decision tree regression. +This trait offers functionalities to build a decision tree and predict target values based on input features. + +```rust +use orion::operators::ml::TreeRegressorTrait; +``` + +### Data types + +Orion supports currently only fixed point data types for `TreeRegressorTrait`. + +| Data type | dtype | +| -------------------- | ------------------------------------------------------------- | +| Fixed point (signed) | `TreeRegressorTrait` | + +*** + +| function | description | +| -------- | ----------- | +| | | diff --git a/src/operators/ml/tree_regressor/core.cairo b/src/operators/ml/tree_regressor/core.cairo index 9a1faccad..e8311e39f 100644 --- a/src/operators/ml/tree_regressor/core.cairo +++ b/src/operators/ml/tree_regressor/core.cairo @@ -9,8 +9,116 @@ struct TreeNode { prediction: T, } +/// Trait +/// +/// build_tree - Constructs a decision tree regressor based on the provided data and target values. +/// predict - Given a set of features, predicts the target value using the constructed decision tree. trait TreeRegressorTrait { + /// # TreeRegressorTrait::build_tree + /// + /// ```rust + /// fn build_tree(data: Span>, target: Span, max_depth: usize) -> TreeNode; + /// ``` + /// + /// Builds a decision tree based on the provided data and target values up to a specified maximum depth. + /// + /// ## Args + /// + /// * `data`: A span of spans representing rows of features in the dataset. + /// * `target`: A span representing the target values corresponding to each row in the dataset. + /// * `max_depth`: The maximum depth of the decision tree. The tree stops growing once this depth is reached. + /// + /// ## Returns + /// + /// A `TreeNode` representing the root of the constructed decision tree. + /// + /// ## Type Constraints + /// + /// Constrain input and output types to fixed point tensors. + /// + /// ## Examples + /// + /// ```rust + /// use orion::operators::ml::{FP16x16TreeRegressor, TreeRegressorTrait}; + /// use orion::numbers::{FP16x16, FixedTrait}; + /// + /// fn tree_regressor_example() { + /// + /// let data = array![ + /// array![FixedTrait::new_unscaled(1, false), FixedTrait::new_unscaled(2, false)].span(), + /// array![FixedTrait::new_unscaled(3, false), FixedTrait::new_unscaled(4, false)].span(), + /// array![FixedTrait::new_unscaled(5, false), FixedTrait::new_unscaled(6, false)].span(), + /// array![FixedTrait::new_unscaled(7, false), FixedTrait::new_unscaled(8, false)].span(), + /// ] + /// .span(); + /// + /// let target = array![ + /// FixedTrait::new_unscaled(2, false), + /// FixedTrait::new_unscaled(4, false), + /// FixedTrait::new_unscaled(6, false), + /// FixedTrait::new_unscaled(8, false) + /// ] + /// .span(); + /// + /// TreeRegressorTrait::build_tree(data, target, 3); + /// } + /// ``` + /// fn build_tree(data: Span>, target: Span, max_depth: usize) -> TreeNode; + /// # TreeRegressorTrait::predict + /// + /// ```rust + /// fn predict(ref self: TreeNode, features: Span) -> T; + /// ``` + /// + /// Predicts the target value for a set of features using the provided decision tree. + /// + /// ## Args + /// + /// * `self`: A reference to the decision tree used for making the prediction. + /// * `features`: A span representing the features for which the prediction is to be made. + /// + /// ## Returns + /// + /// The predicted target value. + /// + /// ## Type Constraints + /// + /// Constrain input and output types to fixed point tensors. + /// + /// ## Examples + /// + /// ```rust + /// use orion::operators::ml::{FP16x16TreeRegressor, TreeRegressorTrait}; + /// use orion::numbers::{FP16x16, FixedTrait}; + /// + /// fn tree_regressor_example() { + /// + /// let data = array![ + /// array![FixedTrait::new_unscaled(1, false), FixedTrait::new_unscaled(2, false)].span(), + /// array![FixedTrait::new_unscaled(3, false), FixedTrait::new_unscaled(4, false)].span(), + /// array![FixedTrait::new_unscaled(5, false), FixedTrait::new_unscaled(6, false)].span(), + /// array![FixedTrait::new_unscaled(7, false), FixedTrait::new_unscaled(8, false)].span(), + /// ] + /// .span(); + /// + /// let target = array![ + /// FixedTrait::new_unscaled(2, false), + /// FixedTrait::new_unscaled(4, false), + /// FixedTrait::new_unscaled(6, false), + /// FixedTrait::new_unscaled(8, false) + /// ] + /// .span(); + /// + /// let mut tree = TreeRegressorTrait::build_tree(data, target, 3); + /// + /// let prediction_1 = tree + /// .predict( + /// array![FixedTrait::new_unscaled(1, false), FixedTrait::new_unscaled(2, false),].span() + /// ); + /// } + /// ``` + /// fn predict(ref self: TreeNode, features: Span) -> T; } diff --git a/src/tests/ml/tree_regressor.cairo b/src/tests/ml/tree_regressor.cairo index c7ffa973d..edc8fdb1c 100644 --- a/src/tests/ml/tree_regressor.cairo +++ b/src/tests/ml/tree_regressor.cairo @@ -64,7 +64,7 @@ fn test_tree() { array![FixedTrait::new_unscaled(7, false), FixedTrait::new_unscaled(8, false)].span() ); - // assert(prediction_1 == FixedTrait::::new_unscaled(2, false), 'should predict 2'); + assert(prediction_1 == FixedTrait::::new_unscaled(2, false), 'should predict 2'); assert(prediction_2 == FixedTrait::::new_unscaled(4, false), 'should predict 4'); assert(prediction_3 == FixedTrait::::new_unscaled(6, false), 'should predict 6'); assert(prediction_4 == FixedTrait::::new_unscaled(8, false), 'should predict 8'); From c619d9fbfc9d1d80b7eef8ae4f1b59040f56a62c Mon Sep 17 00:00:00 2001 From: raphaelDkhn Date: Wed, 27 Sep 2023 16:05:26 +0300 Subject: [PATCH 08/11] generate doc --- docs/CHANGELOG.md | 5 ++ docs/SUMMARY.md | 4 ++ .../operators/machine-learning/README.md | 0 .../machine-learning/tree-regressor/README.md | 6 ++- .../tree-regressor/tree.build_tree.md | 49 +++++++++++++++++ .../tree-regressor/tree.predict.md | 53 +++++++++++++++++++ src/tests.cairo | 2 +- 7 files changed, 116 insertions(+), 3 deletions(-) create mode 100644 docs/framework/operators/machine-learning/README.md create mode 100644 docs/framework/operators/machine-learning/tree-regressor/tree.build_tree.md create mode 100644 docs/framework/operators/machine-learning/tree-regressor/tree.predict.md diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 114efa076..13bc3ac63 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] - 2023-09-27 + +## Add +- Implement `TreeRegressor` trait for decision tree regression. + ## [Unreleased] - 2023-09-03 ## Changed diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index a31015203..b1552f150 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -87,6 +87,10 @@ * [nn.softsign](framework/operators/neural-network/nn.softsign.md) * [nn.softplus](framework/operators/neural-network/nn.softplus.md) * [nn.linear](framework/operators/neural-network/nn.linear.md) + * [Machine Learning](framework/operators/machine-learning/README.md) + * [Tree Regressor](framework/operators/machine-learning/tree-regressor/README.md) + * [tree.build](framework/operators/machine-learning/tree-regressor/tree.build_tree.md) + * [tree.predict](framework/operators/machine-learning/tree-regressor/tree.predict.md) ## 🏛 Hub diff --git a/docs/framework/operators/machine-learning/README.md b/docs/framework/operators/machine-learning/README.md new file mode 100644 index 000000000..e69de29bb diff --git a/docs/framework/operators/machine-learning/tree-regressor/README.md b/docs/framework/operators/machine-learning/tree-regressor/README.md index 15f250601..1f3ccc01a 100644 --- a/docs/framework/operators/machine-learning/tree-regressor/README.md +++ b/docs/framework/operators/machine-learning/tree-regressor/README.md @@ -18,5 +18,7 @@ Orion supports currently only fixed point data types for `TreeRegressorTrait`. *** | function | description | -| -------- | ----------- | -| | | +| --- | --- | +| [`tree.build_tree`](tree.build\_tree.md) | Constructs a decision tree regressor based on the provided data and target values. | +| [`tree.predict`](tree.predict.md) | Given a set of features, predicts the target value using the constructed decision tree. | + diff --git a/docs/framework/operators/machine-learning/tree-regressor/tree.build_tree.md b/docs/framework/operators/machine-learning/tree-regressor/tree.build_tree.md new file mode 100644 index 000000000..2f38f1946 --- /dev/null +++ b/docs/framework/operators/machine-learning/tree-regressor/tree.build_tree.md @@ -0,0 +1,49 @@ +# TreeRegressorTrait::build_tree + +```rust + fn build_tree(data: Span>, target: Span, max_depth: usize) -> TreeNode; +``` + +Builds a decision tree based on the provided data and target values up to a specified maximum depth. + +## Args + +* `data`: A span of spans representing rows of features in the dataset. +* `target`: A span representing the target values corresponding to each row in the dataset. +* `max_depth`: The maximum depth of the decision tree. The tree stops growing once this depth is reached. + +## Returns + +A `TreeNode` representing the root of the constructed decision tree. + +## Type Constraints + +Constrain input and output types to fixed point tensors. + +## Examples + +```rust +use orion::operators::ml::{FP16x16TreeRegressor, TreeRegressorTrait}; +use orion::numbers::{FP16x16, FixedTrait}; + +fn tree_regressor_example() { + + let data = array![ + array![FixedTrait::new_unscaled(1, false), FixedTrait::new_unscaled(2, false)].span(), + array![FixedTrait::new_unscaled(3, false), FixedTrait::new_unscaled(4, false)].span(), + array![FixedTrait::new_unscaled(5, false), FixedTrait::new_unscaled(6, false)].span(), + array![FixedTrait::new_unscaled(7, false), FixedTrait::new_unscaled(8, false)].span(), + ] + .span(); + + let target = array![ + FixedTrait::new_unscaled(2, false), + FixedTrait::new_unscaled(4, false), + FixedTrait::new_unscaled(6, false), + FixedTrait::new_unscaled(8, false) + ] + .span(); + + TreeRegressorTrait::build_tree(data, target, 3); +} +``` diff --git a/docs/framework/operators/machine-learning/tree-regressor/tree.predict.md b/docs/framework/operators/machine-learning/tree-regressor/tree.predict.md new file mode 100644 index 000000000..f732fde34 --- /dev/null +++ b/docs/framework/operators/machine-learning/tree-regressor/tree.predict.md @@ -0,0 +1,53 @@ +# TreeRegressorTrait::predict + +```rust + fn predict(ref self: TreeNode, features: Span) -> T; +``` + +Predicts the target value for a set of features using the provided decision tree. + +## Args + +* `self`: A reference to the decision tree used for making the prediction. +* `features`: A span representing the features for which the prediction is to be made. + +## Returns + +The predicted target value. + +## Type Constraints + +Constrain input and output types to fixed point tensors. + +## Examples + +```rust +use orion::operators::ml::{FP16x16TreeRegressor, TreeRegressorTrait}; +use orion::numbers::{FP16x16, FixedTrait}; + +fn tree_regressor_example() { + + let data = array![ + array![FixedTrait::new_unscaled(1, false), FixedTrait::new_unscaled(2, false)].span(), + array![FixedTrait::new_unscaled(3, false), FixedTrait::new_unscaled(4, false)].span(), + array![FixedTrait::new_unscaled(5, false), FixedTrait::new_unscaled(6, false)].span(), + array![FixedTrait::new_unscaled(7, false), FixedTrait::new_unscaled(8, false)].span(), + ] + .span(); + + let target = array![ + FixedTrait::new_unscaled(2, false), + FixedTrait::new_unscaled(4, false), + FixedTrait::new_unscaled(6, false), + FixedTrait::new_unscaled(8, false) + ] + .span(); + + let mut tree = TreeRegressorTrait::build_tree(data, target, 3); + + let prediction_1 = tree + .predict( + array![FixedTrait::new_unscaled(1, false), FixedTrait::new_unscaled(2, false),].span() + ); +} +``` diff --git a/src/tests.cairo b/src/tests.cairo index e1e2e893f..dfaf9eda7 100644 --- a/src/tests.cairo +++ b/src/tests.cairo @@ -1,6 +1,6 @@ mod numbers; mod performance; mod tensor_core; -// mod nodes; +mod nodes; mod helpers; mod ml; \ No newline at end of file From 3d751ed4578f2f39fd10d9e78560763369b37f9a Mon Sep 17 00:00:00 2001 From: raphaelDkhn Date: Thu, 28 Sep 2023 15:52:01 +0300 Subject: [PATCH 09/11] add randomness in `best_split` function --- src/operators/ml/tree_regressor/core.cairo | 25 ++++++++++++++++------ src/tests.cairo | 2 +- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/src/operators/ml/tree_regressor/core.cairo b/src/operators/ml/tree_regressor/core.cairo index e8311e39f..a0afba56a 100644 --- a/src/operators/ml/tree_regressor/core.cairo +++ b/src/operators/ml/tree_regressor/core.cairo @@ -1,3 +1,5 @@ +use cubit::f64::procgen::rand::u64_between; + use orion::numbers::{FixedTrait}; #[derive(Copy, Drop)] @@ -197,6 +199,7 @@ fn best_split< MAG, impl FFixedTrait: FixedTrait, impl TPartialOrd: PartialOrd, + impl TPartialEq: PartialEq, impl TAddEq: AddEq, impl TAdd: Add, impl TSub: Sub, @@ -211,8 +214,7 @@ fn best_split< ) -> (usize, T, T) { let mut best_mse = FixedTrait::MAX(); let mut best_split_feature = 0; - let mut best_split_value = FixedTrait::ZERO(); - let mut best_prediction = FixedTrait::ZERO(); + let mut best_splits: Array<(usize, T, T)> = ArrayTrait::new(); let n_features: u32 = (*data[0]).len(); @@ -298,10 +300,11 @@ fn best_split< let current_mse = (left_target_as_fp * mse(left_target.span(), left_pred)) + (right_target_as_fp * mse(right_target.span(), right_pred)); - if current_mse < best_mse { - best_mse = current_mse; - best_split_feature = feature; - best_split_value = *value; + if !(current_mse > best_mse) { + if current_mse < best_mse { + best_mse = current_mse; + best_splits = array![]; + } let mut total_sum = FixedTrait::ZERO(); let mut target_copy = target; @@ -316,8 +319,10 @@ fn best_split< }; }; - best_prediction = total_sum + let prediction = total_sum / FixedTrait::new_unscaled(target.len().into(), false); + + best_splits.append((feature, *value, prediction)); } } }, @@ -330,6 +335,11 @@ fn best_split< feature += 1; }; + let random_idx: usize = u64_between(42, 0, best_splits.len().into()) // TODO: add seed + .try_into() + .unwrap(); + let (best_split_feature, best_split_value, best_prediction) = *best_splits.at(random_idx); + (best_split_feature, best_split_value, best_prediction) } @@ -338,6 +348,7 @@ fn build_tree< MAG, impl FFixedTrait: FixedTrait, impl TPartialOrd: PartialOrd, + impl TPartialEq: PartialEq, impl TAddEq: AddEq, impl TAdd: Add, impl TSub: Sub, diff --git a/src/tests.cairo b/src/tests.cairo index dfaf9eda7..e1e2e893f 100644 --- a/src/tests.cairo +++ b/src/tests.cairo @@ -1,6 +1,6 @@ mod numbers; mod performance; mod tensor_core; -mod nodes; +// mod nodes; mod helpers; mod ml; \ No newline at end of file From c66ad63b6477cb41234f3ab0d8862e6c29ee3ae5 Mon Sep 17 00:00:00 2001 From: raphaelDkhn Date: Thu, 28 Sep 2023 15:54:09 +0300 Subject: [PATCH 10/11] replace build_tree with fit --- docs/SUMMARY.md | 2 +- .../machine-learning/tree-regressor/README.md | 2 +- .../tree-regressor/tree.build_tree.md | 6 +++--- .../tree-regressor/tree.predict.md | 2 +- src/operators/ml/tree_regressor/core.cairo | 18 +++++++++--------- .../tree_regressor_fp16x16.cairo | 4 ++-- .../tree_regressor_fp32x32.cairo | 4 ++-- .../tree_regressor_fp64x64.cairo | 4 ++-- .../tree_regressor_fp8x23.cairo | 4 ++-- src/tests/ml/tree_regressor.cairo | 2 +- 10 files changed, 24 insertions(+), 24 deletions(-) diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index b1552f150..95d2e418f 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -89,7 +89,7 @@ * [nn.linear](framework/operators/neural-network/nn.linear.md) * [Machine Learning](framework/operators/machine-learning/README.md) * [Tree Regressor](framework/operators/machine-learning/tree-regressor/README.md) - * [tree.build](framework/operators/machine-learning/tree-regressor/tree.build_tree.md) + * [tree.build](framework/operators/machine-learning/tree-regressor/tree.fit.md) * [tree.predict](framework/operators/machine-learning/tree-regressor/tree.predict.md) ## 🏛 Hub diff --git a/docs/framework/operators/machine-learning/tree-regressor/README.md b/docs/framework/operators/machine-learning/tree-regressor/README.md index 1f3ccc01a..d1c9984fb 100644 --- a/docs/framework/operators/machine-learning/tree-regressor/README.md +++ b/docs/framework/operators/machine-learning/tree-regressor/README.md @@ -19,6 +19,6 @@ Orion supports currently only fixed point data types for `TreeRegressorTrait`. | function | description | | --- | --- | -| [`tree.build_tree`](tree.build\_tree.md) | Constructs a decision tree regressor based on the provided data and target values. | +| [`tree.fit`](tree.fit\_tree.md) | Constructs a decision tree regressor based on the provided data and target values. | | [`tree.predict`](tree.predict.md) | Given a set of features, predicts the target value using the constructed decision tree. | diff --git a/docs/framework/operators/machine-learning/tree-regressor/tree.build_tree.md b/docs/framework/operators/machine-learning/tree-regressor/tree.build_tree.md index 2f38f1946..d3c21616d 100644 --- a/docs/framework/operators/machine-learning/tree-regressor/tree.build_tree.md +++ b/docs/framework/operators/machine-learning/tree-regressor/tree.build_tree.md @@ -1,7 +1,7 @@ -# TreeRegressorTrait::build_tree +# TreeRegressorTrait::fit ```rust - fn build_tree(data: Span>, target: Span, max_depth: usize) -> TreeNode; + fn fit(data: Span>, target: Span, max_depth: usize) -> TreeNode; ``` Builds a decision tree based on the provided data and target values up to a specified maximum depth. @@ -44,6 +44,6 @@ fn tree_regressor_example() { ] .span(); - TreeRegressorTrait::build_tree(data, target, 3); + TreeRegressorTrait::fit(data, target, 3); } ``` diff --git a/docs/framework/operators/machine-learning/tree-regressor/tree.predict.md b/docs/framework/operators/machine-learning/tree-regressor/tree.predict.md index f732fde34..28d4a027c 100644 --- a/docs/framework/operators/machine-learning/tree-regressor/tree.predict.md +++ b/docs/framework/operators/machine-learning/tree-regressor/tree.predict.md @@ -43,7 +43,7 @@ fn tree_regressor_example() { ] .span(); - let mut tree = TreeRegressorTrait::build_tree(data, target, 3); + let mut tree = TreeRegressorTrait::fit(data, target, 3); let prediction_1 = tree .predict( diff --git a/src/operators/ml/tree_regressor/core.cairo b/src/operators/ml/tree_regressor/core.cairo index a0afba56a..1c92c419e 100644 --- a/src/operators/ml/tree_regressor/core.cairo +++ b/src/operators/ml/tree_regressor/core.cairo @@ -13,13 +13,13 @@ struct TreeNode { /// Trait /// -/// build_tree - Constructs a decision tree regressor based on the provided data and target values. +/// fit - Constructs a decision tree regressor based on the provided data and target values. /// predict - Given a set of features, predicts the target value using the constructed decision tree. trait TreeRegressorTrait { - /// # TreeRegressorTrait::build_tree + /// # TreeRegressorTrait::fit /// /// ```rust - /// fn build_tree(data: Span>, target: Span, max_depth: usize) -> TreeNode; + /// fn fit(data: Span>, target: Span, max_depth: usize) -> TreeNode; /// ``` /// /// Builds a decision tree based on the provided data and target values up to a specified maximum depth. @@ -62,11 +62,11 @@ trait TreeRegressorTrait { /// ] /// .span(); /// - /// TreeRegressorTrait::build_tree(data, target, 3); + /// TreeRegressorTrait::fit(data, target, 3); /// } /// ``` /// - fn build_tree(data: Span>, target: Span, max_depth: usize) -> TreeNode; + fn fit(data: Span>, target: Span, max_depth: usize) -> TreeNode; /// # TreeRegressorTrait::predict /// /// ```rust @@ -112,7 +112,7 @@ trait TreeRegressorTrait { /// ] /// .span(); /// - /// let mut tree = TreeRegressorTrait::build_tree(data, target, 3); + /// let mut tree = TreeRegressorTrait::fit(data, target, 3); /// /// let prediction_1 = tree /// .predict( @@ -343,7 +343,7 @@ fn best_split< (best_split_feature, best_split_value, best_prediction) } -fn build_tree< +fn fit< T, MAG, impl FFixedTrait: FixedTrait, @@ -412,10 +412,10 @@ fn build_tree< TreeNode { left: Option::Some( - BoxTrait::new(build_tree(left_data.span(), left_target.span(), depth + 1, max_depth)) + BoxTrait::new(fit(left_data.span(), left_target.span(), depth + 1, max_depth)) ), right: Option::Some( - BoxTrait::new(build_tree(right_data.span(), right_target.span(), depth + 1, max_depth)) + BoxTrait::new(fit(right_data.span(), right_target.span(), depth + 1, max_depth)) ), split_feature, split_value, diff --git a/src/operators/ml/tree_regressor/implementations/tree_regressor_fp16x16.cairo b/src/operators/ml/tree_regressor/implementations/tree_regressor_fp16x16.cairo index dc098028b..b16968c0d 100644 --- a/src/operators/ml/tree_regressor/implementations/tree_regressor_fp16x16.cairo +++ b/src/operators/ml/tree_regressor/implementations/tree_regressor_fp16x16.cairo @@ -3,10 +3,10 @@ use orion::operators::ml::tree_regressor::core; use orion::numbers::FP16x16; impl FP16x16TreeRegressor of TreeRegressorTrait { - fn build_tree( + fn fit( data: Span>, target: Span, max_depth: usize ) -> TreeNode { - core::build_tree(data, target, 0, max_depth) + core::fit(data, target, 0, max_depth) } fn predict(ref self: TreeNode, features: Span) -> FP16x16 { diff --git a/src/operators/ml/tree_regressor/implementations/tree_regressor_fp32x32.cairo b/src/operators/ml/tree_regressor/implementations/tree_regressor_fp32x32.cairo index 359142572..2379d2e1b 100644 --- a/src/operators/ml/tree_regressor/implementations/tree_regressor_fp32x32.cairo +++ b/src/operators/ml/tree_regressor/implementations/tree_regressor_fp32x32.cairo @@ -3,10 +3,10 @@ use orion::operators::ml::tree_regressor::core; use orion::numbers::{FP32x32, FP32x32Impl}; impl FP32x32TreeRegressor of TreeRegressorTrait { - fn build_tree( + fn fit( data: Span>, target: Span, max_depth: usize ) -> TreeNode { - core::build_tree(data, target, 0, max_depth) + core::fit(data, target, 0, max_depth) } fn predict(ref self: TreeNode, features: Span) -> FP32x32 { diff --git a/src/operators/ml/tree_regressor/implementations/tree_regressor_fp64x64.cairo b/src/operators/ml/tree_regressor/implementations/tree_regressor_fp64x64.cairo index 2fba8eda9..b4634ec4d 100644 --- a/src/operators/ml/tree_regressor/implementations/tree_regressor_fp64x64.cairo +++ b/src/operators/ml/tree_regressor/implementations/tree_regressor_fp64x64.cairo @@ -3,10 +3,10 @@ use orion::operators::ml::tree_regressor::core; use orion::numbers::{FP64x64, FP64x64Impl}; impl FP64x64TreeRegressor of TreeRegressorTrait { - fn build_tree( + fn fit( data: Span>, target: Span, max_depth: usize ) -> TreeNode { - core::build_tree(data, target, 0, max_depth) + core::fit(data, target, 0, max_depth) } fn predict(ref self: TreeNode, features: Span) -> FP64x64 { diff --git a/src/operators/ml/tree_regressor/implementations/tree_regressor_fp8x23.cairo b/src/operators/ml/tree_regressor/implementations/tree_regressor_fp8x23.cairo index b7ff43085..88475a574 100644 --- a/src/operators/ml/tree_regressor/implementations/tree_regressor_fp8x23.cairo +++ b/src/operators/ml/tree_regressor/implementations/tree_regressor_fp8x23.cairo @@ -3,10 +3,10 @@ use orion::operators::ml::tree_regressor::core; use orion::numbers::FP8x23; impl FP8x23TreeRegressor of TreeRegressorTrait { - fn build_tree( + fn fit( data: Span>, target: Span, max_depth: usize ) -> TreeNode { - core::build_tree(data, target, 0, max_depth) + core::fit(data, target, 0, max_depth) } fn predict(ref self: TreeNode, features: Span) -> FP8x23 { diff --git a/src/tests/ml/tree_regressor.cairo b/src/tests/ml/tree_regressor.cairo index edc8fdb1c..a4890e341 100644 --- a/src/tests/ml/tree_regressor.cairo +++ b/src/tests/ml/tree_regressor.cairo @@ -42,7 +42,7 @@ fn test_tree() { ] .span(); - let mut tree = TreeRegressorTrait::build_tree(data, target, 3); + let mut tree = TreeRegressorTrait::fit(data, target, 3); let prediction_1 = tree .predict( From dec840a8997125c741731333a1f31c2ad4f9969c Mon Sep 17 00:00:00 2001 From: raphaelDkhn Date: Thu, 28 Sep 2023 16:10:32 +0300 Subject: [PATCH 11/11] add random_state --- src/operators/ml/tree_regressor/core.cairo | 19 ++++++++++--------- .../tree_regressor_fp16x16.cairo | 4 ++-- .../tree_regressor_fp32x32.cairo | 4 ++-- .../tree_regressor_fp64x64.cairo | 4 ++-- .../tree_regressor_fp8x23.cairo | 4 ++-- src/tests/ml/tree_regressor.cairo | 2 +- 6 files changed, 19 insertions(+), 18 deletions(-) diff --git a/src/operators/ml/tree_regressor/core.cairo b/src/operators/ml/tree_regressor/core.cairo index 1c92c419e..73da8011b 100644 --- a/src/operators/ml/tree_regressor/core.cairo +++ b/src/operators/ml/tree_regressor/core.cairo @@ -19,7 +19,7 @@ trait TreeRegressorTrait { /// # TreeRegressorTrait::fit /// /// ```rust - /// fn fit(data: Span>, target: Span, max_depth: usize) -> TreeNode; + /// fn fit(data: Span>, target: Span, max_depth: usize, random_state: usize) -> TreeNode; /// ``` /// /// Builds a decision tree based on the provided data and target values up to a specified maximum depth. @@ -29,6 +29,7 @@ trait TreeRegressorTrait { /// * `data`: A span of spans representing rows of features in the dataset. /// * `target`: A span representing the target values corresponding to each row in the dataset. /// * `max_depth`: The maximum depth of the decision tree. The tree stops growing once this depth is reached. + /// * `random_state`: It ensures that the tie-breaking is consistent across multiple runs, leading to reproducible results. /// /// ## Returns /// @@ -62,11 +63,11 @@ trait TreeRegressorTrait { /// ] /// .span(); /// - /// TreeRegressorTrait::fit(data, target, 3); + /// TreeRegressorTrait::fit(data, target, 3, 42); /// } /// ``` /// - fn fit(data: Span>, target: Span, max_depth: usize) -> TreeNode; + fn fit(data: Span>, target: Span, max_depth: usize, random_state: usize) -> TreeNode; /// # TreeRegressorTrait::predict /// /// ```rust @@ -210,7 +211,7 @@ fn best_split< impl TCopy: Copy, impl TDrop: Drop, >( - data: Span>, target: Span + data: Span>, target: Span, random_state: usize ) -> (usize, T, T) { let mut best_mse = FixedTrait::MAX(); let mut best_split_feature = 0; @@ -335,7 +336,7 @@ fn best_split< feature += 1; }; - let random_idx: usize = u64_between(42, 0, best_splits.len().into()) // TODO: add seed + let random_idx: usize = u64_between(random_state.into(), 0, best_splits.len().into()) .try_into() .unwrap(); let (best_split_feature, best_split_value, best_prediction) = *best_splits.at(random_idx); @@ -359,7 +360,7 @@ fn fit< impl TCopy: Copy, impl TDrop: Drop, >( - data: Span>, target: Span, depth: usize, max_depth: usize + data: Span>, target: Span, depth: usize, max_depth: usize, random_state: usize ) -> TreeNode { if depth == max_depth || data.len() < 2 { let mut total = FixedTrait::ZERO(); @@ -383,7 +384,7 @@ fn fit< }; } - let (split_feature, split_value, prediction) = best_split(data, target); + let (split_feature, split_value, prediction) = best_split(data, target, random_state); let mut left_data = ArrayTrait::new(); let mut left_target = ArrayTrait::new(); @@ -412,10 +413,10 @@ fn fit< TreeNode { left: Option::Some( - BoxTrait::new(fit(left_data.span(), left_target.span(), depth + 1, max_depth)) + BoxTrait::new(fit(left_data.span(), left_target.span(), depth + 1, max_depth, random_state)) ), right: Option::Some( - BoxTrait::new(fit(right_data.span(), right_target.span(), depth + 1, max_depth)) + BoxTrait::new(fit(right_data.span(), right_target.span(), depth + 1, max_depth, random_state)) ), split_feature, split_value, diff --git a/src/operators/ml/tree_regressor/implementations/tree_regressor_fp16x16.cairo b/src/operators/ml/tree_regressor/implementations/tree_regressor_fp16x16.cairo index b16968c0d..3cb35ab13 100644 --- a/src/operators/ml/tree_regressor/implementations/tree_regressor_fp16x16.cairo +++ b/src/operators/ml/tree_regressor/implementations/tree_regressor_fp16x16.cairo @@ -4,9 +4,9 @@ use orion::numbers::FP16x16; impl FP16x16TreeRegressor of TreeRegressorTrait { fn fit( - data: Span>, target: Span, max_depth: usize + data: Span>, target: Span, max_depth: usize, random_state: usize ) -> TreeNode { - core::fit(data, target, 0, max_depth) + core::fit(data, target, 0, max_depth, random_state) } fn predict(ref self: TreeNode, features: Span) -> FP16x16 { diff --git a/src/operators/ml/tree_regressor/implementations/tree_regressor_fp32x32.cairo b/src/operators/ml/tree_regressor/implementations/tree_regressor_fp32x32.cairo index 2379d2e1b..d1791a9c9 100644 --- a/src/operators/ml/tree_regressor/implementations/tree_regressor_fp32x32.cairo +++ b/src/operators/ml/tree_regressor/implementations/tree_regressor_fp32x32.cairo @@ -4,9 +4,9 @@ use orion::numbers::{FP32x32, FP32x32Impl}; impl FP32x32TreeRegressor of TreeRegressorTrait { fn fit( - data: Span>, target: Span, max_depth: usize + data: Span>, target: Span, max_depth: usize, random_state: usize ) -> TreeNode { - core::fit(data, target, 0, max_depth) + core::fit(data, target, 0, max_depth, random_state) } fn predict(ref self: TreeNode, features: Span) -> FP32x32 { diff --git a/src/operators/ml/tree_regressor/implementations/tree_regressor_fp64x64.cairo b/src/operators/ml/tree_regressor/implementations/tree_regressor_fp64x64.cairo index b4634ec4d..54adb6ce4 100644 --- a/src/operators/ml/tree_regressor/implementations/tree_regressor_fp64x64.cairo +++ b/src/operators/ml/tree_regressor/implementations/tree_regressor_fp64x64.cairo @@ -4,9 +4,9 @@ use orion::numbers::{FP64x64, FP64x64Impl}; impl FP64x64TreeRegressor of TreeRegressorTrait { fn fit( - data: Span>, target: Span, max_depth: usize + data: Span>, target: Span, max_depth: usize, random_state: usize ) -> TreeNode { - core::fit(data, target, 0, max_depth) + core::fit(data, target, 0, max_depth, random_state) } fn predict(ref self: TreeNode, features: Span) -> FP64x64 { diff --git a/src/operators/ml/tree_regressor/implementations/tree_regressor_fp8x23.cairo b/src/operators/ml/tree_regressor/implementations/tree_regressor_fp8x23.cairo index 88475a574..baf61096c 100644 --- a/src/operators/ml/tree_regressor/implementations/tree_regressor_fp8x23.cairo +++ b/src/operators/ml/tree_regressor/implementations/tree_regressor_fp8x23.cairo @@ -4,9 +4,9 @@ use orion::numbers::FP8x23; impl FP8x23TreeRegressor of TreeRegressorTrait { fn fit( - data: Span>, target: Span, max_depth: usize + data: Span>, target: Span, max_depth: usize, random_state: usize ) -> TreeNode { - core::fit(data, target, 0, max_depth) + core::fit(data, target, 0, max_depth, random_state) } fn predict(ref self: TreeNode, features: Span) -> FP8x23 { diff --git a/src/tests/ml/tree_regressor.cairo b/src/tests/ml/tree_regressor.cairo index a4890e341..057fa58b0 100644 --- a/src/tests/ml/tree_regressor.cairo +++ b/src/tests/ml/tree_regressor.cairo @@ -42,7 +42,7 @@ fn test_tree() { ] .span(); - let mut tree = TreeRegressorTrait::fit(data, target, 3); + let mut tree = TreeRegressorTrait::fit(data, target, 3, 42); let prediction_1 = tree .predict(