From 44bfceeacf3ffc9267f2aa335a37046ea2fb8d8d Mon Sep 17 00:00:00 2001 From: Pablo Duboue Date: Wed, 6 Dec 2023 09:28:14 -0800 Subject: [PATCH 1/2] Leaky ReLUs with test case Original patch by @echoline (https://github.com/libfann/fann/pull/105#issuecomment-766208665) plus a test case. --- src/fann.c | 17 ++++++++++++----- src/fann_cascade.c | 2 ++ src/fann_train.c | 6 ++++++ src/include/fann_activation.h | 6 ++++++ src/include/fann_data.h | 18 ++++++++++++++++-- src/include/fann_data_cpp.h | 14 +++++++++++++- tests/fann_test_train.cpp | 13 +++++++++++++ 7 files changed, 68 insertions(+), 8 deletions(-) diff --git a/src/fann.c b/src/fann.c index 2f195dee..b8e3b0a0 100644 --- a/src/fann.c +++ b/src/fann.c @@ -683,13 +683,20 @@ FANN_EXTERNAL fann_type *FANN_API fann_run(struct fann *ann, fann_type *input) { neuron_it->value = neuron_sum; break; case FANN_LINEAR_PIECE: - neuron_it->value = (fann_type)( - (neuron_sum < 0) ? 0 : (neuron_sum > multiplier) ? multiplier : neuron_sum); + neuron_it->value = (fann_type)((neuron_sum < 0) ? 0 + : (neuron_sum > multiplier) ? multiplier + : neuron_sum); break; case FANN_LINEAR_PIECE_SYMMETRIC: - neuron_it->value = (fann_type)((neuron_sum < -multiplier) - ? -multiplier - : (neuron_sum > multiplier) ? multiplier : neuron_sum); + neuron_it->value = (fann_type)((neuron_sum < -multiplier) ? -multiplier + : (neuron_sum > multiplier) ? multiplier + : neuron_sum); + break; + case FANN_LINEAR_PIECE_LEAKY: + neuron_it->value = (fann_type)((neuron_sum < 0) ? 0.01 * neuron_sum : neuron_sum); + break; + case FANN_LINEAR_PIECE_RECT: + neuron_it->value = (fann_type)((neuron_sum < 0) ? 0 : neuron_sum); break; case FANN_ELLIOT: case FANN_ELLIOT_SYMMETRIC: diff --git a/src/fann_cascade.c b/src/fann_cascade.c index 75abd81d..9dea5a27 100644 --- a/src/fann_cascade.c +++ b/src/fann_cascade.c @@ -680,6 +680,8 @@ fann_type fann_train_candidates_epoch(struct fann *ann, struct fann_train_data * case FANN_GAUSSIAN_STEPWISE: case FANN_ELLIOT: case FANN_LINEAR_PIECE: + case FANN_LINEAR_PIECE_LEAKY: + case FANN_LINEAR_PIECE_RECT: case FANN_SIN: case FANN_COS: break; diff --git a/src/fann_train.c b/src/fann_train.c index f4f9e70a..786eb97c 100644 --- a/src/fann_train.c +++ b/src/fann_train.c @@ -40,6 +40,10 @@ fann_type fann_activation_derived(unsigned int activation_function, fann_type st case FANN_LINEAR_PIECE: case FANN_LINEAR_PIECE_SYMMETRIC: return (fann_type)fann_linear_derive(steepness, value); + case FANN_LINEAR_PIECE_LEAKY: + return (fann_type)((value < 0) ? steepness * 0.01 : steepness); + case FANN_LINEAR_PIECE_RECT: + return (fann_type)((value < 0) ? 0 : steepness); case FANN_SIGMOID: case FANN_SIGMOID_STEPWISE: value = fann_clip(value, 0.01f, 0.99f); @@ -125,6 +129,8 @@ fann_type fann_update_MSE(struct fann *ann, struct fann_neuron *neuron, fann_typ case FANN_GAUSSIAN_STEPWISE: case FANN_ELLIOT: case FANN_LINEAR_PIECE: + case FANN_LINEAR_PIECE_LEAKY: + case FANN_LINEAR_PIECE_RECT: case FANN_SIN: case FANN_COS: break; diff --git a/src/include/fann_activation.h b/src/include/fann_activation.h index 9c634eef..6e99d684 100644 --- a/src/include/fann_activation.h +++ b/src/include/fann_activation.h @@ -178,6 +178,12 @@ __doublefann_h__ is not defined case FANN_GAUSSIAN_STEPWISE: \ result = 0; \ break; \ + case FANN_LINEAR_PIECE_LEAKY: \ + result = (fann_type)((value < 0) ? value * 0.01 : value); \ + break; \ + case FANN_LINEAR_PIECE_RECT: \ + result = (fann_type)((value < 0) ? 0 : value); \ + break; \ } #endif diff --git a/src/include/fann_data.h b/src/include/fann_data.h index 14253059..a5dc91a1 100644 --- a/src/include/fann_data.h +++ b/src/include/fann_data.h @@ -196,6 +196,16 @@ static char const *const FANN_TRAIN_NAMES[] = {"FANN_TRAIN_INCREMENTAL", "FANN_T * y = cos(x*s)/2+0.5 * d = s*-sin(x*s)/2 + FANN_LINEAR_PIECE_LEAKY - leaky ReLU + * span: -inf < y < inf + * y = x<0? 0.01*x: x + * d = x<0? 0.01: 1 + + FANN_LINEAR_PIECE_RECT - ReLU + * span: -inf < y < inf + * y = x<0? 0: x + * d = x<0? 0: 1 + See also: , , , , @@ -223,7 +233,9 @@ enum fann_activationfunc_enum { FANN_SIN_SYMMETRIC, FANN_COS_SYMMETRIC, FANN_SIN, - FANN_COS + FANN_COS, + FANN_LINEAR_PIECE_LEAKY, + FANN_LINEAR_PIECE_RECT }; /* Constant: FANN_ACTIVATIONFUNC_NAMES @@ -254,7 +266,9 @@ static char const *const FANN_ACTIVATIONFUNC_NAMES[] = {"FANN_LINEAR", "FANN_SIN_SYMMETRIC", "FANN_COS_SYMMETRIC", "FANN_SIN", - "FANN_COS"}; + "FANN_COS", + "FANN_LINEAR_PIECE_LEAKY", + "FANN_LINEAR_PIECE_RECT"}; /* Enum: fann_errorfunc_enum Error function used during training. diff --git a/src/include/fann_data_cpp.h b/src/include/fann_data_cpp.h index 62ebf347..766f9b62 100644 --- a/src/include/fann_data_cpp.h +++ b/src/include/fann_data_cpp.h @@ -200,6 +200,16 @@ enum training_algorithm_enum { * y = cos(x*s) * d = s*-sin(x*s) + FANN_LINEAR_PIECE_LEAKY - leaky ReLU + * span: -inf < y < inf + * y = x<0? 0.01*x: x + * d = x<0? 0.01: 1 + + FANN_LINEAR_PIECE_RECT - ReLU + * span: -inf < y < inf + * y = x<0? 0: x + * d = x<0? 0: 1 + See also: , @@ -220,7 +230,9 @@ enum activation_function_enum { LINEAR_PIECE, LINEAR_PIECE_SYMMETRIC, SIN_SYMMETRIC, - COS_SYMMETRIC + COS_SYMMETRIC, + LINEAR_PIECE_LEAKY, + LINEAR_PIECE_RECT }; /* Enum: network_type_enum diff --git a/tests/fann_test_train.cpp b/tests/fann_test_train.cpp index f521e5e1..ee3f83f5 100644 --- a/tests/fann_test_train.cpp +++ b/tests/fann_test_train.cpp @@ -21,6 +21,18 @@ TEST_F(FannTestTrain, TrainOnDateSimpleXor) { EXPECT_LT(net.test_data(data), 0.001); } +TEST_F(FannTestTrain, TrainOnReLUSimpleXor) { + neural_net net(LAYER, 3, 2, 3, 1); + + data.set_train_data(4, 2, xorInput, 1, xorOutput); + net.set_activation_function_hidden(FANN::LINEAR_PIECE_RECT); + net.set_activation_steepness_hidden(1.0); + net.train_on_data(data, 100, 100, 0.001); + + EXPECT_LT(net.get_MSE(), 0.001); + EXPECT_LT(net.test_data(data), 0.001); +} + TEST_F(FannTestTrain, TrainSimpleIncrementalXor) { neural_net net(LAYER, 3, 2, 3, 1); @@ -41,3 +53,4 @@ TEST_F(FannTestTrain, TrainSimpleIncrementalXor) { EXPECT_LT(net.get_MSE(), 0.01); } + From 23987ef4b78fac6f4f5946f4e9d00c493c2b4b82 Mon Sep 17 00:00:00 2001 From: Pablo Duboue Date: Wed, 3 Apr 2024 02:38:47 -0700 Subject: [PATCH 2/2] Changes from review: - Renamed FANN_LINEAR_PIECE_LEAKY to FANN_LINEAR_PIECE_RECT_LEAKY - Added test case for FANN_LINEAR_PIECE_RECT_LEAKY --- src/fann.c | 6 +++--- src/fann_cascade.c | 2 +- src/fann_train.c | 6 +++--- src/include/fann_activation.h | 6 +++--- src/include/fann_data.h | 18 +++++++++--------- src/include/fann_data_cpp.h | 14 +++++++------- tests/fann_test_train.cpp | 12 ++++++++++++ 7 files changed, 38 insertions(+), 26 deletions(-) diff --git a/src/fann.c b/src/fann.c index b8e3b0a0..f225ab42 100644 --- a/src/fann.c +++ b/src/fann.c @@ -692,12 +692,12 @@ FANN_EXTERNAL fann_type *FANN_API fann_run(struct fann *ann, fann_type *input) { : (neuron_sum > multiplier) ? multiplier : neuron_sum); break; - case FANN_LINEAR_PIECE_LEAKY: - neuron_it->value = (fann_type)((neuron_sum < 0) ? 0.01 * neuron_sum : neuron_sum); - break; case FANN_LINEAR_PIECE_RECT: neuron_it->value = (fann_type)((neuron_sum < 0) ? 0 : neuron_sum); break; + case FANN_LINEAR_PIECE_RECT_LEAKY: + neuron_it->value = (fann_type)((neuron_sum < 0) ? 0.01 * neuron_sum : neuron_sum); + break; case FANN_ELLIOT: case FANN_ELLIOT_SYMMETRIC: case FANN_GAUSSIAN: diff --git a/src/fann_cascade.c b/src/fann_cascade.c index 9dea5a27..0935a4fa 100644 --- a/src/fann_cascade.c +++ b/src/fann_cascade.c @@ -680,8 +680,8 @@ fann_type fann_train_candidates_epoch(struct fann *ann, struct fann_train_data * case FANN_GAUSSIAN_STEPWISE: case FANN_ELLIOT: case FANN_LINEAR_PIECE: - case FANN_LINEAR_PIECE_LEAKY: case FANN_LINEAR_PIECE_RECT: + case FANN_LINEAR_PIECE_RECT_LEAKY: case FANN_SIN: case FANN_COS: break; diff --git a/src/fann_train.c b/src/fann_train.c index 786eb97c..8f974bdf 100644 --- a/src/fann_train.c +++ b/src/fann_train.c @@ -40,10 +40,10 @@ fann_type fann_activation_derived(unsigned int activation_function, fann_type st case FANN_LINEAR_PIECE: case FANN_LINEAR_PIECE_SYMMETRIC: return (fann_type)fann_linear_derive(steepness, value); - case FANN_LINEAR_PIECE_LEAKY: - return (fann_type)((value < 0) ? steepness * 0.01 : steepness); case FANN_LINEAR_PIECE_RECT: return (fann_type)((value < 0) ? 0 : steepness); + case FANN_LINEAR_PIECE_RECT_LEAKY: + return (fann_type)((value < 0) ? steepness * 0.01 : steepness); case FANN_SIGMOID: case FANN_SIGMOID_STEPWISE: value = fann_clip(value, 0.01f, 0.99f); @@ -129,8 +129,8 @@ fann_type fann_update_MSE(struct fann *ann, struct fann_neuron *neuron, fann_typ case FANN_GAUSSIAN_STEPWISE: case FANN_ELLIOT: case FANN_LINEAR_PIECE: - case FANN_LINEAR_PIECE_LEAKY: case FANN_LINEAR_PIECE_RECT: + case FANN_LINEAR_PIECE_RECT_LEAKY: case FANN_SIN: case FANN_COS: break; diff --git a/src/include/fann_activation.h b/src/include/fann_activation.h index 6e99d684..1de138c8 100644 --- a/src/include/fann_activation.h +++ b/src/include/fann_activation.h @@ -178,12 +178,12 @@ __doublefann_h__ is not defined case FANN_GAUSSIAN_STEPWISE: \ result = 0; \ break; \ - case FANN_LINEAR_PIECE_LEAKY: \ - result = (fann_type)((value < 0) ? value * 0.01 : value); \ - break; \ case FANN_LINEAR_PIECE_RECT: \ result = (fann_type)((value < 0) ? 0 : value); \ break; \ + case FANN_LINEAR_PIECE_RECT_LEAKY: \ + result = (fann_type)((value < 0) ? value * 0.01 : value); \ + break; \ } #endif diff --git a/src/include/fann_data.h b/src/include/fann_data.h index a5dc91a1..f9040c27 100644 --- a/src/include/fann_data.h +++ b/src/include/fann_data.h @@ -196,16 +196,16 @@ static char const *const FANN_TRAIN_NAMES[] = {"FANN_TRAIN_INCREMENTAL", "FANN_T * y = cos(x*s)/2+0.5 * d = s*-sin(x*s)/2 - FANN_LINEAR_PIECE_LEAKY - leaky ReLU - * span: -inf < y < inf - * y = x<0? 0.01*x: x - * d = x<0? 0.01: 1 - FANN_LINEAR_PIECE_RECT - ReLU * span: -inf < y < inf * y = x<0? 0: x * d = x<0? 0: 1 + FANN_LINEAR_PIECE_RECT_LEAKY - leaky ReLU + * span: -inf < y < inf + * y = x<0? 0.01*x: x + * d = x<0? 0.01: 1 + See also: , , , , @@ -234,8 +234,8 @@ enum fann_activationfunc_enum { FANN_COS_SYMMETRIC, FANN_SIN, FANN_COS, - FANN_LINEAR_PIECE_LEAKY, - FANN_LINEAR_PIECE_RECT + FANN_LINEAR_PIECE_RECT, + FANN_LINEAR_PIECE_RECT_LEAKY }; /* Constant: FANN_ACTIVATIONFUNC_NAMES @@ -267,8 +267,8 @@ static char const *const FANN_ACTIVATIONFUNC_NAMES[] = {"FANN_LINEAR", "FANN_COS_SYMMETRIC", "FANN_SIN", "FANN_COS", - "FANN_LINEAR_PIECE_LEAKY", - "FANN_LINEAR_PIECE_RECT"}; + "FANN_LINEAR_PIECE_RECT", + "FANN_LINEAR_PIECE_RECT_LEAKY"}; /* Enum: fann_errorfunc_enum Error function used during training. diff --git a/src/include/fann_data_cpp.h b/src/include/fann_data_cpp.h index 766f9b62..2006df27 100644 --- a/src/include/fann_data_cpp.h +++ b/src/include/fann_data_cpp.h @@ -200,16 +200,16 @@ enum training_algorithm_enum { * y = cos(x*s) * d = s*-sin(x*s) - FANN_LINEAR_PIECE_LEAKY - leaky ReLU - * span: -inf < y < inf - * y = x<0? 0.01*x: x - * d = x<0? 0.01: 1 - FANN_LINEAR_PIECE_RECT - ReLU * span: -inf < y < inf * y = x<0? 0: x * d = x<0? 0: 1 + FANN_LINEAR_PIECE_RECT_LEAKY - leaky ReLU + * span: -inf < y < inf + * y = x<0? 0.01*x: x + * d = x<0? 0.01: 1 + See also: , @@ -231,8 +231,8 @@ enum activation_function_enum { LINEAR_PIECE_SYMMETRIC, SIN_SYMMETRIC, COS_SYMMETRIC, - LINEAR_PIECE_LEAKY, - LINEAR_PIECE_RECT + LINEAR_PIECE_RECT, + LINEAR_PIECE_RECT_LEAKY }; /* Enum: network_type_enum diff --git a/tests/fann_test_train.cpp b/tests/fann_test_train.cpp index ee3f83f5..167a2098 100644 --- a/tests/fann_test_train.cpp +++ b/tests/fann_test_train.cpp @@ -33,6 +33,18 @@ TEST_F(FannTestTrain, TrainOnReLUSimpleXor) { EXPECT_LT(net.test_data(data), 0.001); } +TEST_F(FannTestTrain, TrainOnReLULeakySimpleXor) { + neural_net net(LAYER, 3, 2, 3, 1); + + data.set_train_data(4, 2, xorInput, 1, xorOutput); + net.set_activation_function_hidden(FANN::LINEAR_PIECE_RECT_LEAKY); + net.set_activation_steepness_hidden(1.0); + net.train_on_data(data, 100, 100, 0.001); + + EXPECT_LT(net.get_MSE(), 0.001); + EXPECT_LT(net.test_data(data), 0.001); +} + TEST_F(FannTestTrain, TrainSimpleIncrementalXor) { neural_net net(LAYER, 3, 2, 3, 1);