Changes from review:

- Renamed FANN_LINEAR_PIECE_LEAKY to FANN_LINEAR_PIECE_RECT_LEAKY - Added test case for FANN_LINEAR_PIECE_RECT_LEAKY
libfann · Apr 3, 2024 · 23987ef · 23987ef
1 parent 0c95d0d
commit 23987ef
Show file tree

Hide file tree

Showing 7 changed files with 38 additions and 26 deletions.
diff --git a/src/fann.c b/src/fann.c
@@ -692,12 +692,12 @@ FANN_EXTERNAL fann_type *FANN_API fann_run(struct fann *ann, fann_type *input) {
                                          : (neuron_sum > multiplier) ? multiplier
                                                                      : neuron_sum);
           break;
-        case FANN_LINEAR_PIECE_LEAKY:
-          neuron_it->value = (fann_type)((neuron_sum < 0) ? 0.01 * neuron_sum : neuron_sum);
-          break;
         case FANN_LINEAR_PIECE_RECT:
           neuron_it->value = (fann_type)((neuron_sum < 0) ? 0 : neuron_sum);
           break;
+        case FANN_LINEAR_PIECE_RECT_LEAKY:
+          neuron_it->value = (fann_type)((neuron_sum < 0) ? 0.01 * neuron_sum : neuron_sum);
+          break;
         case FANN_ELLIOT:
         case FANN_ELLIOT_SYMMETRIC:
         case FANN_GAUSSIAN:

diff --git a/src/fann_cascade.c b/src/fann_cascade.c
@@ -680,8 +680,8 @@ fann_type fann_train_candidates_epoch(struct fann *ann, struct fann_train_data *
         case FANN_GAUSSIAN_STEPWISE:
         case FANN_ELLIOT:
         case FANN_LINEAR_PIECE:
-        case FANN_LINEAR_PIECE_LEAKY:
         case FANN_LINEAR_PIECE_RECT:
+        case FANN_LINEAR_PIECE_RECT_LEAKY:
         case FANN_SIN:
         case FANN_COS:
           break;

diff --git a/src/fann_train.c b/src/fann_train.c
@@ -40,10 +40,10 @@ fann_type fann_activation_derived(unsigned int activation_function, fann_type st
     case FANN_LINEAR_PIECE:
     case FANN_LINEAR_PIECE_SYMMETRIC:
       return (fann_type)fann_linear_derive(steepness, value);
-    case FANN_LINEAR_PIECE_LEAKY:
-      return (fann_type)((value < 0) ? steepness * 0.01 : steepness);
     case FANN_LINEAR_PIECE_RECT:
       return (fann_type)((value < 0) ? 0 : steepness);
+    case FANN_LINEAR_PIECE_RECT_LEAKY:
+      return (fann_type)((value < 0) ? steepness * 0.01 : steepness);
     case FANN_SIGMOID:
     case FANN_SIGMOID_STEPWISE:
       value = fann_clip(value, 0.01f, 0.99f);
@@ -129,8 +129,8 @@ fann_type fann_update_MSE(struct fann *ann, struct fann_neuron *neuron, fann_typ
     case FANN_GAUSSIAN_STEPWISE:
     case FANN_ELLIOT:
     case FANN_LINEAR_PIECE:
-    case FANN_LINEAR_PIECE_LEAKY:
     case FANN_LINEAR_PIECE_RECT:
+    case FANN_LINEAR_PIECE_RECT_LEAKY:
     case FANN_SIN:
     case FANN_COS:
       break;

diff --git a/src/include/fann_activation.h b/src/include/fann_activation.h
@@ -178,12 +178,12 @@ __doublefann_h__ is not defined
     case FANN_GAUSSIAN_STEPWISE:                                                                   \
       result = 0;                                                                                  \
       break;                                                                                       \
-    case FANN_LINEAR_PIECE_LEAKY:                                                                  \
-      result = (fann_type)((value < 0) ? value * 0.01 : value);                                    \
-      break;                                                                                       \
     case FANN_LINEAR_PIECE_RECT:                                                                   \
       result = (fann_type)((value < 0) ? 0 : value);                                               \
       break;                                                                                       \
+    case FANN_LINEAR_PIECE_RECT_LEAKY:                                                             \
+      result = (fann_type)((value < 0) ? value * 0.01 : value);                                    \
+      break;                                                                                       \
   }
 
 #endif
diff --git a/src/include/fann_data.h b/src/include/fann_data.h
@@ -196,16 +196,16 @@ static char const *const FANN_TRAIN_NAMES[] = {"FANN_TRAIN_INCREMENTAL", "FANN_T
          * y = cos(x*s)/2+0.5
          * d = s*-sin(x*s)/2
 
-        FANN_LINEAR_PIECE_LEAKY - leaky ReLU
-         * span: -inf < y < inf
-         * y = x<0? 0.01*x: x
-         * d = x<0? 0.01: 1
-
         FANN_LINEAR_PIECE_RECT - ReLU
          * span: -inf < y < inf
          * y = x<0? 0: x
          * d = x<0? 0: 1
 
+        FANN_LINEAR_PIECE_RECT_LEAKY - leaky ReLU
+         * span: -inf < y < inf
+         * y = x<0? 0.01*x: x
+         * d = x<0? 0.01: 1
+
         See also:
         <fann_set_activation_function_layer>, <fann_set_activation_function_hidden>,
         <fann_set_activation_function_output>, <fann_set_activation_steepness>,
@@ -234,8 +234,8 @@ enum fann_activationfunc_enum {
   FANN_COS_SYMMETRIC,
   FANN_SIN,
   FANN_COS,
-  FANN_LINEAR_PIECE_LEAKY,
-  FANN_LINEAR_PIECE_RECT
+  FANN_LINEAR_PIECE_RECT,
+  FANN_LINEAR_PIECE_RECT_LEAKY
 };
 
 /* Constant: FANN_ACTIVATIONFUNC_NAMES
@@ -267,8 +267,8 @@ static char const *const FANN_ACTIVATIONFUNC_NAMES[] = {"FANN_LINEAR",
                                                         "FANN_COS_SYMMETRIC",
                                                         "FANN_SIN",
                                                         "FANN_COS",
-                                                        "FANN_LINEAR_PIECE_LEAKY",
-                                                        "FANN_LINEAR_PIECE_RECT"};
+                                                        "FANN_LINEAR_PIECE_RECT",
+                                                        "FANN_LINEAR_PIECE_RECT_LEAKY"};
 
 /* Enum: fann_errorfunc_enum
         Error function used during training.

diff --git a/src/include/fann_data_cpp.h b/src/include/fann_data_cpp.h
@@ -200,16 +200,16 @@ enum training_algorithm_enum {
      * y = cos(x*s)
      * d = s*-sin(x*s)
 
-    FANN_LINEAR_PIECE_LEAKY - leaky ReLU
-     * span: -inf < y < inf
-     * y = x<0? 0.01*x: x
-     * d = x<0? 0.01: 1
-
     FANN_LINEAR_PIECE_RECT - ReLU
      * span: -inf < y < inf
      * y = x<0? 0: x
      * d = x<0? 0: 1
 
+    FANN_LINEAR_PIECE_RECT_LEAKY - leaky ReLU
+     * span: -inf < y < inf
+     * y = x<0? 0.01*x: x
+     * d = x<0? 0.01: 1
+
         See also:
                 <neural_net::set_activation_function_hidden>,
                 <neural_net::set_activation_function_output>
@@ -231,8 +231,8 @@ enum activation_function_enum {
   LINEAR_PIECE_SYMMETRIC,
   SIN_SYMMETRIC,
   COS_SYMMETRIC,
-  LINEAR_PIECE_LEAKY,
-  LINEAR_PIECE_RECT
+  LINEAR_PIECE_RECT,
+  LINEAR_PIECE_RECT_LEAKY
 };
 
 /* Enum: network_type_enum

diff --git a/tests/fann_test_train.cpp b/tests/fann_test_train.cpp
@@ -33,6 +33,18 @@ TEST_F(FannTestTrain, TrainOnReLUSimpleXor) {
     EXPECT_LT(net.test_data(data), 0.001);
 }
 
+TEST_F(FannTestTrain, TrainOnReLULeakySimpleXor) {
+    neural_net net(LAYER, 3, 2, 3, 1);
+
+    data.set_train_data(4, 2, xorInput, 1, xorOutput);
+    net.set_activation_function_hidden(FANN::LINEAR_PIECE_RECT_LEAKY);
+    net.set_activation_steepness_hidden(1.0);
+    net.train_on_data(data, 100, 100, 0.001);
+
+    EXPECT_LT(net.get_MSE(), 0.001);
+    EXPECT_LT(net.test_data(data), 0.001);
+}
+
 TEST_F(FannTestTrain, TrainSimpleIncrementalXor) {
     neural_net net(LAYER, 3, 2, 3, 1);