From 4c626099abb0c137271a3ffeecbaabc0c87c38e0 Mon Sep 17 00:00:00 2001 From: Matthieu Zimmer Date: Fri, 21 Aug 2015 16:56:19 +0200 Subject: [PATCH 1/9] lecun sigmoid function --- src/fann_train.c | 2 ++ src/include/fann_activation.h | 6 ++++++ src/include/fann_data.h | 6 ++++-- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/fann_train.c b/src/fann_train.c index 3aad0845..133c5aab 100644 --- a/src/fann_train.c +++ b/src/fann_train.c @@ -46,6 +46,8 @@ fann_type fann_activation_derived(unsigned int activation_function, case FANN_SIGMOID_STEPWISE: value = fann_clip(value, 0.01f, 0.99f); return (fann_type) fann_sigmoid_derive(steepness, value); + case FANN_SIGMOID_SYMMETRIC_LECUN: + return (fann_type) fann_sigmoid_symmetric_lecun_derive(steepness, value); case FANN_SIGMOID_SYMMETRIC: case FANN_SIGMOID_SYMMETRIC_STEPWISE: value = fann_clip(value, -0.98f, 0.98f); diff --git a/src/include/fann_activation.h b/src/include/fann_activation.h index ae1443f5..6f09cb72 100644 --- a/src/include/fann_activation.h +++ b/src/include/fann_activation.h @@ -46,6 +46,10 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA #define fann_sigmoid_symmetric_real(sum) (2.0f/(1.0f + exp(-2.0f * sum)) - 1.0f) #define fann_sigmoid_symmetric_derive(steepness, value) steepness * (1.0f - (value*value)) +/* FANN_SIGMOID_SYMMETRIC_LECUN */ +#define fann_sigmoid_symmetric_lecun_real(sum) sqrt(3.f) * (2.0f/(1.0f + exp(-2.0f * sum)) - 1.0f) +#define fann_sigmoid_symmetric_lecun_derive(steepness, value) steepness * ( sqrt(3.f) - (value*value)/sqrt(3.f) ) + /* FANN_GAUSSIAN */ /* #define fann_gaussian(steepness, sum) (exp(-sum * steepness * sum * steepness)) */ #define fann_gaussian_real(sum) (exp(-sum * sum)) @@ -97,6 +101,8 @@ switch(activation_function) \ case FANN_SIGMOID: \ result = (fann_type)fann_sigmoid_real(value); \ break; \ + case FANN_SIGMOID_SYMMETRIC_LECUN: \ + result = (fann_type)fann_sigmoid_symmetric_lecun_real(value); \ case FANN_SIGMOID_SYMMETRIC: \ result = (fann_type)fann_sigmoid_symmetric_real(value); \ break; \ diff --git a/src/include/fann_data.h b/src/include/fann_data.h index c8701ef5..90d6cf29 100644 --- a/src/include/fann_data.h +++ b/src/include/fann_data.h @@ -224,7 +224,8 @@ enum fann_activationfunc_enum FANN_SIN_SYMMETRIC, FANN_COS_SYMMETRIC, FANN_SIN, - FANN_COS + FANN_COS, + FANN_SIGMOID_SYMMETRIC_LECUN }; /* Constant: FANN_ACTIVATIONFUNC_NAMES @@ -256,7 +257,8 @@ static char const *const FANN_ACTIVATIONFUNC_NAMES[] = { "FANN_SIN_SYMMETRIC", "FANN_COS_SYMMETRIC", "FANN_SIN", - "FANN_COS" + "FANN_COS", + "FANN_SIGMOID_SYMMETRIC_LECUN" }; /* Enum: fann_errorfunc_enum From 1b2a009c14bd3ea41ebab630c0eddfb4d7d6b9e2 Mon Sep 17 00:00:00 2001 From: Matthieu Zimmer Date: Fri, 21 Aug 2015 17:11:35 +0200 Subject: [PATCH 2/9] cannot found this file --- src/include/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/CMakeLists.txt b/src/include/CMakeLists.txt index 3a69da41..0599e05f 100644 --- a/src/include/CMakeLists.txt +++ b/src/include/CMakeLists.txt @@ -1,4 +1,4 @@ ########### install files ############### -INSTALL_FILES( /include FILES fann.h doublefann.h fann_internal.h floatfann.h fann_data.h fixedfann.h compat_time.h fann_activation.h fann_cascade.h fann_error.h fann_train.h fann_io.h fann_cpp.h ) +INSTALL_FILES( /include/fann FILES fann.h doublefann.h fann_internal.h floatfann.h fann_data.h fixedfann.h fann_activation.h fann_cascade.h fann_error.h fann_train.h fann_io.h fann_cpp.h ) From 553a9a5f046961cde99037254d908b051858d7dd Mon Sep 17 00:00:00 2001 From: Matthieu Zimmer Date: Fri, 2 Oct 2015 16:08:59 +0200 Subject: [PATCH 3/9] fix irprop bug ( sign(0) = 0 ) --- src/fann_train.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fann_train.c b/src/fann_train.c index 133c5aab..cdecf30f 100644 --- a/src/fann_train.c +++ b/src/fann_train.c @@ -750,7 +750,7 @@ void fann_update_weights_irpropm(struct fann *ann, unsigned int first_weight, un if(weights[i] < -1500) weights[i] = -1500; } - else + else if(slope > 0) //if no error, no update sign(0) = 0 { weights[i] += next_step; if(weights[i] > 1500) From 32cb0434607bc2b0b19b2ad61c9e8c06435da0c3 Mon Sep 17 00:00:00 2001 From: Matthieu Zimmer Date: Mon, 2 Nov 2015 13:08:47 +0100 Subject: [PATCH 4/9] fix bug in lecun activation --- src/include/fann_activation.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/include/fann_activation.h b/src/include/fann_activation.h index 6f09cb72..88910c70 100644 --- a/src/include/fann_activation.h +++ b/src/include/fann_activation.h @@ -103,6 +103,7 @@ switch(activation_function) \ break; \ case FANN_SIGMOID_SYMMETRIC_LECUN: \ result = (fann_type)fann_sigmoid_symmetric_lecun_real(value); \ + break; \ case FANN_SIGMOID_SYMMETRIC: \ result = (fann_type)fann_sigmoid_symmetric_real(value); \ break; \ From a3863a70672fa238ccd65a7314d12161dc0ae1e4 Mon Sep 17 00:00:00 2001 From: Matthieu Zimmer Date: Wed, 23 Dec 2015 17:42:51 +0100 Subject: [PATCH 5/9] new feature : labels can be associated with a weight for their importance --- src/fann_train.c | 109 +++++++++++++++++++++++++++++++++++++++ src/fann_train_data.c | 44 ++++++++++++++++ src/include/fann_train.h | 1 + 3 files changed, 154 insertions(+) diff --git a/src/fann_train.c b/src/fann_train.c index cdecf30f..56598e90 100644 --- a/src/fann_train.c +++ b/src/fann_train.c @@ -156,6 +156,55 @@ fann_type fann_update_MSE(struct fann *ann, struct fann_neuron* neuron, fann_typ return neuron_diff; } +fann_type fann_update_MSE_lw(struct fann *ann, struct fann_neuron* neuron, fann_type neuron_diff, fann_type label_weight) +{ + float neuron_diff2; + + switch (neuron->activation_function) + { + case FANN_LINEAR_PIECE_SYMMETRIC: + case FANN_THRESHOLD_SYMMETRIC: + case FANN_SIGMOID_SYMMETRIC: + case FANN_SIGMOID_SYMMETRIC_STEPWISE: + case FANN_ELLIOT_SYMMETRIC: + case FANN_GAUSSIAN_SYMMETRIC: + case FANN_SIN_SYMMETRIC: + case FANN_COS_SYMMETRIC: + neuron_diff /= (fann_type)2.0; + break; + case FANN_THRESHOLD: + case FANN_LINEAR: + case FANN_SIGMOID: + case FANN_SIGMOID_STEPWISE: + case FANN_GAUSSIAN: + case FANN_GAUSSIAN_STEPWISE: + case FANN_ELLIOT: + case FANN_LINEAR_PIECE: + case FANN_SIN: + case FANN_COS: + break; + } + +#ifdef FIXEDFANN + neuron_diff2 = + (neuron_diff / (float) ann->multiplier) * (neuron_diff / (float) ann->multiplier); +#else + neuron_diff2 = (float) (neuron_diff * neuron_diff); +#endif + + ann->MSE_value += neuron_diff2 * label_weight; + + /*printf("neuron_diff %f = (%f - %f)[/2], neuron_diff2=%f, sum=%f, MSE_value=%f, num_MSE=%d\n", neuron_diff, *desired_output, neuron_value, neuron_diff2, last_layer_begin->sum, ann->MSE_value, ann->num_MSE); */ + if(fann_abs(neuron_diff) >= ann->bit_fail_limit) + { + ann->num_bit_fail++; + } + + return neuron_diff; +} + + + /* Tests the network. */ FANN_EXTERNAL fann_type *FANN_API fann_test(struct fann *ann, fann_type * input, @@ -284,6 +333,66 @@ void fann_compute_MSE(struct fann *ann, fann_type * desired_output) } } +void fann_compute_MSE_lw(struct fann *ann, fann_type * desired_output, fann_type label_weight) +{ + fann_type neuron_value, neuron_diff, *error_it = 0, *error_begin = 0; + struct fann_neuron *last_layer_begin = (ann->last_layer - 1)->first_neuron; + const struct fann_neuron *last_layer_end = last_layer_begin + ann->num_output; + const struct fann_neuron *first_neuron = ann->first_layer->first_neuron; + + /* if no room allocated for the error variabels, allocate it now */ + if(ann->train_errors == NULL) + { + ann->train_errors = (fann_type *) calloc(ann->total_neurons, sizeof(fann_type)); + if(ann->train_errors == NULL) + { + fann_error((struct fann_error *) ann, FANN_E_CANT_ALLOCATE_MEM); + return; + } + } + else + { + /* clear the error variabels */ + memset(ann->train_errors, 0, (ann->total_neurons) * sizeof(fann_type)); + } + error_begin = ann->train_errors; + +#ifdef DEBUGTRAIN + printf("\ncalculate errors\n"); +#endif + /* calculate the error and place it in the output layer */ + error_it = error_begin + (last_layer_begin - first_neuron); + + for(; last_layer_begin != last_layer_end; last_layer_begin++) + { + neuron_value = last_layer_begin->value; + neuron_diff = *desired_output - neuron_value; + + neuron_diff = fann_update_MSE_lw(ann, last_layer_begin, neuron_diff, label_weight); + + if(ann->train_error_function) + { /* TODO make switch when more functions */ + if(neuron_diff < -.9999999) + neuron_diff = -17.0; + else if(neuron_diff > .9999999) + neuron_diff = 17.0; + else + neuron_diff = (fann_type) log((1.0 + neuron_diff) / (1.0 - neuron_diff)); + } + + *error_it = fann_activation_derived(last_layer_begin->activation_function, + last_layer_begin->activation_steepness, neuron_value, + last_layer_begin->sum) * neuron_diff * label_weight; + + desired_output++; + error_it++; + + ann->num_MSE++; + } +} + + + /* INTERNAL FUNCTION Propagate the error backwards from the output layer. diff --git a/src/fann_train_data.c b/src/fann_train_data.c index d39bfb34..ddd51295 100644 --- a/src/fann_train_data.c +++ b/src/fann_train_data.c @@ -152,6 +152,32 @@ float fann_train_epoch_irpropm(struct fann *ann, struct fann_train_data *data) return fann_get_MSE(ann); } +float fann_train_epoch_irpropm_lw(struct fann *ann, struct fann_train_data *data, fann_type* label_weight) +{ + unsigned int i; + + if(ann->prev_train_slopes == NULL) + { + fann_clear_train_arrays(ann); + } + + fann_reset_MSE(ann); + + for(i = 0; i < data->num_data; i++) + { + fann_run(ann, data->input[i]); + fann_compute_MSE_lw(ann, data->output[i], label_weight[i]); + fann_backpropagate_MSE(ann); + fann_update_slopes_batch(ann, ann->first_layer + 1, ann->last_layer - 1); + } + + fann_update_weights_irpropm(ann, 0, ann->total_connections); + + return fann_get_MSE(ann); +} + + + /* * Internal train function */ @@ -244,6 +270,24 @@ FANN_EXTERNAL float FANN_API fann_train_epoch(struct fann *ann, struct fann_trai return 0; } +FANN_EXTERNAL float FANN_API fann_train_epoch_lw(struct fann *ann, struct fann_train_data *data, +fann_type* label_weight) +{ + if(fann_check_input_output_sizes(ann, data) == -1) + return 0; + + switch (ann->training_algorithm) + { + case FANN_TRAIN_RPROP: + return fann_train_epoch_irpropm_lw(ann, data, label_weight); + default: + printf("FANN : fann_train_epoch_lw not implemented with others algo\n"); + } + return 0; +} + + + FANN_EXTERNAL void FANN_API fann_train_on_data(struct fann *ann, struct fann_train_data *data, unsigned int max_epochs, unsigned int epochs_between_reports, diff --git a/src/include/fann_train.h b/src/include/fann_train.h index b802d70a..3c198eff 100644 --- a/src/include/fann_train.h +++ b/src/include/fann_train.h @@ -212,6 +212,7 @@ FANN_EXTERNAL void FANN_API fann_train_on_file(struct fann *ann, const char *fil This function appears in FANN >= 1.2.0. */ FANN_EXTERNAL float FANN_API fann_train_epoch(struct fann *ann, struct fann_train_data *data); +FANN_EXTERNAL float FANN_API fann_train_epoch_lw(struct fann *ann, struct fann_train_data *data, fann_type* label_weight); #endif /* NOT FIXEDFANN */ /* Function: fann_test_data From cc68429013849943df4f3426e9e9f5f82d3311aa Mon Sep 17 00:00:00 2001 From: Matthieu Zimmer Date: Thu, 12 May 2016 19:56:22 +0200 Subject: [PATCH 6/9] possibility to define own error function --- src/fann_train.c | 138 +++++++++++++++++++++++++++++++++++++++ src/fann_train_data.c | 26 ++++++++ src/include/fann_train.h | 3 + 3 files changed, 167 insertions(+) diff --git a/src/fann_train.c b/src/fann_train.c index 56598e90..38c95f73 100644 --- a/src/fann_train.c +++ b/src/fann_train.c @@ -333,6 +333,57 @@ void fann_compute_MSE(struct fann *ann, fann_type * desired_output) } } +void fann_compute_MSE_gradient(struct fann *ann, fann_type * input, fann_type (*errorFunction)(fann_type*,fann_type*,int,void*), void* errorFuncdata) +{ + fann_type neuron_value, neuron_diff, *error_it = 0, *error_begin = 0; + struct fann_neuron *last_layer_begin = (ann->last_layer - 1)->first_neuron; + const struct fann_neuron *last_layer_end = last_layer_begin + ann->num_output; + const struct fann_neuron *first_neuron = ann->first_layer->first_neuron; + + /* if no room allocated for the error variabels, allocate it now */ + if(ann->train_errors == NULL) + { + ann->train_errors = (fann_type *) calloc(ann->total_neurons, sizeof(fann_type)); + if(ann->train_errors == NULL) + { + fann_error((struct fann_error *) ann, FANN_E_CANT_ALLOCATE_MEM); + return; + } + } + else + { + /* clear the error variabels */ + memset(ann->train_errors, 0, (ann->total_neurons) * sizeof(fann_type)); + } + error_begin = ann->train_errors; + +#ifdef DEBUGTRAIN + printf("\ncalculate errors\n"); +#endif + /* calculate the error and place it in the output layer */ + error_it = error_begin + (last_layer_begin - first_neuron); + + int i=0; + for(; last_layer_begin != last_layer_end; last_layer_begin++) + { + neuron_value = last_layer_begin->value; + //neuron_diff = -(2.f*neuron_value + *desired_output); + neuron_diff = errorFunction(input, ann->output, i, errorFuncdata); + + *error_it = fann_activation_derived(last_layer_begin->activation_function, + last_layer_begin->activation_steepness, neuron_value, + last_layer_begin->sum) * neuron_diff; + //printf("DEBUG _mse_grad %lf %lf %lf %lf\n", *error_it, neuron_diff, last_layer_begin->activation_function, last_layer_begin->activation_steepness ); + //desired_output++; + error_it++; + + ann->num_MSE++; + i++; + } +} + + + void fann_compute_MSE_lw(struct fann *ann, fann_type * desired_output, fann_type label_weight) { fann_type neuron_value, neuron_diff, *error_it = 0, *error_begin = 0; @@ -475,6 +526,93 @@ void fann_backpropagate_MSE(struct fann *ann) } } +void fann_backpropagate_MSE_firstlayer(struct fann *ann) +{ + fann_type tmp_error; + unsigned int i; + struct fann_layer *layer_it; + struct fann_neuron *neuron_it, *last_neuron; + struct fann_neuron **connections; + + fann_type *error_begin = ann->train_errors; + fann_type *error_prev_layer; + fann_type *weights; + const struct fann_neuron *first_neuron = ann->first_layer->first_neuron; + const struct fann_layer *second_layer = ann->first_layer + 1; + const struct fann_layer *first_layer = ann->first_layer; + struct fann_layer *last_layer = ann->last_layer; + + /* go through all the layers, from last to first. + * And propagate the error backwards */ + for(layer_it = last_layer - 1; layer_it > first_layer; --layer_it) + { + last_neuron = layer_it->last_neuron; + + /* for each connection in this layer, propagate the error backwards */ + if(ann->connection_rate >= 1) + { + if(ann->network_type == FANN_NETTYPE_LAYER) + { + error_prev_layer = error_begin + ((layer_it - 1)->first_neuron - first_neuron); + } + else + { + error_prev_layer = error_begin; + } + + for(neuron_it = layer_it->first_neuron; neuron_it != last_neuron; neuron_it++) + { + + tmp_error = error_begin[neuron_it - first_neuron]; + weights = ann->weights + neuron_it->first_con; + for(i = neuron_it->last_con - neuron_it->first_con; i--;) + { + /*printf("i = %d\n", i); + * printf("error_prev_layer[%d] = %f\n", i, error_prev_layer[i]); + * printf("weights[%d] = %f\n", i, weights[i]); */ + error_prev_layer[i] += tmp_error * weights[i]; + } + } + } + else + { + for(neuron_it = layer_it->first_neuron; neuron_it != last_neuron; neuron_it++) + { + + tmp_error = error_begin[neuron_it - first_neuron]; + weights = ann->weights + neuron_it->first_con; + connections = ann->connections + neuron_it->first_con; + for(i = neuron_it->last_con - neuron_it->first_con; i--;) + { + error_begin[connections[i] - first_neuron] += tmp_error * weights[i]; + } + } + } + + /* then calculate the actual errors in the previous layer */ + error_prev_layer = error_begin + ((layer_it - 1)->first_neuron - first_neuron); + last_neuron = (layer_it - 1)->last_neuron; + + if(layer_it != second_layer){ + for(neuron_it = (layer_it - 1)->first_neuron; neuron_it != last_neuron; neuron_it++) + { + *error_prev_layer *= fann_activation_derived(neuron_it->activation_function, + neuron_it->activation_steepness, neuron_it->value, neuron_it->sum); + error_prev_layer++; + } + } else { + //for(neuron_it = (layer_it - 1)->first_neuron; neuron_it != last_neuron; neuron_it++) + //{ + // printf("\n%f %f %d\n", neuron_it->value, neuron_it->activation_steepness, neuron_it->activation_function); + //} + + } + + } +} + + + /* INTERNAL FUNCTION Update weights for incremental training */ diff --git a/src/fann_train_data.c b/src/fann_train_data.c index ddd51295..a0c9dd5d 100644 --- a/src/fann_train_data.c +++ b/src/fann_train_data.c @@ -152,6 +152,32 @@ float fann_train_epoch_irpropm(struct fann *ann, struct fann_train_data *data) return fann_get_MSE(ann); } +float fann_train_epoch_irpropm_gradient(struct fann *ann, struct fann_train_data *data, fann_type (*errorFunction)(fann_type*,fann_type*,int,void*), void* errorFuncdata) +{ + unsigned int i; + + if(ann->prev_train_slopes == NULL) + { + fann_clear_train_arrays(ann); + } + + fann_reset_MSE(ann); + + for(i = 0; i < data->num_data; i++) + { + fann_run(ann, data->input[i]); + fann_compute_MSE_gradient(ann, data->input[i], errorFunction, errorFuncdata); + fann_backpropagate_MSE(ann); + fann_update_slopes_batch(ann, ann->first_layer + 1, ann->last_layer - 1); + } + + fann_update_weights_irpropm(ann, 0, ann->total_connections); + + return fann_get_MSE(ann); +} + + + float fann_train_epoch_irpropm_lw(struct fann *ann, struct fann_train_data *data, fann_type* label_weight) { unsigned int i; diff --git a/src/include/fann_train.h b/src/include/fann_train.h index 3c198eff..801286ec 100644 --- a/src/include/fann_train.h +++ b/src/include/fann_train.h @@ -213,6 +213,9 @@ FANN_EXTERNAL void FANN_API fann_train_on_file(struct fann *ann, const char *fil */ FANN_EXTERNAL float FANN_API fann_train_epoch(struct fann *ann, struct fann_train_data *data); FANN_EXTERNAL float FANN_API fann_train_epoch_lw(struct fann *ann, struct fann_train_data *data, fann_type* label_weight); +FANN_EXTERNAL float FANN_API fann_train_epoch_irpropm_gradient(struct fann *ann, struct fann_train_data *data, fann_type (*errorFunction)(fann_type*,fann_type*,int,void*),void*); +FANN_EXTERNAL void FANN_API fann_compute_MSE_gradient(struct fann *, fann_type *, fann_type (*errorFunction)(fann_type*,fann_type*,int,void*), void*); +FANN_EXTERNAL void FANN_API fann_backpropagate_MSE_firstlayer(struct fann *); #endif /* NOT FIXEDFANN */ /* Function: fann_test_data From e0c83d03c1079a1bbe440719121246478899df52 Mon Sep 17 00:00:00 2001 From: Matthieu Zimmer Date: Sun, 15 May 2016 18:08:48 +0200 Subject: [PATCH 7/9] compute l1/l2 norm --- src/fann.c | 68 ++++++++++++++++++++++++++++++++++++++++++++++ src/include/fann.h | 4 +++ 2 files changed, 72 insertions(+) diff --git a/src/fann.c b/src/fann.c index 2ca7c062..ea3e3903 100644 --- a/src/fann.c +++ b/src/fann.c @@ -1446,6 +1446,74 @@ FANN_EXTERNAL void FANN_API fann_get_connection_array(struct fann *ann, struct f } } +FANN_EXTERNAL fann_type FANN_API fann_get_l1_norm(struct fann *ann) +{ + struct fann_neuron *first_neuron; + struct fann_layer *layer_it; + struct fann_neuron *neuron_it; + unsigned int idx; + unsigned int source_index; + fann_type l1_term; + + first_neuron = ann->first_layer->first_neuron; + + source_index = 0; + l1_term = 0.f; + + /* The following assumes that the last unused bias has no connections */ + + /* for each layer */ + for(layer_it = ann->first_layer; layer_it != ann->last_layer; layer_it++){ + /* for each neuron */ + for(neuron_it = layer_it->first_neuron; neuron_it != layer_it->last_neuron; neuron_it++){ + /* for each connection */ + for (idx = neuron_it->first_con; idx < neuron_it->last_con; idx++){ + /* Assign the source, destination and weight */ + l1_term += fabs(ann->weights[source_index]); + + source_index++; + } + } + } + + return l1_term; +} + +FANN_EXTERNAL fann_type FANN_API fann_get_l2_norm(struct fann *ann) +{ + struct fann_neuron *first_neuron; + struct fann_layer *layer_it; + struct fann_neuron *neuron_it; + unsigned int idx; + unsigned int source_index; + fann_type l2_term; + + first_neuron = ann->first_layer->first_neuron; + + source_index = 0; + l2_term = 0.f; + + /* The following assumes that the last unused bias has no connections */ + + /* for each layer */ + for(layer_it = ann->first_layer; layer_it != ann->last_layer; layer_it++){ + /* for each neuron */ + for(neuron_it = layer_it->first_neuron; neuron_it != layer_it->last_neuron; neuron_it++){ + /* for each connection */ + for (idx = neuron_it->first_con; idx < neuron_it->last_con; idx++){ + /* Assign the source, destination and weight */ + l2_term += ann->weights[source_index] * ann->weights[source_index]; + + source_index++; + } + } + } + + return sqrt(l2_term); +} + + + FANN_EXTERNAL void FANN_API fann_set_weight_array(struct fann *ann, struct fann_connection *connections, unsigned int num_connections) { diff --git a/src/include/fann.h b/src/include/fann.h index 8531dd37..2b26c80f 100644 --- a/src/include/fann.h +++ b/src/include/fann.h @@ -493,6 +493,10 @@ FANN_EXTERNAL void FANN_API fann_get_bias_array(struct fann *ann, unsigned int * FANN_EXTERNAL void FANN_API fann_get_connection_array(struct fann *ann, struct fann_connection *connections); +FANN_EXTERNAL fann_type FANN_API fann_get_l1_norm(struct fann *ann); + +FANN_EXTERNAL fann_type FANN_API fann_get_l2_norm(struct fann *ann); + /* Function: fann_set_weight_array Set connections in the network. From c169f27d23e61d13ceb1172ead8513b64efab9d0 Mon Sep 17 00:00:00 2001 From: Matthieu Zimmer Date: Tue, 12 Jul 2016 16:36:44 +0200 Subject: [PATCH 8/9] l2 normalization --- src/fann.c | 3 +++ src/fann_io.c | 2 ++ src/fann_train.c | 4 +++- src/include/fann_cpp.h | 17 +++++++++++++++++ src/include/fann_data.h | 1 + src/include/fann_train.h | 3 ++- 6 files changed, 28 insertions(+), 2 deletions(-) diff --git a/src/fann.c b/src/fann.c index ea3e3903..6a959be4 100644 --- a/src/fann.c +++ b/src/fann.c @@ -900,6 +900,7 @@ FANN_EXTERNAL struct fann* FANN_API fann_copy(struct fann* orig) copy->learning_rate = orig->learning_rate; copy->learning_momentum = orig->learning_momentum; + copy->learning_l2_norm = orig->learning_l2_norm; copy->connection_rate = orig->connection_rate; copy->network_type = orig->network_type; copy->num_MSE = orig->num_MSE; @@ -1288,6 +1289,7 @@ FANN_EXTERNAL void FANN_API fann_print_parameters(struct fann *ann) printf("Bit fail limit :%8.3f\n", ann->bit_fail_limit); printf("Learning rate :%8.3f\n", ann->learning_rate); printf("Learning momentum :%8.3f\n", ann->learning_momentum); + printf("Learning l2 norm :%8.3f\n", ann->learning_l2_norm); printf("Quickprop decay :%11.6f\n", ann->quickprop_decay); printf("Quickprop mu :%8.3f\n", ann->quickprop_mu); printf("RPROP increase factor :%8.3f\n", ann->rprop_increase_factor); @@ -1647,6 +1649,7 @@ struct fann *fann_allocate_structure(unsigned int num_layers) ann->errstr = NULL; ann->learning_rate = 0.7f; ann->learning_momentum = 0.0; + ann->learning_l2_norm = 0.0; ann->total_neurons = 0; ann->total_connections = 0; ann->num_input = 0; diff --git a/src/fann_io.c b/src/fann_io.c index 08871c2c..7718adb3 100644 --- a/src/fann_io.c +++ b/src/fann_io.c @@ -181,6 +181,7 @@ int fann_save_internal_fd(struct fann *ann, FILE * conf, const char *configurati fprintf(conf, "network_type=%u\n", ann->network_type); fprintf(conf, "learning_momentum=%f\n", ann->learning_momentum); + fprintf(conf, "learning_l2_norm=%f\n", ann->learning_l2_norm); fprintf(conf, "training_algorithm=%u\n", ann->training_algorithm); fprintf(conf, "train_error_function=%u\n", ann->train_error_function); fprintf(conf, "train_stop_function=%u\n", ann->train_stop_function); @@ -443,6 +444,7 @@ struct fann *fann_create_from_fd(FILE * conf, const char *configuration_file) fann_scanf("%u", "network_type", &tmpVal); ann->network_type = (enum fann_nettype_enum)tmpVal; fann_scanf("%f", "learning_momentum", &ann->learning_momentum); + fann_scanf("%f", "learning_l2_norm", &ann->learning_l2_norm); fann_scanf("%u", "training_algorithm", &tmpVal); ann->training_algorithm = (enum fann_train_enum)tmpVal; fann_scanf("%u", "train_error_function", &tmpVal); diff --git a/src/fann_train.c b/src/fann_train.c index 38c95f73..c3570d31 100644 --- a/src/fann_train.c +++ b/src/fann_train.c @@ -978,7 +978,7 @@ void fann_update_weights_irpropm(struct fann *ann, unsigned int first_weight, un for(; i != past_end; i++) { prev_step = fann_max(prev_steps[i], (fann_type) 0.0001); /* prev_step may not be zero because then the training will stop */ - slope = train_slopes[i]; + slope = train_slopes[i] - ann->learning_l2_norm * weights[i]; prev_slope = prev_train_slopes[i]; same_sign = prev_slope * slope; @@ -1294,3 +1294,5 @@ FANN_GET_SET(float, sarprop_temperature) FANN_GET_SET(enum fann_stopfunc_enum, train_stop_function) FANN_GET_SET(fann_type, bit_fail_limit) FANN_GET_SET(float, learning_momentum) +FANN_GET_SET(float, learning_l2_norm) + diff --git a/src/include/fann_cpp.h b/src/include/fann_cpp.h index 9cefd84c..e2359951 100644 --- a/src/include/fann_cpp.h +++ b/src/include/fann_cpp.h @@ -2686,6 +2686,23 @@ namespace FANN } } + float get_learning_l2_norm() + { + float learning_l2_norm = 0.0f; + if (ann != NULL) + { + learning_l2_norm = fann_get_learning_l2_norm(ann); + } + return learning_l2_norm; + } + + void set_learning_l2_norm(float learning_l2_norm) + { + if (ann != NULL) + { + fann_set_learning_l2_norm(ann, learning_l2_norm); + } + } /* Method: get_train_stop_function Returns the the stop function used during training. diff --git a/src/include/fann_data.h b/src/include/fann_data.h index 90d6cf29..181d5116 100644 --- a/src/include/fann_data.h +++ b/src/include/fann_data.h @@ -491,6 +491,7 @@ struct fann /* The learning momentum used for backpropagation algorithm. */ float learning_momentum; + float learning_l2_norm; /* the connection rate of the network * between 0 and 1, 1 meaning fully connected diff --git a/src/include/fann_train.h b/src/include/fann_train.h index 801286ec..72bda101 100644 --- a/src/include/fann_train.h +++ b/src/include/fann_train.h @@ -769,7 +769,8 @@ FANN_EXTERNAL float FANN_API fann_get_learning_momentum(struct fann *ann); This function appears in FANN >= 2.0.0. */ FANN_EXTERNAL void FANN_API fann_set_learning_momentum(struct fann *ann, float learning_momentum); - +FANN_EXTERNAL float FANN_API fann_get_learning_l2_norm(struct fann *ann); +FANN_EXTERNAL void FANN_API fann_set_learning_l2_norm(struct fann *ann, float learning_l2_norm); /* Function: fann_get_activation_function From 4fbe9c85dcba4fd9312b1adb73300748e1c5dd91 Mon Sep 17 00:00:00 2001 From: Matthieu Zimmer Date: Thu, 15 Sep 2016 10:00:09 +0200 Subject: [PATCH 9/9] relu --- src/fann.c | 6 ++++++ src/fann_train.c | 6 ++++++ src/include/fann_activation.h | 13 +++++++++++++ src/include/fann_data.h | 8 ++++++-- 4 files changed, 31 insertions(+), 2 deletions(-) diff --git a/src/fann.c b/src/fann.c index 6a959be4..7eabf15f 100644 --- a/src/fann.c +++ b/src/fann.c @@ -775,6 +775,12 @@ FANN_EXTERNAL fann_type *FANN_API fann_run(struct fann * ann, fann_type * input) case FANN_LINEAR_PIECE_SYMMETRIC: neuron_it->value = (fann_type)((neuron_sum < -multiplier) ? -multiplier : (neuron_sum > multiplier) ? multiplier : neuron_sum); break; + case FANN_RELU: + neuron_it->value = (fann_type)(neuron_sum > 0 ? neuron_sum : 0); + break; + case FANN_LEAKY_RELU: + neuron_it->value = (fann_type)(neuron_sum > 0 ? neuron_sum : neuron_sum / 100.0); + break; case FANN_ELLIOT: case FANN_ELLIOT_SYMMETRIC: case FANN_GAUSSIAN: diff --git a/src/fann_train.c b/src/fann_train.c index c3570d31..2103ab22 100644 --- a/src/fann_train.c +++ b/src/fann_train.c @@ -74,6 +74,10 @@ fann_type fann_activation_derived(unsigned int activation_function, return (fann_type) fann_cos_derive(steepness, sum); case FANN_THRESHOLD: fann_error(NULL, FANN_E_CANT_TRAIN_ACTIVATION); + case FANN_RELU: + return (fann_type) fann_relu_derive(steepness, sum); + case FANN_LEAKY_RELU: + return (fann_type) fann_leaky_relu_derive(steepness, sum); } return 0; } @@ -135,6 +139,8 @@ fann_type fann_update_MSE(struct fann *ann, struct fann_neuron* neuron, fann_typ case FANN_LINEAR_PIECE: case FANN_SIN: case FANN_COS: + case FANN_RELU: + case FANN_LEAKY_RELU: break; } diff --git a/src/include/fann_activation.h b/src/include/fann_activation.h index 88910c70..2dd8b6ff 100644 --- a/src/include/fann_activation.h +++ b/src/include/fann_activation.h @@ -86,6 +86,13 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA #define fann_cos_real(sum) (cos(sum)/2.0f+0.5f) #define fann_cos_derive(steepness, sum) (steepness*-sin(steepness*sum)/2.0f) +/* FANN_RELU */ +#define fann_relu_real(sum) (sum > 0.0 ? sum : 0.0) +#define fann_relu_derive(steepness, sum) (sum > 0.0 ? steepness : 0.0) + +#define fann_leaky_relu_real(sum) (sum > 0.0 ? sum : (sum / 100.0)) +#define fann_leaky_relu_derive(steepness, sum) (sum > 0.0 ? steepness : (steepness / 100.0)) + #define fann_activation_switch(activation_function, value, result) \ switch(activation_function) \ { \ @@ -146,6 +153,12 @@ switch(activation_function) \ case FANN_GAUSSIAN_STEPWISE: \ result = 0; \ break; \ + case FANN_RELU: \ + result = (fann_type)fann_relu_real(value); \ + break; \ + case FANN_LEAKY_RELU: \ + result = (fann_type)fann_leaky_relu_real(value); \ + break; \ } #endif diff --git a/src/include/fann_data.h b/src/include/fann_data.h index 181d5116..b6c7ae99 100644 --- a/src/include/fann_data.h +++ b/src/include/fann_data.h @@ -225,7 +225,9 @@ enum fann_activationfunc_enum FANN_COS_SYMMETRIC, FANN_SIN, FANN_COS, - FANN_SIGMOID_SYMMETRIC_LECUN + FANN_SIGMOID_SYMMETRIC_LECUN, + FANN_RELU, + FANN_LEAKY_RELU }; /* Constant: FANN_ACTIVATIONFUNC_NAMES @@ -258,7 +260,9 @@ static char const *const FANN_ACTIVATIONFUNC_NAMES[] = { "FANN_COS_SYMMETRIC", "FANN_SIN", "FANN_COS", - "FANN_SIGMOID_SYMMETRIC_LECUN" + "FANN_SIGMOID_SYMMETRIC_LECUN", + "FANN_RELU", + "FANN_LEAKY_RELU" }; /* Enum: fann_errorfunc_enum