Dropout layers always in training mode; except when is called, when they are in inference mode

Author: milancurcic

src/nf/nf_dropout_layer.f90

@@ -22,7 +22,7 @@ module nf_dropout_layer
 
     real :: dropout_rate ! probability of dropping a neuron
     real :: scale ! scale factor to preserve the input sum
-    logical :: training = .false. ! set to .true. in training mode
+    logical :: training = .true. ! set to .false. for inference
 
   contains
 
@@ -33,13 +33,11 @@ module nf_dropout_layer
   end type dropout_layer
 
   interface dropout_layer
-    module function dropout_layer_cons(rate, training) &
+    module function dropout_layer_cons(rate) &
       result(res)
       !! This function returns the `dropout_layer` instance.
       real, intent(in) :: rate
         !! Dropout rate
-      logical, intent(in), optional :: training
-        !! Training mode (default .false.)
       type(dropout_layer) :: res
         !! dropout_layer instance
     end function dropout_layer_cons

src/nf/nf_dropout_layer_submodule.f90

@@ -4,12 +4,10 @@
 
 contains
 
-  module function dropout_layer_cons(rate, training) result(res)
+  module function dropout_layer_cons(rate) result(res)
     real, intent(in) :: rate
-    logical, intent(in), optional :: training
     type(dropout_layer) :: res
     res % dropout_rate = rate
-    if (present(training)) res % training = training
   end function dropout_layer_cons
 
 

src/nf/nf_layer_constructors.f90

@@ -104,7 +104,7 @@ module function dense(layer_size, activation) result(res)
         !! Resulting layer instance
     end function dense
 
-    module function dropout(rate, training) result(res)
+    module function dropout(rate) result(res)
       !! Create a dropout layer with a given dropout rate.
       !!
       !! This layer is for randomly disabling neurons during training.
@@ -118,8 +118,6 @@ module function dropout(rate, training) result(res)
       !! ```
       real, intent(in) :: rate
         !! Dropout rate - fraction of neurons to randomly disable during training
-      logical, intent(in), optional :: training
-        !! Training mode (default .false.)
       type(layer) :: res
         !! Resulting layer instance
     end function dropout

src/nf/nf_layer_constructors_submodule.f90

@@ -65,12 +65,11 @@ module function dense(layer_size, activation) result(res)
   end function dense
 
 
-  module function dropout(rate, training) result(res)
+  module function dropout(rate) result(res)
     real, intent(in) :: rate
-    logical, intent(in), optional :: training
     type(layer) :: res
     res % name = 'dropout'
-    allocate(res % p, source=dropout_layer(rate, training))
+    allocate(res % p, source=dropout_layer(rate))
   end function dropout
 
 

src/nf/nf_network.f90

@@ -26,6 +26,7 @@ module nf_network
     procedure :: get_params
     procedure :: print_info
     procedure :: set_params
+    procedure :: set_training_mode
     procedure :: train
     procedure :: update
 
@@ -223,6 +224,15 @@ module subroutine set_params(self, params)
         !! Network parameters to set
     end subroutine set_params
 
+    module subroutine set_training_mode(self, training)
+      !! Set the mode to training (.true.) or inference (.false.).
+      !! Used internally to enable/disable the dropout layers in the network.
+      class(network), intent(in out) :: self
+        !! Network instance
+      logical, intent(in) :: training
+        !! .true. for training mode, .false. for inference.
+    end subroutine set_training_mode
+
     module subroutine print_info(self)
       !! Prints a brief summary of the network and its layers to the screen.
       class(network), intent(in) :: self

src/nf/nf_network_submodule.f90

@@ -251,15 +251,11 @@ module function predict_1d(self, input) result(res)
     num_layers = size(self % layers)
 
     ! predict is run in inference mode only;
-    ! set all dropout layers' training mode to false.
-    do n = 2, num_layers
-      select type(this_layer => self % layers(n) % p)
-        type is(dropout_layer)
-          this_layer % training = .false.
-      end select
-    end do
-
+    ! set all dropout layers' training mode to false, and
+    ! return to training mode after inference.
+    call self % set_training_mode(.false.)
     call self % forward(input)
+    call self % set_training_mode(.true.)
 
     select type(output_layer => self % layers(num_layers) % p)
       type is(dense_layer)
@@ -269,7 +265,8 @@ module function predict_1d(self, input) result(res)
       type is(flatten_layer)
         res = output_layer % output
       class default
-        error stop 'network % output not implemented for this output layer'
+        error stop 'network % output not implemented for ' // &
+          trim(self % layers(num_layers) % name) // ' layer'
     end select
 
   end function predict_1d
@@ -279,15 +276,25 @@ module function predict_2d(self, input) result(res)
     class(network), intent(in out) :: self
     real, intent(in) :: input(:,:)
     real, allocatable :: res(:)
-    integer :: num_layers
+    integer :: n, num_layers
 
     num_layers = size(self % layers)
 
+    ! predict is run in inference mode only;
+    ! set all dropout layers' training mode to false, and
+    ! return to training mode after inference.
+    call self % set_training_mode(.false.)
     call self % forward(input)
+    call self % set_training_mode(.true.)
 
     select type(output_layer => self % layers(num_layers) % p)
       type is(dense_layer)
         res = output_layer % output
+      type is(flatten_layer)
+        res = output_layer % output
+      class default
+        error stop 'network % output not implemented for ' // &
+          trim(self % layers(num_layers) % name) // ' layer'
     end select
 
   end function predict_2d
@@ -302,15 +309,11 @@ module function predict_3d(self, input) result(res)
     num_layers = size(self % layers)
 
     ! predict is run in inference mode only;
-    ! set all dropout layers' training mode to false.
-    do n = 2, num_layers
-      select type(this_layer => self % layers(n) % p)
-        type is(dropout_layer)
-          this_layer % training = .false.
-      end select
-    end do
-
+    ! set all dropout layers' training mode to false, and
+    ! return to training mode after inference.
+    call self % set_training_mode(.false.)
     call self % forward(input)
+    call self % set_training_mode(.true.)
 
     select type(output_layer => self % layers(num_layers) % p)
       type is(conv2d_layer)
@@ -321,7 +324,8 @@ module function predict_3d(self, input) result(res)
       type is(flatten_layer)
         res = output_layer % output
       class default
-        error stop 'network % output not implemented for this output layer'
+        error stop 'network % output not implemented for ' // &
+          trim(self % layers(num_layers) % name) // ' layer'
     end select
 
   end function predict_3d
@@ -338,13 +342,9 @@ module function predict_batch_1d(self, input) result(res)
     output_size = product(self % layers(num_layers) % layer_shape)
 
     ! predict is run in inference mode only;
-    ! set all dropout layers' training mode to false.
-    do n = 2, num_layers
-      select type(this_layer => self % layers(n) % p)
-        type is(dropout_layer)
-          this_layer % training = .false.
-      end select
-    end do
+    ! set all dropout layers' training mode to false, and
+    ! return to training mode after inference.
+    call self % set_training_mode(.false.)
 
     allocate(res(output_size, batch_size))
 
@@ -358,11 +358,16 @@ module function predict_batch_1d(self, input) result(res)
         type is(flatten_layer)
           res(:,i) = output_layer % output
         class default
-          error stop 'network % output not implemented for this output layer'
+          error stop 'network % output not implemented for ' // &
+            trim(self % layers(num_layers) % name) // ' layer'
       end select
 
     end do batch
 
+    ! We are now done with inference;
+    ! return to training mode for dropout layers.
+    call self % set_training_mode(.true.)
+
   end function predict_batch_1d
 
 
@@ -377,13 +382,9 @@ module function predict_batch_3d(self, input) result(res)
     output_size = product(self % layers(num_layers) % layer_shape)
 
     ! predict is run in inference mode only;
-    ! set all dropout layers' training mode to false.
-    do n = 2, num_layers
-      select type(this_layer => self % layers(n) % p)
-        type is(dropout_layer)
-          this_layer % training = .false.
-      end select
-    end do
+    ! set all dropout layers' training mode to false, and
+    ! return to training mode after inference.
+    call self % set_training_mode(.false.)
 
     allocate(res(output_size, batch_size))
 
@@ -400,11 +401,16 @@ module function predict_batch_3d(self, input) result(res)
         type is(flatten_layer)
           res(:,i) = output_layer % output
         class default
-          error stop 'network % output not implemented for this output layer'
+          error stop 'network % output not implemented for ' // &
+            trim(self % layers(num_layers) % name) // ' layer'
       end select
 
     end do batch
 
+    ! We are now done with inference;
+    ! return to training mode for dropout layers.
+    call self % set_training_mode(.true.)
+
   end function predict_batch_3d
 
 
@@ -484,6 +490,18 @@ module subroutine set_params(self, params)
   end subroutine set_params
 
 
+  module subroutine set_training_mode(self, training)
+    class(network), intent(in out) :: self
+    logical, intent(in) :: training
+    integer :: n
+    do n = 2, size(self % layers)
+      select type(this_layer => self % layers(n) % p); type is(dropout_layer)
+        this_layer % training = training
+      end select
+    end do
+  end subroutine set_training_mode
+
+
   module subroutine train(self, input_data, output_data, batch_size, &
                           epochs, optimizer, loss)
     class(network), intent(in out) :: self

test/test_dropout_layer.f90

@@ -24,9 +24,9 @@ program test_dropout_layer
         write(stderr, '(a)') 'dropout layer dropout rate should be 0.5.. failed'
       end if
 
-      if (layer1_p % training) then
+      if (.not. layer1_p % training) then
         ok = .false.
-        write(stderr, '(a)') 'dropout layer default training mode should be false.. failed'
+        write(stderr, '(a)') 'dropout layer default training mode should be true.. failed'
       end if
 
       if (layer1_p % input_size /= 0) then
@@ -42,25 +42,6 @@ program test_dropout_layer
 
   end select
 
-  ! Test setting training mode explicitly.
-  layer1 = dropout(0.5, training=.true.)
-  select type(layer1_p => layer1 % p)
-    type is(dropout_layer)
-      if (.not. layer1_p % training) then
-        ok = .false.
-        write(stderr, '(a)') 'dropout layer training mode should be true.. failed'
-      end if
-  end select
-
-  layer1 = dropout(0.5, training=.false.)
-  select type(layer1_p => layer1 % p)
-    type is(dropout_layer)
-      if (layer1_p % training) then
-        ok = .false.
-        write(stderr, '(a)') 'dropout layer training mode should be false.. failed'
-      end if
-  end select
-
   ! Now we're gonna initialize a minimal network with an input layer and a
   ! dropout that follows and we'll check that the dropout layer has expected
   ! state.
@@ -122,31 +103,27 @@ program test_dropout_layer
 
 
   training: block
-    real :: x(10), y(5)
+    real :: x(100), y(5)
     real :: tolerance = 1e-3
     integer :: n
-    integer, parameter :: num_iterations = 100000
+    integer, parameter :: num_iterations = 10000
 
     call random_number(x)
-    y = [0.1234, 0.2345, 0.3456, 0.4567, 0.5678]
+    y = [0.12345, 0.23456, 0.34567, 0.45678, 0.56789]
 
     net = network([ &
-      input(10), &
-      dropout(0.5, training=.true.), &
+      input(100), &
+      dropout(0.5), &
       dense(5) &
     ])
 
     do n = 1, num_iterations
-      !select type(dropout_l => net % layers(2) % p)
-      !  type is(dropout_layer)
-      !    print *, dropout_l % training, dropout_l % mask
-      !end select
       call net % forward(x)
       call net % backward(y)
       call net % update()
-      !print *, n, net % predict(x)
 
       if (all(abs(net % predict(x) - y) < tolerance)) exit
+
     end do
 
     if (.not. n <= num_iterations) then