Skip to content

Commit 09bccec

Browse files
jalveszperazz
jalvesz
and
perazz
authored
feat: activation intrinsics for neural networks (#860)
* start working on activations module * softmax for ranks from 1 to 4 * move activations to specialfunctions, add specs * fix float constant definition * update src CMakeLists * add tests for activations * add tests for sigmoid and gelu * missing module procedure * missing interface and change of kind definition for elemental module functions * add SiLU activation * add any rank support for softmax and logsoftmax * add selu activation * Add SELU documentation * add tests * examples * fix relu example * fix tests * improve specs * examples bugfix * replace ifs with merge * add leaky relu activation * Update src/stdlib_specialfunctions.fypp Co-authored-by: Federico Perini <[email protected]> * lowercase procedure names * single shape macro * refactor tanh, add docs, tests on all real precisions * use stdlib_sum * remove unused dim variable --------- Co-authored-by: Federico Perini <[email protected]>
1 parent 64cbb85 commit 09bccec

19 files changed

+2229
-35
lines changed

‎doc/specs/stdlib_specialfunctions_activations.md

+846
Large diffs are not rendered by default.

‎example/specialfunctions_activations/CMakeLists.txt

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
ADD_EXAMPLE(elu)
2+
ADD_EXAMPLE(gaussian)
3+
ADD_EXAMPLE(gelu)
4+
ADD_EXAMPLE(leaky_relu)
5+
ADD_EXAMPLE(relu)
6+
ADD_EXAMPLE(selu)
7+
ADD_EXAMPLE(silu)
8+
ADD_EXAMPLE(softmax)
9+
ADD_EXAMPLE(logsoftmax)
10+
ADD_EXAMPLE(softplus)
11+
ADD_EXAMPLE(step)

‎example/specialfunctions_activations/example_elu.f90

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
program example_elu
2+
use stdlib_kinds, only: sp
3+
use stdlib_math, only: linspace
4+
use stdlib_specialfunctions, only: elu
5+
implicit none
6+
integer, parameter :: n = 10
7+
real(sp) :: x(n), y(n)
8+
9+
x = linspace(-2._sp, 2._sp, n)
10+
y = elu( x , 1.0 )
11+
print *, y
12+
end program example_elu
13+

‎example/specialfunctions_activations/example_gaussian.f90

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
program example_gaussian
2+
use stdlib_kinds, only: sp
3+
use stdlib_math, only: linspace
4+
use stdlib_specialfunctions, only: gaussian
5+
implicit none
6+
integer, parameter :: n = 10
7+
real(sp) :: x(n), y(n)
8+
9+
x = linspace(-2._sp, 2._sp, n)
10+
y = gaussian( x )
11+
print *, y
12+
end program example_gaussian
13+

‎example/specialfunctions_activations/example_gelu.f90

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
program example_gelu
2+
use stdlib_kinds, only: sp
3+
use stdlib_math, only: linspace
4+
use stdlib_specialfunctions, only: gelu
5+
implicit none
6+
integer, parameter :: n = 10
7+
real(sp) :: x(n), y(n)
8+
9+
x = linspace(-2._sp, 2._sp, n)
10+
y = gelu( x )
11+
print *, y
12+
end program example_gelu
13+

‎example/specialfunctions_activations/example_leaky_relu.f90

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
program example_gelu
2+
use stdlib_kinds, only: sp
3+
use stdlib_math, only: linspace
4+
use stdlib_specialfunctions, only: leaky_relu
5+
implicit none
6+
integer, parameter :: n = 10
7+
real(sp) :: x(n), y(n)
8+
9+
x = linspace(-2._sp, 2._sp, n)
10+
y = leaky_relu( x , 0.1_sp )
11+
print *, y
12+
end program example_gelu
13+

‎example/specialfunctions_activations/example_logsoftmax.f90

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
program example_logsoftmax
2+
use stdlib_kinds, only: sp
3+
use stdlib_math, only: linspace
4+
use stdlib_specialfunctions, only: logsoftmax
5+
implicit none
6+
integer, parameter :: n = 10
7+
real(sp) :: x(n), y(n)
8+
9+
x = linspace(-2._sp, 2._sp, n)
10+
y = logsoftmax( x )
11+
print *, y
12+
end program example_logsoftmax
13+

‎example/specialfunctions_activations/example_relu.f90

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
program example_relu
2+
use stdlib_kinds, only: sp
3+
use stdlib_math, only: linspace
4+
use stdlib_specialfunctions, only: relu
5+
implicit none
6+
integer, parameter :: n = 10
7+
real(sp) :: x(n), y(n)
8+
9+
x = linspace(-2._sp, 2._sp, n)
10+
y = relu( x )
11+
print *, y
12+
end program example_relu
13+

‎example/specialfunctions_activations/example_selu.f90

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
program example_selu
2+
use stdlib_kinds, only: sp
3+
use stdlib_math, only: linspace
4+
use stdlib_specialfunctions, only: selu
5+
implicit none
6+
integer, parameter :: n = 10
7+
real(sp) :: x(n), y(n)
8+
9+
x = linspace(-2._sp, 2._sp, n)
10+
y = selu( x )
11+
print *, y
12+
end program example_selu
13+

‎example/specialfunctions_activations/example_silu.f90

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
program example_silu
2+
use stdlib_kinds, only: sp
3+
use stdlib_math, only: linspace
4+
use stdlib_specialfunctions, only: silu
5+
implicit none
6+
integer, parameter :: n = 10
7+
real(sp) :: x(n), y(n)
8+
9+
x = linspace(-2._sp, 2._sp, n)
10+
y = silu( x )
11+
print *, y
12+
end program example_silu
13+

‎example/specialfunctions_activations/example_softmax.f90

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
program example_softmax
2+
use stdlib_kinds, only: sp
3+
use stdlib_math, only: linspace
4+
use stdlib_specialfunctions, only: softmax
5+
implicit none
6+
integer, parameter :: n = 10
7+
real(sp) :: x(n), y(n)
8+
9+
x = linspace(-2._sp, 2._sp, n)
10+
y = softmax( x )
11+
print *, y
12+
end program example_softmax
13+

‎example/specialfunctions_activations/example_softplus.f90

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
program example_softplus
2+
use stdlib_kinds, only: sp
3+
use stdlib_math, only: linspace
4+
use stdlib_specialfunctions, only: softplus
5+
implicit none
6+
integer, parameter :: n = 10
7+
real(sp) :: x(n), y(n)
8+
9+
x = linspace(-2._sp, 2._sp, n)
10+
y = softplus( x )
11+
print *, y
12+
end program example_softplus
13+

‎example/specialfunctions_activations/example_step.f90

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
program example_step
2+
use stdlib_kinds, only: sp
3+
use stdlib_math, only: linspace
4+
use stdlib_specialfunctions, only: step
5+
implicit none
6+
integer, parameter :: n = 10
7+
real(sp) :: x(n), y(n)
8+
9+
x = linspace(-2._sp, 2._sp, n)
10+
y = step( x )
11+
print *, y
12+
end program example_step
13+

‎src/CMakeLists.txt

+2-1
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,9 @@ set(fppFiles
5252
stdlib_sparse_conversion.fypp
5353
stdlib_sparse_kinds.fypp
5454
stdlib_sparse_spmv.fypp
55+
stdlib_specialfunctions_activations.fypp
5556
stdlib_specialfunctions_gamma.fypp
57+
stdlib_specialfunctions.fypp
5658
stdlib_stats.fypp
5759
stdlib_stats_corr.fypp
5860
stdlib_stats_cov.fypp
@@ -116,7 +118,6 @@ set(SRC
116118
stdlib_system_subprocess.F90
117119
stdlib_system.F90
118120
stdlib_sparse.f90
119-
stdlib_specialfunctions.f90
120121
stdlib_specialfunctions_legendre.f90
121122
stdlib_quadrature_gauss.f90
122123
stdlib_stringlist_type.f90

‎src/stdlib_specialfunctions.f90

-34
This file was deleted.

0 commit comments

Comments
 (0)