From ab8d1f92146e5f37c8a2cd89eb97c27c78ed178e Mon Sep 17 00:00:00 2001 From: jatin Date: Thu, 18 Jan 2024 23:38:57 -0800 Subject: [PATCH 1/3] Add sigmoid_exp approximation --- include/math_approx/src/sigmoid_approx.hpp | 21 +++-- test/src/sigmoid_approx_test.cpp | 98 ++++++++++++++++++---- tools/bench/sigmoid_bench.cpp | 14 +++- tools/plotter/plotter.cpp | 13 ++- 4 files changed, 116 insertions(+), 30 deletions(-) diff --git a/include/math_approx/src/sigmoid_approx.hpp b/include/math_approx/src/sigmoid_approx.hpp index 7605f57..68a9ffe 100644 --- a/include/math_approx/src/sigmoid_approx.hpp +++ b/include/math_approx/src/sigmoid_approx.hpp @@ -75,11 +75,18 @@ T sigmoid (T x) return (S) 0.5 * x_poly * rsqrt (x_poly * x_poly + (S) 1) + (S) 0.5; } -// So far this has tested slower than the above approx (for equivalent error), -// but maybe it will be useful for someone! -// template -// T sigmoid_exp (T x) -// { -// return (T) 1 / ((T) 1 + math_approx::exp (-x)); -// } + +/** + * Approximation of sigmoid(x) := 1 / (1 + e^-x), + * using math_approx::exp (x). + * + * So far this has tested slower than the above approximation + * for similar absolute error, but has better relative error + * characteristics. + */ +template +T sigmoid_exp (T x) +{ + return (T) 1 / ((T) 1 + math_approx::exp (-x)); +} } // namespace math_approx diff --git a/test/src/sigmoid_approx_test.cpp b/test/src/sigmoid_approx_test.cpp index b444325..c0dce6f 100644 --- a/test/src/sigmoid_approx_test.cpp +++ b/test/src/sigmoid_approx_test.cpp @@ -4,7 +4,54 @@ #include -TEST_CASE ("Sigmoid Approx Test") +// TEST_CASE ("Sigmoid Approx Test") +// { +// #if ! defined(WIN32) +// const auto all_floats = test_helpers::all_32_bit_floats (-10.0f, 10.0f, 1.0e-3f); +// #else +// const auto all_floats = test_helpers::all_32_bit_floats (-10.0f, 10.0f, 1.0e-1f); +// #endif +// const auto y_exact = test_helpers::compute_all (all_floats, [] (auto x) +// { return 1.0f / (1.0f + std::exp (-x)); }); +// +// const auto test_approx = [&all_floats, &y_exact] (auto&& f_approx, float err_bound) +// { +// const auto y_approx = test_helpers::compute_all (all_floats, f_approx); +// +// const auto error = test_helpers::compute_error (y_exact, y_approx); +// const auto max_error = test_helpers::abs_max (error); +// +// std::cout << max_error << std::endl; +// REQUIRE (std::abs (max_error) < err_bound); +// }; +// +// SECTION ("9th-Order") +// { +// test_approx ([] (auto x) +// { return math_approx::sigmoid<9> (x); }, +// 6.5e-7f); +// } +// SECTION ("7th-Order") +// { +// test_approx ([] (auto x) +// { return math_approx::sigmoid<7> (x); }, +// 7.0e-6f); +// } +// SECTION ("5th-Order") +// { +// test_approx ([] (auto x) +// { return math_approx::sigmoid<5> (x); }, +// 1.0e-4f); +// } +// SECTION ("3th-Order") +// { +// test_approx ([] (auto x) +// { return math_approx::sigmoid<3> (x); }, +// 2.0e-3f); +// } +// } + +TEST_CASE ("Sigmoid (Exp) Approx Test") { #if ! defined(WIN32) const auto all_floats = test_helpers::all_32_bit_floats (-10.0f, 10.0f, 1.0e-3f); @@ -12,41 +59,60 @@ TEST_CASE ("Sigmoid Approx Test") const auto all_floats = test_helpers::all_32_bit_floats (-10.0f, 10.0f, 1.0e-1f); #endif const auto y_exact = test_helpers::compute_all (all_floats, [] (auto x) - { return 1.0f / (1.0f + std::exp (-x)); }); + { return 1.0f / (1.0f + std::exp (-x)); }); - const auto test_approx = [&all_floats, &y_exact] (auto&& f_approx, float err_bound) + const auto test_approx = [&all_floats, &y_exact] (auto&& f_approx, float err_bound, float rel_err_bound, uint32_t ulp_bound) { const auto y_approx = test_helpers::compute_all (all_floats, f_approx); const auto error = test_helpers::compute_error (y_exact, y_approx); + const auto rel_error = test_helpers::compute_rel_error (y_exact, y_approx); + const auto ulp_error = test_helpers::compute_ulp_error (y_exact, y_approx); + const auto max_error = test_helpers::abs_max (error); + const auto max_rel_error = test_helpers::abs_max (rel_error); + const auto max_ulp_error = *std::max_element (ulp_error.begin(), ulp_error.end()); - std::cout << max_error << std::endl; + std::cout << max_error << ", " << max_rel_error << ", " << max_ulp_error << std::endl; REQUIRE (std::abs (max_error) < err_bound); + REQUIRE (std::abs (max_rel_error) < rel_err_bound); + if (ulp_bound > 0) + REQUIRE (max_ulp_error < ulp_bound); }; - SECTION ("9th-Order") + SECTION ("6th-Order (Exp)") { test_approx ([] (auto x) - { return math_approx::sigmoid<9> (x); }, - 6.5e-7f); + { return math_approx::sigmoid_exp<6> (x); }, + 1.5e-7f, + 6.5e-7f, + 12); } - SECTION ("7th-Order") + + SECTION ("5th-Order (Exp)") { test_approx ([] (auto x) - { return math_approx::sigmoid<7> (x); }, - 7.0e-6f); + { return math_approx::sigmoid_exp<5> (x); }, + 1.5e-7f, + 7.0e-7f, + 12); } - SECTION ("5th-Order") + + SECTION ("4th-Order (Exp)") { test_approx ([] (auto x) - { return math_approx::sigmoid<5> (x); }, - 1.0e-4f); + { return math_approx::sigmoid_exp<4> (x); }, + 9.5e-7f, + 4.5e-6f, + 65); } - SECTION ("3th-Order") + + SECTION ("3rd-Order (Exp)") { test_approx ([] (auto x) - { return math_approx::sigmoid<3> (x); }, - 2.0e-3f); + { return math_approx::sigmoid_exp<3> (x); }, + 3.0e-4f, + 1.5e-4f, + 0); } } diff --git a/tools/bench/sigmoid_bench.cpp b/tools/bench/sigmoid_bench.cpp index 5f868f5..d3e1aba 100644 --- a/tools/bench/sigmoid_bench.cpp +++ b/tools/bench/sigmoid_bench.cpp @@ -24,10 +24,13 @@ benchmark::DoNotOptimize (y); \ } \ } \ BENCHMARK (name); -SIGMOID_BENCH (sigmoid_std, [] (auto x) { return 1.0f / (1.0f + std::exp (-x)); }) -SIGMOID_BENCH (sigmoid_approx9, math_approx::sigmoid<9>) -SIGMOID_BENCH (sigmoid_approx7, math_approx::sigmoid<7>) -SIGMOID_BENCH (sigmoid_approx5, math_approx::sigmoid<5>) +// SIGMOID_BENCH (sigmoid_std, [] (auto x) { return 1.0f / (1.0f + std::exp (-x)); }) +// SIGMOID_BENCH (sigmoid_approx9, math_approx::sigmoid<9>) +// SIGMOID_BENCH (sigmoid_approx7, math_approx::sigmoid<7>) +// SIGMOID_BENCH (sigmoid_approx5, math_approx::sigmoid<5>) +// SIGMOID_BENCH (sigmoid_exp_approx6, math_approx::sigmoid_exp<6>) +// SIGMOID_BENCH (sigmoid_exp_approx5, math_approx::sigmoid_exp<5>) +// SIGMOID_BENCH (sigmoid_exp_approx4, math_approx::sigmoid_exp<4>) #define SIGMOID_SIMD_BENCH(name, func) \ void name (benchmark::State& state) \ @@ -47,5 +50,8 @@ SIGMOID_SIMD_BENCH (sigmoid_xsimd, [] (auto x) { return 1.0f / (1.0f + xsimd::ex SIGMOID_SIMD_BENCH (sigmoid_simd_approx9, math_approx::tanh<9>) SIGMOID_SIMD_BENCH (sigmoid_simd_approx7, math_approx::tanh<7>) SIGMOID_SIMD_BENCH (sigmoid_simd_approx5, math_approx::tanh<5>) +SIGMOID_SIMD_BENCH (sigmoid_exp_simd_approx6, math_approx::sigmoid_exp<6>) +SIGMOID_SIMD_BENCH (sigmoid_exp_simd_approx5, math_approx::sigmoid_exp<5>) +SIGMOID_SIMD_BENCH (sigmoid_exp_simd_approx4, math_approx::sigmoid_exp<4>) BENCHMARK_MAIN(); diff --git a/tools/plotter/plotter.cpp b/tools/plotter/plotter.cpp index be727cb..1021eef 100644 --- a/tools/plotter/plotter.cpp +++ b/tools/plotter/plotter.cpp @@ -56,17 +56,24 @@ void plot_function (std::span all_floats, plt::named_plot (name, all_floats, y_approx); } +template +T sigmoid_ref (T x) +{ + return (T) 1 / ((T) 1 + std::exp (-x)); +} + #define FLOAT_FUNC(func) [] (float x) { return func (x); } int main() { plt::figure(); - const auto range = std::make_pair (1.0f, 10.0f); + const auto range = std::make_pair (-10.0f, 10.0f); static constexpr auto tol = 1.0e-2f; const auto all_floats = test_helpers::all_32_bit_floats (range.first, range.second, tol); - const auto y_exact = test_helpers::compute_all (all_floats, FLOAT_FUNC (std::acosh)); - plot_error (all_floats, y_exact, FLOAT_FUNC ((math_approx::acosh<5>) ), "acosh-5"); + const auto y_exact = test_helpers::compute_all (all_floats, FLOAT_FUNC (sigmoid_ref)); + plot_ulp_error (all_floats, y_exact, FLOAT_FUNC ((math_approx::sigmoid_exp<5, true>) ), "sigmoid_exp-5_c1"); + plot_ulp_error (all_floats, y_exact, FLOAT_FUNC ((math_approx::sigmoid_exp<6, true>) ), "sigmoid_exp-6_c1"); plt::legend ({ { "loc", "upper right" } }); plt::xlim (range.first, range.second); From dbfbbe3a0a597c38d4e8ccb1f975504aae9eb323 Mon Sep 17 00:00:00 2001 From: jatin Date: Thu, 18 Jan 2024 23:40:09 -0800 Subject: [PATCH 2/3] Undo comments --- test/src/sigmoid_approx_test.cpp | 92 ++++++++++++++++---------------- tools/bench/sigmoid_bench.cpp | 14 ++--- 2 files changed, 53 insertions(+), 53 deletions(-) diff --git a/test/src/sigmoid_approx_test.cpp b/test/src/sigmoid_approx_test.cpp index c0dce6f..66a6192 100644 --- a/test/src/sigmoid_approx_test.cpp +++ b/test/src/sigmoid_approx_test.cpp @@ -4,52 +4,52 @@ #include -// TEST_CASE ("Sigmoid Approx Test") -// { -// #if ! defined(WIN32) -// const auto all_floats = test_helpers::all_32_bit_floats (-10.0f, 10.0f, 1.0e-3f); -// #else -// const auto all_floats = test_helpers::all_32_bit_floats (-10.0f, 10.0f, 1.0e-1f); -// #endif -// const auto y_exact = test_helpers::compute_all (all_floats, [] (auto x) -// { return 1.0f / (1.0f + std::exp (-x)); }); -// -// const auto test_approx = [&all_floats, &y_exact] (auto&& f_approx, float err_bound) -// { -// const auto y_approx = test_helpers::compute_all (all_floats, f_approx); -// -// const auto error = test_helpers::compute_error (y_exact, y_approx); -// const auto max_error = test_helpers::abs_max (error); -// -// std::cout << max_error << std::endl; -// REQUIRE (std::abs (max_error) < err_bound); -// }; -// -// SECTION ("9th-Order") -// { -// test_approx ([] (auto x) -// { return math_approx::sigmoid<9> (x); }, -// 6.5e-7f); -// } -// SECTION ("7th-Order") -// { -// test_approx ([] (auto x) -// { return math_approx::sigmoid<7> (x); }, -// 7.0e-6f); -// } -// SECTION ("5th-Order") -// { -// test_approx ([] (auto x) -// { return math_approx::sigmoid<5> (x); }, -// 1.0e-4f); -// } -// SECTION ("3th-Order") -// { -// test_approx ([] (auto x) -// { return math_approx::sigmoid<3> (x); }, -// 2.0e-3f); -// } -// } +TEST_CASE ("Sigmoid Approx Test") +{ +#if ! defined(WIN32) + const auto all_floats = test_helpers::all_32_bit_floats (-10.0f, 10.0f, 1.0e-3f); +#else + const auto all_floats = test_helpers::all_32_bit_floats (-10.0f, 10.0f, 1.0e-1f); +#endif + const auto y_exact = test_helpers::compute_all (all_floats, [] (auto x) + { return 1.0f / (1.0f + std::exp (-x)); }); + + const auto test_approx = [&all_floats, &y_exact] (auto&& f_approx, float err_bound) + { + const auto y_approx = test_helpers::compute_all (all_floats, f_approx); + + const auto error = test_helpers::compute_error (y_exact, y_approx); + const auto max_error = test_helpers::abs_max (error); + + std::cout << max_error << std::endl; + REQUIRE (std::abs (max_error) < err_bound); + }; + + SECTION ("9th-Order") + { + test_approx ([] (auto x) + { return math_approx::sigmoid<9> (x); }, + 6.5e-7f); + } + SECTION ("7th-Order") + { + test_approx ([] (auto x) + { return math_approx::sigmoid<7> (x); }, + 7.0e-6f); + } + SECTION ("5th-Order") + { + test_approx ([] (auto x) + { return math_approx::sigmoid<5> (x); }, + 1.0e-4f); + } + SECTION ("3th-Order") + { + test_approx ([] (auto x) + { return math_approx::sigmoid<3> (x); }, + 2.0e-3f); + } +} TEST_CASE ("Sigmoid (Exp) Approx Test") { diff --git a/tools/bench/sigmoid_bench.cpp b/tools/bench/sigmoid_bench.cpp index d3e1aba..606321e 100644 --- a/tools/bench/sigmoid_bench.cpp +++ b/tools/bench/sigmoid_bench.cpp @@ -24,13 +24,13 @@ benchmark::DoNotOptimize (y); \ } \ } \ BENCHMARK (name); -// SIGMOID_BENCH (sigmoid_std, [] (auto x) { return 1.0f / (1.0f + std::exp (-x)); }) -// SIGMOID_BENCH (sigmoid_approx9, math_approx::sigmoid<9>) -// SIGMOID_BENCH (sigmoid_approx7, math_approx::sigmoid<7>) -// SIGMOID_BENCH (sigmoid_approx5, math_approx::sigmoid<5>) -// SIGMOID_BENCH (sigmoid_exp_approx6, math_approx::sigmoid_exp<6>) -// SIGMOID_BENCH (sigmoid_exp_approx5, math_approx::sigmoid_exp<5>) -// SIGMOID_BENCH (sigmoid_exp_approx4, math_approx::sigmoid_exp<4>) +SIGMOID_BENCH (sigmoid_std, [] (auto x) { return 1.0f / (1.0f + std::exp (-x)); }) +SIGMOID_BENCH (sigmoid_approx9, math_approx::sigmoid<9>) +SIGMOID_BENCH (sigmoid_approx7, math_approx::sigmoid<7>) +SIGMOID_BENCH (sigmoid_approx5, math_approx::sigmoid<5>) +SIGMOID_BENCH (sigmoid_exp_approx6, math_approx::sigmoid_exp<6>) +SIGMOID_BENCH (sigmoid_exp_approx5, math_approx::sigmoid_exp<5>) +SIGMOID_BENCH (sigmoid_exp_approx4, math_approx::sigmoid_exp<4>) #define SIGMOID_SIMD_BENCH(name, func) \ void name (benchmark::State& state) \ From 53d7d70884cd13624e821a0aed63a87adc652267 Mon Sep 17 00:00:00 2001 From: jatin Date: Thu, 18 Jan 2024 23:49:31 -0800 Subject: [PATCH 3/3] Tweaking error bounds --- test/src/sigmoid_approx_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/src/sigmoid_approx_test.cpp b/test/src/sigmoid_approx_test.cpp index 66a6192..2cddbdd 100644 --- a/test/src/sigmoid_approx_test.cpp +++ b/test/src/sigmoid_approx_test.cpp @@ -94,7 +94,7 @@ TEST_CASE ("Sigmoid (Exp) Approx Test") test_approx ([] (auto x) { return math_approx::sigmoid_exp<5> (x); }, 1.5e-7f, - 7.0e-7f, + 7.5e-7f, 12); }