Standardising Benchmarks, with support for nanobench as an option for its backend

.github/workflows/linux_with_bench_test.yml

@@ -0,0 +1,46 @@
+# Copyright (c) 2024 Vedant Nimje
+#
+# SPDX-License-Identifier: BSL-1.0
+# Distributed under the Boost Software License, Version 1.0. (See accompanying
+# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+name: Linux CI (Debug) with Benchmark Test
+
+on: [pull_request]
+
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    container: stellargroup/build_env:14
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Configure
+      shell: bash
+      run: |
+          cmake \
+              . \
+              -Bbuild \
+              -GNinja \
+              -DCMAKE_BUILD_TYPE=Debug \
+              -DHPX_WITH_MALLOC=system \
+              -DHPX_WITH_FETCH_ASIO=ON \
+              -DHPX_WITH_EXAMPLES=ON \
+              -DHPX_WITH_TESTS=ON \
+              -DHPX_WITH_TESTS_MAX_THREADS_PER_LOCALITY=2 \
+              -DHPX_WITH_CHECK_MODULE_DEPENDENCIES=On
+    - name: Build
+      shell: bash
+      run: |
+          cmake --build build/ --target tests.performance.modules.segmented_algorithms.minmax_element_performance
+    - name: Test
+      shell: bash
+      run: |
+          cd build
+          ./bin/minmax_element_performance_test
+    - name: Test with detailed output
+      shell: bash
+      run: |
+          cd build
+          ./bin/minmax_element_performance_test --detailed_bench

.github/workflows/linux_with_nanobench_test.yml

@@ -0,0 +1,48 @@
+# Copyright (c) 2024 Vedant Nimje
+#
+# SPDX-License-Identifier: BSL-1.0
+# Distributed under the Boost Software License, Version 1.0. (See accompanying
+# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+name: Linux CI (Debug) with Nanobench Benchmark Test
+
+on: [pull_request]
+
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    container: stellargroup/build_env:14
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Configure
+      shell: bash
+      run: |
+          cmake \
+              . \
+              -Bbuild \
+              -GNinja \
+              -DCMAKE_BUILD_TYPE=Debug \
+              -DHPX_WITH_MALLOC=system \
+              -DHPX_WITH_FETCH_ASIO=ON \
+              -DHPX_WITH_EXAMPLES=ON \
+              -DHPX_WITH_TESTS=ON \
+              -DHPX_WITH_NANOBENCH=ON \
+              -DHPX_WITH_TESTS_MAX_THREADS_PER_LOCALITY=2 \
+              -DHPX_WITH_CHECK_MODULE_DEPENDENCIES=On
+    - name: Build
+      shell: bash
+      run: |
+          cmake --build build/ --target tests.performance.modules.segmented_algorithms.minmax_element_performance
+    - name: Test
+      shell: bash
+      run: |
+          cd build
+          ./bin/minmax_element_performance_test
+
+    - name: Test with detailed output
+      shell: bash
+      run: |
+          cd build
+          ./bin/minmax_element_performance_test --detailed_bench

.jenkins/lsu-perftests/launch_perftests.sh

@@ -16,13 +16,13 @@ hpx_targets=(
 hpx_test_options=(
     "--hpx:ini=hpx.thread_queue.init_threads_count=100 \
     --hpx:threads=4 --vector_size=104857 --work_delay=1 \
-    --chunk_size=0 --test_count=200"
+    --chunk_size=0 --test_count=200 --detailed_bench"
     "--hpx:ini=hpx.thread_queue.init_threads_count=100 \
     --hpx:queuing=local-priority --hpx:threads=4 --test-all \
-    --repetitions=40 --futures=207270"
+    --repetitions=40 --futures=207270 --detailed_bench"
     "--hpx:ini=hpx.thread_queue.init_threads_count=100 \
     --vector_size=518176 --hpx:threads=4 --iterations=200 \
-    --warmup_iterations=20")
+    --warmup_iterations=20 --detailed_bench")
 
 # Build binaries for performance tests
 ${perftests_dir}/driver.py -v -l $logfile build -b release -o build \

CMakeLists.txt

@@ -1027,6 +1027,15 @@ hpx_option(
   ADVANCED
 )
 
+hpx_option(
+  HPX_WITH_NANOBENCH
+  BOOL
+  "Use Nanobench for performance tests. Nanobench will be fetched using FetchContent (defualt: OFF)"
+  OFF
+  CATEGORY "Build Targets"
+  ADVANCED
+)
+
 # cmake-format: off
 # LibCDS option
 # NOTE: The libcds option is disabled for the 1.5.0 release as it is not ready
@@ -1263,6 +1272,10 @@ if(HPX_WITH_NETWORKING)
   if(HPX_WITH_PARCELPORT_ACTION_COUNTERS)
     hpx_add_config_define(HPX_HAVE_PARCELPORT_ACTION_COUNTERS)
   endif()
+  if(HPX_WITH_NANOBENCH)
+    hpx_add_config_define(HPX_HAVE_NANOBENCH)
+    include(HPX_SetupNanobench)
+  endif()
 else(HPX_WITH_NETWORKING)
   # if networking is off,  then allow the option of using our asynchronous MPI
   # features
@@ -2550,6 +2563,10 @@ if(HPX_WITH_PRECOMPILED_HEADERS)
       <boost/spirit/home/x3.hpp>
   )
 
+  if(HPX_WITH_NANOBENCH)
+    set(system_precompiled_headers ${system_precompiled_headers} <nanobench.h>)
+  endif()
+
   if(HPX_WITH_CXX17_FILESYSTEM)
     list(APPEND system_precompiled_headers <filesystem>)
   endif()

cmake/HPX_SetupNanobench.cmake

@@ -0,0 +1,47 @@
+include(FetchContent)
+
+fetchcontent_declare(
+  nanobench
+  GIT_REPOSITORY https://github.com/martinus/nanobench.git
+  GIT_TAG v4.3.11
+  GIT_SHALLOW TRUE
+)
+
+if(NOT nanobench_POPULATED)
+  fetchcontent_populate(nanobench)
+endif()
+set(Nanobench_ROOT ${nanobench_SOURCE_DIR})
+
+add_library(nanobench INTERFACE)
+target_include_directories(
+  nanobench SYSTEM INTERFACE $<BUILD_INTERFACE:${Nanobench_ROOT}/src/include/>
+                             $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
+)
+
+install(
+  TARGETS nanobench
+  EXPORT HPXNanobenchTarget
+  COMPONENT core
+)
+
+install(
+  FILES ${NANOBENCH_ROOT}/include/nanobench.h
+  DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
+  COMPONENT core
+)
+
+export(
+  TARGETS nanobench
+  NAMESPACE nanobench::
+  FILE "${CMAKE_CURRENT_BINARY_DIR}/lib/cmake/${HPX_PACKAGE_NAME}/HPXNanobenchTarget.cmake"
+)
+
+install(
+  EXPORT HPXNanobenchTarget
+  NAMESPACE nanobench::
+  FILE HPXNanobenchTarget.cmake
+  DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${HPX_PACKAGE_NAME}
+  COMPONENT cmake
+)
+
+add_library(nanobench::nanobench ALIAS nanobench)

libs/core/algorithms/tests/performance/foreach_report.cpp

@@ -41,6 +41,8 @@ int hpx_main(hpx::program_options::variables_map& vm)
     disable_stealing = vm.count("disable_stealing");
     fast_idle_mode = vm.count("fast_idle_mode");
 
+    hpx::util::perftests_init(vm);
+
     // verify that input is within domain of program
     if (test_count == 0 || test_count < 0)
     {
@@ -122,6 +124,7 @@ int main(int argc, char* argv[])
         ;
     // clang-format on
 
+    hpx::util::perftests_cfg(cmdline);
     hpx::local::init_params init_args;
     init_args.desc_cmdline = cmdline;
     init_args.cfg = {"hpx.os_threads=all"};

libs/core/command_line_handling_local/src/parse_command_line_local.cpp

@@ -614,6 +614,7 @@ namespace hpx::local::detail {
             ("hpx:debug-app-log", value<std::string>()->implicit_value("cout"),
                 "enable all messages on the application log channel and send all "
                 "application logs to the target destination")
+            // ("hpx:verbose_bench", "For logging benchmarks in detail")
         ;
 
         all_options[options_type::hidden_options].add_options()

libs/core/datastructures/tests/performance/small_vector_benchmark.cpp

@@ -9,6 +9,7 @@
 #include <hpx/datastructures/detail/small_vector.hpp>
 #include <hpx/functional.hpp>
 #include <hpx/init.hpp>
+#include <hpx/modules/testing.hpp>
 
 #include <cstddef>
 #include <cstdint>
@@ -19,61 +20,85 @@
 
 ///////////////////////////////////////////////////////////////////////////////
 template <typename Container>
-std::uint64_t measure(std::size_t repeat, std::size_t size)
+void fill(std::size_t size)
 {
-    std::uint64_t start = hpx::chrono::high_resolution_clock::now();
-    for (std::size_t i = 0; i != repeat; ++i)
+    Container cont;
+    for (std::size_t i = 0; i != size; ++i)
     {
-        Container cont;
-        for (std::size_t i = 0; i != size; ++i)
-        {
-            cont.push_back(typename Container::value_type{});
-        }
+        cont.push_back(typename Container::value_type{});
     }
-    return (hpx::chrono::high_resolution_clock::now() - start) / repeat;
 }
 
-template <typename T, std::size_t N>
-void compare(std::size_t repeat, std::size_t size)
-{
-    std::uint64_t time = measure<hpx::detail::small_vector<T, N>>(repeat, size);
+// template <typename T, std::size_t N>
+// void compare(std::size_t repeat, std::size_t size)
+// {
+//     std::uint64_t time = measure<hpx::detail::small_vector<T, N>>(repeat, size);
 
-    std::cout << "-----Average-(hpx::small_vector<" << typeid(T).name() << ", "
-              << N << ">)------ \n"
-              << std::left << "Average execution time : " << std::right
-              << std::setw(8) << time / 1e9 << "\n";
-}
+//     std::cout << "-----Average-(hpx::small_vector<" << typeid(T).name() << ", "
+//               << N << ">)------ \n"
+//               << std::left << "Average execution time : " << std::right
+//               << std::setw(8) << time / 1e9 << "\n";
+// }
 
 int hpx_main(hpx::program_options::variables_map& vm)
 {
     // pull values from cmd
     std::size_t repeat = vm["test_count"].as<std::size_t>();
     std::size_t size = vm["vector_size"].as<std::size_t>();
 
-    std::cout << std::left
-              << "----------------Parameters---------------------\n"
-              << std::left
-              << "Vector size                       : " << std::right
-              << std::setw(8) << size << "\n"
-              << std::left
-              << "Number of tests                   : " << std::right
-              << std::setw(8) << repeat << "\n"
-              << std::left
-              << "Display time in                   : " << std::right
-              << std::setw(8) << "Seconds\n"
-              << std::flush;
-
-    compare<int, 1>(repeat, size);
-    compare<int, 2>(repeat, size);
-    compare<int, 4>(repeat, size);
-    compare<int, 8>(repeat, size);
-    compare<int, 16>(repeat, size);
-
-    compare<hpx::move_only_function<void()>, 1>(repeat, size);
-    compare<hpx::move_only_function<void()>, 2>(repeat, size);
-    compare<hpx::move_only_function<void()>, 4>(repeat, size);
-    compare<hpx::move_only_function<void()>, 8>(repeat, size);
-    compare<hpx::move_only_function<void()>, 16>(repeat, size);
+    hpx::util::perftests_init(vm);
+
+    // int
+
+    hpx::util::perftests_report("hpx::small_vector", "<int, 1>", repeat,
+        [&] { fill<hpx::detail::small_vector<int, 1>>(size); });
+
+    hpx::util::perftests_report("hpx::small_vector", "<int, 2>", repeat,
+        [&] { fill<hpx::detail::small_vector<int, 2>>(size); });
+
+    hpx::util::perftests_report("hpx::small_vector", "<int, 4>", repeat,
+        [&] { fill<hpx::detail::small_vector<int, 4>>(size); });
+
+    hpx::util::perftests_report("hpx::small_vector", "<int, 8>", repeat,
+        [&] { fill<hpx::detail::small_vector<int, 8>>(size); });
+
+    hpx::util::perftests_report("hpx::small_vector", "<int, 16>", repeat,
+        [&] { fill<hpx::detail::small_vector<int, 16>>(size); });
+
+    // hpx::move_only_function<void()>
+
+    hpx::util::perftests_report(
+        "hpx::small_vector", "<fxn<void()>, 1>", repeat, [&] {
+            fill<hpx::detail::small_vector<hpx::move_only_function<void()>, 1>>(
+                size);
+        });
+
+    hpx::util::perftests_report(
+        "hpx::small_vector", "<fxn<void()>, 2>", repeat, [&] {
+            fill<hpx::detail::small_vector<hpx::move_only_function<void()>, 2>>(
+                size);
+        });
+
+    hpx::util::perftests_report(
+        "hpx::small_vector", "<fxn<void()>, 4>", repeat, [&] {
+            fill<hpx::detail::small_vector<hpx::move_only_function<void()>, 4>>(
+                size);
+        });
+
+    hpx::util::perftests_report(
+        "hpx::small_vector", "<fxn<void()>, 8>", repeat, [&] {
+            fill<hpx::detail::small_vector<hpx::move_only_function<void()>, 8>>(
+                size);
+        });
+
+    hpx::util::perftests_report(
+        "hpx::small_vector", "<fxn<void()>, 16>", repeat, [&] {
+            fill<
+                hpx::detail::small_vector<hpx::move_only_function<void()>, 16>>(
+                size);
+        });
+
+    hpx::util::perftests_print_times();
 
     return hpx::local::finalize();
 }
@@ -97,6 +122,8 @@ int main(int argc, char* argv[])
         ;
     // clang-format on
 
+    hpx::util::perftests_cfg(cmdline);
+
     hpx::local::init_params init_args;
     init_args.desc_cmdline = cmdline;
     init_args.cfg = cfg;

libs/core/testing/CMakeLists.txt

@@ -20,14 +20,27 @@ set(testing_compat_headers
 # Default location is $HPX_ROOT/libs/testing/src
 set(testing_sources testing.cpp performance.cpp)
 
+set(module_deps
+    hpx_assertion
+    hpx_config
+    hpx_format
+    hpx_functional
+    hpx_preprocessor
+    hpx_util
+    hpx_command_line_handling_local
+)
+
+if(HPX_WITH_NANOBENCH)
+  set(module_deps ${module_deps} nanobench)
+endif()
+
 include(HPX_AddModule)
 add_hpx_module(
   core testing
   GLOBAL_HEADER_GEN OFF
   SOURCES ${testing_sources}
   HEADERS ${testing_headers}
   COMPAT_HEADERS ${testing_compat_headers}
-  MODULE_DEPENDENCIES hpx_assertion hpx_config hpx_format hpx_functional
-                      hpx_preprocessor hpx_util
+  MODULE_DEPENDENCIES ${module_deps}
   CMAKE_SUBDIRS examples tests
 )