From 5167e12faad628641d5739546a62632321a0255f Mon Sep 17 00:00:00 2001 From: Vedant Date: Fri, 16 Feb 2024 15:20:56 +0530 Subject: [PATCH 01/24] Add initial nanobench benchmark config Signed-off-by: Vedant --- CMakeLists.txt | 13 ++ cmake/HPX_SetupNanobench.cmake | 10 ++ .../tests/performance/CMakeLists.txt | 11 +- .../tests/performance/minmax_nanobench.cpp | 122 ++++++++++++++++++ 4 files changed, 155 insertions(+), 1 deletion(-) create mode 100644 cmake/HPX_SetupNanobench.cmake create mode 100644 libs/full/segmented_algorithms/tests/performance/minmax_nanobench.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 4eeebf15fc7b..240d8bccf594 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1027,6 +1027,15 @@ hpx_option( ADVANCED ) +hpx_option( + HPX_WITH_NANOBENCH_BENCHMARKS + BOOL + "Use Nanobench for performance tests. Nanobench will be fetched using FetchContent (defualt: OFF)" + OFF + CATEGORY "Build Targets" + ADVANCED +) + # cmake-format: off # LibCDS option # NOTE: The libcds option is disabled for the 1.5.0 release as it is not ready @@ -2219,6 +2228,9 @@ endif() # should be found separately in the appropriate subdirectory. include(HPX_SetupThreads) +if (HPX_WITH_NANOBENCH_BENCHMARKS) + include(HPX_SetupNanobench) +endif() # Setup our required Boost libraries. include(HPX_SetupBoost) @@ -2548,6 +2560,7 @@ if(HPX_WITH_PRECOMPILED_HEADERS) + ) if(HPX_WITH_CXX17_FILESYSTEM) diff --git a/cmake/HPX_SetupNanobench.cmake b/cmake/HPX_SetupNanobench.cmake new file mode 100644 index 000000000000..632f82751d1c --- /dev/null +++ b/cmake/HPX_SetupNanobench.cmake @@ -0,0 +1,10 @@ +include(FetchContent) + +FetchContent_Declare( + nanobench + GIT_REPOSITORY https://github.com/martinus/nanobench.git + GIT_TAG v4.1.0 + GIT_SHALLOW TRUE) + +FetchContent_MakeAvailable(nanobench) + diff --git a/libs/full/segmented_algorithms/tests/performance/CMakeLists.txt b/libs/full/segmented_algorithms/tests/performance/CMakeLists.txt index e8db7f041b00..159c688711cf 100644 --- a/libs/full/segmented_algorithms/tests/performance/CMakeLists.txt +++ b/libs/full/segmented_algorithms/tests/performance/CMakeLists.txt @@ -6,16 +6,25 @@ set(benchmarks minmax_element_performance) +if (HPX_WITH_NANOBENCH_BENCHMARKS) + set(benchmarks ${benchmarks} minmax_nanobench) +endif() + foreach(benchmark ${benchmarks}) set(sources ${benchmark}.cpp) source_group("Source Files" FILES ${sources}) + set(deps iostreams_component partitioned_vector_component) + if (HPX_WITH_NANOBENCH_BENCHMARKS) + set(deps ${deps} nanobench) + endif() + # add example executable add_hpx_executable( ${benchmark}_test INTERNAL_FLAGS SOURCES ${sources} ${${benchmark}_FLAGS} - DEPENDENCIES iostreams_component partitioned_vector_component + DEPENDENCIES ${deps} EXCLUDE_FROM_ALL HPX_PREFIX ${HPX_BUILD_PREFIX} FOLDER "Benchmarks/Modules/Full/SegmentedAlgorithms" diff --git a/libs/full/segmented_algorithms/tests/performance/minmax_nanobench.cpp b/libs/full/segmented_algorithms/tests/performance/minmax_nanobench.cpp new file mode 100644 index 000000000000..2e96037fb353 --- /dev/null +++ b/libs/full/segmented_algorithms/tests/performance/minmax_nanobench.cpp @@ -0,0 +1,122 @@ +// Copyright (c) 2016 Hartmut Kaiser +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#include +#if !defined(HPX_COMPUTE_DEVICE_CODE) +#include +#include + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +/////////////////////////////////////////////////////////////////////////////// +// Define the vector types to be used. +HPX_REGISTER_PARTITIONED_VECTOR(int) +unsigned int seed = (unsigned int) std::random_device{}(); + +/////////////////////////////////////////////////////////////////////////////// +struct random_fill +{ + random_fill() + : gen(seed) + , dist(0, RAND_MAX) + { + } + + int operator()() + { + return dist(gen); + } + + std::mt19937 gen; + std::uniform_int_distribution<> dist; + + template + void serialize(Archive&, unsigned) + { + } +}; + +/////////////////////////////////////////////////////////////////////////////// +int hpx_main(hpx::program_options::variables_map& vm) +{ + if (hpx::get_locality_id() == 0) + { + // pull values from cmd + std::size_t size = vm["vector_size"].as(); + //bool csvoutput = vm.count("csv_output") != 0; + int test_count = vm["test_count"].as(); + + // create as many partitions as we have localities + hpx::partitioned_vector v( + size, hpx::container_layout(hpx::find_all_localities())); + + // initialize data + hpx::generate(hpx::execution::par, v.begin(), v.end(), random_fill()); + + ankerl::nanobench::Bench().minEpochIterations(test_count).run("min_element", [&]{ + hpx::min_element(hpx::execution::par, v.begin(), v.end()); + ankerl::nanobench::doNotOptimizeAway(v); + }); + + ankerl::nanobench::Bench().minEpochIterations(test_count).run("max_element", [&]{ + hpx::max_element(hpx::execution::par, v.begin(), v.end()); + ankerl::nanobench::doNotOptimizeAway(v); + }); + + ankerl::nanobench::Bench().minEpochIterations(test_count).run("minmax_element", [&]{ + hpx::minmax_element(hpx::execution::par, v.begin(), v.end()); + ankerl::nanobench::doNotOptimizeAway(v); + }); + + return hpx::finalize(); + } + + return 0; +} + +int main(int argc, char** argv) +{ + // ::benchmark::Initialize(&argc, argv); + + std::srand((unsigned int) std::time(nullptr)); + + // initialize program + std::vector const cfg = { + "hpx.os_threads=all", "hpx.run_hpx_main!=1"}; + + hpx::program_options::options_description cmdline( + "usage: " HPX_APPLICATION_STRING " [options]"); + + cmdline.add_options()("vector_size", + hpx::program_options::value()->default_value(1000), + "size of vector (default: 1000)")("test_count", + hpx::program_options::value()->default_value(1000), + "number of tests to be averaged (default: 100)")( + "csv_output", "print results in csv format")("seed,s", + hpx::program_options::value(), + "the random number generator seed to use for this run"); + + hpx::init_params init_args; + init_args.desc_cmdline = cmdline; + init_args.cfg = cfg; + return hpx::init(argc, argv, init_args); + return 0; +} +#endif From 16e42eaffdfb47b4e3aab1a498051321bab7f00d Mon Sep 17 00:00:00 2001 From: Vedant Date: Wed, 21 Feb 2024 22:37:45 +0530 Subject: [PATCH 02/24] Pyperf integration for testing puporses Signed-off-by: Vedant --- cmake/HPX_SetupNanobench.cmake | 2 +- .../tests/performance/minmax_nanobench.cpp | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/cmake/HPX_SetupNanobench.cmake b/cmake/HPX_SetupNanobench.cmake index 632f82751d1c..7fe43d784667 100644 --- a/cmake/HPX_SetupNanobench.cmake +++ b/cmake/HPX_SetupNanobench.cmake @@ -3,7 +3,7 @@ include(FetchContent) FetchContent_Declare( nanobench GIT_REPOSITORY https://github.com/martinus/nanobench.git - GIT_TAG v4.1.0 + GIT_TAG v4.3.11 GIT_SHALLOW TRUE) FetchContent_MakeAvailable(nanobench) diff --git a/libs/full/segmented_algorithms/tests/performance/minmax_nanobench.cpp b/libs/full/segmented_algorithms/tests/performance/minmax_nanobench.cpp index 2e96037fb353..37b8204adc67 100644 --- a/libs/full/segmented_algorithms/tests/performance/minmax_nanobench.cpp +++ b/libs/full/segmented_algorithms/tests/performance/minmax_nanobench.cpp @@ -72,18 +72,16 @@ int hpx_main(hpx::program_options::variables_map& vm) ankerl::nanobench::Bench().minEpochIterations(test_count).run("min_element", [&]{ hpx::min_element(hpx::execution::par, v.begin(), v.end()); - ankerl::nanobench::doNotOptimizeAway(v); }); ankerl::nanobench::Bench().minEpochIterations(test_count).run("max_element", [&]{ hpx::max_element(hpx::execution::par, v.begin(), v.end()); - ankerl::nanobench::doNotOptimizeAway(v); }); ankerl::nanobench::Bench().minEpochIterations(test_count).run("minmax_element", [&]{ hpx::minmax_element(hpx::execution::par, v.begin(), v.end()); - ankerl::nanobench::doNotOptimizeAway(v); - }); + }) + .render(ankerl::nanobench::templates::pyperf(), std::cout); return hpx::finalize(); } @@ -107,7 +105,7 @@ int main(int argc, char** argv) cmdline.add_options()("vector_size", hpx::program_options::value()->default_value(1000), "size of vector (default: 1000)")("test_count", - hpx::program_options::value()->default_value(1000), + hpx::program_options::value()->default_value(100), "number of tests to be averaged (default: 100)")( "csv_output", "print results in csv format")("seed,s", hpx::program_options::value(), From 3d388184cfe0fac44437cbbb841a4b4428ddc063 Mon Sep 17 00:00:00 2001 From: Vedant Date: Sun, 25 Feb 2024 23:18:08 +0530 Subject: [PATCH 03/24] Added interface for nanbench backend, and tested interface with minmax perf test Signed-off-by: Vedant --- CMakeLists.txt | 14 +- cmake/HPX_SetupNanobench.cmake | 39 +++++- libs/core/testing/CMakeLists.txt | 16 ++- .../include/hpx/testing/performance.hpp | 10 +- libs/core/testing/src/performance.cpp | 75 +++++++++++ .../tests/performance/CMakeLists.txt | 12 +- .../minmax_element_performance.cpp | 27 ++-- .../tests/performance/minmax_nanobench.cpp | 120 ------------------ 8 files changed, 168 insertions(+), 145 deletions(-) delete mode 100644 libs/full/segmented_algorithms/tests/performance/minmax_nanobench.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 240d8bccf594..418428ac173b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1028,7 +1028,7 @@ hpx_option( ) hpx_option( - HPX_WITH_NANOBENCH_BENCHMARKS + HPX_WITH_NANOBENCH BOOL "Use Nanobench for performance tests. Nanobench will be fetched using FetchContent (defualt: OFF)" OFF @@ -1272,6 +1272,10 @@ if(HPX_WITH_NETWORKING) if(HPX_WITH_PARCELPORT_ACTION_COUNTERS) hpx_add_config_define(HPX_HAVE_PARCELPORT_ACTION_COUNTERS) endif() + if (HPX_WITH_NANOBENCH) + hpx_add_config_define(HPX_HAVE_NANOBENCH) + include(HPX_SetupNanobench) + endif() else(HPX_WITH_NETWORKING) # if networking is off, then allow the option of using our asynchronous MPI # features @@ -2228,9 +2232,6 @@ endif() # should be found separately in the appropriate subdirectory. include(HPX_SetupThreads) -if (HPX_WITH_NANOBENCH_BENCHMARKS) - include(HPX_SetupNanobench) -endif() # Setup our required Boost libraries. include(HPX_SetupBoost) @@ -2560,9 +2561,12 @@ if(HPX_WITH_PRECOMPILED_HEADERS) - ) + if (HPX_WITH_NANOBENCH) + set(system_precompiled_headers ${system_precompiled_headers} ) + endif() + if(HPX_WITH_CXX17_FILESYSTEM) list(APPEND system_precompiled_headers ) endif() diff --git a/cmake/HPX_SetupNanobench.cmake b/cmake/HPX_SetupNanobench.cmake index 7fe43d784667..3e78625f3d23 100644 --- a/cmake/HPX_SetupNanobench.cmake +++ b/cmake/HPX_SetupNanobench.cmake @@ -6,5 +6,42 @@ FetchContent_Declare( GIT_TAG v4.3.11 GIT_SHALLOW TRUE) -FetchContent_MakeAvailable(nanobench) +# fetchcontent_makeavailable(nanobench) +if(NOT nanobench_POPULATED) + fetchcontent_populate(nanobench) +endif() +set(NANOBENCH_ROOT ${nanobench_SOURCE_DIR}) +add_library(nanobench INTERFACE) +target_include_directories( + nanobench SYSTEM INTERFACE $ + $ +) + +install( + TARGETS nanobench + EXPORT HPXNanobenchTarget + COMPONENT core +) + +install( + FILES ${NANOBENCH_ROOT}/include/nanobench.h + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + COMPONENT core +) + +export( + TARGETS nanobench + # NAMESPACE nanobench:: + FILE "${CMAKE_CURRENT_BINARY_DIR}/lib/cmake/${HPX_PACKAGE_NAME}/HPXNanobenchTarget.cmake" +) + +install( + EXPORT HPXNanobenchTarget + # NAMESPACE nanobench:: + FILE HPXNanobenchTarget.cmake + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${HPX_PACKAGE_NAME} + COMPONENT cmake +) + +# add_library(nanobench::nanobench ALIAS nanobench) \ No newline at end of file diff --git a/libs/core/testing/CMakeLists.txt b/libs/core/testing/CMakeLists.txt index de9e4c969fe9..0884c1abadef 100644 --- a/libs/core/testing/CMakeLists.txt +++ b/libs/core/testing/CMakeLists.txt @@ -20,6 +20,19 @@ set(testing_compat_headers # Default location is $HPX_ROOT/libs/testing/src set(testing_sources testing.cpp performance.cpp) +set(module_deps + hpx_assertion + hpx_config + hpx_format + hpx_functional + hpx_preprocessor + hpx_util +) + +if(HPX_WITH_NANOBENCH) + set(module_deps ${module_deps} nanobench) +endif() + include(HPX_AddModule) add_hpx_module( core testing @@ -27,7 +40,6 @@ add_hpx_module( SOURCES ${testing_sources} HEADERS ${testing_headers} COMPAT_HEADERS ${testing_compat_headers} - MODULE_DEPENDENCIES hpx_assertion hpx_config hpx_format hpx_functional - hpx_preprocessor hpx_util + MODULE_DEPENDENCIES ${module_deps} CMAKE_SUBDIRS examples tests ) diff --git a/libs/core/testing/include/hpx/testing/performance.hpp b/libs/core/testing/include/hpx/testing/performance.hpp index d957ac6f2b14..5b084ddde200 100644 --- a/libs/core/testing/include/hpx/testing/performance.hpp +++ b/libs/core/testing/include/hpx/testing/performance.hpp @@ -13,10 +13,18 @@ #include namespace hpx::util { - +#if defined(HPX_HAVE_NANOBENCH) + HPX_CORE_EXPORT void perftests_report(std::string const& name, + std::string const& exec, std::size_t const steps, + hpx::function&& test); + HPX_CORE_EXPORT void perftests_print_times(std::ostream& strm, char const* templ); + HPX_CORE_EXPORT void perftests_print_times(std::ostream& strm); + HPX_CORE_EXPORT void perftests_print_times(); +#else HPX_CORE_EXPORT void perftests_report(std::string const& name, std::string const& exec, std::size_t const steps, hpx::function&& test); HPX_CORE_EXPORT void perftests_print_times(); +#endif } // namespace hpx::util diff --git a/libs/core/testing/src/performance.cpp b/libs/core/testing/src/performance.cpp index 5cd224ff1497..d043a6df451e 100644 --- a/libs/core/testing/src/performance.cpp +++ b/libs/core/testing/src/performance.cpp @@ -15,10 +15,46 @@ #include #include +#if defined(HPX_HAVE_NANOBENCH) +#define ANKERL_NANOBENCH_IMPLEMENT +#include +#endif + namespace hpx::util { namespace detail { +#if defined(HPX_HAVE_NANOBENCH) + constexpr int nanobench_epochs = 24; + constexpr int nanobench_warmup = 40; + + char const* nanobench_hpx_template() noexcept + { + return R"DELIM({ + "outputs": [ +{{#result}} { + "name": "{{name}}", + "executor": "{{context(executor)}}", + "series": [ + {{#measurement}}{{elapsed}}{{^-last}}, + {{/-last}}{{/measurement}} + ] + }{{^-last}},{{/-last}} +{{/result}} ] +})DELIM"; + } + + ankerl::nanobench::Bench& bench() + { + static ankerl::nanobench::Bench b; + static ankerl::nanobench::Config cfg; + + cfg.mWarmup = nanobench_warmup; + cfg.mNumEpochs = nanobench_epochs; + + return b.config(cfg); + } +#else // Json output for performance reports class json_perf_times { @@ -90,8 +126,46 @@ namespace hpx::util { { m_map[key_t(name, executor)].push_back(time); } +#endif + } // namespace detail +#if defined(HPX_HAVE_NANOBENCH) + void perftests_report(std::string const& name, std::string const& exec, + std::size_t const steps, hpx::function&& test) + { + if (steps == 0) + return; + + std::size_t const steps_per_epoch = steps / detail::nanobench_epochs + 1; + + detail::bench() + .name(name) + .context("executor", exec) + .minEpochIterations(steps_per_epoch) + .run(test); + } + + // Print all collected results to the provided stream, + // formatted the json according to the provided + // "mustache-style" template + void perftests_print_times(char const* templ, std::ostream& strm) + { + detail::bench().render(templ, strm); + } + + // Overload that uses a default nanobench template + void perftests_print_times(std::ostream& strm) + { + perftests_print_times(detail::nanobench_hpx_template(), strm); + } + + // Overload that uses a default nanobench template and prints to std::cout + void perftests_print_times() + { + perftests_print_times(detail::nanobench_hpx_template(), std::cout); + } +#else void perftests_report(std::string const& name, std::string const& exec, std::size_t const steps, hpx::function&& test) { @@ -119,4 +193,5 @@ namespace hpx::util { { std::cout << detail::times(); } +#endif } // namespace hpx::util diff --git a/libs/full/segmented_algorithms/tests/performance/CMakeLists.txt b/libs/full/segmented_algorithms/tests/performance/CMakeLists.txt index 159c688711cf..1901f356ab51 100644 --- a/libs/full/segmented_algorithms/tests/performance/CMakeLists.txt +++ b/libs/full/segmented_algorithms/tests/performance/CMakeLists.txt @@ -6,9 +6,9 @@ set(benchmarks minmax_element_performance) -if (HPX_WITH_NANOBENCH_BENCHMARKS) - set(benchmarks ${benchmarks} minmax_nanobench) -endif() +# if (HPX_WITH_NANOBENCH_BENCHMARKS) +# set(benchmarks ${benchmarks} minmax_nanobench) +# endif() foreach(benchmark ${benchmarks}) set(sources ${benchmark}.cpp) @@ -16,9 +16,9 @@ foreach(benchmark ${benchmarks}) source_group("Source Files" FILES ${sources}) set(deps iostreams_component partitioned_vector_component) - if (HPX_WITH_NANOBENCH_BENCHMARKS) - set(deps ${deps} nanobench) - endif() + # if (HPX_WITH_NANOBENCH_BENCHMARKS) + # set(deps ${deps} nanobench) + # endif() # add example executable add_hpx_executable( diff --git a/libs/full/segmented_algorithms/tests/performance/minmax_element_performance.cpp b/libs/full/segmented_algorithms/tests/performance/minmax_element_performance.cpp index dfbc603a7e92..8cd3df818caf 100644 --- a/libs/full/segmented_algorithms/tests/performance/minmax_element_performance.cpp +++ b/libs/full/segmented_algorithms/tests/performance/minmax_element_performance.cpp @@ -14,8 +14,8 @@ #include #include #include - #include +#include #include #include @@ -124,17 +124,24 @@ int hpx_main(hpx::program_options::variables_map& vm) hpx::generate(hpx::execution::par, v.begin(), v.end(), random_fill()); // run benchmark - double time_minmax = run_minmax_element_benchmark(test_count, v); - double time_min = run_min_element_benchmark(test_count, v); - double time_max = run_max_element_benchmark(test_count, v); + hpx::util::perftests_report("hpx::minmax", "par", test_count, [&]{ + hpx::minmax_element(hpx::execution::par, v.begin(), v.end());}); + + hpx::util::perftests_report("hpx::min", "par", test_count, [&]{hpx::min_element( + hpx::execution::par, v.begin(), v.end());}); + + hpx::util::perftests_report("hpx::max", "par", test_count, [&]{hpx::max_element( + hpx::execution::par, v.begin(), v.end());}); + + hpx::util::perftests_print_times(); // if (csvoutput) - { - std::cout << "minmax" << test_count << "," << time_minmax - << std::endl; - std::cout << "min" << test_count << "," << time_min << std::endl; - std::cout << "max" << test_count << "," << time_max << std::endl; - } + // { + // std::cout << "minmax" << test_count << "," << time_minmax + // << std::endl; + // std::cout << "min" << test_count << "," << time_min << std::endl; + // std::cout << "max" << test_count << "," << time_max << std::endl; + // } return hpx::finalize(); } diff --git a/libs/full/segmented_algorithms/tests/performance/minmax_nanobench.cpp b/libs/full/segmented_algorithms/tests/performance/minmax_nanobench.cpp deleted file mode 100644 index 37b8204adc67..000000000000 --- a/libs/full/segmented_algorithms/tests/performance/minmax_nanobench.cpp +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright (c) 2016 Hartmut Kaiser -// -// SPDX-License-Identifier: BSL-1.0 -// Distributed under the Boost Software License, Version 1.0. (See accompanying -// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -#include -#if !defined(HPX_COMPUTE_DEVICE_CODE) -#include -#include - -#include -#include -#include -#include -#include - -#include - -#include -#include -#include -#include -#include -#include -#include - -/////////////////////////////////////////////////////////////////////////////// -// Define the vector types to be used. -HPX_REGISTER_PARTITIONED_VECTOR(int) -unsigned int seed = (unsigned int) std::random_device{}(); - -/////////////////////////////////////////////////////////////////////////////// -struct random_fill -{ - random_fill() - : gen(seed) - , dist(0, RAND_MAX) - { - } - - int operator()() - { - return dist(gen); - } - - std::mt19937 gen; - std::uniform_int_distribution<> dist; - - template - void serialize(Archive&, unsigned) - { - } -}; - -/////////////////////////////////////////////////////////////////////////////// -int hpx_main(hpx::program_options::variables_map& vm) -{ - if (hpx::get_locality_id() == 0) - { - // pull values from cmd - std::size_t size = vm["vector_size"].as(); - //bool csvoutput = vm.count("csv_output") != 0; - int test_count = vm["test_count"].as(); - - // create as many partitions as we have localities - hpx::partitioned_vector v( - size, hpx::container_layout(hpx::find_all_localities())); - - // initialize data - hpx::generate(hpx::execution::par, v.begin(), v.end(), random_fill()); - - ankerl::nanobench::Bench().minEpochIterations(test_count).run("min_element", [&]{ - hpx::min_element(hpx::execution::par, v.begin(), v.end()); - }); - - ankerl::nanobench::Bench().minEpochIterations(test_count).run("max_element", [&]{ - hpx::max_element(hpx::execution::par, v.begin(), v.end()); - }); - - ankerl::nanobench::Bench().minEpochIterations(test_count).run("minmax_element", [&]{ - hpx::minmax_element(hpx::execution::par, v.begin(), v.end()); - }) - .render(ankerl::nanobench::templates::pyperf(), std::cout); - - return hpx::finalize(); - } - - return 0; -} - -int main(int argc, char** argv) -{ - // ::benchmark::Initialize(&argc, argv); - - std::srand((unsigned int) std::time(nullptr)); - - // initialize program - std::vector const cfg = { - "hpx.os_threads=all", "hpx.run_hpx_main!=1"}; - - hpx::program_options::options_description cmdline( - "usage: " HPX_APPLICATION_STRING " [options]"); - - cmdline.add_options()("vector_size", - hpx::program_options::value()->default_value(1000), - "size of vector (default: 1000)")("test_count", - hpx::program_options::value()->default_value(100), - "number of tests to be averaged (default: 100)")( - "csv_output", "print results in csv format")("seed,s", - hpx::program_options::value(), - "the random number generator seed to use for this run"); - - hpx::init_params init_args; - init_args.desc_cmdline = cmdline; - init_args.cfg = cfg; - return hpx::init(argc, argv, init_args); - return 0; -} -#endif From 35a3f1db98593cf2d6a0dd843fe90b16d6a30332 Mon Sep 17 00:00:00 2001 From: Vedant Date: Mon, 26 Feb 2024 22:11:41 +0530 Subject: [PATCH 04/24] Added workflows to test out benchmarks Signed-off-by: Vedant --- .github/workflows/linux_with_bench_test.yml | 50 +++++++++++++++++++++ libs/core/testing/src/performance.cpp | 2 +- 2 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/linux_with_bench_test.yml diff --git a/.github/workflows/linux_with_bench_test.yml b/.github/workflows/linux_with_bench_test.yml new file mode 100644 index 000000000000..6886bf465b41 --- /dev/null +++ b/.github/workflows/linux_with_bench_test.yml @@ -0,0 +1,50 @@ +# Copyright (c) 2024 Vedant Nimje +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +name: Linux CI (Debug) with Benchmark Test + +on: + pull_request: + branches: [ master ] + push: + branches: [ nano_test ] + + +jobs: + build: + runs-on: ubuntu-latest + container: stellargroup/build_env:14 + + steps: + - uses: actions/checkout@v4 + - name: Configure + shell: bash + run: | + cmake \ + . \ + -Bbuild \ + -GNinja \ + -DCMAKE_BUILD_TYPE=Debug \ + -DHPX_WITH_MALLOC=system \ + -DHPX_WITH_FETCH_ASIO=ON \ + -DHPX_WITH_EXAMPLES=ON \ + -DHPX_WITH_TESTS=ON \ + -DHPX_WITH_NANOBENCH=ON \ + -DHPX_WITH_TESTS_MAX_THREADS_PER_LOCALITY=2 \ + -DHPX_WITH_CHECK_MODULE_DEPENDENCIES=On + - name: Build + shell: bash + run: | + cmake --build build --target all + cmake --build build --target examples + - name: Test + shell: bash + run: | + cd build + ctest \ + --output-on-failure \ + --tests-regex tests.examples \ + --exclude-regex tests.examples.transpose.transpose_block_numa diff --git a/libs/core/testing/src/performance.cpp b/libs/core/testing/src/performance.cpp index d043a6df451e..58a66ab07f79 100644 --- a/libs/core/testing/src/performance.cpp +++ b/libs/core/testing/src/performance.cpp @@ -31,7 +31,7 @@ namespace hpx::util { char const* nanobench_hpx_template() noexcept { return R"DELIM({ - "outputs": [ + "outputs (with nanobench)": [ {{#result}} { "name": "{{name}}", "executor": "{{context(executor)}}", From 59363bde3b522ca51dc65d5c97ee5c91fc677156 Mon Sep 17 00:00:00 2001 From: Vedant Date: Mon, 26 Feb 2024 22:21:52 +0530 Subject: [PATCH 05/24] Modified CI to build and test minmax benchmark only Signed-off-by: Vedant --- .github/workflows/linux_with_bench_test.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/linux_with_bench_test.yml b/.github/workflows/linux_with_bench_test.yml index 6886bf465b41..73703dc848e8 100644 --- a/.github/workflows/linux_with_bench_test.yml +++ b/.github/workflows/linux_with_bench_test.yml @@ -38,13 +38,9 @@ jobs: - name: Build shell: bash run: | - cmake --build build --target all - cmake --build build --target examples + cmake --build build/ --target tests.performance.modules.segmented_algorithms.minmax_element_performance - name: Test shell: bash run: | cd build - ctest \ - --output-on-failure \ - --tests-regex tests.examples \ - --exclude-regex tests.examples.transpose.transpose_block_numa + ./bin/minmax_element_performance_test From 506e240715591e656a659199b48cb8e38ea0e58a Mon Sep 17 00:00:00 2001 From: Vedant Date: Mon, 26 Feb 2024 22:58:16 +0530 Subject: [PATCH 06/24] Added separate CI for benchmark without nanobench Signed-off-by: Vedant --- .github/workflows/linux_with_bench_test.yml | 1 - .../workflows/linux_with_nanobench_test.yml | 46 +++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/linux_with_nanobench_test.yml diff --git a/.github/workflows/linux_with_bench_test.yml b/.github/workflows/linux_with_bench_test.yml index 73703dc848e8..142c1b418613 100644 --- a/.github/workflows/linux_with_bench_test.yml +++ b/.github/workflows/linux_with_bench_test.yml @@ -32,7 +32,6 @@ jobs: -DHPX_WITH_FETCH_ASIO=ON \ -DHPX_WITH_EXAMPLES=ON \ -DHPX_WITH_TESTS=ON \ - -DHPX_WITH_NANOBENCH=ON \ -DHPX_WITH_TESTS_MAX_THREADS_PER_LOCALITY=2 \ -DHPX_WITH_CHECK_MODULE_DEPENDENCIES=On - name: Build diff --git a/.github/workflows/linux_with_nanobench_test.yml b/.github/workflows/linux_with_nanobench_test.yml new file mode 100644 index 000000000000..73703dc848e8 --- /dev/null +++ b/.github/workflows/linux_with_nanobench_test.yml @@ -0,0 +1,46 @@ +# Copyright (c) 2024 Vedant Nimje +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +name: Linux CI (Debug) with Benchmark Test + +on: + pull_request: + branches: [ master ] + push: + branches: [ nano_test ] + + +jobs: + build: + runs-on: ubuntu-latest + container: stellargroup/build_env:14 + + steps: + - uses: actions/checkout@v4 + - name: Configure + shell: bash + run: | + cmake \ + . \ + -Bbuild \ + -GNinja \ + -DCMAKE_BUILD_TYPE=Debug \ + -DHPX_WITH_MALLOC=system \ + -DHPX_WITH_FETCH_ASIO=ON \ + -DHPX_WITH_EXAMPLES=ON \ + -DHPX_WITH_TESTS=ON \ + -DHPX_WITH_NANOBENCH=ON \ + -DHPX_WITH_TESTS_MAX_THREADS_PER_LOCALITY=2 \ + -DHPX_WITH_CHECK_MODULE_DEPENDENCIES=On + - name: Build + shell: bash + run: | + cmake --build build/ --target tests.performance.modules.segmented_algorithms.minmax_element_performance + - name: Test + shell: bash + run: | + cd build + ./bin/minmax_element_performance_test From cbee8b1683ec65e9484f5edfa8a8234fd4d53a92 Mon Sep 17 00:00:00 2001 From: Vedant Date: Mon, 26 Feb 2024 22:59:20 +0530 Subject: [PATCH 07/24] Changing new CI name Signed-off-by: Vedant --- .github/workflows/linux_with_bench_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux_with_bench_test.yml b/.github/workflows/linux_with_bench_test.yml index 142c1b418613..de03182dfe4d 100644 --- a/.github/workflows/linux_with_bench_test.yml +++ b/.github/workflows/linux_with_bench_test.yml @@ -4,7 +4,7 @@ # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -name: Linux CI (Debug) with Benchmark Test +name: Linux CI (Debug) with Nanobench Benchmark Test on: pull_request: From 297900120f52f8d5c47038e41c83c87f9c9ffba0 Mon Sep 17 00:00:00 2001 From: Vedant Date: Mon, 26 Feb 2024 23:00:12 +0530 Subject: [PATCH 08/24] Correcting CI names Signed-off-by: Vedant --- .github/workflows/linux_with_bench_test.yml | 2 +- .github/workflows/linux_with_nanobench_test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux_with_bench_test.yml b/.github/workflows/linux_with_bench_test.yml index de03182dfe4d..142c1b418613 100644 --- a/.github/workflows/linux_with_bench_test.yml +++ b/.github/workflows/linux_with_bench_test.yml @@ -4,7 +4,7 @@ # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -name: Linux CI (Debug) with Nanobench Benchmark Test +name: Linux CI (Debug) with Benchmark Test on: pull_request: diff --git a/.github/workflows/linux_with_nanobench_test.yml b/.github/workflows/linux_with_nanobench_test.yml index 73703dc848e8..49acb3994c3f 100644 --- a/.github/workflows/linux_with_nanobench_test.yml +++ b/.github/workflows/linux_with_nanobench_test.yml @@ -4,7 +4,7 @@ # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -name: Linux CI (Debug) with Benchmark Test +name: Linux CI (Debug) with Nanobench Benchmark Test on: pull_request: From 4683bea2ece7574cb1bcb4f38d5095fc7e48763c Mon Sep 17 00:00:00 2001 From: Vedant Date: Tue, 27 Feb 2024 21:40:16 +0530 Subject: [PATCH 09/24] Changing CI activations to PR only Signed-off-by: Vedant --- .github/workflows/linux_with_bench_test.yml | 6 +----- .github/workflows/linux_with_nanobench_test.yml | 6 +----- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/.github/workflows/linux_with_bench_test.yml b/.github/workflows/linux_with_bench_test.yml index 142c1b418613..a43544d43423 100644 --- a/.github/workflows/linux_with_bench_test.yml +++ b/.github/workflows/linux_with_bench_test.yml @@ -6,11 +6,7 @@ name: Linux CI (Debug) with Benchmark Test -on: - pull_request: - branches: [ master ] - push: - branches: [ nano_test ] +on: [pull_request] jobs: diff --git a/.github/workflows/linux_with_nanobench_test.yml b/.github/workflows/linux_with_nanobench_test.yml index 49acb3994c3f..5d6ba22afaa3 100644 --- a/.github/workflows/linux_with_nanobench_test.yml +++ b/.github/workflows/linux_with_nanobench_test.yml @@ -6,11 +6,7 @@ name: Linux CI (Debug) with Nanobench Benchmark Test -on: - pull_request: - branches: [ master ] - push: - branches: [ nano_test ] +on: [pull_request] jobs: From 1f2ae1c8cc55cf7554048a79ca7eb30cccf3d597 Mon Sep 17 00:00:00 2001 From: Vedant Date: Thu, 7 Mar 2024 10:58:01 +0530 Subject: [PATCH 10/24] Adding back namespace defs and add_library call for nanobench Signed-off-by: Vedant --- CMakeLists.txt | 4 +-- cmake/HPX_SetupNanobench.cmake | 50 +++++++++++++++++----------------- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 418428ac173b..3be765feec99 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1272,7 +1272,7 @@ if(HPX_WITH_NETWORKING) if(HPX_WITH_PARCELPORT_ACTION_COUNTERS) hpx_add_config_define(HPX_HAVE_PARCELPORT_ACTION_COUNTERS) endif() - if (HPX_WITH_NANOBENCH) + if(HPX_WITH_NANOBENCH) hpx_add_config_define(HPX_HAVE_NANOBENCH) include(HPX_SetupNanobench) endif() @@ -2563,7 +2563,7 @@ if(HPX_WITH_PRECOMPILED_HEADERS) ) - if (HPX_WITH_NANOBENCH) + if(HPX_WITH_NANOBENCH) set(system_precompiled_headers ${system_precompiled_headers} ) endif() diff --git a/cmake/HPX_SetupNanobench.cmake b/cmake/HPX_SetupNanobench.cmake index 3e78625f3d23..14ba9880a209 100644 --- a/cmake/HPX_SetupNanobench.cmake +++ b/cmake/HPX_SetupNanobench.cmake @@ -1,47 +1,47 @@ include(FetchContent) -FetchContent_Declare( - nanobench - GIT_REPOSITORY https://github.com/martinus/nanobench.git - GIT_TAG v4.3.11 - GIT_SHALLOW TRUE) +fetchcontent_declare( + nanobench + GIT_REPOSITORY https://github.com/martinus/nanobench.git + GIT_TAG v4.3.11 + GIT_SHALLOW TRUE +) -# fetchcontent_makeavailable(nanobench) if(NOT nanobench_POPULATED) - fetchcontent_populate(nanobench) + fetchcontent_populate(nanobench) endif() -set(NANOBENCH_ROOT ${nanobench_SOURCE_DIR}) +set(Nanobench_ROOT ${nanobench_SOURCE_DIR}) add_library(nanobench INTERFACE) target_include_directories( - nanobench SYSTEM INTERFACE $ - $ + nanobench SYSTEM INTERFACE $ + $ ) install( - TARGETS nanobench - EXPORT HPXNanobenchTarget - COMPONENT core + TARGETS nanobench + EXPORT HPXNanobenchTarget + COMPONENT core ) install( - FILES ${NANOBENCH_ROOT}/include/nanobench.h - DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} - COMPONENT core + FILES ${NANOBENCH_ROOT}/include/nanobench.h + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + COMPONENT core ) export( - TARGETS nanobench - # NAMESPACE nanobench:: - FILE "${CMAKE_CURRENT_BINARY_DIR}/lib/cmake/${HPX_PACKAGE_NAME}/HPXNanobenchTarget.cmake" + TARGETS nanobench + NAMESPACE nanobench:: + FILE "${CMAKE_CURRENT_BINARY_DIR}/lib/cmake/${HPX_PACKAGE_NAME}/HPXNanobenchTarget.cmake" ) install( - EXPORT HPXNanobenchTarget - # NAMESPACE nanobench:: - FILE HPXNanobenchTarget.cmake - DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${HPX_PACKAGE_NAME} - COMPONENT cmake + EXPORT HPXNanobenchTarget + NAMESPACE nanobench:: + FILE HPXNanobenchTarget.cmake + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${HPX_PACKAGE_NAME} + COMPONENT cmake ) -# add_library(nanobench::nanobench ALIAS nanobench) \ No newline at end of file +add_library(nanobench::nanobench ALIAS nanobench) From ee41a7942a9ea18c185d132c7cc90ef693322ae7 Mon Sep 17 00:00:00 2001 From: Vedant Date: Sun, 17 Mar 2024 00:00:08 +0530 Subject: [PATCH 11/24] Clean up minmax test Signed-off-by: Vedant --- .../minmax_element_performance.cpp | 62 ------------------- 1 file changed, 62 deletions(-) diff --git a/libs/full/segmented_algorithms/tests/performance/minmax_element_performance.cpp b/libs/full/segmented_algorithms/tests/performance/minmax_element_performance.cpp index 8cd3df818caf..de0242854807 100644 --- a/libs/full/segmented_algorithms/tests/performance/minmax_element_performance.cpp +++ b/libs/full/segmented_algorithms/tests/performance/minmax_element_performance.cpp @@ -52,60 +52,6 @@ struct random_fill } }; -/////////////////////////////////////////////////////////////////////////////// -double run_min_element_benchmark( - int test_count, hpx::partitioned_vector const& v) -{ - std::uint64_t time = hpx::chrono::high_resolution_clock::now(); - - for (int i = 0; i != test_count; ++i) - { - // invoke minmax - /*auto iters = */ hpx::min_element( - hpx::execution::par, v.begin(), v.end()); - } - - time = hpx::chrono::high_resolution_clock::now() - time; - - return (time * 1e-9) / test_count; -} - -/////////////////////////////////////////////////////////////////////////////// -double run_max_element_benchmark( - int test_count, hpx::partitioned_vector const& v) -{ - std::uint64_t time = hpx::chrono::high_resolution_clock::now(); - - for (int i = 0; i != test_count; ++i) - { - // invoke minmax - /*auto iters = */ hpx::max_element( - hpx::execution::par, v.begin(), v.end()); - } - - time = hpx::chrono::high_resolution_clock::now() - time; - - return (time * 1e-9) / test_count; -} - -/////////////////////////////////////////////////////////////////////////////// -double run_minmax_element_benchmark( - int test_count, hpx::partitioned_vector const& v) -{ - std::uint64_t time = hpx::chrono::high_resolution_clock::now(); - - for (int i = 0; i != test_count; ++i) - { - // invoke minmax - /*auto iters = */ hpx::minmax_element( - hpx::execution::par, v.begin(), v.end()); - } - - time = hpx::chrono::high_resolution_clock::now() - time; - - return (time * 1e-9) / test_count; -} - /////////////////////////////////////////////////////////////////////////////// int hpx_main(hpx::program_options::variables_map& vm) { @@ -135,14 +81,6 @@ int hpx_main(hpx::program_options::variables_map& vm) hpx::util::perftests_print_times(); - // if (csvoutput) - // { - // std::cout << "minmax" << test_count << "," << time_minmax - // << std::endl; - // std::cout << "min" << test_count << "," << time_min << std::endl; - // std::cout << "max" << test_count << "," << time_max << std::endl; - // } - return hpx::finalize(); } From 4bb7d3d1c607069f101a81c13a1ea6a653639499 Mon Sep 17 00:00:00 2001 From: Vedant Date: Sun, 17 Mar 2024 01:02:31 +0530 Subject: [PATCH 12/24] Adding command line argument for detailed bench output Signed-off-by: Vedant --- .../workflows/linux_with_nanobench_test.yml | 6 ++++++ .../command_line_handling_local.hpp | 2 ++ .../src/command_line_handling_local.cpp | 4 ++++ .../src/parse_command_line_local.cpp | 1 + libs/core/testing/CMakeLists.txt | 1 + .../include/hpx/testing/performance.hpp | 1 + libs/core/testing/src/performance.cpp | 18 +++++++++++++++++- 7 files changed, 32 insertions(+), 1 deletion(-) diff --git a/.github/workflows/linux_with_nanobench_test.yml b/.github/workflows/linux_with_nanobench_test.yml index 5d6ba22afaa3..4d29ad983d1d 100644 --- a/.github/workflows/linux_with_nanobench_test.yml +++ b/.github/workflows/linux_with_nanobench_test.yml @@ -40,3 +40,9 @@ jobs: run: | cd build ./bin/minmax_element_performance_test + + name: Test + shell: bash + run: | + cd build + ./bin/minmax_element_performance_test --hpx:verbose_bench diff --git a/libs/core/command_line_handling_local/include/hpx/command_line_handling_local/command_line_handling_local.hpp b/libs/core/command_line_handling_local/include/hpx/command_line_handling_local/command_line_handling_local.hpp index 83bb82d0ed27..38dbf65385eb 100644 --- a/libs/core/command_line_handling_local/include/hpx/command_line_handling_local/command_line_handling_local.hpp +++ b/libs/core/command_line_handling_local/include/hpx/command_line_handling_local/command_line_handling_local.hpp @@ -88,6 +88,8 @@ namespace hpx::local::detail { std::vector& ini_config) const; }; + HPX_CORE_EXPORT inline bool verbose_; + /////////////////////////////////////////////////////////////////////////// HPX_CORE_EXPORT std::string runtime_configuration_string( command_line_handling const& cfg); diff --git a/libs/core/command_line_handling_local/src/command_line_handling_local.cpp b/libs/core/command_line_handling_local/src/command_line_handling_local.cpp index 4437042ace83..ba419e6fe201 100644 --- a/libs/core/command_line_handling_local/src/command_line_handling_local.cpp +++ b/libs/core/command_line_handling_local/src/command_line_handling_local.cpp @@ -633,6 +633,10 @@ namespace hpx::local::detail { { print_config(ini_config); } + + if (vm.count("hpx:verbose_bench")) { + verbose_ = true; + } return true; } diff --git a/libs/core/command_line_handling_local/src/parse_command_line_local.cpp b/libs/core/command_line_handling_local/src/parse_command_line_local.cpp index dceab26cb24e..5b01d3121204 100644 --- a/libs/core/command_line_handling_local/src/parse_command_line_local.cpp +++ b/libs/core/command_line_handling_local/src/parse_command_line_local.cpp @@ -614,6 +614,7 @@ namespace hpx::local::detail { ("hpx:debug-app-log", value()->implicit_value("cout"), "enable all messages on the application log channel and send all " "application logs to the target destination") + ("hpx:verbose_bench", "For logging benchmarks in detail") ; all_options[options_type::hidden_options].add_options() diff --git a/libs/core/testing/CMakeLists.txt b/libs/core/testing/CMakeLists.txt index 0884c1abadef..e4e4d60f84e9 100644 --- a/libs/core/testing/CMakeLists.txt +++ b/libs/core/testing/CMakeLists.txt @@ -27,6 +27,7 @@ set(module_deps hpx_functional hpx_preprocessor hpx_util + hpx_command_line_handling_local ) if(HPX_WITH_NANOBENCH) diff --git a/libs/core/testing/include/hpx/testing/performance.hpp b/libs/core/testing/include/hpx/testing/performance.hpp index 5b084ddde200..e232d8285768 100644 --- a/libs/core/testing/include/hpx/testing/performance.hpp +++ b/libs/core/testing/include/hpx/testing/performance.hpp @@ -8,6 +8,7 @@ #include #include +#include #include #include diff --git a/libs/core/testing/src/performance.cpp b/libs/core/testing/src/performance.cpp index 58a66ab07f79..ce8eaf78aecf 100644 --- a/libs/core/testing/src/performance.cpp +++ b/libs/core/testing/src/performance.cpp @@ -28,6 +28,19 @@ namespace hpx::util { constexpr int nanobench_epochs = 24; constexpr int nanobench_warmup = 40; + char const* nanobench_hpx_simple_template() noexcept + { + return R"DELIM({ + "outputs (with nanobench)": [ +{{#result}} { + "name": "{{name}}", + "executor": "{{context(executor)}}", + "median": "{{median(elapsed)}}" + }{{^-last}},{{/-last}} +{{/result}} ] +})DELIM"; + } + char const* nanobench_hpx_template() noexcept { return R"DELIM({ @@ -163,7 +176,10 @@ namespace hpx::util { // Overload that uses a default nanobench template and prints to std::cout void perftests_print_times() { - perftests_print_times(detail::nanobench_hpx_template(), std::cout); + if (local::detail::verbose_) + perftests_print_times(detail::nanobench_hpx_template(), std::cout); + else + perftests_print_times(detail::nanobench_hpx_simple_template(), std::cout); } #else void perftests_report(std::string const& name, std::string const& exec, From 77fa1975c59d3262106fa7208c87c827245bde8c Mon Sep 17 00:00:00 2001 From: Vedant Date: Sun, 17 Mar 2024 01:15:24 +0530 Subject: [PATCH 13/24] Fixing CI format Signed-off-by: Vedant --- .github/workflows/linux_with_nanobench_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux_with_nanobench_test.yml b/.github/workflows/linux_with_nanobench_test.yml index 4d29ad983d1d..4409c64040e3 100644 --- a/.github/workflows/linux_with_nanobench_test.yml +++ b/.github/workflows/linux_with_nanobench_test.yml @@ -41,7 +41,7 @@ jobs: cd build ./bin/minmax_element_performance_test - name: Test + - name: Test with detailed output shell: bash run: | cd build From 2e6258a377efe6934f8489d518ca752a2f72a028 Mon Sep 17 00:00:00 2001 From: Vedant Date: Sun, 17 Mar 2024 15:59:42 +0530 Subject: [PATCH 14/24] Adding initial benchmark visualisation script Signed-off-by: Vedant --- libs/core/testing/src/performance.cpp | 4 +-- tools/perftests_plot.py | 35 +++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 tools/perftests_plot.py diff --git a/libs/core/testing/src/performance.cpp b/libs/core/testing/src/performance.cpp index ce8eaf78aecf..74f601198351 100644 --- a/libs/core/testing/src/performance.cpp +++ b/libs/core/testing/src/performance.cpp @@ -31,7 +31,7 @@ namespace hpx::util { char const* nanobench_hpx_simple_template() noexcept { return R"DELIM({ - "outputs (with nanobench)": [ + "outputs": [ {{#result}} { "name": "{{name}}", "executor": "{{context(executor)}}", @@ -44,7 +44,7 @@ namespace hpx::util { char const* nanobench_hpx_template() noexcept { return R"DELIM({ - "outputs (with nanobench)": [ + "outputs": [ {{#result}} { "name": "{{name}}", "executor": "{{context(executor)}}", diff --git a/tools/perftests_plot.py b/tools/perftests_plot.py new file mode 100644 index 000000000000..756a1c1d7e36 --- /dev/null +++ b/tools/perftests_plot.py @@ -0,0 +1,35 @@ +import sys +import os +import subprocess +import json +import matplotlib.pyplot as plt + +if len(sys.argv) != 2: + print("No benchmark selected") + print("Usage: python perftests_plot.py [path_to_benchmark_binary]") +else: + test_name = sys.argv[1] + + contents = subprocess.run([test_name, "--hpx:verbose_bench"], capture_output=True) + + # print(contents) + json_obj = json.loads(contents.stdout.decode('utf-8')) + + test_names = [] + samples = [] + + for test in json_obj["outputs"]: + test_names.append(test["name"] + "," + test["executor"]) + samples.append(test["series"]) + + fig = plt.figure() + ax = fig.add_subplot() + bp = ax.boxplot(samples) + + ax.set_xticklabels(test_names) + + plt.show() + + + + \ No newline at end of file From 794de41bce0ea90082da054a86ebebc398f367a7 Mon Sep 17 00:00:00 2001 From: Vedant Date: Wed, 27 Mar 2024 12:37:56 +0530 Subject: [PATCH 15/24] Added benchmark output specialisations for normal case as well Signed-off-by: Vedant --- libs/core/testing/src/performance.cpp | 28 +++++++++++++++++---------- tools/perftests_plot.py | 9 +++------ 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/libs/core/testing/src/performance.cpp b/libs/core/testing/src/performance.cpp index 74f601198351..908a6da277d5 100644 --- a/libs/core/testing/src/performance.cpp +++ b/libs/core/testing/src/performance.cpp @@ -35,7 +35,7 @@ namespace hpx::util { {{#result}} { "name": "{{name}}", "executor": "{{context(executor)}}", - "median": "{{median(elapsed)}}" + "average": "{{average(elapsed)}}" }{{^-last}},{{/-last}} {{/result}} ] })DELIM"; @@ -72,7 +72,7 @@ namespace hpx::util { class json_perf_times { using key_t = std::tuple; - using value_t = std::vector; + using value_t = std::vector; using map_t = std::map; map_t m_map; @@ -111,19 +111,27 @@ namespace hpx::util { << "\",\n"; strm << R"( "executor" : ")" << std::get<1>(item.first) << "\",\n"; - strm << R"( "series" : [)"; - double average = 0.0; + + if (local::detail::verbose_) + strm << R"( "series" : [)"; + long double average = 0.0; int series = 0; - for (auto const val : item.second) + strm.precision(std::numeric_limits::max_digits10 - 1); + for (long double const val : item.second) { - if (series) - strm << ", "; - strm << val; + if (local::detail::verbose_) + { + if (series) + strm << ", "; + strm << std::scientific << val; + } ++series; average += val; } - strm << "],\n"; - strm << " \"average\" : " << average / series << "\n"; + if (local::detail::verbose_) + strm << "]\n"; + else + strm << " \"average\" : " << average / series << "\n"; strm << " }"; ++outputs; } diff --git a/tools/perftests_plot.py b/tools/perftests_plot.py index 756a1c1d7e36..73b33b68087a 100644 --- a/tools/perftests_plot.py +++ b/tools/perftests_plot.py @@ -3,6 +3,7 @@ import subprocess import json import matplotlib.pyplot as plt +import matplotlib.ticker as ticker if len(sys.argv) != 2: print("No benchmark selected") @@ -12,24 +13,20 @@ contents = subprocess.run([test_name, "--hpx:verbose_bench"], capture_output=True) - # print(contents) json_obj = json.loads(contents.stdout.decode('utf-8')) test_names = [] samples = [] + print(json_obj) for test in json_obj["outputs"]: test_names.append(test["name"] + "," + test["executor"]) samples.append(test["series"]) fig = plt.figure() ax = fig.add_subplot() - bp = ax.boxplot(samples) + bp = ax.boxplot(samples, showfliers=False) ax.set_xticklabels(test_names) - plt.show() - - - \ No newline at end of file From af5726b863e3b7c9d7b5001e8e2592f6d37938be Mon Sep 17 00:00:00 2001 From: Vedant Date: Fri, 29 Mar 2024 00:44:53 +0530 Subject: [PATCH 16/24] Standardised modules.datastructures.small_vector_benchmark, and made minor changes to plotting script * Further simplified the default template for the benchmark Signed-off-by: Vedant --- .../performance/small_vector_benchmark.cpp | 126 ++++++++++++------ libs/core/testing/src/performance.cpp | 16 +-- .../tests/performance/CMakeLists.txt | 7 +- tools/perftests_plot.py | 8 +- 4 files changed, 95 insertions(+), 62 deletions(-) diff --git a/libs/core/datastructures/tests/performance/small_vector_benchmark.cpp b/libs/core/datastructures/tests/performance/small_vector_benchmark.cpp index 964a3aa75c0a..6b1f98056afa 100644 --- a/libs/core/datastructures/tests/performance/small_vector_benchmark.cpp +++ b/libs/core/datastructures/tests/performance/small_vector_benchmark.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -19,30 +20,25 @@ /////////////////////////////////////////////////////////////////////////////// template -std::uint64_t measure(std::size_t repeat, std::size_t size) +void fill(std::size_t size) { - std::uint64_t start = hpx::chrono::high_resolution_clock::now(); - for (std::size_t i = 0; i != repeat; ++i) + Container cont; + for (std::size_t i = 0; i != size; ++i) { - Container cont; - for (std::size_t i = 0; i != size; ++i) - { - cont.push_back(typename Container::value_type{}); - } + cont.push_back(typename Container::value_type{}); } - return (hpx::chrono::high_resolution_clock::now() - start) / repeat; } -template -void compare(std::size_t repeat, std::size_t size) -{ - std::uint64_t time = measure>(repeat, size); +// template +// void compare(std::size_t repeat, std::size_t size) +// { +// std::uint64_t time = measure>(repeat, size); - std::cout << "-----Average-(hpx::small_vector<" << typeid(T).name() << ", " - << N << ">)------ \n" - << std::left << "Average execution time : " << std::right - << std::setw(8) << time / 1e9 << "\n"; -} +// std::cout << "-----Average-(hpx::small_vector<" << typeid(T).name() << ", " +// << N << ">)------ \n" +// << std::left << "Average execution time : " << std::right +// << std::setw(8) << time / 1e9 << "\n"; +// } int hpx_main(hpx::program_options::variables_map& vm) { @@ -50,30 +46,76 @@ int hpx_main(hpx::program_options::variables_map& vm) std::size_t repeat = vm["test_count"].as(); std::size_t size = vm["vector_size"].as(); - std::cout << std::left - << "----------------Parameters---------------------\n" - << std::left - << "Vector size : " << std::right - << std::setw(8) << size << "\n" - << std::left - << "Number of tests : " << std::right - << std::setw(8) << repeat << "\n" - << std::left - << "Display time in : " << std::right - << std::setw(8) << "Seconds\n" - << std::flush; - - compare(repeat, size); - compare(repeat, size); - compare(repeat, size); - compare(repeat, size); - compare(repeat, size); - - compare, 1>(repeat, size); - compare, 2>(repeat, size); - compare, 4>(repeat, size); - compare, 8>(repeat, size); - compare, 16>(repeat, size); + // std::cout << std::left + // << "----------------Parameters---------------------\n" + // << std::left + // << "Vector size : " << std::right + // << std::setw(8) << size << "\n" + // << std::left + // << "Number of tests : " << std::right + // << std::setw(8) << repeat << "\n" + // << std::left + // << "Display time in : " << std::right + // << std::setw(8) << "Seconds\n" + // << std::flush; + + // compare(repeat, size); + // compare(repeat, size); + // compare(repeat, size); + // compare(repeat, size); + // compare(repeat, size); + + // compare, 1>(repeat, size); + // compare, 2>(repeat, size); + // compare, 4>(repeat, size); + // compare, 8>(repeat, size); + // compare, 16>(repeat, size); + + // int + + hpx::util::perftests_report("hpx::small_vector", "", repeat, [&] { + fill>(size); + }); + + hpx::util::perftests_report("hpx::small_vector", "", repeat, [&] { + fill>(size); + }); + + hpx::util::perftests_report("hpx::small_vector", "", repeat, [&] { + fill>(size); + }); + + hpx::util::perftests_report("hpx::small_vector", "", repeat, [&] { + fill>(size); + }); + + hpx::util::perftests_report("hpx::small_vector", "", repeat, [&] { + fill>(size); + }); + + // hpx::move_only_function + + hpx::util::perftests_report("hpx::small_vector", ", 1>", repeat, [&] { + fill, 1>>(size); + }); + + hpx::util::perftests_report("hpx::small_vector", ", 2>", repeat, [&] { + fill, 2>>(size); + }); + + hpx::util::perftests_report("hpx::small_vector", ", 4>", repeat, [&] { + fill, 4>>(size); + }); + + hpx::util::perftests_report("hpx::small_vector", ", 8>", repeat, [&] { + fill, 8>>(size); + }); + + hpx::util::perftests_report("hpx::small_vector", ", 16>", repeat, [&] { + fill, 16>>(size); + }); + + hpx::util::perftests_print_times(); return hpx::local::finalize(); } diff --git a/libs/core/testing/src/performance.cpp b/libs/core/testing/src/performance.cpp index 908a6da277d5..1a6d43824714 100644 --- a/libs/core/testing/src/performance.cpp +++ b/libs/core/testing/src/performance.cpp @@ -30,15 +30,13 @@ namespace hpx::util { char const* nanobench_hpx_simple_template() noexcept { - return R"DELIM({ - "outputs": [ -{{#result}} { - "name": "{{name}}", - "executor": "{{context(executor)}}", - "average": "{{average(elapsed)}}" - }{{^-last}},{{/-last}} -{{/result}} ] -})DELIM"; + return R"DELIM(Results: +{{#result}} +name: {{name}}, +executor: {{context(executor)}}, +average": {{average(elapsed)}} +{{/result}} +)DELIM"; } char const* nanobench_hpx_template() noexcept diff --git a/libs/full/segmented_algorithms/tests/performance/CMakeLists.txt b/libs/full/segmented_algorithms/tests/performance/CMakeLists.txt index 1901f356ab51..7ef40a433fd1 100644 --- a/libs/full/segmented_algorithms/tests/performance/CMakeLists.txt +++ b/libs/full/segmented_algorithms/tests/performance/CMakeLists.txt @@ -15,16 +15,11 @@ foreach(benchmark ${benchmarks}) source_group("Source Files" FILES ${sources}) - set(deps iostreams_component partitioned_vector_component) - # if (HPX_WITH_NANOBENCH_BENCHMARKS) - # set(deps ${deps} nanobench) - # endif() - # add example executable add_hpx_executable( ${benchmark}_test INTERNAL_FLAGS SOURCES ${sources} ${${benchmark}_FLAGS} - DEPENDENCIES ${deps} + DEPENDENCIES iostreams_component partitioned_vector_component EXCLUDE_FROM_ALL HPX_PREFIX ${HPX_BUILD_PREFIX} FOLDER "Benchmarks/Modules/Full/SegmentedAlgorithms" diff --git a/tools/perftests_plot.py b/tools/perftests_plot.py index 73b33b68087a..8291c1b463cb 100644 --- a/tools/perftests_plot.py +++ b/tools/perftests_plot.py @@ -18,15 +18,13 @@ test_names = [] samples = [] - print(json_obj) for test in json_obj["outputs"]: - test_names.append(test["name"] + "," + test["executor"]) + test_names.append(test["name"] + ",\n" + test["executor"]) samples.append(test["series"]) - fig = plt.figure() + fig = plt.figure(figsize=(20, 6)) ax = fig.add_subplot() bp = ax.boxplot(samples, showfliers=False) - - ax.set_xticklabels(test_names) + plt.setp(ax.set_xticklabels(test_names), fontsize=7) plt.show() \ No newline at end of file From 14374eda360954537c1df89ba103b21b0e6925d0 Mon Sep 17 00:00:00 2001 From: Vedant Date: Thu, 11 Apr 2024 23:15:41 +0530 Subject: [PATCH 17/24] Updated cmdline switching to local config, and refactored benchmarks accordingly Signed-off-by: Vedant --- .../command_line_handling_local.hpp | 2 - .../src/command_line_handling_local.cpp | 4 - .../src/parse_command_line_local.cpp | 2 +- .../performance/small_vector_benchmark.cpp | 33 +----- .../include/hpx/testing/performance.hpp | 3 + libs/core/testing/src/performance.cpp | 106 +++++++++++------- .../minmax_element_performance.cpp | 4 + tools/perftests_plot.py | 9 +- 8 files changed, 87 insertions(+), 76 deletions(-) diff --git a/libs/core/command_line_handling_local/include/hpx/command_line_handling_local/command_line_handling_local.hpp b/libs/core/command_line_handling_local/include/hpx/command_line_handling_local/command_line_handling_local.hpp index 38dbf65385eb..83bb82d0ed27 100644 --- a/libs/core/command_line_handling_local/include/hpx/command_line_handling_local/command_line_handling_local.hpp +++ b/libs/core/command_line_handling_local/include/hpx/command_line_handling_local/command_line_handling_local.hpp @@ -88,8 +88,6 @@ namespace hpx::local::detail { std::vector& ini_config) const; }; - HPX_CORE_EXPORT inline bool verbose_; - /////////////////////////////////////////////////////////////////////////// HPX_CORE_EXPORT std::string runtime_configuration_string( command_line_handling const& cfg); diff --git a/libs/core/command_line_handling_local/src/command_line_handling_local.cpp b/libs/core/command_line_handling_local/src/command_line_handling_local.cpp index ba419e6fe201..4437042ace83 100644 --- a/libs/core/command_line_handling_local/src/command_line_handling_local.cpp +++ b/libs/core/command_line_handling_local/src/command_line_handling_local.cpp @@ -633,10 +633,6 @@ namespace hpx::local::detail { { print_config(ini_config); } - - if (vm.count("hpx:verbose_bench")) { - verbose_ = true; - } return true; } diff --git a/libs/core/command_line_handling_local/src/parse_command_line_local.cpp b/libs/core/command_line_handling_local/src/parse_command_line_local.cpp index 5b01d3121204..f4788e8896fd 100644 --- a/libs/core/command_line_handling_local/src/parse_command_line_local.cpp +++ b/libs/core/command_line_handling_local/src/parse_command_line_local.cpp @@ -614,7 +614,7 @@ namespace hpx::local::detail { ("hpx:debug-app-log", value()->implicit_value("cout"), "enable all messages on the application log channel and send all " "application logs to the target destination") - ("hpx:verbose_bench", "For logging benchmarks in detail") + // ("hpx:verbose_bench", "For logging benchmarks in detail") ; all_options[options_type::hidden_options].add_options() diff --git a/libs/core/datastructures/tests/performance/small_vector_benchmark.cpp b/libs/core/datastructures/tests/performance/small_vector_benchmark.cpp index 6b1f98056afa..0c659c3cfab6 100644 --- a/libs/core/datastructures/tests/performance/small_vector_benchmark.cpp +++ b/libs/core/datastructures/tests/performance/small_vector_benchmark.cpp @@ -45,34 +45,11 @@ int hpx_main(hpx::program_options::variables_map& vm) // pull values from cmd std::size_t repeat = vm["test_count"].as(); std::size_t size = vm["vector_size"].as(); + + hpx::util::perftests_init(vm); - // std::cout << std::left - // << "----------------Parameters---------------------\n" - // << std::left - // << "Vector size : " << std::right - // << std::setw(8) << size << "\n" - // << std::left - // << "Number of tests : " << std::right - // << std::setw(8) << repeat << "\n" - // << std::left - // << "Display time in : " << std::right - // << std::setw(8) << "Seconds\n" - // << std::flush; - - // compare(repeat, size); - // compare(repeat, size); - // compare(repeat, size); - // compare(repeat, size); - // compare(repeat, size); - - // compare, 1>(repeat, size); - // compare, 2>(repeat, size); - // compare, 4>(repeat, size); - // compare, 8>(repeat, size); - // compare, 16>(repeat, size); - - // int - + // int + hpx::util::perftests_report("hpx::small_vector", "", repeat, [&] { fill>(size); }); @@ -139,6 +116,8 @@ int main(int argc, char* argv[]) ; // clang-format on + hpx::util::perftests_cfg(cmdline); + hpx::local::init_params init_args; init_args.desc_cmdline = cmdline; init_args.cfg = cfg; diff --git a/libs/core/testing/include/hpx/testing/performance.hpp b/libs/core/testing/include/hpx/testing/performance.hpp index e232d8285768..207dca784d8a 100644 --- a/libs/core/testing/include/hpx/testing/performance.hpp +++ b/libs/core/testing/include/hpx/testing/performance.hpp @@ -14,6 +14,9 @@ #include namespace hpx::util { + HPX_CORE_EXPORT inline bool detailed_ = false; + HPX_CORE_EXPORT void perftests_cfg(hpx::program_options::options_description& cmdline); + HPX_CORE_EXPORT void perftests_init(const hpx::program_options::variables_map& vm); #if defined(HPX_HAVE_NANOBENCH) HPX_CORE_EXPORT void perftests_report(std::string const& name, std::string const& exec, std::size_t const steps, diff --git a/libs/core/testing/src/performance.cpp b/libs/core/testing/src/performance.cpp index 1a6d43824714..1a3192278c15 100644 --- a/libs/core/testing/src/performance.cpp +++ b/libs/core/testing/src/performance.cpp @@ -22,6 +22,19 @@ namespace hpx::util { + void perftests_cfg(hpx::program_options::options_description& cmdline) { + cmdline.add_options() + ("detailed_bench", + "Use if detailed benchmarks are required, showing the execution time taken for each epoch") + ; + } + + void perftests_init(const hpx::program_options::variables_map& vm) { + if (vm.count("detailed_bench")) { + detailed_ = true; + } + } + namespace detail { #if defined(HPX_HAVE_NANOBENCH) @@ -34,9 +47,9 @@ namespace hpx::util { {{#result}} name: {{name}}, executor: {{context(executor)}}, -average": {{average(elapsed)}} -{{/result}} -)DELIM"; +average: {{average(elapsed)}}{{^-last}} +{{/-last}} +{{/result}})DELIM"; } char const* nanobench_hpx_template() noexcept @@ -52,7 +65,8 @@ average": {{average(elapsed)}} ] }{{^-last}},{{/-last}} {{/result}} ] -})DELIM"; +} +)DELIM"; } ankerl::nanobench::Bench& bench() @@ -97,46 +111,62 @@ average": {{average(elapsed)}} std::ostream& operator<<(std::ostream& strm, json_perf_times const& obj) { - strm << "{\n"; - strm << " \"outputs\" : ["; - int outputs = 0; - for (auto&& item : obj.m_map) + + if (detailed_) { - if (outputs) - strm << ","; - strm << "\n {\n"; - strm << R"( "name" : ")" << std::get<0>(item.first) - << "\",\n"; - strm << R"( "executor" : ")" << std::get<1>(item.first) - << "\",\n"; - - if (local::detail::verbose_) - strm << R"( "series" : [)"; - long double average = 0.0; - int series = 0; - strm.precision(std::numeric_limits::max_digits10 - 1); - for (long double const val : item.second) + strm << "{\n"; + strm << " \"outputs\" : ["; + int outputs = 0; + for (auto&& item : obj.m_map) { - if (local::detail::verbose_) + if (outputs) + strm << ","; + strm << "\n {\n"; + strm << R"( "name" : ")" << std::get<0>(item.first) + << "\",\n"; + strm << R"( "executor" : ")" << std::get<1>(item.first) + << "\",\n"; + strm << R"( "series" : [)" << "\n"; + int series = 0; + strm.precision(std::numeric_limits::max_digits10 - 1); + for (long double const val : item.second) { if (series) - strm << ", "; - strm << std::scientific << val; + { + strm << ",\n"; + } + strm << R"( )" << std::scientific << val; + ++series; + } + + strm << "\n ]\n"; + + strm << " }"; + ++outputs; + } + if (outputs) + strm << "\n"; + strm << "]\n"; + strm << "}\n"; + } + else + { + strm << "Results:\n\n"; + for (auto&& item: obj.m_map) + { + long double average = 0.0; + int series = 0; + strm << "name: " << std::get<0>(item.first) << "\n"; + strm << "executor: " << std::get<1>(item.first) << "\n"; + for (long double const val : item.second) + { + ++series; + average += val; } - ++series; - average += val; + strm.precision(std::numeric_limits::max_digits10 - 1); + strm << std::scientific << "average: " << average / series << "\n\n"; } - if (local::detail::verbose_) - strm << "]\n"; - else - strm << " \"average\" : " << average / series << "\n"; - strm << " }"; - ++outputs; } - if (outputs) - strm << "\n "; - strm << "]\n"; - strm << "}\n"; return strm; } @@ -182,7 +212,7 @@ average": {{average(elapsed)}} // Overload that uses a default nanobench template and prints to std::cout void perftests_print_times() { - if (local::detail::verbose_) + if (detailed_) perftests_print_times(detail::nanobench_hpx_template(), std::cout); else perftests_print_times(detail::nanobench_hpx_simple_template(), std::cout); diff --git a/libs/full/segmented_algorithms/tests/performance/minmax_element_performance.cpp b/libs/full/segmented_algorithms/tests/performance/minmax_element_performance.cpp index de0242854807..0160adb1d06f 100644 --- a/libs/full/segmented_algorithms/tests/performance/minmax_element_performance.cpp +++ b/libs/full/segmented_algorithms/tests/performance/minmax_element_performance.cpp @@ -69,6 +69,8 @@ int hpx_main(hpx::program_options::variables_map& vm) // initialize data hpx::generate(hpx::execution::par, v.begin(), v.end(), random_fill()); + hpx::util::perftests_init(vm); + // run benchmark hpx::util::perftests_report("hpx::minmax", "par", test_count, [&]{ hpx::minmax_element(hpx::execution::par, v.begin(), v.end());}); @@ -107,6 +109,8 @@ int main(int argc, char* argv[]) hpx::program_options::value(), "the random number generator seed to use for this run"); + hpx::util::perftests_cfg(cmdline); + hpx::init_params init_args; init_args.desc_cmdline = cmdline; init_args.cfg = cfg; diff --git a/tools/perftests_plot.py b/tools/perftests_plot.py index 8291c1b463cb..7163f2a64d88 100644 --- a/tools/perftests_plot.py +++ b/tools/perftests_plot.py @@ -1,17 +1,18 @@ import sys -import os import subprocess import json import matplotlib.pyplot as plt -import matplotlib.ticker as ticker if len(sys.argv) != 2: - print("No benchmark selected") + if len(sys.argv) == 1: + print("No benchmark selected!") + else: + print("Too many arguments!") print("Usage: python perftests_plot.py [path_to_benchmark_binary]") else: test_name = sys.argv[1] - contents = subprocess.run([test_name, "--hpx:verbose_bench"], capture_output=True) + contents = subprocess.run([test_name, "--detailed_bench"], capture_output=True) json_obj = json.loads(contents.stdout.decode('utf-8')) From 21fc400e91fa22c19c556ac168dd59ada9586922 Mon Sep 17 00:00:00 2001 From: Vedant Date: Fri, 12 Apr 2024 11:38:01 +0530 Subject: [PATCH 18/24] Removing definition of inline variable, and modified CIs to use the correct option Signed-off-by: Vedant --- .github/workflows/linux_with_bench_test.yml | 5 +++++ .github/workflows/linux_with_nanobench_test.yml | 2 +- libs/core/testing/include/hpx/testing/performance.hpp | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux_with_bench_test.yml b/.github/workflows/linux_with_bench_test.yml index a43544d43423..0dd7522661d6 100644 --- a/.github/workflows/linux_with_bench_test.yml +++ b/.github/workflows/linux_with_bench_test.yml @@ -39,3 +39,8 @@ jobs: run: | cd build ./bin/minmax_element_performance_test + - name: Test with detailed output + shell: bash + run: | + cd build + ./bin/minmax_element_performance_test --detailed_bench diff --git a/.github/workflows/linux_with_nanobench_test.yml b/.github/workflows/linux_with_nanobench_test.yml index 4409c64040e3..f8cc8213a5f3 100644 --- a/.github/workflows/linux_with_nanobench_test.yml +++ b/.github/workflows/linux_with_nanobench_test.yml @@ -45,4 +45,4 @@ jobs: shell: bash run: | cd build - ./bin/minmax_element_performance_test --hpx:verbose_bench + ./bin/minmax_element_performance_test --detailed_bench diff --git a/libs/core/testing/include/hpx/testing/performance.hpp b/libs/core/testing/include/hpx/testing/performance.hpp index 207dca784d8a..27006f552485 100644 --- a/libs/core/testing/include/hpx/testing/performance.hpp +++ b/libs/core/testing/include/hpx/testing/performance.hpp @@ -14,7 +14,7 @@ #include namespace hpx::util { - HPX_CORE_EXPORT inline bool detailed_ = false; + HPX_CORE_EXPORT inline bool detailed_; HPX_CORE_EXPORT void perftests_cfg(hpx::program_options::options_description& cmdline); HPX_CORE_EXPORT void perftests_init(const hpx::program_options::variables_map& vm); #if defined(HPX_HAVE_NANOBENCH) From 1d18a5c5758e7f8ed51853d1e221d75f7d4d5cec Mon Sep 17 00:00:00 2001 From: Vedant Date: Fri, 19 Apr 2024 23:25:59 +0530 Subject: [PATCH 19/24] Applying cmake-format and clang-format Signed-off-by: Vedant --- .../performance/small_vector_benchmark.cpp | 78 ++++++++++--------- libs/core/testing/CMakeLists.txt | 16 ++-- .../include/hpx/testing/performance.hpp | 9 ++- libs/core/testing/src/performance.cpp | 51 ++++++------ .../tests/performance/CMakeLists.txt | 4 - .../minmax_element_performance.cpp | 17 ++-- 6 files changed, 94 insertions(+), 81 deletions(-) diff --git a/libs/core/datastructures/tests/performance/small_vector_benchmark.cpp b/libs/core/datastructures/tests/performance/small_vector_benchmark.cpp index 0c659c3cfab6..10f7576409b6 100644 --- a/libs/core/datastructures/tests/performance/small_vector_benchmark.cpp +++ b/libs/core/datastructures/tests/performance/small_vector_benchmark.cpp @@ -45,52 +45,58 @@ int hpx_main(hpx::program_options::variables_map& vm) // pull values from cmd std::size_t repeat = vm["test_count"].as(); std::size_t size = vm["vector_size"].as(); - + hpx::util::perftests_init(vm); // int - - hpx::util::perftests_report("hpx::small_vector", "", repeat, [&] { - fill>(size); - }); - - hpx::util::perftests_report("hpx::small_vector", "", repeat, [&] { - fill>(size); - }); - - hpx::util::perftests_report("hpx::small_vector", "", repeat, [&] { - fill>(size); - }); - hpx::util::perftests_report("hpx::small_vector", "", repeat, [&] { - fill>(size); - }); + hpx::util::perftests_report("hpx::small_vector", "", repeat, + [&] { fill>(size); }); - hpx::util::perftests_report("hpx::small_vector", "", repeat, [&] { - fill>(size); - }); + hpx::util::perftests_report("hpx::small_vector", "", repeat, + [&] { fill>(size); }); - // hpx::move_only_function - - hpx::util::perftests_report("hpx::small_vector", ", 1>", repeat, [&] { - fill, 1>>(size); - }); + hpx::util::perftests_report("hpx::small_vector", "", repeat, + [&] { fill>(size); }); - hpx::util::perftests_report("hpx::small_vector", ", 2>", repeat, [&] { - fill, 2>>(size); - }); + hpx::util::perftests_report("hpx::small_vector", "", repeat, + [&] { fill>(size); }); - hpx::util::perftests_report("hpx::small_vector", ", 4>", repeat, [&] { - fill, 4>>(size); - }); + hpx::util::perftests_report("hpx::small_vector", "", repeat, + [&] { fill>(size); }); - hpx::util::perftests_report("hpx::small_vector", ", 8>", repeat, [&] { - fill, 8>>(size); - }); + // hpx::move_only_function - hpx::util::perftests_report("hpx::small_vector", ", 16>", repeat, [&] { - fill, 16>>(size); - }); + hpx::util::perftests_report( + "hpx::small_vector", ", 1>", repeat, [&] { + fill, 1>>( + size); + }); + + hpx::util::perftests_report( + "hpx::small_vector", ", 2>", repeat, [&] { + fill, 2>>( + size); + }); + + hpx::util::perftests_report( + "hpx::small_vector", ", 4>", repeat, [&] { + fill, 4>>( + size); + }); + + hpx::util::perftests_report( + "hpx::small_vector", ", 8>", repeat, [&] { + fill, 8>>( + size); + }); + + hpx::util::perftests_report( + "hpx::small_vector", ", 16>", repeat, [&] { + fill< + hpx::detail::small_vector, 16>>( + size); + }); hpx::util::perftests_print_times(); diff --git a/libs/core/testing/CMakeLists.txt b/libs/core/testing/CMakeLists.txt index e4e4d60f84e9..b7f8c2a0347e 100644 --- a/libs/core/testing/CMakeLists.txt +++ b/libs/core/testing/CMakeLists.txt @@ -20,14 +20,14 @@ set(testing_compat_headers # Default location is $HPX_ROOT/libs/testing/src set(testing_sources testing.cpp performance.cpp) -set(module_deps - hpx_assertion - hpx_config - hpx_format - hpx_functional - hpx_preprocessor - hpx_util - hpx_command_line_handling_local +set(module_deps + hpx_assertion + hpx_config + hpx_format + hpx_functional + hpx_preprocessor + hpx_util + hpx_command_line_handling_local ) if(HPX_WITH_NANOBENCH) diff --git a/libs/core/testing/include/hpx/testing/performance.hpp b/libs/core/testing/include/hpx/testing/performance.hpp index 27006f552485..1470a6b51aa8 100644 --- a/libs/core/testing/include/hpx/testing/performance.hpp +++ b/libs/core/testing/include/hpx/testing/performance.hpp @@ -15,13 +15,16 @@ namespace hpx::util { HPX_CORE_EXPORT inline bool detailed_; - HPX_CORE_EXPORT void perftests_cfg(hpx::program_options::options_description& cmdline); - HPX_CORE_EXPORT void perftests_init(const hpx::program_options::variables_map& vm); + HPX_CORE_EXPORT void perftests_cfg( + hpx::program_options::options_description& cmdline); + HPX_CORE_EXPORT void perftests_init( + const hpx::program_options::variables_map& vm); #if defined(HPX_HAVE_NANOBENCH) HPX_CORE_EXPORT void perftests_report(std::string const& name, std::string const& exec, std::size_t const steps, hpx::function&& test); - HPX_CORE_EXPORT void perftests_print_times(std::ostream& strm, char const* templ); + HPX_CORE_EXPORT void perftests_print_times( + std::ostream& strm, char const* templ); HPX_CORE_EXPORT void perftests_print_times(std::ostream& strm); HPX_CORE_EXPORT void perftests_print_times(); #else diff --git a/libs/core/testing/src/performance.cpp b/libs/core/testing/src/performance.cpp index 1a3192278c15..8ed90f46f56d 100644 --- a/libs/core/testing/src/performance.cpp +++ b/libs/core/testing/src/performance.cpp @@ -22,15 +22,17 @@ namespace hpx::util { - void perftests_cfg(hpx::program_options::options_description& cmdline) { - cmdline.add_options() - ("detailed_bench", - "Use if detailed benchmarks are required, showing the execution time taken for each epoch") - ; + void perftests_cfg(hpx::program_options::options_description& cmdline) + { + cmdline.add_options()("detailed_bench", + "Use if detailed benchmarks are required, showing the execution " + "time taken for each epoch"); } - void perftests_init(const hpx::program_options::variables_map& vm) { - if (vm.count("detailed_bench")) { + void perftests_init(const hpx::program_options::variables_map& vm) + { + if (vm.count("detailed_bench")) + { detailed_ = true; } } @@ -111,8 +113,7 @@ average: {{average(elapsed)}}{{^-last}} std::ostream& operator<<(std::ostream& strm, json_perf_times const& obj) { - - if (detailed_) + if (detailed_) { strm << "{\n"; strm << " \"outputs\" : ["; @@ -123,12 +124,14 @@ average: {{average(elapsed)}}{{^-last}} strm << ","; strm << "\n {\n"; strm << R"( "name" : ")" << std::get<0>(item.first) - << "\",\n"; + << "\",\n"; strm << R"( "executor" : ")" << std::get<1>(item.first) - << "\",\n"; - strm << R"( "series" : [)" << "\n"; + << "\",\n"; + strm << R"( "series" : [)" + << "\n"; int series = 0; - strm.precision(std::numeric_limits::max_digits10 - 1); + strm.precision( + std::numeric_limits::max_digits10 - 1); for (long double const val : item.second) { if (series) @@ -138,9 +141,9 @@ average: {{average(elapsed)}}{{^-last}} strm << R"( )" << std::scientific << val; ++series; } - + strm << "\n ]\n"; - + strm << " }"; ++outputs; } @@ -149,10 +152,10 @@ average: {{average(elapsed)}}{{^-last}} strm << "]\n"; strm << "}\n"; } - else + else { strm << "Results:\n\n"; - for (auto&& item: obj.m_map) + for (auto&& item : obj.m_map) { long double average = 0.0; int series = 0; @@ -163,8 +166,10 @@ average: {{average(elapsed)}}{{^-last}} ++series; average += val; } - strm.precision(std::numeric_limits::max_digits10 - 1); - strm << std::scientific << "average: " << average / series << "\n\n"; + strm.precision( + std::numeric_limits::max_digits10 - 1); + strm << std::scientific << "average: " << average / series + << "\n\n"; } } return strm; @@ -186,7 +191,8 @@ average: {{average(elapsed)}}{{^-last}} if (steps == 0) return; - std::size_t const steps_per_epoch = steps / detail::nanobench_epochs + 1; + std::size_t const steps_per_epoch = + steps / detail::nanobench_epochs + 1; detail::bench() .name(name) @@ -214,8 +220,9 @@ average: {{average(elapsed)}}{{^-last}} { if (detailed_) perftests_print_times(detail::nanobench_hpx_template(), std::cout); - else - perftests_print_times(detail::nanobench_hpx_simple_template(), std::cout); + else + perftests_print_times( + detail::nanobench_hpx_simple_template(), std::cout); } #else void perftests_report(std::string const& name, std::string const& exec, diff --git a/libs/full/segmented_algorithms/tests/performance/CMakeLists.txt b/libs/full/segmented_algorithms/tests/performance/CMakeLists.txt index 7ef40a433fd1..e8db7f041b00 100644 --- a/libs/full/segmented_algorithms/tests/performance/CMakeLists.txt +++ b/libs/full/segmented_algorithms/tests/performance/CMakeLists.txt @@ -6,10 +6,6 @@ set(benchmarks minmax_element_performance) -# if (HPX_WITH_NANOBENCH_BENCHMARKS) -# set(benchmarks ${benchmarks} minmax_nanobench) -# endif() - foreach(benchmark ${benchmarks}) set(sources ${benchmark}.cpp) diff --git a/libs/full/segmented_algorithms/tests/performance/minmax_element_performance.cpp b/libs/full/segmented_algorithms/tests/performance/minmax_element_performance.cpp index 0160adb1d06f..85e1293e0a71 100644 --- a/libs/full/segmented_algorithms/tests/performance/minmax_element_performance.cpp +++ b/libs/full/segmented_algorithms/tests/performance/minmax_element_performance.cpp @@ -13,9 +13,9 @@ #include #include #include -#include #include #include +#include #include #include @@ -72,15 +72,16 @@ int hpx_main(hpx::program_options::variables_map& vm) hpx::util::perftests_init(vm); // run benchmark - hpx::util::perftests_report("hpx::minmax", "par", test_count, [&]{ - hpx::minmax_element(hpx::execution::par, v.begin(), v.end());}); + hpx::util::perftests_report("hpx::minmax", "par", test_count, [&] { + hpx::minmax_element(hpx::execution::par, v.begin(), v.end()); + }); + + hpx::util::perftests_report("hpx::min", "par", test_count, + [&] { hpx::min_element(hpx::execution::par, v.begin(), v.end()); }); - hpx::util::perftests_report("hpx::min", "par", test_count, [&]{hpx::min_element( - hpx::execution::par, v.begin(), v.end());}); + hpx::util::perftests_report("hpx::max", "par", test_count, + [&] { hpx::max_element(hpx::execution::par, v.begin(), v.end()); }); - hpx::util::perftests_report("hpx::max", "par", test_count, [&]{hpx::max_element( - hpx::execution::par, v.begin(), v.end());}); - hpx::util::perftests_print_times(); return hpx::finalize(); From 995ae0eab1b4aa6d0bf1e0872653e1fa30fffedb Mon Sep 17 00:00:00 2001 From: Vedant Date: Mon, 22 Apr 2024 23:54:59 +0530 Subject: [PATCH 20/24] Adding explicit double to long double conversions for the default case Signed-off-by: Vedant --- libs/core/testing/src/performance.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/libs/core/testing/src/performance.cpp b/libs/core/testing/src/performance.cpp index 8ed90f46f56d..ea61f7a55294 100644 --- a/libs/core/testing/src/performance.cpp +++ b/libs/core/testing/src/performance.cpp @@ -96,7 +96,7 @@ average: {{average(elapsed)}}{{^-last}} public: HPX_CORE_EXPORT void add(std::string const& name, - std::string const& executor, double time); + std::string const& executor, long double time); }; json_perf_times& times() @@ -157,7 +157,7 @@ average: {{average(elapsed)}}{{^-last}} strm << "Results:\n\n"; for (auto&& item : obj.m_map) { - long double average = 0.0; + long double average = static_cast(0.0); int series = 0; strm << "name: " << std::get<0>(item.first) << "\n"; strm << "executor: " << std::get<1>(item.first) << "\n"; @@ -175,8 +175,8 @@ average: {{average(elapsed)}}{{^-last}} return strm; } - void json_perf_times::add( - std::string const& name, std::string const& executor, double time) + void json_perf_times::add(std::string const& name, + std::string const& executor, long double time) { m_map[key_t(name, executor)].push_back(time); } @@ -234,7 +234,7 @@ average: {{average(elapsed)}}{{^-last}} // First iteration to cache the data test(); using timer = std::chrono::high_resolution_clock; - for (size_t i = 0; i != steps; ++i) + for (std::size_t i = 0; i != steps; ++i) { // For now we don't flush the cache //flush_cache(); From cc25fb10890392278faac1045217dd6c9a357a67 Mon Sep 17 00:00:00 2001 From: Vedant Date: Tue, 23 Apr 2024 00:20:17 +0530 Subject: [PATCH 21/24] Fixed implicit conversion issue while using add_time function Signed-off-by: Vedant --- libs/core/testing/src/performance.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/core/testing/src/performance.cpp b/libs/core/testing/src/performance.cpp index ea61f7a55294..dfb5ffa56061 100644 --- a/libs/core/testing/src/performance.cpp +++ b/libs/core/testing/src/performance.cpp @@ -106,7 +106,7 @@ average: {{average(elapsed)}}{{^-last}} } void add_time(std::string const& test_name, std::string const& executor, - double time) + long double time) { times().add(test_name, executor, time); } @@ -242,7 +242,7 @@ average: {{average(elapsed)}}{{^-last}} test(); // default is in seconds auto time = - std::chrono::duration_cast>( + std::chrono::duration_cast>( timer::now() - start); detail::add_time(name, exec, time.count()); } From 33ea494374c4cba52c73e8439f79b8e6b395dd13 Mon Sep 17 00:00:00 2001 From: Vedant Date: Tue, 11 Jun 2024 00:24:22 +0530 Subject: [PATCH 22/24] Standardised foreach_report, stream_report and future_overhead_report, and made minor changes to perftests ci Signed-off-by: Vedant --- .jenkins/lsu-perftests/launch_perftests.sh | 6 +++--- .../tests/performance/foreach_report.cpp | 3 +++ libs/core/testing/src/performance.cpp | 14 ++++++++------ tests/performance/local/future_overhead_report.cpp | 4 ++++ tests/performance/local/stream_report.cpp | 2 ++ 5 files changed, 20 insertions(+), 9 deletions(-) mode change 100644 => 100755 .jenkins/lsu-perftests/launch_perftests.sh diff --git a/.jenkins/lsu-perftests/launch_perftests.sh b/.jenkins/lsu-perftests/launch_perftests.sh old mode 100644 new mode 100755 index 5e8c6e0a57e1..b744f600d1cf --- a/.jenkins/lsu-perftests/launch_perftests.sh +++ b/.jenkins/lsu-perftests/launch_perftests.sh @@ -16,13 +16,13 @@ hpx_targets=( hpx_test_options=( "--hpx:ini=hpx.thread_queue.init_threads_count=100 \ --hpx:threads=4 --vector_size=104857 --work_delay=1 \ - --chunk_size=0 --test_count=200" + --chunk_size=0 --test_count=200 --detailed_bench" "--hpx:ini=hpx.thread_queue.init_threads_count=100 \ --hpx:queuing=local-priority --hpx:threads=4 --test-all \ - --repetitions=40 --futures=207270" + --repetitions=40 --futures=207270 --detailed_bench" "--hpx:ini=hpx.thread_queue.init_threads_count=100 \ --vector_size=518176 --hpx:threads=4 --iterations=200 \ - --warmup_iterations=20") + --warmup_iterations=20 --detailed_bench") # Build binaries for performance tests ${perftests_dir}/driver.py -v -l $logfile build -b release -o build \ diff --git a/libs/core/algorithms/tests/performance/foreach_report.cpp b/libs/core/algorithms/tests/performance/foreach_report.cpp index 9d91491271ec..0ee6030a1f70 100644 --- a/libs/core/algorithms/tests/performance/foreach_report.cpp +++ b/libs/core/algorithms/tests/performance/foreach_report.cpp @@ -41,6 +41,8 @@ int hpx_main(hpx::program_options::variables_map& vm) disable_stealing = vm.count("disable_stealing"); fast_idle_mode = vm.count("fast_idle_mode"); + hpx::util::perftests_init(vm); + // verify that input is within domain of program if (test_count == 0 || test_count < 0) { @@ -122,6 +124,7 @@ int main(int argc, char* argv[]) ; // clang-format on + hpx::util::perftests_cfg(cmdline); hpx::local::init_params init_args; init_args.desc_cmdline = cmdline; init_args.cfg = {"hpx.os_threads=all"}; diff --git a/libs/core/testing/src/performance.cpp b/libs/core/testing/src/performance.cpp index dfb5ffa56061..357af43516bf 100644 --- a/libs/core/testing/src/performance.cpp +++ b/libs/core/testing/src/performance.cpp @@ -120,14 +120,15 @@ average: {{average(elapsed)}}{{^-last}} int outputs = 0; for (auto&& item : obj.m_map) { + long double average = static_cast(0.0); if (outputs) strm << ","; strm << "\n {\n"; - strm << R"( "name" : ")" << std::get<0>(item.first) + strm << R"( "name": ")" << std::get<0>(item.first) << "\",\n"; - strm << R"( "executor" : ")" << std::get<1>(item.first) + strm << R"( "executor": ")" << std::get<1>(item.first) << "\",\n"; - strm << R"( "series" : [)" + strm << R"( "series": [)" << "\n"; int series = 0; strm.precision( @@ -140,10 +141,11 @@ average: {{average(elapsed)}}{{^-last}} } strm << R"( )" << std::scientific << val; ++series; + average += val; } - - strm << "\n ]\n"; - + strm << "\n ],\n"; + strm << std::scientific << R"( "average": )"<< average / series + << "\n"; strm << " }"; ++outputs; } diff --git a/tests/performance/local/future_overhead_report.cpp b/tests/performance/local/future_overhead_report.cpp index 05acfd7af2eb..3744e72b408c 100644 --- a/tests/performance/local/future_overhead_report.cpp +++ b/tests/performance/local/future_overhead_report.cpp @@ -165,6 +165,9 @@ int hpx_main(variables_map& vm) num_iterations = vm["delay-iterations"].as(); const std::uint64_t count = vm["futures"].as(); + + hpx::util::perftests_init(vm); + if (HPX_UNLIKELY(0 == count)) throw std::logic_error("error: count of 0 futures specified\n"); @@ -201,6 +204,7 @@ int main(int argc, char* argv[]) // clang-format on // Initialize and run HPX. + hpx::util::perftests_cfg(cmdline); hpx::local::init_params init_args; init_args.desc_cmdline = cmdline; diff --git a/tests/performance/local/stream_report.cpp b/tests/performance/local/stream_report.cpp index c693daa89681..7cef564a6ab8 100644 --- a/tests/performance/local/stream_report.cpp +++ b/tests/performance/local/stream_report.cpp @@ -442,6 +442,7 @@ int hpx_main(hpx::program_options::variables_map& vm) std::size_t iterations = vm["iterations"].as(); std::size_t warmup_iterations = vm["warmup_iterations"].as(); std::size_t chunk_size = vm["chunk_size"].as(); + hpx::util::perftests_init(vm); std::size_t executor; header = vm.count("header") > 0; @@ -548,6 +549,7 @@ int main(int argc, char* argv[]) "hpx.numa_sensitive=2" // no-cross NUMA stealing }; + hpx::util::perftests_cfg(cmdline); hpx::init_params init_args; init_args.desc_cmdline = cmdline; init_args.cfg = cfg; From 526e0486114a864157d45b5e866e4a1183badaad Mon Sep 17 00:00:00 2001 From: Vedant Date: Wed, 3 Jul 2024 23:30:37 +0530 Subject: [PATCH 23/24] Fixing clang-format for performance.cpp and future_overhead_report Signed-off-by: Vedant --- libs/core/testing/src/performance.cpp | 4 ++-- tests/performance/local/future_overhead_report.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libs/core/testing/src/performance.cpp b/libs/core/testing/src/performance.cpp index 357af43516bf..0227a776785a 100644 --- a/libs/core/testing/src/performance.cpp +++ b/libs/core/testing/src/performance.cpp @@ -144,8 +144,8 @@ average: {{average(elapsed)}}{{^-last}} average += val; } strm << "\n ],\n"; - strm << std::scientific << R"( "average": )"<< average / series - << "\n"; + strm << std::scientific << R"( "average": )" + << average / series << "\n"; strm << " }"; ++outputs; } diff --git a/tests/performance/local/future_overhead_report.cpp b/tests/performance/local/future_overhead_report.cpp index 3744e72b408c..0212360596b8 100644 --- a/tests/performance/local/future_overhead_report.cpp +++ b/tests/performance/local/future_overhead_report.cpp @@ -167,7 +167,7 @@ int hpx_main(variables_map& vm) const std::uint64_t count = vm["futures"].as(); hpx::util::perftests_init(vm); - + if (HPX_UNLIKELY(0 == count)) throw std::logic_error("error: count of 0 futures specified\n"); From c35e4e4f3ca2332c6f08323c9e3504175bede47d Mon Sep 17 00:00:00 2001 From: Vedant Date: Sun, 14 Jul 2024 23:29:50 +0530 Subject: [PATCH 24/24] Removed perftests_plot.py, and made minor clean-up changes Signed-off-by: Vedant --- CMakeLists.txt | 2 +- .../performance/small_vector_benchmark.cpp | 11 ------- tools/perftests_plot.py | 31 ------------------- 3 files changed, 1 insertion(+), 43 deletions(-) delete mode 100644 tools/perftests_plot.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 3be765feec99..63cc5bd760b8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1030,7 +1030,7 @@ hpx_option( hpx_option( HPX_WITH_NANOBENCH BOOL - "Use Nanobench for performance tests. Nanobench will be fetched using FetchContent (defualt: OFF)" + "Use Nanobench for performance tests. Nanobench will be fetched using FetchContent (default: OFF)" OFF CATEGORY "Build Targets" ADVANCED diff --git a/libs/core/datastructures/tests/performance/small_vector_benchmark.cpp b/libs/core/datastructures/tests/performance/small_vector_benchmark.cpp index 10f7576409b6..afee73c2878b 100644 --- a/libs/core/datastructures/tests/performance/small_vector_benchmark.cpp +++ b/libs/core/datastructures/tests/performance/small_vector_benchmark.cpp @@ -29,17 +29,6 @@ void fill(std::size_t size) } } -// template -// void compare(std::size_t repeat, std::size_t size) -// { -// std::uint64_t time = measure>(repeat, size); - -// std::cout << "-----Average-(hpx::small_vector<" << typeid(T).name() << ", " -// << N << ">)------ \n" -// << std::left << "Average execution time : " << std::right -// << std::setw(8) << time / 1e9 << "\n"; -// } - int hpx_main(hpx::program_options::variables_map& vm) { // pull values from cmd diff --git a/tools/perftests_plot.py b/tools/perftests_plot.py deleted file mode 100644 index 7163f2a64d88..000000000000 --- a/tools/perftests_plot.py +++ /dev/null @@ -1,31 +0,0 @@ -import sys -import subprocess -import json -import matplotlib.pyplot as plt - -if len(sys.argv) != 2: - if len(sys.argv) == 1: - print("No benchmark selected!") - else: - print("Too many arguments!") - print("Usage: python perftests_plot.py [path_to_benchmark_binary]") -else: - test_name = sys.argv[1] - - contents = subprocess.run([test_name, "--detailed_bench"], capture_output=True) - - json_obj = json.loads(contents.stdout.decode('utf-8')) - - test_names = [] - samples = [] - - for test in json_obj["outputs"]: - test_names.append(test["name"] + ",\n" + test["executor"]) - samples.append(test["series"]) - - fig = plt.figure(figsize=(20, 6)) - ax = fig.add_subplot() - bp = ax.boxplot(samples, showfliers=False) - plt.setp(ax.set_xticklabels(test_names), fontsize=7) - plt.show() - \ No newline at end of file