From 34b56818830d6435202fb423a687e3663b728ffc Mon Sep 17 00:00:00 2001 From: Dominik Drexler Date: Thu, 4 Jan 2024 16:10:25 +0100 Subject: [PATCH] added more benchmarks --- .github/workflows/benchmarks.yml | 17 +-- CMakeLists.txt | 4 +- benchmarks/persistent_factory.cpp | 159 +++++++++++++++++++++-------- include/loki/common/pddl/types.hpp | 4 +- 4 files changed, 124 insertions(+), 60 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 5b34022b..2b91f557 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -1,9 +1,8 @@ name: Googlebenchmark -on: [push, pull_request] -#on: -# push: -# branches: -# - main +on: + push: + branches: + - main permissions: @@ -27,7 +26,7 @@ jobs: echo "BOOST_ROOT=${{runner.workspace}}/Loki/boost_1_84_0" >> "$GITHUB_OUTPUT" - name: Configure CMake - run: cmake -DENABLE_BENCHMARKING:BOOL=TRUE -S $GITHUB_WORKSPACE -B ${{runner.workspace}}/build + run: cmake -DCMAKE_BUILD_TYPE=Release -DENABLE_BENCHMARKING:BOOL=TRUE -S $GITHUB_WORKSPACE -B ${{runner.workspace}}/build env: BOOST_ROOT: ${{ steps.install-boost.outputs.BOOST_ROOT }} @@ -38,12 +37,6 @@ jobs: # Run benchmark and store the output to a file - name: Run benchmark run: ${{runner.workspace}}/build/benchmarks/persistent_factory --benchmark_format=json | tee benchmark_result.json - # Download previous benchmark result from cache (if exists) - #- name: Download previous benchmark data - # uses: actions/cache@v1 - # with: - # path: ./cache - # key: ${{ runner.os }}-benchmark # Run `github-action-benchmark` action - name: Store benchmark result uses: benchmark-action/github-action-benchmark@v1 diff --git a/CMakeLists.txt b/CMakeLists.txt index 37249c0e..07425c70 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -74,12 +74,12 @@ add_subdirectory(exe) add_subdirectory(examples) -set(ENABLE_TESTING True) +#set(ENABLE_TESTING True) if (ENABLE_TESTING) add_subdirectory(tests) endif() -set(ENABLE_BENCHMARKING True) +#set(ENABLE_BENCHMARKING True) if (ENABLE_BENCHMARKING) add_subdirectory(benchmarks) endif() diff --git a/benchmarks/persistent_factory.cpp b/benchmarks/persistent_factory.cpp index 5b1b4fab..23ae63bb 100644 --- a/benchmarks/persistent_factory.cpp +++ b/benchmarks/persistent_factory.cpp @@ -20,64 +20,135 @@ #include -#include #include #include +static loki::pddl::AtomList create_atoms( + size_t num_objects, + size_t num_predicates, + loki::CollectionOfPDDLFactories& factories) { + // Create num_objects-many objects with name object_1,...,object_ + auto objects = loki::pddl::ObjectList(); + for (size_t i = 1; i <= num_objects; ++i) { + objects.push_back(factories.objects.get_or_create( + ("object_" + std::to_string(i))) + ); + } + + // Create num_predicates-many binary predicates with name predicate_1,...,predicate_ + auto parameters = loki::pddl::ParameterList{ + factories.parameters.get_or_create( + factories.variables.get_or_create("?variable_left"), + loki::pddl::TypeList{}), + factories.parameters.get_or_create( + factories.variables.get_or_create("?variable_right"), + loki::pddl::TypeList{}) + }; + + auto predicates = loki::pddl::PredicateList(); + for (size_t i = 1; i <= num_predicates; ++i) { + predicates.push_back(factories.predicates.get_or_create( + ("predicate_" + std::to_string(i)), + parameters)); + } + + auto atoms = loki::pddl::AtomList(); + // Construct num_objects^2 * num_predicates many atoms + for (const auto& predicate : predicates) { + for (const auto& object_left : objects) { + for (const auto& object_right : objects) { + atoms.push_back(factories.problem_atoms.get_or_create( + predicate, + loki::pddl::TermList{ + factories.terms.get_or_create(object_left), + factories.terms.get_or_create(object_right) + })); + } + } + } + return atoms; +} + + /// @brief In this benchmark, we evaluate the performance of constructing atoms. static void BM_ConstructAtoms(benchmark::State& state) { const size_t num_objects = 100; - const size_t num_predicates = 100; + const size_t num_predicates = 10; for (auto _ : state) { - // Create num_objects-many objects with name object_1,...,object_ - auto object_factory = loki::ObjectFactory(); - auto objects = loki::pddl::ObjectList(); - for (size_t i = 1; i <= num_objects; ++i) { - objects.push_back(object_factory.get_or_create( - ("object_" + std::to_string(i))) - ); - } + auto factories = loki::CollectionOfPDDLFactories(); + + auto atoms = create_atoms(num_objects, num_predicates, factories); + } +} +BENCHMARK(BM_ConstructAtoms); + - // Create num_predicates-many binary predicates with name predicate_1,...,predicate_ - auto predicate_factory = loki::PredicateFactory(); - auto parameter_factory = loki::ParameterFactory(); - auto variable_factory = loki::VariableFactory(); - auto parameters = loki::pddl::ParameterList{ - parameter_factory.get_or_create( - variable_factory.get_or_create("?variable_left"), - loki::pddl::TypeList{}), - parameter_factory.get_or_create( - variable_factory.get_or_create("?variable_right"), - loki::pddl::TypeList{}) - }; - - auto predicates = loki::pddl::PredicateList(); - for (size_t i = 1; i <= num_predicates; ++i) { - predicates.push_back(predicate_factory.get_or_create( - ("predicate_" + std::to_string(i)), - parameters)); +auto identifier_visitor = [](const auto& arg) -> int { + return arg.get_identifier(); +}; +auto name_visitor = [](const auto& arg) -> int { + return arg.get_identifier(); +}; + +static void access_atom_data(const loki::pddl::Atom& atom) { + const auto& atom_identifier = atom->get_identifier(); + const auto& atom_predicate = atom->get_predicate(); + const auto& atom_terms = atom->get_terms(); + + const auto& predicate_identifier = atom_predicate->get_identifier(); + const auto& predicate_name = atom_predicate->get_name(); + const auto& predicate_parameters = atom_predicate->get_parameters(); + + for (const auto& term : atom_terms) { + int term_identifier = std::visit(identifier_visitor, *term); + const auto& term_name = std::visit(name_visitor, *term); + } +} + + +/// @brief In this benchmark, we evaluate the performance of accessing data in sequence +static void BM_IterateAtoms(benchmark::State& state) { + const size_t num_objects = 100; + const size_t num_predicates = 10; + + auto factories = loki::CollectionOfPDDLFactories(); + + auto atoms = create_atoms(num_objects, num_predicates, factories); + + for (auto _ : state) { + for (const auto& atom : atoms) { + access_atom_data(atom); } + } +} +BENCHMARK(BM_IterateAtoms); - auto atom_factory = loki::AtomFactory(); - auto term_factory = loki::TermFactory(); - auto atoms = loki::pddl::AtomList(); - // Construct num_objects^2 * num_predicates many atoms - for (const auto& predicate : predicates) { - for (const auto& object_left : objects) { - for (const auto& object_right : objects) { - atoms.push_back(atom_factory.get_or_create( - predicate, - loki::pddl::TermList{ - term_factory.get_or_create(object_left), - term_factory.get_or_create(object_right) - })); - } - } + +static void BM_RandomlyIterateAtoms(benchmark::State& state) { + const size_t num_objects = 100; + const size_t num_predicates = 10; + + auto factories = loki::CollectionOfPDDLFactories(); + + auto atoms = create_atoms(num_objects, num_predicates, factories); + + std::random_device rd; // Obtain a random number from hardware + std::mt19937 eng(rd()); // Seed the generator + + for (auto _ : state) { + + std::vector indices(atoms.size()); + std::iota(indices.begin(), indices.end(), 0); + std::shuffle(indices.begin(), indices.end(), eng); + + for (int index : indices) { + access_atom_data(atoms[index]); } } } -BENCHMARK(BM_ConstructAtoms); +BENCHMARK(BM_RandomlyIterateAtoms); + BENCHMARK_MAIN(); diff --git a/include/loki/common/pddl/types.hpp b/include/loki/common/pddl/types.hpp index afbc52b8..7f0c05a4 100644 --- a/include/loki/common/pddl/types.hpp +++ b/include/loki/common/pddl/types.hpp @@ -44,9 +44,9 @@ namespace loki { -// The segmented sizes are chosen sufficiently large to avoid +// The segmented sizes are chosen sufficiently large to avoid // to avoid allocations and for continuous storage. -// The values are just educated guesses based on the knowledge +// The values are just educated guesses based on the knowledge // that cache line size is 64 Bytes. using RequirementFactory = PersistentFactory; using TypeFactory = PersistentFactory;