From 5270e34f138fa7230c10423e694a261cf33bd289 Mon Sep 17 00:00:00 2001 From: Enrico Seiler Date: Tue, 28 Nov 2023 10:48:33 +0100 Subject: [PATCH 1/5] [FIX] bit_vector::clear --- include/hibf/misc/bit_vector.hpp | 7 +++++++ test/unit/hibf/bit_vector_test.cpp | 18 ++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/include/hibf/misc/bit_vector.hpp b/include/hibf/misc/bit_vector.hpp index 5afc55af..d4b80a49 100644 --- a/include/hibf/misc/bit_vector.hpp +++ b/include/hibf/misc/bit_vector.hpp @@ -705,6 +705,13 @@ class bit_vector : std::ranges::fill(begin() + old_size, end(), bit); } + //!\brief Erases all elements. After this call, size() returns zero. capacity() remains unchanged. + HIBF_CONSTEXPR_VECTOR void clear() noexcept + { + base_t::clear(); + _size = 0u; + } + //!\brief Performs binary AND between `this` and `rhs`. constexpr bit_vector & operator&=(bit_vector const & rhs) noexcept { diff --git a/test/unit/hibf/bit_vector_test.cpp b/test/unit/hibf/bit_vector_test.cpp index a4ea9006..03d100ed 100644 --- a/test/unit/hibf/bit_vector_test.cpp +++ b/test/unit/hibf/bit_vector_test.cpp @@ -404,6 +404,24 @@ TEST(bit_vector_test, resize) EXPECT_TRUE(test_vector.none()); } +TEST(bit_vector_test, clear) +{ + seqan::hibf::bit_vector test_vector{}; + size_t capacity = test_vector.capacity(); + + test_vector.clear(); + EXPECT_EQ(test_vector.size(), 0u); + EXPECT_EQ(test_vector.capacity(), capacity); + + test_vector.resize(128, true); + capacity = test_vector.capacity(); + EXPECT_EQ(test_vector.size(), 128u); + EXPECT_GE(test_vector.capacity(), 128u); + test_vector.clear(); + EXPECT_EQ(test_vector.size(), 0u); + EXPECT_EQ(test_vector.capacity(), capacity); +} + TEST(bit_vector_test, push_back) { seqan::hibf::bit_vector test_vector{}; From 7bc261304097f0413c3cf046285167acfe422397 Mon Sep 17 00:00:00 2001 From: Enrico Seiler Date: Tue, 28 Nov 2023 11:09:15 +0100 Subject: [PATCH 2/5] [MISC] Move counting_vector into separate header To make review easier --- include/hibf/interleaved_bloom_filter.hpp | 155 +----------------- include/hibf/misc/counting_vector.hpp | 190 ++++++++++++++++++++++ 2 files changed, 191 insertions(+), 154 deletions(-) create mode 100644 include/hibf/misc/counting_vector.hpp diff --git a/include/hibf/interleaved_bloom_filter.hpp b/include/hibf/interleaved_bloom_filter.hpp index b29d6ab8..1f6a65b3 100644 --- a/include/hibf/interleaved_bloom_filter.hpp +++ b/include/hibf/interleaved_bloom_filter.hpp @@ -26,6 +26,7 @@ #include // for config #include // for aligned_allocator #include // for bit_vector +#include // for counting_vector #include // for CEREAL_SERIALIZE_FUNCTION_NAME #include // for base_class @@ -494,160 +495,6 @@ inline interleaved_bloom_filter::membership_agent_type interleaved_bloom_filter: return interleaved_bloom_filter::membership_agent_type{*this}; } -/*!\brief A data structure that behaves like a std::vector and can be used to consolidate the results of multiple calls - * to seqan::hibf::interleaved_bloom_filter::membership_agent_type::bulk_contains. - * \ingroup ibf - * \tparam value_t The type of the count. Must model std::integral. - * - * \details - * - * When using the seqan::hibf::interleaved_bloom_filter::membership_agent_type::bulk_contains operation, a common use case is to - * add up, for example, the results for all k-mers in a query. This yields, for each bin, the number of k-mers of a - * query that are in the respective bin. Such information can be used to apply further filtering or abundance estimation - * based on the k-mer counts. - * - * The seqan::hibf::counting_vector offers an easy way to add up the individual - * seqan::hibf::bit_vector by offering an `+=` operator. - * - * The `value_t` template parameter should be chosen in a way that no overflow occurs if all calls to `bulk_contains` - * return a hit for a specific bin. For example, `uint8_t` will suffice when processing short Illumina reads, whereas - * long reads will require at least `uint32_t`. - * - * ### Example - * - * \include test/snippet/ibf/counting_vector.cpp - */ -template -class counting_vector : public std::vector -{ -private: - //!\brief The base type. - using base_t = std::vector; - -public: - /*!\name Constructors, destructor and assignment - * \{ - */ - counting_vector() = default; //!< Defaulted. - counting_vector(counting_vector const &) = default; //!< Defaulted. - counting_vector & operator=(counting_vector const &) = default; //!< Defaulted. - counting_vector(counting_vector &&) = default; //!< Defaulted. - counting_vector & operator=(counting_vector &&) = default; //!< Defaulted. - ~counting_vector() = default; //!< Defaulted. - - using base_t::base_t; - //!\} - - /*!\brief Bin-wise adds the bits of a seqan::hibf::bit_vector. - * \param bit_vector The seqan::hibf::bit_vector. - * \attention The counting_vector must be at least as big as `bit_vector`. - * - * \details - * - * ### Example - * - * \include test/snippet/ibf/counting_vector.cpp - */ - counting_vector & operator+=(bit_vector const & bit_vector) - { - for_each_set_bin(bit_vector, - [this](size_t const bin) - { - ++(*this)[bin]; - }); - return *this; - } - - /*!\brief Bin-wise subtracts the bits of a seqan::hibf::bit_vector. - * \param bit_vector The seqan::hibf::bit_vector. - * \attention The counting_vector must be at least as big as `bit_vector`. - */ - counting_vector & operator-=(bit_vector const & bit_vector) - { - for_each_set_bin(bit_vector, - [this](size_t const bin) - { - assert((*this)[bin] > 0); - --(*this)[bin]; - }); - return *this; - } - - /*!\brief Bin-wise addition of two `seqan::hibf::counting_vector`s. - * \param rhs The other seqan::hibf::counting_vector. - * \attention The seqan::hibf::counting_vector must be at least as big as `rhs`. - * - * \details - * - * ### Example - * - * \include test/snippet/ibf/counting_vector.cpp - */ - counting_vector & operator+=(counting_vector const & rhs) - { - assert(this->size() >= rhs.size()); // The counting vector may be bigger than what we need. - - std::transform(this->begin(), this->end(), rhs.begin(), this->begin(), std::plus()); - - return *this; - } - - /*!\brief Bin-wise substraction of two `seqan::hibf::counting_vector`s. - * \param rhs The other seqan::hibf::counting_vector. - * \attention The seqan::hibf::counting_vector must be at least as big as `rhs`. - */ - counting_vector & operator-=(counting_vector const & rhs) - { - assert(this->size() >= rhs.size()); // The counting vector may be bigger than what we need. - - std::transform(this->begin(), - this->end(), - rhs.begin(), - this->begin(), - [](auto a, auto b) - { - assert(a >= b); - return a - b; - }); - - return *this; - } - -private: - //!\brief Enumerates all bins of a seqan::hibf::bit_vector. - template - void for_each_set_bin(bit_vector const & bit_vector, on_bin_fn_t && on_bin_fn) - { - assert(this->size() >= bit_vector.size()); // The counting vector may be bigger than what we need. - size_t const words = (bit_vector.size() + 63u) >> 6; - uint64_t const * const bitvector_raw = bit_vector.data(); - - // Jump to the next 1 and return the number of places jumped in the bit_sequence - auto jump_to_next_1bit = [](uint64_t & x) - { - auto const zeros = std::countr_zero(x); - x >>= zeros; // skip number of zeros - return zeros; - }; - - // Each iteration can handle 64 bits - for (size_t batch = 0; batch < words; ++batch) - { - // get 64 bits starting at position `bit_pos` - uint64_t bit_sequence = bitvector_raw[batch]; - - // process each relative bin inside the bit_sequence - for (size_t bin = batch << 6; bit_sequence != 0u; ++bin, bit_sequence >>= 1) - { - // Jump to the next 1 and - bin += jump_to_next_1bit(bit_sequence); - - on_bin_fn(bin); - } - } - } -}; - /*!\brief Manages counting ranges of values for the seqan::hibf::interleaved_bloom_filter. * \attention Calling seqan::hibf::interleaved_bloom_filter::increase_bin_number_to invalidates the counting_agent_type. * diff --git a/include/hibf/misc/counting_vector.hpp b/include/hibf/misc/counting_vector.hpp new file mode 100644 index 00000000..e0eeba31 --- /dev/null +++ b/include/hibf/misc/counting_vector.hpp @@ -0,0 +1,190 @@ +// SPDX-FileCopyrightText: 2006-2023, Knut Reinert & Freie Universität Berlin +// SPDX-FileCopyrightText: 2016-2023, Knut Reinert & MPI für molekulare Genetik +// SPDX-License-Identifier: BSD-3-Clause + +/*!\file + * \author Enrico Seiler + * \brief Provides seqan::hibf::counting_vector. + */ + +#pragma once + +#include // for fill +#include // for array +#include // for countr_zero +#include // for assert +#include // for uint64_t, uint16_t +#include // for integral, same_as, unsigned_integral +#include // for size_t +#include // for plus +#include // for range, forward_range, input_range, range_reference_t, range_value_t +#include // for remove_cvref_t +#include // for addressof +#include // for vector + +#include // for cereal_archive +#include // for config +#include // for aligned_allocator +#include // for bit_vector + +#include // for CEREAL_SERIALIZE_FUNCTION_NAME +#include // for base_class + +namespace seqan::hibf +{ + +/*!\brief A data structure that behaves like a std::vector and can be used to consolidate the results of multiple calls + * to seqan::hibf::interleaved_bloom_filter::membership_agent_type::bulk_contains. + * \ingroup ibf + * \tparam value_t The type of the count. Must model std::integral. + * + * \details + * + * When using the seqan::hibf::interleaved_bloom_filter::membership_agent_type::bulk_contains operation, a common use + * case is to add up, for example, the results for all k-mers in a query. This yields, for each bin, the number of + * k-mers of a query that are in the respective bin. Such information can be used to apply further filtering or + * abundance estimation based on the k-mer counts. + * + * The seqan::hibf::counting_vector offers an easy way to add up the individual + * seqan::hibf::bit_vector by offering an `+=` operator. + * + * The `value_t` template parameter should be chosen in a way that no overflow occurs if all calls to `bulk_contains` + * return a hit for a specific bin. For example, `uint8_t` will suffice when processing short Illumina reads, whereas + * long reads will require at least `uint32_t`. + * + * ### Example + * + * \include test/snippet/ibf/counting_vector.cpp + */ +template +class counting_vector : public std::vector +{ +private: + //!\brief The base type. + using base_t = std::vector; + +public: + /*!\name Constructors, destructor and assignment + * \{ + */ + counting_vector() = default; //!< Defaulted. + counting_vector(counting_vector const &) = default; //!< Defaulted. + counting_vector & operator=(counting_vector const &) = default; //!< Defaulted. + counting_vector(counting_vector &&) = default; //!< Defaulted. + counting_vector & operator=(counting_vector &&) = default; //!< Defaulted. + ~counting_vector() = default; //!< Defaulted. + + using base_t::base_t; + //!\} + + /*!\brief Bin-wise adds the bits of a seqan::hibf::bit_vector. + * \param bit_vector The seqan::hibf::bit_vector. + * \attention The counting_vector must be at least as big as `bit_vector`. + * + * \details + * + * ### Example + * + * \include test/snippet/ibf/counting_vector.cpp + */ + counting_vector & operator+=(bit_vector const & bit_vector) + { + for_each_set_bin(bit_vector, + [this](size_t const bin) + { + ++(*this)[bin]; + }); + return *this; + } + + /*!\brief Bin-wise subtracts the bits of a seqan::hibf::bit_vector. + * \param bit_vector The seqan::hibf::bit_vector. + * \attention The counting_vector must be at least as big as `bit_vector`. + */ + counting_vector & operator-=(bit_vector const & bit_vector) + { + for_each_set_bin(bit_vector, + [this](size_t const bin) + { + assert((*this)[bin] > 0); + --(*this)[bin]; + }); + return *this; + } + + /*!\brief Bin-wise addition of two `seqan::hibf::counting_vector`s. + * \param rhs The other seqan::hibf::counting_vector. + * \attention The seqan::hibf::counting_vector must be at least as big as `rhs`. + * + * \details + * + * ### Example + * + * \include test/snippet/ibf/counting_vector.cpp + */ + counting_vector & operator+=(counting_vector const & rhs) + { + assert(this->size() >= rhs.size()); // The counting vector may be bigger than what we need. + + std::transform(this->begin(), this->end(), rhs.begin(), this->begin(), std::plus()); + + return *this; + } + + /*!\brief Bin-wise substraction of two `seqan::hibf::counting_vector`s. + * \param rhs The other seqan::hibf::counting_vector. + * \attention The seqan::hibf::counting_vector must be at least as big as `rhs`. + */ + counting_vector & operator-=(counting_vector const & rhs) + { + assert(this->size() >= rhs.size()); // The counting vector may be bigger than what we need. + + std::transform(this->begin(), + this->end(), + rhs.begin(), + this->begin(), + [](auto a, auto b) + { + assert(a >= b); + return a - b; + }); + + return *this; + } + +private: + //!\brief Enumerates all bins of a seqan::hibf::bit_vector. + template + void for_each_set_bin(bit_vector const & bit_vector, on_bin_fn_t && on_bin_fn) + { + assert(this->size() >= bit_vector.size()); // The counting vector may be bigger than what we need. + size_t const words = (bit_vector.size() + 63u) >> 6; + uint64_t const * const bitvector_raw = bit_vector.data(); + + // Jump to the next 1 and return the number of places jumped in the bit_sequence + auto jump_to_next_1bit = [](uint64_t & x) + { + auto const zeros = std::countr_zero(x); + x >>= zeros; // skip number of zeros + return zeros; + }; + + // Each iteration can handle 64 bits + for (size_t batch = 0; batch < words; ++batch) + { + // get 64 bits starting at position `bit_pos` + uint64_t bit_sequence = bitvector_raw[batch]; + + // process each relative bin inside the bit_sequence + for (size_t bin = batch << 6; bit_sequence != 0u; ++bin, bit_sequence >>= 1) + { + // Jump to the next 1 and + bin += jump_to_next_1bit(bit_sequence); + + on_bin_fn(bin); + } + } + } +}; + +} // namespace seqan::hibf From 2826ef57154810c6e6efc73e1e1b461cba9cc054 Mon Sep 17 00:00:00 2001 From: Enrico Seiler Date: Tue, 28 Nov 2023 10:49:27 +0100 Subject: [PATCH 3/5] [FEATURE] SIMD add --- include/hibf/interleaved_bloom_filter.hpp | 17 ++ include/hibf/misc/counting_vector.hpp | 217 ++++++++++++++---- include/hibf/platform.hpp | 12 + .../hibf/interleaved_bloom_filter_test.cpp | 21 +- 4 files changed, 206 insertions(+), 61 deletions(-) diff --git a/include/hibf/interleaved_bloom_filter.hpp b/include/hibf/interleaved_bloom_filter.hpp index 1f6a65b3..29a805a9 100644 --- a/include/hibf/interleaved_bloom_filter.hpp +++ b/include/hibf/interleaved_bloom_filter.hpp @@ -535,8 +535,25 @@ class interleaved_bloom_filter::counting_agent_type explicit counting_agent_type(interleaved_bloom_filter const & ibf) : ibf_ptr(std::addressof(ibf)), membership_agent(ibf), +#if !HIBF_HAS_AVX512 result_buffer(ibf.bin_count()) {} +#else + // AVX512 will access result_buffer's memory in chunks, so we need to make sure that we allocate enough memory + // such that the last chunk is not out of bounds. + result_buffer(next_multiple_of_64(ibf.bin_count())) // Ensure large enough capacity. + { + result_buffer.resize(ibf.bin_count()); // Resize to actual requested size. + // Silences llvm's ASAN container-overflow warning. +# if defined(_LIBCPP_VERSION) && !defined(_LIBCPP_HAS_NO_ASAN) + __sanitizer_annotate_contiguous_container(result_buffer.data(), + result_buffer.data() + result_buffer.capacity(), + result_buffer.data() + result_buffer.size(), + result_buffer.data() + result_buffer.capacity()); +# endif + } +#endif + //!\} /*!\name Counting diff --git a/include/hibf/misc/counting_vector.hpp b/include/hibf/misc/counting_vector.hpp index e0eeba31..4cb50e3e 100644 --- a/include/hibf/misc/counting_vector.hpp +++ b/include/hibf/misc/counting_vector.hpp @@ -26,13 +26,103 @@ #include // for config #include // for aligned_allocator #include // for bit_vector +#include +#include #include // for CEREAL_SERIALIZE_FUNCTION_NAME #include // for base_class +#if HIBF_HAS_AVX512 +# include +# include +# include +# include +# include +# include +#endif + namespace seqan::hibf { +#if HIBF_HAS_AVX512 +//!\cond +// Since the counting_vector can have different value types, we need specific SIMD instructions for each value type. +template +struct simd_mapping +{}; + +// CRTP base class for the simd_mapping, containg common functionality. +template + requires std::same_as> +struct simd_mapping_crtp +{ + // Let `B = sizeof(integral_t) * CHAR_BIT`, e.g. 8 for (u)int8_t, and 16 for (u)int16_t. + // We can process `512 / B` bits of the bit_vector at once. + static inline constexpr size_t bits_per_iterations = 512u / (sizeof(integral_t) * CHAR_BIT); + // clang-format off + // The type that is need to represent `bits_per_iterations` bits. + using bits_type = std::conditional_t>>>; + // clang-format on + static_assert(!std::same_as, "Unsupported bits_type."); + + // Takes B bits from the bit_vector and expands them to a bits_type. + // E.g., B = 8 : [1,0,1,1,0,0,1,0] -> [0...01, 0...00, 0...01, 0...01, 0...00, 0...00, 0...01, 0...00], where + // each element is 64 bits wide. + static inline constexpr auto expand_bits(bits_type const * const bits) + { + return derived_t::mm512_maskz_mov_epi(*bits, derived_t::mm512_set1_epi(1)); + } +}; + +// SIMD instructions for int8_t and uint8_t. +template + requires (sizeof(integral_t) == 1) +struct simd_mapping : simd_mapping_crtp, integral_t> +{ + static inline constexpr auto mm512_maskz_mov_epi = simde_mm512_maskz_mov_epi8; + static inline constexpr auto mm512_set1_epi = simde_mm512_set1_epi8; + static inline constexpr auto mm512_add_epi = simde_mm512_add_epi8; + static inline constexpr auto mm512_sub_epi = simde_mm512_sub_epi8; +}; + +// SIMD instructions for int16_t and uint16_t. +template + requires (sizeof(integral_t) == 2) +struct simd_mapping : simd_mapping_crtp, integral_t> +{ + static inline constexpr auto mm512_maskz_mov_epi = simde_mm512_maskz_mov_epi16; + static inline constexpr auto mm512_set1_epi = simde_mm512_set1_epi16; + static inline constexpr auto mm512_add_epi = simde_mm512_add_epi16; + static inline constexpr auto mm512_sub_epi = simde_mm512_sub_epi16; +}; + +// SIMD instructions for int32_t and uint32_t. +template + requires (sizeof(integral_t) == 4) +struct simd_mapping : simd_mapping_crtp, integral_t> +{ + static inline constexpr auto mm512_maskz_mov_epi = simde_mm512_maskz_mov_epi32; + static inline constexpr auto mm512_set1_epi = simde_mm512_set1_epi32; + static inline constexpr auto mm512_add_epi = simde_mm512_add_epi32; + static inline constexpr auto mm512_sub_epi = simde_mm512_sub_epi32; +}; + +// SIMD instructions for int64_t and uint64_t. +template + requires (sizeof(integral_t) == 8) +struct simd_mapping : simd_mapping_crtp, integral_t> +{ + static inline constexpr auto mm512_maskz_mov_epi = simde_mm512_maskz_mov_epi64; + static inline constexpr auto mm512_set1_epi = simde_mm512_set1_epi64; + static inline constexpr auto mm512_add_epi = simde_mm512_add_epi64; + static inline constexpr auto mm512_sub_epi = simde_mm512_sub_epi64; +}; +//!\endcond +#endif + /*!\brief A data structure that behaves like a std::vector and can be used to consolidate the results of multiple calls * to seqan::hibf::interleaved_bloom_filter::membership_agent_type::bulk_contains. * \ingroup ibf @@ -57,11 +147,11 @@ namespace seqan::hibf * \include test/snippet/ibf/counting_vector.cpp */ template -class counting_vector : public std::vector +class counting_vector : public std::vector> { private: //!\brief The base type. - using base_t = std::vector; + using base_t = std::vector>; public: /*!\name Constructors, destructor and assignment @@ -78,46 +168,37 @@ class counting_vector : public std::vector //!\} /*!\brief Bin-wise adds the bits of a seqan::hibf::bit_vector. - * \param bit_vector The seqan::hibf::bit_vector. - * \attention The counting_vector must be at least as big as `bit_vector`. - * + * \copydetails operator-=(bit_vector const &) * \details - * * ### Example * * \include test/snippet/ibf/counting_vector.cpp */ counting_vector & operator+=(bit_vector const & bit_vector) { - for_each_set_bin(bit_vector, - [this](size_t const bin) - { - ++(*this)[bin]; - }); + impl(bit_vector); return *this; } /*!\brief Bin-wise subtracts the bits of a seqan::hibf::bit_vector. * \param bit_vector The seqan::hibf::bit_vector. - * \attention The counting_vector must be at least as big as `bit_vector`. + * \pre `counting_vector.size() >= bit_vector.size()` and either: + * * `bit_vector.size() % 64 == 0` + * \pre or + * * `counting_vector.capacity() >= seqan::hibf::next_multiple_of_64(bit_vector.size())` and + * * ∀ bit ∈ [`bit_vector.size()`, `bit_vector.capacity()`) : `bit_vector[bit] == 0`
+ * In practive, this condition might be violated by setting bits in a bit_vector and then shrinking it via + * \ref bit_vector::resize() "resize()". */ counting_vector & operator-=(bit_vector const & bit_vector) { - for_each_set_bin(bit_vector, - [this](size_t const bin) - { - assert((*this)[bin] > 0); - --(*this)[bin]; - }); + impl(bit_vector); return *this; } /*!\brief Bin-wise addition of two `seqan::hibf::counting_vector`s. - * \param rhs The other seqan::hibf::counting_vector. - * \attention The seqan::hibf::counting_vector must be at least as big as `rhs`. - * + * \copydetails operator-=(counting_vector const &) * \details - * * ### Example * * \include test/snippet/ibf/counting_vector.cpp @@ -131,59 +212,93 @@ class counting_vector : public std::vector return *this; } - /*!\brief Bin-wise substraction of two `seqan::hibf::counting_vector`s. + /*!\brief Bin-wise subtraction of two `seqan::hibf::counting_vector`s. * \param rhs The other seqan::hibf::counting_vector. - * \attention The seqan::hibf::counting_vector must be at least as big as `rhs`. + * \pre `counting_vector.size() >= rhs.size()` */ counting_vector & operator-=(counting_vector const & rhs) { assert(this->size() >= rhs.size()); // The counting vector may be bigger than what we need. - std::transform(this->begin(), - this->end(), - rhs.begin(), - this->begin(), - [](auto a, auto b) - { - assert(a >= b); - return a - b; - }); + std::transform(this->begin(), this->end(), rhs.begin(), this->begin(), std::minus()); return *this; } private: - //!\brief Enumerates all bins of a seqan::hibf::bit_vector. - template - void for_each_set_bin(bit_vector const & bit_vector, on_bin_fn_t && on_bin_fn) + //!\brief The operation to perform. + enum class operation + { + add, + sub + }; + + //!\brief Bin-wise adds or subtracts the bits of a seqan::hibf::bit_vector. + template + inline void impl(bit_vector const & bit_vector) { assert(this->size() >= bit_vector.size()); // The counting vector may be bigger than what we need. - size_t const words = (bit_vector.size() + 63u) >> 6; - uint64_t const * const bitvector_raw = bit_vector.data(); +#if HIBF_HAS_AVX512 + // AVX512BW: mm512_maskz_mov_epi, mm512_add_epi + // AVX512F: mm512_set1_epi, _mm512_load_si512, _mm512_store_si512 + using simd = simd_mapping; + using bits_type = typename simd::bits_type; + + bits_type const * bit_vector_ptr = reinterpret_cast(bit_vector.data()); + value_t * counting_vector_ptr = base_t::data(); + + size_t const bits = next_multiple_of_64(bit_vector.size()); + assert(bits <= this->capacity()); // Not enough memory reserved for AVX512 chunk access. + size_t const iterations = bits / simd::bits_per_iterations; + + for (size_t iteration = 0; iteration < iterations; + ++iteration, ++bit_vector_ptr, counting_vector_ptr += simd::bits_per_iterations) + { + simde__m512i load = simde_mm512_load_si512(counting_vector_ptr); + if constexpr (op == operation::add) + { + load = simd::mm512_add_epi(load, simd::expand_bits(bit_vector_ptr)); + } + else + { + load = simd::mm512_sub_epi(load, simd::expand_bits(bit_vector_ptr)); + } + simde_mm512_store_si512(counting_vector_ptr, load); + } +#else + size_t const words = divide_and_ceil(bit_vector.size(), 64u); + uint64_t const * const bit_vector_ptr = bit_vector.data(); - // Jump to the next 1 and return the number of places jumped in the bit_sequence - auto jump_to_next_1bit = [](uint64_t & x) + // Jump to the next 1 and return the number of jumped bits in value. + auto jump_to_next_1bit = [](uint64_t & value) { - auto const zeros = std::countr_zero(x); - x >>= zeros; // skip number of zeros + auto const zeros = std::countr_zero(value); + value >>= zeros; // skip number of zeros return zeros; }; - // Each iteration can handle 64 bits - for (size_t batch = 0; batch < words; ++batch) + // Each iteration can handle 64 bits, i.e., one word. + for (size_t iteration = 0; iteration < words; ++iteration) { - // get 64 bits starting at position `bit_pos` - uint64_t bit_sequence = bitvector_raw[batch]; + uint64_t current_word = bit_vector_ptr[iteration]; - // process each relative bin inside the bit_sequence - for (size_t bin = batch << 6; bit_sequence != 0u; ++bin, bit_sequence >>= 1) + // For each set bit in the current word, add/subtract 1 to the corresponding bin. + for (size_t bin = iteration * 64u; current_word != 0u; ++bin, current_word >>= 1) { - // Jump to the next 1 and - bin += jump_to_next_1bit(bit_sequence); + // Jump to the next 1 + bin += jump_to_next_1bit(current_word); - on_bin_fn(bin); + if constexpr (op == operation::add) + { + ++(*this)[bin]; + } + else + { + --(*this)[bin]; + } } } +#endif } }; diff --git a/include/hibf/platform.hpp b/include/hibf/platform.hpp index 58b284f4..75dc5c66 100644 --- a/include/hibf/platform.hpp +++ b/include/hibf/platform.hpp @@ -49,6 +49,18 @@ # define HIBF_DISABLE_COMPILER_CHECK #endif // HIBF_DOXYGEN_ONLY(1)0 +/*!\def HIBF_HAS_AVX512 + * \brief Whether AVX512F and AVX512BW are available. + * \ingroup core + */ +#ifndef HIBF_HAS_AVX512 +# if __AVX512F__ && __AVX512BW__ +# define HIBF_HAS_AVX512 1 +# else +# define HIBF_HAS_AVX512 0 +# endif +#endif + // ============================================================================ // Compiler support GCC // ============================================================================ diff --git a/test/unit/hibf/interleaved_bloom_filter_test.cpp b/test/unit/hibf/interleaved_bloom_filter_test.cpp index e87c4b45..c0936f03 100644 --- a/test/unit/hibf/interleaved_bloom_filter_test.cpp +++ b/test/unit/hibf/interleaved_bloom_filter_test.cpp @@ -238,23 +238,23 @@ TEST(ibf_test, counting) counting += agent.bulk_contains(hash); } std::vector expected(128, 128); - EXPECT_EQ(counting, expected); + EXPECT_RANGE_EQ(counting, expected); // Counting vectors can be added together std::vector expected2(128, 256); counting += counting; - EXPECT_EQ(counting, expected2); + EXPECT_RANGE_EQ(counting, expected2); // minus bit_vector for (size_t hash : std::views::iota(0, 128)) // test correct resize for each bin individually { counting -= agent.bulk_contains(hash); } - EXPECT_EQ(counting, std::vector(128, 128)); + EXPECT_RANGE_EQ(counting, std::vector(128, 128)); // minus other counting vector counting -= seqan::hibf::counting_vector(128, 128 - 42); - EXPECT_EQ(counting, std::vector(128, 42)); + EXPECT_RANGE_EQ(counting, std::vector(128, 42)); } TEST(ibf_test, counting_agent) @@ -299,25 +299,26 @@ TEST(ibf_test, counting_no_ub) std::vector expected(128, 0); expected[63] = 128; expected[127] = 128; - EXPECT_EQ(counting, expected); + EXPECT_RANGE_EQ(counting, expected); // Counting vectors can be added together std::vector expected2(128, 0); expected2[63] = 256; expected2[127] = 256; counting += counting; - EXPECT_EQ(counting, expected2); + EXPECT_RANGE_EQ(counting, expected2); } // Check special case where there is only one `1` in the bitvector. +// Also checks that counting with `b` bins and `b % 64 != 0` works. TEST(ibf_test, counting_agent_no_ub) { // 1. Construct and emplace - seqan::hibf::interleaved_bloom_filter ibf{seqan::hibf::bin_count{128u}, + seqan::hibf::interleaved_bloom_filter ibf{seqan::hibf::bin_count{127u}, seqan::hibf::bin_size{1024u}, seqan::hibf::hash_function_count{2u}}; - for (size_t bin_idx : std::array{63, 127}) + for (size_t bin_idx : std::array{63, 126}) for (size_t hash : std::views::iota(0, 128)) ibf.emplace(hash, seqan::hibf::bin_index{bin_idx}); @@ -325,9 +326,9 @@ TEST(ibf_test, counting_agent_no_ub) auto agent = ibf.counting_agent(); auto agent2 = ibf.template counting_agent(); - std::vector expected(128, 0); + std::vector expected(127, 0); expected[63] = 128; - expected[127] = 128; + expected[126] = 128; EXPECT_RANGE_EQ(agent.bulk_count(std::views::iota(0u, 128u)), expected); EXPECT_RANGE_EQ(agent2.bulk_count(std::views::iota(0u, 128u)), expected); } From 2de04e71e6bbb8d8bb13b8cde60623f5fbb3d515 Mon Sep 17 00:00:00 2001 From: Enrico Seiler Date: Tue, 28 Nov 2023 11:01:54 +0100 Subject: [PATCH 4/5] [TEST] Add counting_vector_test And AVX512 tests. Uses intrinsics if available, simde's simulation otherwise --- test/unit/hibf/CMakeLists.txt | 3 + .../unit/hibf/counting_vector_avx512_test.cpp | 7 + test/unit/hibf/counting_vector_test.cpp | 194 ++++++++++++++++++ .../interleaved_bloom_filter_avx512_test.cpp | 7 + .../hibf/interleaved_bloom_filter_test.cpp | 4 + 5 files changed, 215 insertions(+) create mode 100644 test/unit/hibf/counting_vector_avx512_test.cpp create mode 100644 test/unit/hibf/counting_vector_test.cpp create mode 100644 test/unit/hibf/interleaved_bloom_filter_avx512_test.cpp diff --git a/test/unit/hibf/CMakeLists.txt b/test/unit/hibf/CMakeLists.txt index f0f65446..b4aeaacb 100644 --- a/test/unit/hibf/CMakeLists.txt +++ b/test/unit/hibf/CMakeLists.txt @@ -6,8 +6,11 @@ add_subdirectories () hibf_test (bit_vector_test.cpp) hibf_test (config_test.cpp) +hibf_test (counting_vector_test.cpp) +hibf_test (counting_vector_avx512_test.cpp) hibf_test (hierarchical_interleaved_bloom_filter_test.cpp) hibf_test (interleaved_bloom_filter_test.cpp) +hibf_test (interleaved_bloom_filter_avx512_test.cpp) hibf_test (path_test.cpp) hibf_test (print_test.cpp) hibf_test (timer_test.cpp) diff --git a/test/unit/hibf/counting_vector_avx512_test.cpp b/test/unit/hibf/counting_vector_avx512_test.cpp new file mode 100644 index 00000000..e0c65de3 --- /dev/null +++ b/test/unit/hibf/counting_vector_avx512_test.cpp @@ -0,0 +1,7 @@ +// SPDX-FileCopyrightText: 2006-2023, Knut Reinert & Freie Universität Berlin +// SPDX-FileCopyrightText: 2016-2023, Knut Reinert & MPI für molekulare Genetik +// SPDX-License-Identifier: BSD-3-Clause + +#define HIBF_HAS_AVX512 1 + +#include "counting_vector_test.cpp" diff --git a/test/unit/hibf/counting_vector_test.cpp b/test/unit/hibf/counting_vector_test.cpp new file mode 100644 index 00000000..95cc50be --- /dev/null +++ b/test/unit/hibf/counting_vector_test.cpp @@ -0,0 +1,194 @@ +// SPDX-FileCopyrightText: 2006-2023, Knut Reinert & Freie Universität Berlin +// SPDX-FileCopyrightText: 2016-2023, Knut Reinert & MPI für molekulare Genetik +// SPDX-License-Identifier: BSD-3-Clause + +#ifndef HIBF_HAS_AVX512 +# define HIBF_HAS_AVX512 0 +#endif + +#include // for EXPECT_EQ, GetCapturedStderr, GetCapturedStdout, Message, TestP... + +#include // for int64_t, int16_t, int32_t, int8_t, uint16_t, uint32_t, uint64_t +#include // for same_as, unsigned_integral +#include // for cerr +#include // for range_value_t +#include // for allocator, vector + +#include + +template +class counting_vector_test : public ::testing::Test +{ +protected: + seqan::hibf::counting_vector counting_vector; + seqan::hibf::bit_vector bit_vector; + seqan::hibf::counting_vector expected; + + void SetUp() override + { + counting_vector.resize(2048); + bit_vector.resize(2048); + expected.resize(2048); + } + + void check_add() + { + counting_vector += bit_vector; + ASSERT_EQ(counting_vector.size(), expected.size()); + EXPECT_EQ(counting_vector, expected); + } + + void check_sub() + { + counting_vector -= bit_vector; + ASSERT_EQ(counting_vector.size(), expected.size()); + EXPECT_EQ(counting_vector, expected); + } + + void annotate_llvm_asan() const + { +#if defined(_LIBCPP_VERSION) && !defined(_LIBCPP_HAS_NO_ASAN) + __sanitizer_annotate_contiguous_container(counting_vector.data(), + counting_vector.data() + counting_vector.capacity(), + counting_vector.data() + counting_vector.size(), + counting_vector.data() + counting_vector.capacity()); +#endif + } +}; + +using test_types = ::testing::Types; + +TYPED_TEST_SUITE(counting_vector_test, test_types); + +TYPED_TEST(counting_vector_test, empty) +{ + this->counting_vector.clear(); + this->bit_vector.clear(); + this->expected.clear(); + + this->check_add(); + this->check_sub(); +} + +TYPED_TEST(counting_vector_test, no_bits_set) +{ + this->check_add(); + this->check_sub(); +} + +TYPED_TEST(counting_vector_test, all_bits_set) +{ + constexpr TypeParam max_value = std::numeric_limits::max(); + + std::ranges::fill(this->counting_vector, max_value - 1u); + std::ranges::fill(this->bit_vector, true); + + std::ranges::fill(this->expected, max_value); + this->check_add(); + + std::ranges::fill(this->expected, max_value - 1u); + this->check_sub(); + + if constexpr (std::signed_integral) + { + std::ranges::fill(this->counting_vector, 0); + + std::ranges::fill(this->expected, -1); + this->check_sub(); + } +} + +TYPED_TEST(counting_vector_test, every_other_bit_set) +{ + for (size_t i = 0; i < this->bit_vector.size(); ++i) + this->bit_vector[i] = i % 2; + + for (size_t i = 0; i < this->expected.size(); ++i) + this->expected[i] = i % 2; + this->check_add(); + + std::ranges::fill(this->expected, TypeParam{}); + this->check_sub(); + + if constexpr (std::signed_integral) + { + std::ranges::fill(this->counting_vector, 0); + + for (size_t i = 0; i < this->expected.size(); ++i) + this->expected[i] = -(i % 2); + this->check_sub(); + } +} + +TYPED_TEST(counting_vector_test, first_and_last_set) +{ + this->bit_vector.front() = this->bit_vector.back() = true; + + ++this->expected.front(); + ++this->expected.back(); + this->check_add(); + + --this->expected.front(); + --this->expected.back(); + this->check_sub(); + + if constexpr (std::signed_integral) + { + --this->expected.front(); + --this->expected.back(); + this->check_sub(); + } +} + +TYPED_TEST(counting_vector_test, counting_vector_is_bigger) +{ + constexpr TypeParam max_value = std::numeric_limits::max(); + + this->counting_vector.resize(4096, max_value); + std::ranges::fill(this->bit_vector, true); + + this->expected.resize(4096, max_value); + std::ranges::fill_n(this->expected.begin(), this->expected.size() / 2u, TypeParam{1u}); + this->check_add(); + + std::ranges::fill_n(this->expected.begin(), this->expected.size() / 2u, TypeParam{}); + this->check_sub(); +} +TYPED_TEST(counting_vector_test, size_not_divisible_by_64) +{ + constexpr TypeParam max_value = std::numeric_limits::max(); + + this->counting_vector.resize(2033); + std::ranges::fill(this->counting_vector, max_value - 1u); + this->annotate_llvm_asan(); + + this->bit_vector.resize(2033); + std::ranges::fill(this->bit_vector, true); + + this->expected.resize(2033); + std::ranges::fill(this->expected, max_value); + this->check_add(); + + std::ranges::fill(this->expected, max_value - 1u); + this->check_sub(); +} + +TYPED_TEST(counting_vector_test, overflow) +{ + constexpr TypeParam max_value = std::numeric_limits::max(); + + std::ranges::fill(this->counting_vector, max_value); + this->counting_vector.resize(2033); + this->annotate_llvm_asan(); + + std::ranges::fill(this->bit_vector, true); + this->bit_vector.resize(2033); + std::ranges::fill(this->bit_vector, false); + + this->expected.resize(2033); + std::ranges::fill(this->expected, max_value); + this->check_add(); // counting_vector[2033...2047] may overflow but it does not affect the actual results + + std::ranges::fill(this->expected, max_value); + this->check_sub(); // counting_vector[2033...2047] may underflow but it does not affect the actual results +} diff --git a/test/unit/hibf/interleaved_bloom_filter_avx512_test.cpp b/test/unit/hibf/interleaved_bloom_filter_avx512_test.cpp new file mode 100644 index 00000000..12ec2353 --- /dev/null +++ b/test/unit/hibf/interleaved_bloom_filter_avx512_test.cpp @@ -0,0 +1,7 @@ +// SPDX-FileCopyrightText: 2006-2023, Knut Reinert & Freie Universität Berlin +// SPDX-FileCopyrightText: 2016-2023, Knut Reinert & MPI für molekulare Genetik +// SPDX-License-Identifier: BSD-3-Clause + +#define HIBF_HAS_AVX512 1 + +#include "interleaved_bloom_filter_test.cpp" diff --git a/test/unit/hibf/interleaved_bloom_filter_test.cpp b/test/unit/hibf/interleaved_bloom_filter_test.cpp index c0936f03..ee60ad5c 100644 --- a/test/unit/hibf/interleaved_bloom_filter_test.cpp +++ b/test/unit/hibf/interleaved_bloom_filter_test.cpp @@ -2,6 +2,10 @@ // SPDX-FileCopyrightText: 2016-2023, Knut Reinert & MPI für molekulare Genetik // SPDX-License-Identifier: BSD-3-Clause +#ifndef HIBF_HAS_AVX512 +# define HIBF_HAS_AVX512 0 +#endif + #include // for Test, Message, TestPartResult, AssertionResult, TestInfo, EXPEC... #include // for __for_each_fn, for_each From 1cebcfc77150f2668a3cd493e9d1f7fb9d6e5ff3 Mon Sep 17 00:00:00 2001 From: Enrico Seiler Date: Tue, 28 Nov 2023 11:02:12 +0100 Subject: [PATCH 5/5] [TEST] Refactor IBF benchmark --- .../interleaved_bloom_filter_benchmark.cpp | 203 ++++++++++-------- 1 file changed, 117 insertions(+), 86 deletions(-) diff --git a/test/performance/ibf/interleaved_bloom_filter_benchmark.cpp b/test/performance/ibf/interleaved_bloom_filter_benchmark.cpp index 659a2072..b9663955 100644 --- a/test/performance/ibf/interleaved_bloom_filter_benchmark.cpp +++ b/test/performance/ibf/interleaved_bloom_filter_benchmark.cpp @@ -13,117 +13,135 @@ #include // for make_tuple #include // for vector, allocator +#include #include // for operator| #include // for operator==, pair #include // for to #include // for zip_view, operator==, zip, zip_fn #include // for bin_index, interleaved_bloom_filter, bin_count, bin_size - -inline benchmark::Counter hashes_per_second(size_t const count) +#include +#include + +using namespace seqan::hibf::test::literals; +static constexpr size_t total_ibf_size_in_bytes{1_MiB}; +static constexpr size_t number_of_hash_functions{2u}; +static constexpr double false_positive_rate{0.05}; +// This computes how many elements need to be inserted into the IBF to achieve the desired false positive rate for the +// given size. +// The `number_of_elements` many generated values are used for both constructing and querying the IBF. +static /* cmath not constexpr in libc++ */ size_t number_of_elements = []() { - return benchmark::Counter(count, benchmark::Counter::kIsIterationInvariantRate, benchmark::Counter::OneK::kIs1000); -} - -#if 1 -static void arguments(benchmark::internal::Benchmark * b) -{ - // Total size: 1MiB - // bins, bin_size, hash_num, sequence_length - b->Args({64, 1LL << 17, 2, 1LL << 17}); - // b->Args({128, 1LL << 16, 2, 1LL << 17}); - // b->Args({192, 1LL << 16, 2, 1LL << 17}); - // b->Args({256, 1LL << 15, 2, 1LL << 17}); - // b->Args({320, 1LL << 15, 2, 1LL << 17}); - // b->Args({384, 1LL << 14, 2, 1LL << 17}); - // b->Args({448, 1LL << 14, 2, 1LL << 17}); - // b->Args({512, 1LL << 13, 2, 1LL << 17}); - b->Args({1024, 1LL << 10, 2, 1LL << 17}); -} -#else -static void arguments(benchmark::internal::Benchmark * b) -{ - // Total size: 1GiB - // bins, bin_size, hash_num, sequence_length - b->Args({64, 1LL << 27, 2, 1LL << 27}); - b->Args({128, 1LL << 26, 2, 1LL << 27}); - b->Args({192, 1LL << 26, 2, 1LL << 27}); - b->Args({256, 1LL << 25, 2, 1LL << 27}); - b->Args({1024, 1LL << 20, 2, 1LL << 27}); -} -#endif + size_t const bits = 8u * total_ibf_size_in_bytes; + double const numerator = -std::log(1 - std::exp(std::log(false_positive_rate) / number_of_hash_functions)) * bits; + return std::ceil(numerator / number_of_hash_functions); +}(); + +// We cache the generated values and constructed IBFs such that they are reused for all benchmarks and multiple runs of +// the same benchmark. Google benchmark may rerun the same benchmark multiple times in order to reach the requested +// minimum benchmark time or when multiple runs are requested via the --benchmark_repetitions flag. +// Caching also means that we access the values via `const &`: `auto const & [values, ibf] = set_up(state);`. +// If a benchmark modifies the IBFs, it needs to make a copy of the IBF (clear_benchmark) or +// construct a new one (emplace_benchmark). +static robin_hood::unordered_map, seqan::hibf::interleaved_bloom_filter>> + cache{}; auto set_up(::benchmark::State const & state) { size_t const bins = state.range(0); - size_t const bits = state.range(1); - size_t const hash_num = state.range(2); - size_t const sequence_length = state.range(3); + size_t const bits = 8u * total_ibf_size_in_bytes / bins; + + if (auto it = cache.find(bins); it != cache.end()) + return it->second; - auto generate = [sequence_length](size_t const max_value = std::numeric_limits::max()) + std::vector const values = []() { - auto generator = [max_value]() + std::mt19937_64 engine{0ULL}; + std::uniform_int_distribution distribution{}; + + auto gen = [&]() { - std::uniform_int_distribution distr{0u, max_value}; - std::mt19937_64 engine{0ULL}; - return distr(engine); + return distribution(engine); }; - std::vector result(sequence_length); - std::ranges::generate(result, generator); - return result; - }; + std::vector result(number_of_elements); + std::ranges::generate(result, gen); - std::vector const bin_indices{generate(bins - 1)}; - std::vector const hash_values{generate()}; + return result; + }(); seqan::hibf::interleaved_bloom_filter ibf{seqan::hibf::bin_count{bins}, seqan::hibf::bin_size{bits}, - seqan::hibf::hash_function_count{hash_num}}; + seqan::hibf::hash_function_count{number_of_hash_functions}}; + + size_t const chunk_size = seqan::hibf::divide_and_ceil(number_of_elements, bins); + size_t bin_index = 0u; + for (auto && chunk : seqan::stl::views::chunk(values, chunk_size)) + { + for (auto value : chunk) + ibf.emplace(value, seqan::hibf::bin_index{bin_index}); + ++bin_index; + } - return std::make_tuple(bin_indices, hash_values, ibf); + auto it = cache.emplace(bins, std::make_tuple(std::move(values), std::move(ibf))); + return it.first->second; +} + +inline benchmark::Counter elements_per_second(size_t const count) +{ + return benchmark::Counter(count, benchmark::Counter::kIsIterationInvariantRate, benchmark::Counter::OneK::kIs1000); } void emplace_benchmark(::benchmark::State & state) { - auto && [bin_indices, hash_values, ibf] = set_up(state); + auto const & [values, original_ibf] = set_up(state); + + size_t const bins = state.range(0); + size_t const chunk_size = seqan::hibf::divide_and_ceil(number_of_elements, bins); + + seqan::hibf::interleaved_bloom_filter ibf{seqan::hibf::bin_count{original_ibf.bin_count()}, + seqan::hibf::bin_size{original_ibf.bin_size()}, + seqan::hibf::hash_function_count{original_ibf.hash_function_count()}}; for (auto _ : state) { - for (auto [hash, bin] : seqan::stl::views::zip(hash_values, bin_indices)) - ibf.emplace(hash, seqan::hibf::bin_index{bin}); + size_t bin_index = 0u; + for (auto && chunk : seqan::stl::views::chunk(values, chunk_size)) + { + for (auto value : chunk) + ibf.emplace(value, seqan::hibf::bin_index{bin_index}); + ++bin_index; + } } - state.counters["hashes/sec"] = hashes_per_second(std::ranges::size(hash_values)); + state.counters["elements"] = elements_per_second(number_of_elements); } void clear_benchmark(::benchmark::State & state) { - auto && [bin_indices, hash_values, ibf] = set_up(state); - (void)bin_indices; - (void)hash_values; + auto const & [values, original_ibf] = set_up(state); + (void)values; - std::vector bin_range = std::views::iota(0u, static_cast(state.range(0))) - | std::views::transform( - [](size_t i) - { - return seqan::hibf::bin_index{i}; - }) - | seqan::stl::ranges::to(); + size_t const bins = state.range(0); + + seqan::hibf::interleaved_bloom_filter ibf{original_ibf}; for (auto _ : state) { - for (auto bin : bin_range) - ibf.clear(bin); + for (size_t i = 0; i < bins; ++i) + ibf.clear(seqan::hibf::bin_index{i}); } - state.counters["bins/sec"] = hashes_per_second(std::ranges::size(bin_range)); + state.counters["bins"] = elements_per_second(bins); } void clear_range_benchmark(::benchmark::State & state) { - auto && [bin_indices, hash_values, ibf] = set_up(state); - (void)bin_indices; - (void)hash_values; + auto const & [values, original_ibf] = set_up(state); + (void)values; + + size_t const bins = state.range(0); + + seqan::hibf::interleaved_bloom_filter ibf{original_ibf}; std::vector bin_range = std::views::iota(0u, static_cast(state.range(0))) | std::views::transform( @@ -138,50 +156,63 @@ void clear_range_benchmark(::benchmark::State & state) ibf.clear(bin_range); } - state.counters["bins/sec"] = hashes_per_second(std::ranges::size(bin_range)); + state.counters["bins"] = elements_per_second(bins); } void bulk_contains_benchmark(::benchmark::State & state) { - auto && [bin_indices, hash_values, ibf] = set_up(state); - - for (auto [hash, bin] : seqan::stl::views::zip(hash_values, bin_indices)) - ibf.emplace(hash, seqan::hibf::bin_index{bin}); + auto const & [values, ibf] = set_up(state); auto agent = ibf.membership_agent(); for (auto _ : state) { - for (auto hash : hash_values) + for (auto hash : values) { [[maybe_unused]] auto & res = agent.bulk_contains(hash); benchmark::ClobberMemory(); } } - state.counters["hashes/sec"] = hashes_per_second(std::ranges::size(hash_values)); + state.counters["elements"] = elements_per_second(number_of_elements); } void bulk_count_benchmark(::benchmark::State & state) { - auto && [bin_indices, hash_values, ibf] = set_up(state); - - for (auto [hash, bin] : seqan::stl::views::zip(hash_values, bin_indices)) - ibf.emplace(hash, seqan::hibf::bin_index{bin}); + auto const & [values, ibf] = set_up(state); auto agent = ibf.counting_agent(); for (auto _ : state) { - [[maybe_unused]] auto & res = agent.bulk_count(hash_values); + [[maybe_unused]] auto & res = agent.bulk_count(values); benchmark::ClobberMemory(); } - state.counters["hashes/sec"] = hashes_per_second(std::ranges::size(hash_values)); + state.counters["elements"] = elements_per_second(number_of_elements); } -BENCHMARK(emplace_benchmark)->Apply(arguments); -BENCHMARK(clear_benchmark)->Apply(arguments); -BENCHMARK(clear_range_benchmark)->Apply(arguments); -BENCHMARK(bulk_contains_benchmark)->Apply(arguments); -BENCHMARK(bulk_count_benchmark)->Apply(arguments); +BENCHMARK(emplace_benchmark)->RangeMultiplier(2)->Range(64, 1024); +BENCHMARK(clear_benchmark)->RangeMultiplier(2)->Range(64, 1024); +BENCHMARK(clear_range_benchmark)->RangeMultiplier(2)->Range(64, 1024); +BENCHMARK(bulk_contains_benchmark)->RangeMultiplier(2)->Range(64, 1024); +BENCHMARK(bulk_count_benchmark)->RangeMultiplier(2)->Range(64, 1024); + +// This is a hack to add custom context information to the benchmark output. +// The alternative would be to do it in the main(). However, this would require +// not using the BENCHMARK_MAIN macro. +[[maybe_unused]] static bool foo = []() +{ + benchmark::AddCustomContext("IBF size in bytes", std::to_string(total_ibf_size_in_bytes)); + benchmark::AddCustomContext("Number of hash functions", std::to_string(number_of_hash_functions)); + benchmark::AddCustomContext("False positive rate", std::to_string(false_positive_rate)); + benchmark::AddCustomContext("Number of elements", std::to_string(number_of_elements)); + benchmark::AddCustomContext("HIBF_HAS_AVX512", HIBF_HAS_AVX512 ? "true" : "false"); + benchmark::AddCustomContext("AVX512 support", +#if __AVX512F__ && __AVX512BW__ + "true"); +#else + "false"); +#endif + return true; +}(); BENCHMARK_MAIN();