From 8b316b287c5633059f282e0dfd765754f7cdcf60 Mon Sep 17 00:00:00 2001 From: Peng Liu <winner245@hotmail.com> Date: Mon, 23 Dec 2024 12:29:59 -0500 Subject: [PATCH] Optimize ranges::copy_backward for vector<bool>::iterator --- libcxx/docs/ReleaseNotes/20.rst | 3 + libcxx/include/__algorithm/copy_backward.h | 131 +++++++++++++++++ libcxx/include/__bit_reference | 136 +----------------- libcxx/include/__vector/vector_bool.h | 1 + libcxx/include/bitset | 2 + .../algorithms/copy_backward.bench.cpp | 55 +++++++ .../alg.copy/copy_backward.pass.cpp | 88 +++++++----- .../alg.copy/ranges.copy_backward.pass.cpp | 104 +++++++++----- 8 files changed, 322 insertions(+), 198 deletions(-) create mode 100644 libcxx/test/benchmarks/algorithms/copy_backward.bench.cpp diff --git a/libcxx/docs/ReleaseNotes/20.rst b/libcxx/docs/ReleaseNotes/20.rst index 2736061544c53..e43a69f0f6ebc 100644 --- a/libcxx/docs/ReleaseNotes/20.rst +++ b/libcxx/docs/ReleaseNotes/20.rst @@ -111,6 +111,9 @@ Improvements and New Features std::errc::not_a_directory``, or use ``err.default_error_condition()`` to map to an ``error_condition``, and then test its ``value()`` and ``category()``. +- The ``std::ranges::copy_backward`` algorithm has been optimized for ``std::vector<bool>::iterator``\s, resulting in + a performance improvement of up to 2000x. + Deprecations and Removals ------------------------- diff --git a/libcxx/include/__algorithm/copy_backward.h b/libcxx/include/__algorithm/copy_backward.h index 48a768f577f55..02ffc14361e6a 100644 --- a/libcxx/include/__algorithm/copy_backward.h +++ b/libcxx/include/__algorithm/copy_backward.h @@ -10,11 +10,14 @@ #define _LIBCPP___ALGORITHM_COPY_BACKWARD_H #include <__algorithm/copy_move_common.h> +#include <__algorithm/copy_n.h> #include <__algorithm/iterator_operations.h> #include <__algorithm/min.h> #include <__config> +#include <__fwd/bit_reference.h> #include <__iterator/iterator_traits.h> #include <__iterator/segmented_iterator.h> +#include <__memory/pointer_traits.h> #include <__type_traits/common_type.h> #include <__type_traits/enable_if.h> #include <__type_traits/is_constructible.h> @@ -34,6 +37,124 @@ template <class _AlgPolicy, class _InIter, class _Sent, class _OutIter> _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_InIter, _OutIter> __copy_backward(_InIter __first, _Sent __last, _OutIter __result); +template <class _Cp, bool _IsConst> +_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __bit_iterator<_Cp, false> __copy_backward_aligned( + __bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __last, __bit_iterator<_Cp, false> __result) { + using _In = __bit_iterator<_Cp, _IsConst>; + using difference_type = typename _In::difference_type; + using __storage_type = typename _In::__storage_type; + + const int __bits_per_word = _In::__bits_per_word; + difference_type __n = __last - __first; + if (__n > 0) { + // do first word + if (__last.__ctz_ != 0) { + difference_type __dn = std::min(static_cast<difference_type>(__last.__ctz_), __n); + __n -= __dn; + unsigned __clz = __bits_per_word - __last.__ctz_; + __storage_type __m = (~__storage_type(0) << (__last.__ctz_ - __dn)) & (~__storage_type(0) >> __clz); + __storage_type __b = *__last.__seg_ & __m; + *__result.__seg_ &= ~__m; + *__result.__seg_ |= __b; + __result.__ctz_ = static_cast<unsigned>(((-__dn & (__bits_per_word - 1)) + __result.__ctz_) % __bits_per_word); + // __last.__ctz_ = 0 + } + // __last.__ctz_ == 0 || __n == 0 + // __result.__ctz_ == 0 || __n == 0 + // do middle words + __storage_type __nw = __n / __bits_per_word; + __result.__seg_ -= __nw; + __last.__seg_ -= __nw; + std::copy_n(std::__to_address(__last.__seg_), __nw, std::__to_address(__result.__seg_)); + __n -= __nw * __bits_per_word; + // do last word + if (__n > 0) { + __storage_type __m = ~__storage_type(0) << (__bits_per_word - __n); + __storage_type __b = *--__last.__seg_ & __m; + *--__result.__seg_ &= ~__m; + *__result.__seg_ |= __b; + __result.__ctz_ = static_cast<unsigned>(-__n & (__bits_per_word - 1)); + } + } + return __result; +} + +template <class _Cp, bool _IsConst> +_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __bit_iterator<_Cp, false> __copy_backward_unaligned( + __bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __last, __bit_iterator<_Cp, false> __result) { + using _In = __bit_iterator<_Cp, _IsConst>; + using difference_type = typename _In::difference_type; + using __storage_type = typename _In::__storage_type; + + const int __bits_per_word = _In::__bits_per_word; + difference_type __n = __last - __first; + if (__n > 0) { + // do first word + if (__last.__ctz_ != 0) { + difference_type __dn = std::min(static_cast<difference_type>(__last.__ctz_), __n); + __n -= __dn; + unsigned __clz_l = __bits_per_word - __last.__ctz_; + __storage_type __m = (~__storage_type(0) << (__last.__ctz_ - __dn)) & (~__storage_type(0) >> __clz_l); + __storage_type __b = *__last.__seg_ & __m; + unsigned __clz_r = __bits_per_word - __result.__ctz_; + __storage_type __ddn = std::min(__dn, static_cast<difference_type>(__result.__ctz_)); + if (__ddn > 0) { + __m = (~__storage_type(0) << (__result.__ctz_ - __ddn)) & (~__storage_type(0) >> __clz_r); + *__result.__seg_ &= ~__m; + if (__result.__ctz_ > __last.__ctz_) + *__result.__seg_ |= __b << (__result.__ctz_ - __last.__ctz_); + else + *__result.__seg_ |= __b >> (__last.__ctz_ - __result.__ctz_); + __result.__ctz_ = static_cast<unsigned>(((-__ddn & (__bits_per_word - 1)) + __result.__ctz_) % __bits_per_word); + __dn -= __ddn; + } + if (__dn > 0) { + // __result.__ctz_ == 0 + --__result.__seg_; + __result.__ctz_ = static_cast<unsigned>(-__dn & (__bits_per_word - 1)); + __m = ~__storage_type(0) << __result.__ctz_; + *__result.__seg_ &= ~__m; + __last.__ctz_ -= __dn + __ddn; + *__result.__seg_ |= __b << (__result.__ctz_ - __last.__ctz_); + } + // __last.__ctz_ = 0 + } + // __last.__ctz_ == 0 || __n == 0 + // __result.__ctz_ != 0 || __n == 0 + // do middle words + unsigned __clz_r = __bits_per_word - __result.__ctz_; + __storage_type __m = ~__storage_type(0) >> __clz_r; + for (; __n >= __bits_per_word; __n -= __bits_per_word) { + __storage_type __b = *--__last.__seg_; + *__result.__seg_ &= ~__m; + *__result.__seg_ |= __b >> __clz_r; + *--__result.__seg_ &= __m; + *__result.__seg_ |= __b << __result.__ctz_; + } + // do last word + if (__n > 0) { + __m = ~__storage_type(0) << (__bits_per_word - __n); + __storage_type __b = *--__last.__seg_ & __m; + __clz_r = __bits_per_word - __result.__ctz_; + __storage_type __dn = std::min(__n, static_cast<difference_type>(__result.__ctz_)); + __m = (~__storage_type(0) << (__result.__ctz_ - __dn)) & (~__storage_type(0) >> __clz_r); + *__result.__seg_ &= ~__m; + *__result.__seg_ |= __b >> (__bits_per_word - __result.__ctz_); + __result.__ctz_ = static_cast<unsigned>(((-__dn & (__bits_per_word - 1)) + __result.__ctz_) % __bits_per_word); + __n -= __dn; + if (__n > 0) { + // __result.__ctz_ == 0 + --__result.__seg_; + __result.__ctz_ = static_cast<unsigned>(-__n & (__bits_per_word - 1)); + __m = ~__storage_type(0) << __result.__ctz_; + *__result.__seg_ &= ~__m; + *__result.__seg_ |= __b << (__result.__ctz_ - (__bits_per_word - __n - __dn)); + } + } + } + return __result; +} + template <class _AlgPolicy> struct __copy_backward_impl { template <class _InIter, class _Sent, class _OutIter> @@ -107,6 +228,16 @@ struct __copy_backward_impl { } } + template <class _Cp, bool _IsConst> + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<__bit_iterator<_Cp, _IsConst>, __bit_iterator<_Cp, false> > + operator()(__bit_iterator<_Cp, _IsConst> __first, + __bit_iterator<_Cp, _IsConst> __last, + __bit_iterator<_Cp, false> __result) { + if (__last.__ctz_ == __result.__ctz_) + return std::make_pair(__last, std::__copy_backward_aligned(__first, __last, __result)); + return std::make_pair(__last, std::__copy_backward_unaligned(__first, __last, __result)); + } + // At this point, the iterators have been unwrapped so any `contiguous_iterator` has been unwrapped to a pointer. template <class _In, class _Out, __enable_if_t<__can_lower_copy_assignment_to_memmove<_In, _Out>::value, int> = 0> _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_In*, _Out*> diff --git a/libcxx/include/__bit_reference b/libcxx/include/__bit_reference index 67abb023122ed..07cc777b8bfb6 100644 --- a/libcxx/include/__bit_reference +++ b/libcxx/include/__bit_reference @@ -10,6 +10,7 @@ #ifndef _LIBCPP___BIT_REFERENCE #define _LIBCPP___BIT_REFERENCE +#include <__algorithm/copy_backward.h> #include <__algorithm/copy_n.h> #include <__algorithm/min.h> #include <__bit/countr.h> @@ -307,134 +308,6 @@ copy(__bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __last return std::__copy_unaligned(__first, __last, __result); } -// copy_backward - -template <class _Cp, bool _IsConst> -_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __bit_iterator<_Cp, false> __copy_backward_aligned( - __bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __last, __bit_iterator<_Cp, false> __result) { - using _In = __bit_iterator<_Cp, _IsConst>; - using difference_type = typename _In::difference_type; - using __storage_type = typename _In::__storage_type; - - const int __bits_per_word = _In::__bits_per_word; - difference_type __n = __last - __first; - if (__n > 0) { - // do first word - if (__last.__ctz_ != 0) { - difference_type __dn = std::min(static_cast<difference_type>(__last.__ctz_), __n); - __n -= __dn; - unsigned __clz = __bits_per_word - __last.__ctz_; - __storage_type __m = (~__storage_type(0) << (__last.__ctz_ - __dn)) & (~__storage_type(0) >> __clz); - __storage_type __b = *__last.__seg_ & __m; - *__result.__seg_ &= ~__m; - *__result.__seg_ |= __b; - __result.__ctz_ = static_cast<unsigned>(((-__dn & (__bits_per_word - 1)) + __result.__ctz_) % __bits_per_word); - // __last.__ctz_ = 0 - } - // __last.__ctz_ == 0 || __n == 0 - // __result.__ctz_ == 0 || __n == 0 - // do middle words - __storage_type __nw = __n / __bits_per_word; - __result.__seg_ -= __nw; - __last.__seg_ -= __nw; - std::copy_n(std::__to_address(__last.__seg_), __nw, std::__to_address(__result.__seg_)); - __n -= __nw * __bits_per_word; - // do last word - if (__n > 0) { - __storage_type __m = ~__storage_type(0) << (__bits_per_word - __n); - __storage_type __b = *--__last.__seg_ & __m; - *--__result.__seg_ &= ~__m; - *__result.__seg_ |= __b; - __result.__ctz_ = static_cast<unsigned>(-__n & (__bits_per_word - 1)); - } - } - return __result; -} - -template <class _Cp, bool _IsConst> -_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __bit_iterator<_Cp, false> __copy_backward_unaligned( - __bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __last, __bit_iterator<_Cp, false> __result) { - using _In = __bit_iterator<_Cp, _IsConst>; - using difference_type = typename _In::difference_type; - using __storage_type = typename _In::__storage_type; - - const int __bits_per_word = _In::__bits_per_word; - difference_type __n = __last - __first; - if (__n > 0) { - // do first word - if (__last.__ctz_ != 0) { - difference_type __dn = std::min(static_cast<difference_type>(__last.__ctz_), __n); - __n -= __dn; - unsigned __clz_l = __bits_per_word - __last.__ctz_; - __storage_type __m = (~__storage_type(0) << (__last.__ctz_ - __dn)) & (~__storage_type(0) >> __clz_l); - __storage_type __b = *__last.__seg_ & __m; - unsigned __clz_r = __bits_per_word - __result.__ctz_; - __storage_type __ddn = std::min(__dn, static_cast<difference_type>(__result.__ctz_)); - if (__ddn > 0) { - __m = (~__storage_type(0) << (__result.__ctz_ - __ddn)) & (~__storage_type(0) >> __clz_r); - *__result.__seg_ &= ~__m; - if (__result.__ctz_ > __last.__ctz_) - *__result.__seg_ |= __b << (__result.__ctz_ - __last.__ctz_); - else - *__result.__seg_ |= __b >> (__last.__ctz_ - __result.__ctz_); - __result.__ctz_ = static_cast<unsigned>(((-__ddn & (__bits_per_word - 1)) + __result.__ctz_) % __bits_per_word); - __dn -= __ddn; - } - if (__dn > 0) { - // __result.__ctz_ == 0 - --__result.__seg_; - __result.__ctz_ = static_cast<unsigned>(-__dn & (__bits_per_word - 1)); - __m = ~__storage_type(0) << __result.__ctz_; - *__result.__seg_ &= ~__m; - __last.__ctz_ -= __dn + __ddn; - *__result.__seg_ |= __b << (__result.__ctz_ - __last.__ctz_); - } - // __last.__ctz_ = 0 - } - // __last.__ctz_ == 0 || __n == 0 - // __result.__ctz_ != 0 || __n == 0 - // do middle words - unsigned __clz_r = __bits_per_word - __result.__ctz_; - __storage_type __m = ~__storage_type(0) >> __clz_r; - for (; __n >= __bits_per_word; __n -= __bits_per_word) { - __storage_type __b = *--__last.__seg_; - *__result.__seg_ &= ~__m; - *__result.__seg_ |= __b >> __clz_r; - *--__result.__seg_ &= __m; - *__result.__seg_ |= __b << __result.__ctz_; - } - // do last word - if (__n > 0) { - __m = ~__storage_type(0) << (__bits_per_word - __n); - __storage_type __b = *--__last.__seg_ & __m; - __clz_r = __bits_per_word - __result.__ctz_; - __storage_type __dn = std::min(__n, static_cast<difference_type>(__result.__ctz_)); - __m = (~__storage_type(0) << (__result.__ctz_ - __dn)) & (~__storage_type(0) >> __clz_r); - *__result.__seg_ &= ~__m; - *__result.__seg_ |= __b >> (__bits_per_word - __result.__ctz_); - __result.__ctz_ = static_cast<unsigned>(((-__dn & (__bits_per_word - 1)) + __result.__ctz_) % __bits_per_word); - __n -= __dn; - if (__n > 0) { - // __result.__ctz_ == 0 - --__result.__seg_; - __result.__ctz_ = static_cast<unsigned>(-__n & (__bits_per_word - 1)); - __m = ~__storage_type(0) << __result.__ctz_; - *__result.__seg_ &= ~__m; - *__result.__seg_ |= __b << (__result.__ctz_ - (__bits_per_word - __n - __dn)); - } - } - } - return __result; -} - -template <class _Cp, bool _IsConst> -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __bit_iterator<_Cp, false> copy_backward( - __bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __last, __bit_iterator<_Cp, false> __result) { - if (__last.__ctz_ == __result.__ctz_) - return std::__copy_backward_aligned(__first, __last, __result); - return std::__copy_backward_unaligned(__first, __last, __result); -} - // move template <class _Cp, bool _IsConst> @@ -997,9 +870,10 @@ private: template <class _Dp, bool _IC> _LIBCPP_CONSTEXPR_SINCE_CXX20 friend __bit_iterator<_Dp, false> __copy_backward_unaligned( __bit_iterator<_Dp, _IC> __first, __bit_iterator<_Dp, _IC> __last, __bit_iterator<_Dp, false> __result); - template <class _Dp, bool _IC> - _LIBCPP_CONSTEXPR_SINCE_CXX20 friend __bit_iterator<_Dp, false> - copy_backward(__bit_iterator<_Dp, _IC> __first, __bit_iterator<_Dp, _IC> __last, __bit_iterator<_Dp, false> __result); + // Note: dependent nested name specifier __copy_backward_impl<_AlgPolicy>::operator() for friend declaration + // is not supported in clang. Thus, we use a friend declaration for the entire class. + template <class _AlgPolicy> + friend struct __copy_backward_impl; template <class _Cl, class _Cr> friend __bit_iterator<_Cr, false> __swap_ranges_aligned(__bit_iterator<_Cl, false>, __bit_iterator<_Cl, false>, __bit_iterator<_Cr, false>); diff --git a/libcxx/include/__vector/vector_bool.h b/libcxx/include/__vector/vector_bool.h index 4f1c442ce0be8..d9b16f2c9489c 100644 --- a/libcxx/include/__vector/vector_bool.h +++ b/libcxx/include/__vector/vector_bool.h @@ -10,6 +10,7 @@ #define _LIBCPP___VECTOR_VECTOR_BOOL_H #include <__algorithm/copy.h> +#include <__algorithm/copy_backward.h> #include <__algorithm/fill_n.h> #include <__algorithm/iterator_operations.h> #include <__algorithm/max.h> diff --git a/libcxx/include/bitset b/libcxx/include/bitset index 10576eb80bf2e..a20842985b3d5 100644 --- a/libcxx/include/bitset +++ b/libcxx/include/bitset @@ -129,6 +129,8 @@ template <size_t N> struct hash<std::bitset<N>>; #if __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS) # include <__cxx03/bitset> #else +# include <__algorithm/copy.h> +# include <__algorithm/copy_backward.h> # include <__algorithm/count.h> # include <__algorithm/fill.h> # include <__algorithm/fill_n.h> diff --git a/libcxx/test/benchmarks/algorithms/copy_backward.bench.cpp b/libcxx/test/benchmarks/algorithms/copy_backward.bench.cpp new file mode 100644 index 0000000000000..c943d9a874b49 --- /dev/null +++ b/libcxx/test/benchmarks/algorithms/copy_backward.bench.cpp @@ -0,0 +1,55 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20 + +#include <algorithm> +#include <benchmark/benchmark.h> +#include <vector> + +static void bm_ranges_copy_backward_vb(benchmark::State& state, bool aligned) { + auto n = state.range(); + std::vector<bool> in(n, true); + std::vector<bool> out(aligned ? n : n + 8); + benchmark::DoNotOptimize(&in); + auto dst = aligned ? out.end() : out.end() - 4; + for (auto _ : state) { + benchmark::DoNotOptimize(std::ranges::copy_backward(in, dst)); + benchmark::DoNotOptimize(&out); + } +} + +static void bm_copy_backward_vb(benchmark::State& state, bool aligned) { + auto n = state.range(); + std::vector<bool> in(n, true); + std::vector<bool> out(aligned ? n : n + 8); + benchmark::DoNotOptimize(&in); + auto beg = in.begin(); + auto end = in.end(); + auto dst = aligned ? out.end() : out.end() - 4; + for (auto _ : state) { + benchmark::DoNotOptimize(std::copy_backward(beg, end, dst)); + benchmark::DoNotOptimize(&out); + } +} + +static void bm_ranges_copy_backward_vb_aligned(benchmark::State& state) { bm_ranges_copy_backward_vb(state, true); } +static void bm_ranges_copy_backward_vb_unaligned(benchmark::State& state) { bm_ranges_copy_backward_vb(state, false); } + +static void bm_copy_backward_vb_aligned(benchmark::State& state) { bm_copy_backward_vb(state, true); } +static void bm_copy_backward_vb_unaligned(benchmark::State& state) { bm_copy_backward_vb(state, false); } + +// Test std::ranges::copy_backward for vector<bool>::iterator +BENCHMARK(bm_ranges_copy_backward_vb_aligned)->Range(8, 1 << 16)->DenseRange(102400, 204800, 4096); +BENCHMARK(bm_ranges_copy_backward_vb_unaligned)->Range(8, 1 << 20); + +// Test std::copy_backward for vector<bool>::iterator +BENCHMARK(bm_copy_backward_vb_aligned)->Range(8, 1 << 20); +BENCHMARK(bm_copy_backward_vb_unaligned)->Range(8, 1 << 20); + +BENCHMARK_MAIN(); diff --git a/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/copy_backward.pass.cpp b/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/copy_backward.pass.cpp index 928903de1ade2..6924f2f85adb5 100644 --- a/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/copy_backward.pass.cpp +++ b/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/copy_backward.pass.cpp @@ -15,6 +15,7 @@ #include <algorithm> #include <cassert> +#include <vector> #include "test_macros.h" #include "test_iterators.h" @@ -36,47 +37,63 @@ class Derived : public PaddedBase { }; template <class InIter, class OutIter> -TEST_CONSTEXPR_CXX20 void -test_copy_backward() -{ +TEST_CONSTEXPR_CXX20 void test_copy_backward() { { const unsigned N = 1000; - int ia[N] = {}; + int ia[N] = {}; for (unsigned i = 0; i < N; ++i) - ia[i] = i; + ia[i] = i; int ib[N] = {0}; - OutIter r = std::copy_backward(InIter(ia), InIter(ia+N), OutIter(ib+N)); + OutIter r = std::copy_backward(InIter(ia), InIter(ia + N), OutIter(ib + N)); assert(base(r) == ib); for (unsigned i = 0; i < N; ++i) - assert(ia[i] == ib[i]); + assert(ia[i] == ib[i]); } } -TEST_CONSTEXPR_CXX20 bool -test() -{ - test_copy_backward<bidirectional_iterator<const int*>, bidirectional_iterator<int*> >(); - test_copy_backward<bidirectional_iterator<const int*>, random_access_iterator<int*> >(); - test_copy_backward<bidirectional_iterator<const int*>, int*>(); +TEST_CONSTEXPR_CXX20 bool test_vector_bool(std::size_t N) { + std::vector<bool> in(N, false); + for (std::size_t i = 0; i < N; i += 2) + in[i] = true; - test_copy_backward<random_access_iterator<const int*>, bidirectional_iterator<int*> >(); - test_copy_backward<random_access_iterator<const int*>, random_access_iterator<int*> >(); - test_copy_backward<random_access_iterator<const int*>, int*>(); + { // Test copy_backward with aligned bytes + std::vector<bool> out(N); + std::copy_backward(in.begin(), in.end(), out.end()); + assert(in == out); + } + { // Test copy_backward with unaligned bytes + std::vector<bool> out(N + 8); + std::copy_backward(in.begin(), in.end(), out.end() - 4); + for (std::size_t i = 0; i < N; ++i) + assert(out[i + 4] == in[i]); + } + + return true; +}; - test_copy_backward<const int*, bidirectional_iterator<int*> >(); - test_copy_backward<const int*, random_access_iterator<int*> >(); - test_copy_backward<const int*, int*>(); +TEST_CONSTEXPR_CXX20 bool test() { + test_copy_backward<bidirectional_iterator<const int*>, bidirectional_iterator<int*> >(); + test_copy_backward<bidirectional_iterator<const int*>, random_access_iterator<int*> >(); + test_copy_backward<bidirectional_iterator<const int*>, int*>(); + + test_copy_backward<random_access_iterator<const int*>, bidirectional_iterator<int*> >(); + test_copy_backward<random_access_iterator<const int*>, random_access_iterator<int*> >(); + test_copy_backward<random_access_iterator<const int*>, int*>(); + + test_copy_backward<const int*, bidirectional_iterator<int*> >(); + test_copy_backward<const int*, random_access_iterator<int*> >(); + test_copy_backward<const int*, int*>(); #if TEST_STD_VER > 17 - test_copy_backward<contiguous_iterator<const int*>, bidirectional_iterator<int*>>(); - test_copy_backward<contiguous_iterator<const int*>, random_access_iterator<int*>>(); - test_copy_backward<contiguous_iterator<const int*>, int*>(); - - test_copy_backward<bidirectional_iterator<const int*>, contiguous_iterator<int*>>(); - test_copy_backward<random_access_iterator<const int*>, contiguous_iterator<int*>>(); - test_copy_backward<contiguous_iterator<const int*>, contiguous_iterator<int*>>(); - test_copy_backward<const int*, contiguous_iterator<int*>>(); + test_copy_backward<contiguous_iterator<const int*>, bidirectional_iterator<int*>>(); + test_copy_backward<contiguous_iterator<const int*>, random_access_iterator<int*>>(); + test_copy_backward<contiguous_iterator<const int*>, int*>(); + + test_copy_backward<bidirectional_iterator<const int*>, contiguous_iterator<int*>>(); + test_copy_backward<random_access_iterator<const int*>, contiguous_iterator<int*>>(); + test_copy_backward<contiguous_iterator<const int*>, contiguous_iterator<int*>>(); + test_copy_backward<const int*, contiguous_iterator<int*>>(); #endif { // Make sure that padding bits aren't copied @@ -96,15 +113,22 @@ test() assert(std::equal(a, a + 10, expected)); } - return true; + { // Test vector<bool>::iterator optimization + assert(test_vector_bool(8)); + assert(test_vector_bool(16)); + assert(test_vector_bool(32)); + assert(test_vector_bool(64)); + assert(test_vector_bool(256)); + } + + return true; } -int main(int, char**) -{ - test(); +int main(int, char**) { + test(); #if TEST_STD_VER > 17 - static_assert(test()); + static_assert(test()); #endif return 0; diff --git a/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/ranges.copy_backward.pass.cpp b/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/ranges.copy_backward.pass.cpp index 343447446ab2d..574d25676666c 100644 --- a/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/ranges.copy_backward.pass.cpp +++ b/libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/ranges.copy_backward.pass.cpp @@ -29,6 +29,7 @@ #include "almost_satisfies_types.h" #include "test_iterators.h" +#include "test_macros.h" template <class In, class Out = In, class Sent = sentinel_wrapper<In>> concept HasCopyBackwardIt = requires(In in, Sent sent, Out out) { std::ranges::copy_backward(in, sent, out); }; @@ -61,16 +62,16 @@ template <class In, class Out, class Sent> constexpr void test_iterators() { { // simple test { - std::array in {1, 2, 3, 4}; + std::array in{1, 2, 3, 4}; std::array<int, 4> out; std::same_as<std::ranges::in_out_result<In, Out>> auto ret = - std::ranges::copy_backward(In(in.data()), Sent(In(in.data() + in.size())), Out(out.data() + out.size())); + std::ranges::copy_backward(In(in.data()), Sent(In(in.data() + in.size())), Out(out.data() + out.size())); assert(in == out); assert(base(ret.in) == in.data() + in.size()); assert(base(ret.out) == out.data()); } { - std::array in {1, 2, 3, 4}; + std::array in{1, 2, 3, 4}; std::array<int, 4> out; auto range = std::ranges::subrange(In(in.data()), Sent(In(in.data() + in.size()))); std::same_as<std::ranges::in_out_result<In, Out>> auto ret = @@ -94,7 +95,7 @@ constexpr void test_iterators() { std::array<int, 0> in; std::array<int, 0> out; auto range = std::ranges::subrange(In(in.data()), Sent(In(in.data() + in.size()))); - auto ret = std::ranges::copy_backward(range, Out(out.data())); + auto ret = std::ranges::copy_backward(range, Out(out.data())); assert(base(ret.in) == in.data() + in.size()); assert(base(ret.out) == out.data()); } @@ -104,16 +105,16 @@ constexpr void test_iterators() { template <class InContainer, class OutContainer, class In, class Out, class Sent = In> constexpr void test_containers() { { - InContainer in {1, 2, 3, 4}; + InContainer in{1, 2, 3, 4}; OutContainer out(4); std::same_as<std::ranges::in_out_result<In, Out>> auto ret = - std::ranges::copy_backward(In(in.begin()), Sent(In(in.end())), Out(out.end())); + std::ranges::copy_backward(In(in.begin()), Sent(In(in.end())), Out(out.end())); assert(std::ranges::equal(in, out)); assert(base(ret.in) == in.end()); assert(base(ret.out) == out.begin()); } { - InContainer in {1, 2, 3, 4}; + InContainer in{1, 2, 3, 4}; OutContainer out(4); auto range = std::ranges::subrange(In(in.begin()), Sent(In(in.end()))); std::same_as<std::ranges::in_out_result<In, Out>> auto ret = std::ranges::copy_backward(range, Out(out.end())); @@ -125,13 +126,12 @@ constexpr void test_containers() { template <class Iter, class Sent> constexpr void test_join_view() { - auto to_subranges = std::views::transform([](auto& vec) { - return std::ranges::subrange(Iter(vec.begin()), Sent(Iter(vec.end()))); - }); + auto to_subranges = + std::views::transform([](auto& vec) { return std::ranges::subrange(Iter(vec.begin()), Sent(Iter(vec.end()))); }); { // segmented -> contiguous std::vector<std::vector<int>> vectors = {}; - auto range = vectors | to_subranges; + auto range = vectors | to_subranges; std::vector<std::ranges::subrange<Iter, Sent>> subrange_vector(range.begin(), range.end()); std::array<int, 0> arr; @@ -140,7 +140,7 @@ constexpr void test_join_view() { } { // segmented -> contiguous std::vector<std::vector<int>> vectors = {{1, 2, 3, 4}, {5, 6, 7, 8}, {9, 10}, {}}; - auto range = vectors | to_subranges; + auto range = vectors | to_subranges; std::vector<std::ranges::subrange<Iter, Sent>> subrange_vector(range.begin(), range.end()); std::array<int, 10> arr; @@ -149,7 +149,7 @@ constexpr void test_join_view() { } { // contiguous -> segmented std::vector<std::vector<int>> vectors = {{0, 0, 0, 0}, {0, 0}, {0, 0, 0, 0}, {}}; - auto range = vectors | to_subranges; + auto range = vectors | to_subranges; std::vector<std::ranges::subrange<Iter, Sent>> subrange_vector(range.begin(), range.end()); std::array arr = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; @@ -158,10 +158,10 @@ constexpr void test_join_view() { } { // segmented -> segmented std::vector<std::vector<int>> vectors = {{1, 2, 3, 4}, {5, 6, 7, 8}, {9, 10}, {}}; - auto range1 = vectors | to_subranges; + auto range1 = vectors | to_subranges; std::vector<std::ranges::subrange<Iter, Sent>> subrange_vector(range1.begin(), range1.end()); std::vector<std::vector<int>> to_vectors = {{0, 0, 0, 0}, {0, 0, 0, 0}, {}, {0, 0}}; - auto range2 = to_vectors | to_subranges; + auto range2 = to_vectors | to_subranges; std::vector<std::ranges::subrange<Iter, Sent>> to_subrange_vector(range2.begin(), range2.end()); std::ranges::copy_backward(subrange_vector | std::views::join, (to_subrange_vector | std::views::join).end()); @@ -224,6 +224,30 @@ constexpr void test_proxy_in_iterators() { test_sentinels<ProxyIterator, Out>(); } +#if TEST_STD_VER >= 23 + +constexpr bool test_vector_bool(std::size_t N) { + std::vector<bool> in(N, false); + for (std::size_t i = 0; i < N; i += 2) + in[i] = true; + + { // Test copy_backward with aligned bytes + std::vector<bool> out(N); + std::ranges::copy_backward(in, out.end()); + assert(in == out); + } + { // Test copy_backward with unaligned bytes + std::vector<bool> out(N + 8); + std::ranges::copy_backward(in, out.end() - 4); + for (std::size_t i = 0; i < N; ++i) + assert(out[i + 4] == in[i]); + } + + return true; +}; + +#endif + constexpr bool test() { test_in_iterators<bidirectional_iterator>(); test_in_iterators<random_access_iterator>(); @@ -237,13 +261,13 @@ constexpr bool test() { { // check that ranges::dangling is returned std::array<int, 4> out; std::same_as<std::ranges::in_out_result<std::ranges::dangling, int*>> auto ret = - std::ranges::copy_backward(std::array {1, 2, 3, 4}, out.data() + out.size()); + std::ranges::copy_backward(std::array{1, 2, 3, 4}, out.data() + out.size()); assert(ret.out == out.data()); assert((out == std::array{1, 2, 3, 4})); } { // check that an iterator is returned with a borrowing range - std::array in {1, 2, 3, 4}; + std::array in{1, 2, 3, 4}; std::array<int, 4> out; std::same_as<std::ranges::in_out_result<std::array<int, 4>::iterator, int*>> auto ret = std::ranges::copy_backward(std::views::all(in), out.data() + out.size()); @@ -254,8 +278,8 @@ constexpr bool test() { { // check that every element is copied exactly once struct CopyOnce { - bool copied = false; - constexpr CopyOnce() = default; + bool copied = false; + constexpr CopyOnce() = default; constexpr CopyOnce(const CopyOnce& other) = delete; constexpr CopyOnce& operator=(const CopyOnce& other) { assert(!other.copied); @@ -264,16 +288,16 @@ constexpr bool test() { } }; { - std::array<CopyOnce, 4> in {}; - std::array<CopyOnce, 4> out {}; + std::array<CopyOnce, 4> in{}; + std::array<CopyOnce, 4> out{}; auto ret = std::ranges::copy_backward(in.begin(), in.end(), out.end()); assert(ret.in == in.end()); assert(ret.out == out.begin()); assert(std::all_of(out.begin(), out.end(), [](const auto& e) { return e.copied; })); } { - std::array<CopyOnce, 4> in {}; - std::array<CopyOnce, 4> out {}; + std::array<CopyOnce, 4> in{}; + std::array<CopyOnce, 4> out{}; auto ret = std::ranges::copy_backward(in, out.end()); assert(ret.in == in.end()); assert(ret.out == out.begin()); @@ -284,8 +308,8 @@ constexpr bool test() { { // check that the range is copied backwards struct OnlyBackwardsCopyable { OnlyBackwardsCopyable* next = nullptr; - bool canCopy = false; - OnlyBackwardsCopyable() = default; + bool canCopy = false; + OnlyBackwardsCopyable() = default; constexpr OnlyBackwardsCopyable& operator=(const OnlyBackwardsCopyable&) { assert(canCopy); if (next != nullptr) @@ -294,12 +318,12 @@ constexpr bool test() { } }; { - std::array<OnlyBackwardsCopyable, 3> in {}; - std::array<OnlyBackwardsCopyable, 3> out {}; - out[1].next = &out[0]; - out[2].next = &out[1]; + std::array<OnlyBackwardsCopyable, 3> in{}; + std::array<OnlyBackwardsCopyable, 3> out{}; + out[1].next = &out[0]; + out[2].next = &out[1]; out[2].canCopy = true; - auto ret = std::ranges::copy_backward(in, out.end()); + auto ret = std::ranges::copy_backward(in, out.end()); assert(ret.in == in.end()); assert(ret.out == out.begin()); assert(out[0].canCopy); @@ -307,12 +331,12 @@ constexpr bool test() { assert(out[2].canCopy); } { - std::array<OnlyBackwardsCopyable, 3> in {}; - std::array<OnlyBackwardsCopyable, 3> out {}; - out[1].next = &out[0]; - out[2].next = &out[1]; + std::array<OnlyBackwardsCopyable, 3> in{}; + std::array<OnlyBackwardsCopyable, 3> out{}; + out[1].next = &out[0]; + out[2].next = &out[1]; out[2].canCopy = true; - auto ret = std::ranges::copy_backward(in.begin(), in.end(), out.end()); + auto ret = std::ranges::copy_backward(in.begin(), in.end(), out.end()); assert(ret.in == in.end()); assert(ret.out == out.begin()); assert(out[0].canCopy); @@ -321,6 +345,16 @@ constexpr bool test() { } } +#if TEST_STD_VER >= 23 + { // Test vector<bool>::iterator optimization + assert(test_vector_bool(8)); + assert(test_vector_bool(16)); + assert(test_vector_bool(32)); + assert(test_vector_bool(64)); + assert(test_vector_bool(256)); + } +#endif + return true; }