[libc++] Optimize std::find_if

Author: philnik777

libcxx/include/CMakeLists.txt

@@ -604,6 +604,7 @@ set(files
   __memory/unique_temporary_buffer.h
   __memory/uses_allocator.h
   __memory/uses_allocator_construction.h
+  __memory/valid_range.h
   __memory_resource/memory_resource.h
   __memory_resource/monotonic_buffer_resource.h
   __memory_resource/polymorphic_allocator.h
@@ -927,7 +928,6 @@ set(files
   __utility/in_place.h
   __utility/integer_sequence.h
   __utility/is_pointer_in_range.h
-  __utility/is_valid_range.h
   __utility/lazy_synth_three_way_comparator.h
   __utility/move.h
   __utility/no_destroy.h

libcxx/include/__algorithm/find_if.h

@@ -11,6 +11,7 @@
 #define _LIBCPP___ALGORITHM_FIND_IF_H
 
 #include <__config>
+#include <__memory/valid_range.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -21,6 +22,8 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 template <class _InputIterator, class _Predicate>
 [[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _InputIterator
 find_if(_InputIterator __first, _InputIterator __last, _Predicate __pred) {
+  std::__assume_valid_range(__first, __last);
+
   for (; __first != __last; ++__first)
     if (__pred(*__first))
       break;

libcxx/include/__utility/is_valid_range.h renamed to libcxx/include/__memory/valid_range.h

@@ -6,11 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___UTILITY_IS_VALID_RANGE_H
-#define _LIBCPP___UTILITY_IS_VALID_RANGE_H
+#ifndef _LIBCPP___MEMORY_VALID_RANGE_H
+#define _LIBCPP___MEMORY_VALID_RANGE_H
 
 #include <__algorithm/comp.h>
+#include <__assert>
 #include <__config>
+#include <__iterator/iterator_traits.h>
+#include <__memory/assume_aligned.h>
+#include <__memory/pointer_traits.h>
 #include <__type_traits/is_constant_evaluated.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
@@ -32,6 +36,20 @@ __is_valid_range(const _Tp* __first, const _Tp* __last) {
   return !__less<>()(__last, __first);
 }
 
+template <class _Iter, class _Sent>
+_LIBCPP_HIDE_FROM_ABI void __assume_valid_range(_Iter&& __first, _Sent&& __last) {
+#if __has_builtin(__builtin_assume_dereferenceable) && !defined(_LIBCPP_CXX03_LANG)
+  if constexpr (__libcpp_is_contiguous_iterator<_Iter>::value && is_same<_Iter, _Sent>::value) {
+    _LIBCPP_ASSERT_INTERNAL(std::__is_valid_range(std::__to_address(__first), std::__to_address(__last)),
+                            "Valid range assumption does not hold");
+    using __value_type = typename iterator_traits<_Iter>::value_type;
+    __builtin_assume_dereferenceable(std::__to_address(__first), (__last - __first) * sizeof(__value_type));
+    (void)std::__assume_aligned<_LIBCPP_ALIGNOF(__value_type)>(std::__to_address(__first));
+    (void)std::__assume_aligned<_LIBCPP_ALIGNOF(__value_type)>(std::__to_address(__last));
+  }
+#endif
+}
+
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___UTILITY_IS_VALID_RANGE_H
+#endif // _LIBCPP___MEMORY_VALID_RANGE_H

libcxx/include/__utility/is_pointer_in_range.h

@@ -12,12 +12,12 @@
 #include <__algorithm/comp.h>
 #include <__assert>
 #include <__config>
+#include <__memory/valid_range.h>
 #include <__type_traits/enable_if.h>
 #include <__type_traits/integral_constant.h>
 #include <__type_traits/is_constant_evaluated.h>
 #include <__type_traits/void_t.h>
 #include <__utility/declval.h>
-#include <__utility/is_valid_range.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header

libcxx/include/streambuf

@@ -117,8 +117,8 @@ protected:
 #    include <__assert>
 #    include <__fwd/streambuf.h>
 #    include <__locale>
+#    include <__memory/valid_range.h>
 #    include <__type_traits/is_same.h>
-#    include <__utility/is_valid_range.h>
 #    include <__utility/scope_guard.h>
 #    include <climits>
 #    include <ios>

libcxx/test/benchmarks/algorithms/nonmodifying/find.bench.cpp

@@ -21,30 +21,18 @@
 int main(int argc, char** argv) {
   auto std_find    = [](auto first, auto last, auto const& value) { return std::find(first, last, value); };
   auto std_find_if = [](auto first, auto last, auto const& value) {
-    return std::find_if(first, last, [&](auto element) {
-      benchmark::DoNotOptimize(element);
-      return element == value;
-    });
+    return std::find_if(first, last, [&](auto element) { return element == value; });
   };
   auto std_find_if_not = [](auto first, auto last, auto const& value) {
-    return std::find_if_not(first, last, [&](auto element) {
-      benchmark::DoNotOptimize(element);
-      return element != value;
-    });
+    return std::find_if_not(first, last, [&](auto element) { return element != value; });
   };
 
   auto ranges_find    = [](auto first, auto last, auto const& value) { return std::ranges::find(first, last, value); };
   auto ranges_find_if = [](auto first, auto last, auto const& value) {
-    return std::ranges::find_if(first, last, [&](auto element) {
-      benchmark::DoNotOptimize(element);
-      return element == value;
-    });
+    return std::ranges::find_if(first, last, [&](auto element) { return element == value; });
   };
   auto ranges_find_if_not = [](auto first, auto last, auto const& value) {
-    return std::ranges::find_if_not(first, last, [&](auto element) {
-      benchmark::DoNotOptimize(element);
-      return element != value;
-    });
+    return std::ranges::find_if_not(first, last, [&](auto element) { return element != value; });
   };
 
   auto register_benchmarks = [&](auto bm, std::string comment) {

libcxx/test/benchmarks/algorithms/nonmodifying/find_if.bench.cpp

@@ -0,0 +1,51 @@
+#include <benchmark/benchmark.h>
+
+template <class Iter, class ValueT>
+Iter my_find(Iter first, Iter last, const ValueT& i) {
+  for (; first != last; ++first) {
+    if (*first == i)
+      break;
+  }
+  return first;
+}
+
+static auto bm_find_if_no_vectorization(benchmark::State& state) {
+  std::size_t const size = 8192;
+  std::vector<short> c(size, 0);
+
+  for ([[maybe_unused]] auto _ : state) {
+    benchmark::DoNotOptimize(c);
+    std::vector<short>::iterator result;
+    [[clang::noinline]] result = my_find(c.begin(), c.end(), 1);
+    benchmark::DoNotOptimize(result);
+  }
+}
+BENCHMARK(bm_find_if_no_vectorization);
+
+static auto bm_find_if_autovectorization(benchmark::State& state) {
+  std::size_t const size = 8192;
+  std::vector<short> c(size, 0);
+
+  for ([[maybe_unused]] auto _ : state) {
+    benchmark::DoNotOptimize(c);
+    std::vector<short>::iterator result;
+    [[clang::noinline]] result = find_if(c.begin(), c.end(), [](short i) { return i == 1; });
+    benchmark::DoNotOptimize(result);
+  }
+}
+BENCHMARK(bm_find_if_autovectorization);
+
+static auto bm_find_manual_vectorization(benchmark::State& state) {
+  std::size_t const size = 8192;
+  std::vector<short> c(size, 0);
+
+  for ([[maybe_unused]] auto _ : state) {
+    benchmark::DoNotOptimize(c);
+    std::vector<short>::iterator result;
+    [[clang::noinline]] result = find(c.begin(), c.end(), 1);
+    benchmark::DoNotOptimize(result);
+  }
+}
+BENCHMARK(bm_find_manual_vectorization);
+
+BENCHMARK_MAIN();