Add lockfree atomic uint128_type CAS for x86

.gitignore

@@ -1,4 +1,5 @@
 # Copyright (c) 2012 Bryce Adelstein-Lelbach
+# Copyright (c) 2024 Jacob Tucker
 #
 # SPDX-License-Identifier: BSL-1.0
 # Distributed under the Boost Software License, Version 1.0. (See accompanying
@@ -21,4 +22,5 @@ packages
 .vs
 .idea
 cmake-build*
-/.cache
+/.cache
+/CMakeFiles

cmake/HPX_AddConfigTest.cmake

@@ -5,6 +5,7 @@
 # Copyright (c) 2017 Taeguk Kwon
 # Copyright (c) 2020 Giannis Gonidelis
 # Copyright (c) 2021-2024 Hartmut Kaiser
+# Copyright (c)      2024 Jacob Tucker
 #
 # SPDX-License-Identifier: BSL-1.0
 # Distributed under the Boost Software License, Version 1.0. (See accompanying
@@ -330,8 +331,19 @@ function(hpx_check_for_cxx11_std_atomic_128bit)
     HPX_WITH_CXX11_ATOMIC_128BIT
     SOURCE cmake/tests/cxx11_std_atomic_128bit.cpp
     LIBRARIES ${HPX_CXX11_STD_ATOMIC_LIBRARIES}
-    FILE EXECUTE
-  )
+    FILE
+    DEFINITIONS HPX_HAVE_CXX11_STD_ATOMIC_128BIT
+  )
+  # Check if lockfree
+  if(HPX_WITH_CXX11_ATOMIC_128BIT)
+    add_hpx_config_test(
+      HPX_WITH_CXX11_ATOMIC_128BIT_LOCKFREE
+      SOURCE cmake/tests/cxx11_std_atomic_128bit.cpp
+      LIBRARIES ${HPX_CXX11_STD_ATOMIC_LIBRARIES}
+      FILE EXECUTE
+      DEFINITIONS HPX_HAVE_CXX11_STD_ATOMIC_128BIT_LOCKFREE
+    )
+  endif()
   if(NOT MSVC)
     # Sometimes linking against libatomic is required, if the platform doesn't
     # support lock-free atomics. We already know that MSVC works
@@ -345,8 +357,19 @@ function(hpx_check_for_cxx11_std_atomic_128bit)
         HPX_WITH_CXX11_ATOMIC_128BIT
         SOURCE cmake/tests/cxx11_std_atomic_128bit.cpp
         LIBRARIES ${HPX_CXX11_STD_ATOMIC_LIBRARIES}
-        FILE EXECUTE
+        FILE
+        DEFINITIONS HPX_HAVE_CXX11_STD_ATOMIC_128BIT
       )
+      # Check if lockfree after adding -latomic
+      if(HPX_WITH_CXX11_ATOMIC_128BIT)
+        add_hpx_config_test(
+          HPX_WITH_CXX11_ATOMIC_128BIT_LOCKFREE
+          SOURCE cmake/tests/cxx11_std_atomic_128bit.cpp
+          LIBRARIES ${HPX_CXX11_STD_ATOMIC_LIBRARIES}
+          FILE EXECUTE
+          DEFINITIONS HPX_HAVE_CXX11_STD_ATOMIC_128BIT_LOCKFREE
+        )
+      endif()
       if(NOT HPX_WITH_CXX11_ATOMIC_128BIT)
         # Adding -latomic did not help, so we don't attempt to link to it later
         # but only if normal atomics don't require it

cmake/HPX_PerformCxxFeatureTests.cmake

@@ -3,6 +3,7 @@
 # Copyright (c) 2013-2016 Agustin Berge
 # Copyright (c)      2017 Taeguk Kwon
 # Copyright (c)      2020 Giannis Gonidelis
+# Copyright (c)      2024 Jacob Tucker
 #
 # SPDX-License-Identifier: BSL-1.0
 # Distributed under the Boost Software License, Version 1.0. (See accompanying
@@ -36,9 +37,7 @@ function(hpx_perform_cxx_feature_tests)
   )
 
   # Separately check for 128 bit atomics
-  hpx_check_for_cxx11_std_atomic_128bit(
-    DEFINITIONS HPX_HAVE_CXX11_STD_ATOMIC_128BIT
-  )
+  hpx_check_for_cxx11_std_atomic_128bit()
 
   hpx_check_for_cxx11_std_quick_exit(DEFINITIONS HPX_HAVE_CXX11_STD_QUICK_EXIT)
 

libs/core/concurrency/CMakeLists.txt

@@ -21,6 +21,8 @@ set(concurrency_headers
     hpx/concurrency/detail/tagged_ptr_dcas.hpp
     hpx/concurrency/detail/tagged_ptr_ptrcompression.hpp
     hpx/concurrency/detail/tagged_ptr_pair.hpp
+    hpx/concurrency/detail/uint128_atomic.hpp
+    hpx/concurrency/detail/uint128_type.hpp
     hpx/concurrency/queue.hpp
     hpx/concurrency/spinlock.hpp
     hpx/concurrency/spinlock_pool.hpp

libs/core/concurrency/include/hpx/concurrency/detail/tagged_ptr_pair.hpp

@@ -1,6 +1,7 @@
 //  Copyright (C) 2008-2011 Tim Blechmann
 //  Copyright (C) 2011      Bryce Lelbach
 //  Copyright (c) 2022-2023 Hartmut Kaiser
+//  Copyright (c) 2024      Jacob Tucker
 //
 //  SPDX-License-Identifier: BSL-1.0
 //  Distributed under the Boost Software License, Version 1.0. (See accompanying
@@ -12,46 +13,13 @@
 #pragma once
 
 #include <hpx/config.hpp>
+#include <hpx/concurrency/detail/uint128_atomic.hpp>
 #include <hpx/type_support/bit_cast.hpp>
 
 #include <cstddef>    // for std::size_t
 #include <cstdint>
 
 namespace hpx::lockfree {
-
-    struct HPX_LOCKFREE_DCAS_ALIGNMENT uint128_type
-    {
-        std::uint64_t left = 0;
-        std::uint64_t right = 0;
-
-        uint128_type() = default;
-
-        constexpr uint128_type(std::size_t l, std::size_t r) noexcept
-          : left(l)
-          , right(r)
-        {
-        }
-
-        uint128_type(uint128_type const&) = default;
-        uint128_type(uint128_type&&) = default;
-        uint128_type& operator=(uint128_type const&) = default;
-        uint128_type& operator=(uint128_type&&) = default;
-
-        ~uint128_type() = default;
-
-        friend constexpr bool operator==(uint128_type const& lhs,    //-V835
-            uint128_type const& rhs) noexcept                        //-V835
-        {
-            return lhs.left == rhs.left && lhs.right == rhs.right;
-        }
-
-        friend constexpr bool operator!=(uint128_type const& lhs,    //-V835
-            uint128_type const& rhs) noexcept                        //-V835
-        {
-            return !(lhs == rhs);
-        }
-    };
-
     namespace detail {
 
         template <std::size_t Size>

libs/core/concurrency/include/hpx/concurrency/detail/uint128_atomic.hpp

@@ -0,0 +1,79 @@
+//  Copyright (c) 2024      Jacob Tucker
+//
+//  SPDX-License-Identifier: BSL-1.0
+//  Distributed under the Boost Software License, Version 1.0. (See accompanying
+//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#pragma once
+
+#include <hpx/concurrency/detail/uint128_type.hpp>
+#include <cstddef>
+
+#include <atomic>
+
+/* Note: currently only supported on Linux */
+#if !defined(HPX_WITH_CXX11_ATOMIC_128BIT_LOCKFREE) && defined(__linux__)
+
+namespace hpx::lockfree {
+    struct uint128_atomic : std::atomic<hpx::lockfree::uint128_type>
+    {
+        constexpr uint128_atomic(std::size_t l, std::size_t r) noexcept
+          : std::atomic<hpx::lockfree::uint128_type>(uint128_type{l, r})
+        {
+        }
+
+        constexpr uint128_atomic(hpx::lockfree::uint128_type value) noexcept
+          : std::atomic<hpx::lockfree::uint128_type>(value)
+        {
+        }
+
+        bool is_lock_free() const noexcept
+        {
+            return true;
+        }
+
+        bool compare_exchange_weak(hpx::lockfree::uint128_type& expected,
+            hpx::lockfree::uint128_type desired,
+            std::memory_order memOrder) noexcept
+        {
+            (void) memOrder;
+            bool result = false;
+            asm("lock cmpxchg16b %1\n\t"
+                "setz %0"
+                : "=r"(result),
+                "+m"(*reinterpret_cast<hpx::lockfree::uint128_type*>(this)),
+                "+d"(expected.right),    // high
+                "+a"(expected.left)      // low
+                : "c"(desired.right),    // high
+                "b"(desired.left)        // low
+                : "cc", "memory");
+            return result;
+        }
+
+        bool compare_exchange_strong(hpx::lockfree::uint128_type& expected,
+            hpx::lockfree::uint128_type desired,
+            std::memory_order memOrder) noexcept
+        {
+            (void) memOrder;
+            bool result = false;
+            asm("lock cmpxchg16b %1\n\t"
+                "setz %0"
+                : "=r"(result),
+                "+m"(*reinterpret_cast<hpx::lockfree::uint128_type*>(this)),
+                "+d"(expected.right),    // high
+                "+a"(expected.left)      // low
+                : "c"(desired.right),    // high
+                "b"(desired.left)        // low
+                : "cc", "memory");
+            return result;
+        }
+    };
+}    // namespace hpx::lockfree
+
+#else
+
+namespace hpx::lockfree {
+    /* Alias std::atomic if custom implementation isn't supported */
+    using uint128_atomic = std::atomic<uint128_type>;
+}    // namespace hpx::lockfree
+#endif    // !defined(HPX_WITH_CXX11_ATOMIC_128BIT_LOCKFREE) && defined(__linux__)

libs/core/concurrency/include/hpx/concurrency/detail/uint128_type.hpp

@@ -0,0 +1,47 @@
+//  Copyright (c) 2024      Jacob Tucker
+//
+//  SPDX-License-Identifier: BSL-1.0
+//  Distributed under the Boost Software License, Version 1.0. (See accompanying
+//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#pragma once
+
+#include <hpx/config.hpp>
+
+#include <cstddef>
+#include <cstdint>
+
+namespace hpx::lockfree {
+    struct HPX_LOCKFREE_DCAS_ALIGNMENT uint128_type
+    {
+        std::uint64_t left = 0;
+        std::uint64_t right = 0;
+
+        uint128_type() = default;
+
+        constexpr uint128_type(std::size_t l, std::size_t r) noexcept
+          : left(l)
+          , right(r)
+        {
+        }
+
+        uint128_type(uint128_type const&) = default;
+        uint128_type(uint128_type&&) = default;
+        uint128_type& operator=(uint128_type const&) = default;
+        uint128_type& operator=(uint128_type&&) = default;
+
+        ~uint128_type() = default;
+
+        friend constexpr bool operator==(uint128_type const& lhs,    //-V835
+            uint128_type const& rhs) noexcept                        //-V835
+        {
+            return lhs.left == rhs.left && lhs.right == rhs.right;
+        }
+
+        friend constexpr bool operator!=(uint128_type const& lhs,    //-V835
+            uint128_type const& rhs) noexcept                        //-V835
+        {
+            return !(lhs == rhs);
+        }
+    };
+}    // namespace hpx::lockfree

libs/core/concurrency/tests/unit/CMakeLists.txt

@@ -16,6 +16,7 @@ set(tests
     stack_destructor
     stack_stress
     tagged_ptr
+    uint128_atomic
 )
 
 set(contiguous_index_queue_PARAMETERS THREADS_PER_LOCALITY 4)

libs/core/concurrency/tests/unit/tagged_ptr.cpp

@@ -1,5 +1,6 @@
 //  Copyright (C) 2011 Tim Blechmann
 //  Copyright (c) 2022 Hartmut Kaiser
+//  Copyright (c) 2024 Jacob Tucker
 //
 //  SPDX-License-Identifier: BSL-1.0
 //  Distributed under the Boost Software License, Version 1.0. (See accompanying
@@ -25,7 +26,7 @@ void tagged_ptr_test()
         i = j;
 
         HPX_TEST_EQ(i.get_ptr(), &b);
-        HPX_TEST_EQ(i.get_tag(), 1);
+        HPX_TEST_EQ(i.get_tag(), 1UL);
     }
 
     {
@@ -43,7 +44,7 @@ void tagged_ptr_test()
 
     {
         tagged_ptr<int> j(&a, max_tag);
-        HPX_TEST_EQ(j.get_next_tag(), 0);
+        HPX_TEST_EQ(j.get_next_tag(), 0UL);
     }
 
     {

libs/core/concurrency/tests/unit/uint128_atomic.cpp

@@ -0,0 +1,34 @@
+//  Copyright (c) 2024 Jacob Tucker
+//
+//  SPDX-License-Identifier: BSL-1.0
+//  Distributed under the Boost Software License, Version 1.0. (See accompanying
+//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#include <hpx/concurrency/detail/uint128_atomic.hpp>
+#include <hpx/modules/testing.hpp>
+
+#include <atomic>
+#include <iostream>
+
+void uint128_atomic_test()
+{
+    hpx::lockfree::uint128_type x(1, 2);
+    hpx::lockfree::uint128_type y(3, 4);
+    hpx::lockfree::uint128_atomic a(x);
+    hpx::lockfree::uint128_atomic b(y);
+
+    HPX_TEST(a.is_lock_free());
+
+    hpx::lockfree::uint128_type expected = x;
+    hpx::lockfree::uint128_type desired = y;
+    bool result =
+        a.compare_exchange_weak(expected, desired, std::memory_order_seq_cst);
+    hpx::lockfree::uint128_type received = a.load();
+    HPX_TEST(result && (received == desired));
+}
+
+int main()
+{
+    uint128_atomic_test();
+    return hpx::util::report_errors();
+}