From bfb43ab80b0272e60c8b43bcd14533ee68a11f13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hana=20Dusi=CC=81kova=CC=81?= Date: Mon, 6 Jan 2025 20:40:01 +0100 Subject: [PATCH] [clang] implement P3309 atomic builtins in constant evaluator --- clang/include/clang/Basic/Builtins.td | 80 +-- clang/lib/AST/ExprConstant.cpp | 541 +++++++++++++++++- .../SemaCXX/atomic-constexpr-c11-builtins.cpp | 326 +++++++++++ .../SemaCXX/atomic-constexpr-gcc-builtins.cpp | 504 ++++++++++++++++ 4 files changed, 1410 insertions(+), 41 deletions(-) create mode 100644 clang/test/SemaCXX/atomic-constexpr-c11-builtins.cpp create mode 100644 clang/test/SemaCXX/atomic-constexpr-gcc-builtins.cpp diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td index 468c16050e2bf08..797191fba888d26 100644 --- a/clang/include/clang/Basic/Builtins.td +++ b/clang/include/clang/Basic/Builtins.td @@ -1744,97 +1744,97 @@ def SyncSwapN : Builtin, SyncBuiltinsTemplate { // C11 _Atomic operations for . def C11AtomicInit : AtomicBuiltin { let Spellings = ["__c11_atomic_init"]; - let Attributes = [CustomTypeChecking]; + let Attributes = [CustomTypeChecking, Constexpr]; let Prototype = "void(...)"; } def C11AtomicLoad : AtomicBuiltin { let Spellings = ["__c11_atomic_load"]; - let Attributes = [CustomTypeChecking]; + let Attributes = [CustomTypeChecking, Constexpr]; let Prototype = "void(...)"; } def C11AtomicStore : AtomicBuiltin { let Spellings = ["__c11_atomic_store"]; - let Attributes = [CustomTypeChecking]; + let Attributes = [CustomTypeChecking, Constexpr]; let Prototype = "void(...)"; } def C11AtomicExchange : AtomicBuiltin { let Spellings = ["__c11_atomic_exchange"]; - let Attributes = [CustomTypeChecking]; + let Attributes = [CustomTypeChecking, Constexpr]; let Prototype = "void(...)"; } def C11AtomicCompareExchangeStrong : AtomicBuiltin { let Spellings = ["__c11_atomic_compare_exchange_strong"]; - let Attributes = [CustomTypeChecking]; + let Attributes = [CustomTypeChecking, Constexpr]; let Prototype = "void(...)"; } def C11AtomicCompareExchangeWeak : AtomicBuiltin { let Spellings = ["__c11_atomic_compare_exchange_weak"]; - let Attributes = [CustomTypeChecking]; + let Attributes = [CustomTypeChecking, Constexpr]; let Prototype = "void(...)"; } def C11AtomicFetchAdd : AtomicBuiltin { let Spellings = ["__c11_atomic_fetch_add"]; - let Attributes = [CustomTypeChecking]; + let Attributes = [CustomTypeChecking, Constexpr]; let Prototype = "void(...)"; } def C11AtomicFetchSub : AtomicBuiltin { let Spellings = ["__c11_atomic_fetch_sub"]; - let Attributes = [CustomTypeChecking]; + let Attributes = [CustomTypeChecking, Constexpr]; let Prototype = "void(...)"; } def C11AtomicFetchAnd : AtomicBuiltin { let Spellings = ["__c11_atomic_fetch_and"]; - let Attributes = [CustomTypeChecking]; + let Attributes = [CustomTypeChecking, Constexpr]; let Prototype = "void(...)"; } def C11AtomicFetchOr : AtomicBuiltin { let Spellings = ["__c11_atomic_fetch_or"]; - let Attributes = [CustomTypeChecking]; + let Attributes = [CustomTypeChecking, Constexpr]; let Prototype = "void(...)"; } def C11AtomicFetchXor : AtomicBuiltin { let Spellings = ["__c11_atomic_fetch_xor"]; - let Attributes = [CustomTypeChecking]; + let Attributes = [CustomTypeChecking, Constexpr]; let Prototype = "void(...)"; } def C11AtomicFetchNand : AtomicBuiltin { let Spellings = ["__c11_atomic_fetch_nand"]; - let Attributes = [CustomTypeChecking]; + let Attributes = [CustomTypeChecking, Constexpr]; let Prototype = "void(...)"; } def C11AtomicFetchMax : AtomicBuiltin { let Spellings = ["__c11_atomic_fetch_max"]; - let Attributes = [CustomTypeChecking]; + let Attributes = [CustomTypeChecking, Constexpr]; let Prototype = "void(...)"; } def C11AtomicFetchMin : AtomicBuiltin { let Spellings = ["__c11_atomic_fetch_min"]; - let Attributes = [CustomTypeChecking]; + let Attributes = [CustomTypeChecking, Constexpr]; let Prototype = "void(...)"; } def C11AtomicThreadFence : Builtin { let Spellings = ["__c11_atomic_thread_fence"]; - let Attributes = [NoThrow]; + let Attributes = [NoThrow, Constexpr]; let Prototype = "void(int)"; } def C11AtomicSignalFence : Builtin { let Spellings = ["__c11_atomic_signal_fence"]; - let Attributes = [NoThrow]; + let Attributes = [NoThrow, Constexpr]; let Prototype = "void(int)"; } @@ -1847,133 +1847,133 @@ def C11AtomicIsLockFree : Builtin { // GNU atomic builtins. def AtomicLoad : AtomicBuiltin { let Spellings = ["__atomic_load"]; - let Attributes = [CustomTypeChecking]; + let Attributes = [CustomTypeChecking, Constexpr]; let Prototype = "void(...)"; } def AtomicLoadN : AtomicBuiltin { let Spellings = ["__atomic_load_n"]; - let Attributes = [CustomTypeChecking]; + let Attributes = [CustomTypeChecking, Constexpr]; let Prototype = "void(...)"; } def AtomicStore : AtomicBuiltin { let Spellings = ["__atomic_store"]; - let Attributes = [CustomTypeChecking]; + let Attributes = [CustomTypeChecking, Constexpr]; let Prototype = "void(...)"; } def AtomicStoreN : AtomicBuiltin { let Spellings = ["__atomic_store_n"]; - let Attributes = [CustomTypeChecking]; + let Attributes = [CustomTypeChecking, Constexpr]; let Prototype = "void(...)"; } def AtomicExchange : AtomicBuiltin { let Spellings = ["__atomic_exchange"]; - let Attributes = [CustomTypeChecking]; + let Attributes = [CustomTypeChecking, Constexpr]; let Prototype = "void(...)"; } def AtomicExchangeN : AtomicBuiltin { let Spellings = ["__atomic_exchange_n"]; - let Attributes = [CustomTypeChecking]; + let Attributes = [CustomTypeChecking, Constexpr]; let Prototype = "void(...)"; } def AtomicCompareExchange : AtomicBuiltin { let Spellings = ["__atomic_compare_exchange"]; - let Attributes = [CustomTypeChecking]; + let Attributes = [CustomTypeChecking, Constexpr]; let Prototype = "void(...)"; } def AtomicCompareExchangeN : AtomicBuiltin { let Spellings = ["__atomic_compare_exchange_n"]; - let Attributes = [CustomTypeChecking]; + let Attributes = [CustomTypeChecking, Constexpr]; let Prototype = "void(...)"; } def AtomicFetchAdd : AtomicBuiltin { let Spellings = ["__atomic_fetch_add"]; - let Attributes = [CustomTypeChecking]; + let Attributes = [CustomTypeChecking, Constexpr]; let Prototype = "void(...)"; } def AtomicFetchSub : AtomicBuiltin { let Spellings = ["__atomic_fetch_sub"]; - let Attributes = [CustomTypeChecking]; + let Attributes = [CustomTypeChecking, Constexpr]; let Prototype = "void(...)"; } def AtomicFetchAnd : AtomicBuiltin { let Spellings = ["__atomic_fetch_and"]; - let Attributes = [CustomTypeChecking]; + let Attributes = [CustomTypeChecking, Constexpr]; let Prototype = "void(...)"; } def AtomicFetchOr : AtomicBuiltin { let Spellings = ["__atomic_fetch_or"]; - let Attributes = [CustomTypeChecking]; + let Attributes = [CustomTypeChecking, Constexpr]; let Prototype = "void(...)"; } def AtomicFetchXor : AtomicBuiltin { let Spellings = ["__atomic_fetch_xor"]; - let Attributes = [CustomTypeChecking]; + let Attributes = [CustomTypeChecking, Constexpr]; let Prototype = "void(...)"; } def AtomicFetchNand : AtomicBuiltin { let Spellings = ["__atomic_fetch_nand"]; - let Attributes = [CustomTypeChecking]; + let Attributes = [CustomTypeChecking, Constexpr]; let Prototype = "void(...)"; } def AtomicAddFetch : AtomicBuiltin { let Spellings = ["__atomic_add_fetch"]; - let Attributes = [CustomTypeChecking]; + let Attributes = [CustomTypeChecking, Constexpr]; let Prototype = "void(...)"; } def AtomicSubFetch : AtomicBuiltin { let Spellings = ["__atomic_sub_fetch"]; - let Attributes = [CustomTypeChecking]; + let Attributes = [CustomTypeChecking, Constexpr]; let Prototype = "void(...)"; } def AtomicAndFetch : AtomicBuiltin { let Spellings = ["__atomic_and_fetch"]; - let Attributes = [CustomTypeChecking]; + let Attributes = [CustomTypeChecking, Constexpr]; let Prototype = "void(...)"; } def AtomicOrFetch : AtomicBuiltin { let Spellings = ["__atomic_or_fetch"]; - let Attributes = [CustomTypeChecking]; + let Attributes = [CustomTypeChecking, Constexpr]; let Prototype = "void(...)"; } def AtomicXorFetch : AtomicBuiltin { let Spellings = ["__atomic_xor_fetch"]; - let Attributes = [CustomTypeChecking]; + let Attributes = [CustomTypeChecking, Constexpr]; let Prototype = "void(...)"; } def AtomicMaxFetch : AtomicBuiltin { let Spellings = ["__atomic_max_fetch"]; - let Attributes = [CustomTypeChecking]; + let Attributes = [CustomTypeChecking, Constexpr]; let Prototype = "void(...)"; } def AtomicMinFetch : AtomicBuiltin { let Spellings = ["__atomic_min_fetch"]; - let Attributes = [CustomTypeChecking]; + let Attributes = [CustomTypeChecking, Constexpr]; let Prototype = "void(...)"; } def AtomicNandFetch : AtomicBuiltin { let Spellings = ["__atomic_nand_fetch"]; - let Attributes = [CustomTypeChecking]; + let Attributes = [CustomTypeChecking, Constexpr]; let Prototype = "void(...)"; } @@ -1991,7 +1991,7 @@ def AtomicClear : Builtin { def AtomicThreadFence : Builtin { let Spellings = ["__atomic_thread_fence"]; - let Attributes = [NoThrow]; + let Attributes = [NoThrow, Constexpr]; let Prototype = "void(int)"; } @@ -2003,7 +2003,7 @@ def ScopedAtomicThreadFence : Builtin { def AtomicSignalFence : Builtin { let Spellings = ["__atomic_signal_fence"]; - let Attributes = [NoThrow]; + let Attributes = [NoThrow, Constexpr]; let Prototype = "void(int)"; } diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp index e220f69b3a4f58b..a5b48be64839ce3 100644 --- a/clang/lib/AST/ExprConstant.cpp +++ b/clang/lib/AST/ExprConstant.cpp @@ -1921,6 +1921,22 @@ static bool EvaluateFixedPointOrInteger(const Expr *E, APFixedPoint &Result, static bool EvaluateFixedPoint(const Expr *E, APFixedPoint &Result, EvalInfo &Info); +/// Support for atomic builtins +static bool EvaluateAtomicOrder(const AtomicExpr *E, EvalInfo &Info); +static bool EvaluateAtomicWeak(const AtomicExpr *E, EvalInfo &Info); +static bool EvaluateAtomicLoad(const AtomicExpr *E, APValue &Result, + EvalInfo &Info); +static bool EvaluatePointerAndStoreValueInto(Expr *ResultPtr, + APValue &ValueToStore, + EvalInfo &Info); +static bool EvaluateAtomicLoadInto(const AtomicExpr *E, EvalInfo &Info); +static bool EvaluateAtomicStore(const AtomicExpr *E, EvalInfo &Info); +static bool EvaluateAtomicExchangeInto(const AtomicExpr *E, EvalInfo &Info); +static bool EvaluateAtomicFetchOp(const AtomicExpr *E, APValue &Result, + EvalInfo &Info, bool StoreToResultAfter); +static bool EvaluateAtomicCompareExchange(const AtomicExpr *E, APValue &Result, + EvalInfo &Info); + //===----------------------------------------------------------------------===// // Misc utilities //===----------------------------------------------------------------------===// @@ -8075,6 +8091,66 @@ class ExprEvaluatorBase return StmtVisitorTy::Visit(Source); } + bool VisitAtomicExpr(const AtomicExpr *E) { + if (!EvaluateAtomicOrder(E, Info)) + return false; + + APValue LocalResult; + switch (E->getOp()) { + default: + return Error(E); + case AtomicExpr::AO__c11_atomic_load: + case AtomicExpr::AO__atomic_load_n: + if (!EvaluateAtomicLoad(E, LocalResult, Info)) + return Error(E); + return DerivedSuccess(LocalResult, E); + case AtomicExpr::AO__c11_atomic_compare_exchange_strong: + case AtomicExpr::AO__c11_atomic_compare_exchange_weak: + case AtomicExpr::AO__atomic_compare_exchange: + case AtomicExpr::AO__atomic_compare_exchange_n: + if (!EvaluateAtomicCompareExchange(E, LocalResult, Info)) + return Error(E); + return DerivedSuccess(LocalResult, E); + case AtomicExpr::AO__c11_atomic_exchange: + case AtomicExpr::AO__atomic_exchange_n: + if (!EvaluateAtomicLoad(E, LocalResult, Info)) + return Error(E); + if (!EvaluateAtomicStore(E, Info)) + return Error(E); + return DerivedSuccess(LocalResult, E); + case AtomicExpr::AO__c11_atomic_fetch_add: + case AtomicExpr::AO__c11_atomic_fetch_sub: + case AtomicExpr::AO__c11_atomic_fetch_and: + case AtomicExpr::AO__c11_atomic_fetch_or: + case AtomicExpr::AO__c11_atomic_fetch_xor: + case AtomicExpr::AO__c11_atomic_fetch_nand: + case AtomicExpr::AO__c11_atomic_fetch_max: + case AtomicExpr::AO__c11_atomic_fetch_min: + case AtomicExpr::AO__atomic_fetch_add: + case AtomicExpr::AO__atomic_fetch_sub: + case AtomicExpr::AO__atomic_fetch_and: + case AtomicExpr::AO__atomic_fetch_xor: + case AtomicExpr::AO__atomic_fetch_or: + case AtomicExpr::AO__atomic_fetch_nand: + case AtomicExpr::AO__atomic_fetch_max: + case AtomicExpr::AO__atomic_fetch_min: + if (!EvaluateAtomicFetchOp(E, LocalResult, Info, false)) + return Error(E); + return DerivedSuccess(LocalResult, E); + case AtomicExpr::AO__atomic_add_fetch: + case AtomicExpr::AO__atomic_sub_fetch: + case AtomicExpr::AO__atomic_and_fetch: + case AtomicExpr::AO__atomic_xor_fetch: + case AtomicExpr::AO__atomic_or_fetch: + case AtomicExpr::AO__atomic_nand_fetch: + case AtomicExpr::AO__atomic_max_fetch: + case AtomicExpr::AO__atomic_min_fetch: + if (!EvaluateAtomicFetchOp(E, LocalResult, Info, true)) + return Error(E); + return DerivedSuccess(LocalResult, E); + } + } + bool VisitPseudoObjectExpr(const PseudoObjectExpr *E) { for (const Expr *SemE : E->semantics()) { if (auto *OVE = dyn_cast(SemE)) { @@ -16268,6 +16344,25 @@ class VoidExprEvaluator } } + bool VisitAtomicExpr(const AtomicExpr *E) { + if (!EvaluateAtomicOrder(E, Info)) + return false; + + switch (E->getOp()) { + default: + return Error(E); + case AtomicExpr::AO__atomic_load: + return EvaluateAtomicLoadInto(E, Info); + case AtomicExpr::AO__atomic_exchange: + return EvaluateAtomicExchangeInto(E, Info); + case AtomicExpr::AO__c11_atomic_init: + case AtomicExpr::AO__c11_atomic_store: + case AtomicExpr::AO__atomic_store: + case AtomicExpr::AO__atomic_store_n: + return EvaluateAtomicStore(E, Info); + } + } + bool VisitCallExpr(const CallExpr *E) { if (!IsConstantEvaluatedBuiltinCall(E)) return ExprEvaluatorBaseTy::VisitCallExpr(E); @@ -16281,6 +16376,14 @@ class VoidExprEvaluator case Builtin::BI__builtin_operator_delete: return HandleOperatorDeleteCall(Info, E); + case Builtin::BI__c11_atomic_thread_fence: + case Builtin::BI__c11_atomic_signal_fence: + case Builtin::BI__atomic_thread_fence: + case Builtin::BI__atomic_signal_fence: { + [[maybe_unused]] APSInt IgnoredAtomicOrdering; + return EvaluateInteger(E->getArg(0), IgnoredAtomicOrdering, Info); + } + default: return false; } @@ -17893,4 +17996,440 @@ std::optional EvaluateBuiltinIsWithinLifetime(IntExprEvaluator &IEE, IsWithinLifetimeHandler handler{Info}; return findSubobject(Info, E, CO, Val.getLValueDesignator(), handler); } -} // namespace + +static bool EvaluateAtomicOrder(const AtomicExpr *E, EvalInfo &Info) { + // we ignore results, but we need to evaluate them + [[maybe_unused]] APSInt OrderIgnoredResult; + + if (E->getOp() != AtomicExpr::AO__c11_atomic_init) { + const Expr *OrderSuccess = E->getOrder(); + if (OrderSuccess && + !EvaluateInteger(OrderSuccess, OrderIgnoredResult, Info)) + return false; + } + + if (E->isCmpXChg()) { + const Expr *OrderFail = E->getOrderFail(); + if (!EvaluateInteger(OrderFail, OrderIgnoredResult, Info)) + return false; + } + + return true; +} + +static bool EvaluateAtomicWeak(const AtomicExpr *E, EvalInfo &Info) { + // we ignore results, but we need to evaluate them + [[maybe_unused]] APSInt WeakIgnoredResult; + + if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n || + E->getOp() == AtomicExpr::AO__atomic_compare_exchange) { + const Expr *Weak = E->getWeak(); + if (!EvaluateInteger(Weak, WeakIgnoredResult, Info)) + return false; + } + + return true; +} + +static bool EvaluateAtomicLoad(const AtomicExpr *E, APValue &Result, + EvalInfo &Info) { + LValue AtomicStorageLV; + + if (!EvaluatePointer(E->getPtr(), AtomicStorageLV, Info)) + return false; + + return handleLValueToRValueConversion(Info, E->getPtr(), E->getValueType(), + AtomicStorageLV, Result); +} + +static bool EvaluatePointerAndStoreValueInto(Expr *ResultPtr, + APValue &ValueToStore, + EvalInfo &Info) { + assert(ResultPtr->getType()->isPointerType()); + QualType PointeeTy = ResultPtr->getType()->getPointeeType(); + LValue PointeeLV; + + if (!EvaluatePointer(ResultPtr, PointeeLV, Info)) + return false; + + return handleAssignment(Info, ResultPtr, PointeeLV, PointeeTy, ValueToStore); +} + +static bool EvaluateAtomicLoadInto(const AtomicExpr *E, EvalInfo &Info) { + APValue LocalResult; + + if (!EvaluateAtomicLoad(E, LocalResult, Info)) + return false; + + if (!EvaluatePointerAndStoreValueInto(E->getVal1(), LocalResult, Info)) + return false; + + return true; +} + +static bool EvaluateAtomicStore(const AtomicExpr *E, EvalInfo &Info) { + LValue AtomicStorageLV; + + if (!EvaluatePointer(E->getPtr(), AtomicStorageLV, Info)) + return false; + + APValue ProvidedValue; + + // GCC's atomic_store takes pointer to value, not value itself + if (E->getOp() == AtomicExpr::AO__atomic_store) { + LValue ProvidedLV; + if (!EvaluatePointer(E->getVal1(), ProvidedLV, Info)) + return false; + + if (!handleLValueToRValueConversion(Info, E->getVal1(), + E->getVal1()->getType(), ProvidedLV, + ProvidedValue)) + return false; + + } else { + if (!Evaluate(ProvidedValue, Info, E->getVal1())) + return false; + } + if (!handleAssignment(Info, E, AtomicStorageLV, E->getValueType(), + ProvidedValue)) + return false; + + return true; +} + +static bool EvaluateAtomicExchangeInto(const AtomicExpr *E, EvalInfo &Info) { + assert(E->getOp() == AtomicExpr::AO__atomic_exchange); + // implementation of GCC's exchange (non _n version) + LValue AtomicStorageLV; + if (!EvaluatePointer(E->getPtr(), AtomicStorageLV, Info)) + return false; + + // read previous value + APValue PreviousValue; + if (!handleLValueToRValueConversion(Info, E->getPtr(), E->getValueType(), + AtomicStorageLV, PreviousValue)) + return false; + + // get provided value from argument (pointer) + LValue ProvidedLV; + if (!EvaluatePointer(E->getVal1(), ProvidedLV, Info)) + return false; + + APValue ProvidedValue; + if (!handleLValueToRValueConversion(Info, E->getVal1(), + E->getVal1()->getType(), ProvidedLV, + ProvidedValue)) + return false; + + // store provided value to atomic value + if (!handleAssignment(Info, E, AtomicStorageLV, E->getValueType(), + ProvidedValue)) + return false; + + // store previous value in output pointer + if (!EvaluatePointerAndStoreValueInto(E->getVal2(), PreviousValue, Info)) + return false; + + return true; +} + +static bool EvaluateAtomicFetchOp(const AtomicExpr *E, APValue &Result, + EvalInfo &Info, bool StoreToResultAfter) { + // read atomic + LValue AtomicStorageLV; + QualType AtomicValueTy = E->getValueType(); + if (!EvaluatePointer(E->getPtr(), AtomicStorageLV, Info)) + return false; + + APValue CurrentValue; + if (!handleLValueToRValueConversion(Info, E->getPtr(), E->getType(), + AtomicStorageLV, CurrentValue)) + return false; + + // store current value for fetch-OP operations + if (!StoreToResultAfter) + Result = CurrentValue; + + // read argument for fetch OP + APValue ArgumentVal; + if (!Evaluate(ArgumentVal, Info, E->getVal1())) + return false; + + // calculate new value + APValue Replacement; + if (AtomicValueTy->isIntegralOrEnumerationType()) { + // both arguments are integers + const APSInt AtomicInt = CurrentValue.getInt(); + const APSInt ArgumentInt = ArgumentVal.getInt(); + + switch (E->getOp()) { + case AtomicExpr::AO__c11_atomic_fetch_add: + case AtomicExpr::AO__atomic_fetch_add: + case AtomicExpr::AO__atomic_add_fetch: + Replacement = APValue(AtomicInt + ArgumentInt); + break; + case AtomicExpr::AO__c11_atomic_fetch_sub: + case AtomicExpr::AO__atomic_fetch_sub: + case AtomicExpr::AO__atomic_sub_fetch: + Replacement = APValue(AtomicInt - ArgumentInt); + break; + case AtomicExpr::AO__c11_atomic_fetch_and: + case AtomicExpr::AO__atomic_fetch_and: + case AtomicExpr::AO__atomic_and_fetch: + Replacement = APValue(AtomicInt & ArgumentInt); + break; + case AtomicExpr::AO__c11_atomic_fetch_or: + case AtomicExpr::AO__atomic_fetch_or: + case AtomicExpr::AO__atomic_or_fetch: + Replacement = APValue(AtomicInt | ArgumentInt); + break; + case AtomicExpr::AO__c11_atomic_fetch_xor: + case AtomicExpr::AO__atomic_fetch_xor: + case AtomicExpr::AO__atomic_xor_fetch: + Replacement = APValue(AtomicInt ^ ArgumentInt); + break; + case AtomicExpr::AO__c11_atomic_fetch_nand: + case AtomicExpr::AO__atomic_fetch_nand: + case AtomicExpr::AO__atomic_nand_fetch: + Replacement = APValue(~(AtomicInt & ArgumentInt)); + break; + case AtomicExpr::AO__c11_atomic_fetch_max: + case AtomicExpr::AO__atomic_fetch_max: + case AtomicExpr::AO__atomic_max_fetch: + Replacement = + APValue((AtomicInt > ArgumentInt) ? AtomicInt : ArgumentInt); + break; + case AtomicExpr::AO__c11_atomic_fetch_min: + case AtomicExpr::AO__atomic_fetch_min: + case AtomicExpr::AO__atomic_min_fetch: + Replacement = + APValue((AtomicInt < ArgumentInt) ? AtomicInt : ArgumentInt); + break; + default: + return false; + } + } else if (AtomicValueTy->isRealFloatingType()) { + // both arguments are float operations + const llvm::RoundingMode RM = getActiveRoundingMode(Info, E); + APFloat AtomicFlt = CurrentValue.getFloat(); + const APFloat ArgumentFlt = ArgumentVal.getFloat(); + APFloat::opStatus St; + + switch (E->getOp()) { + case AtomicExpr::AO__c11_atomic_fetch_add: // GCC atomics doesn't support + // floats + St = AtomicFlt.add(ArgumentFlt, RM); + Replacement = APValue(AtomicFlt); + break; + case AtomicExpr::AO__c11_atomic_fetch_sub: + St = AtomicFlt.subtract(ArgumentFlt, RM); + Replacement = APValue(AtomicFlt); + break; + default: + return false; + } + + if (!checkFloatingPointResult(Info, E, St)) + return false; + + } else if (AtomicValueTy->isPointerType()) { + // pointer + int arguments + LValue AtomicPtr; + AtomicPtr.setFrom(Info.Ctx, CurrentValue); + + APSInt ArgumentInt = ArgumentVal.getInt(); + + // calculate size of pointee object + CharUnits SizeOfPointee; + if (!HandleSizeof(Info, E->getExprLoc(), AtomicValueTy->getPointeeType(), + SizeOfPointee)) + return false; + + // GCC's atomic_fetch add/sub compute new pointer by bytes and not + // sizeof(T) + switch (E->getOp()) { + case AtomicExpr::AO__atomic_fetch_add: + case AtomicExpr::AO__atomic_add_fetch: + case AtomicExpr::AO__atomic_fetch_sub: + case AtomicExpr::AO__atomic_sub_fetch: { + const auto sizeOfOneItem = APSInt( + APInt(ArgumentInt.getBitWidth(), SizeOfPointee.getQuantity(), false), + false); + // incrementing pointer by size which is not dividable by pointee size + // is UB and therefore disallowed + if ((ArgumentInt % sizeOfOneItem) != 0) + return false; + + ArgumentInt /= sizeOfOneItem; + } break; + default: + break; + } + + switch (E->getOp()) { + case AtomicExpr::AO__c11_atomic_fetch_add: + case AtomicExpr::AO__atomic_fetch_add: + case AtomicExpr::AO__atomic_add_fetch: + AtomicPtr.adjustOffsetAndIndex(Info, E, ArgumentInt, SizeOfPointee); + break; + case AtomicExpr::AO__c11_atomic_fetch_sub: + case AtomicExpr::AO__atomic_fetch_sub: + case AtomicExpr::AO__atomic_sub_fetch: + ArgumentInt.negate(); + AtomicPtr.adjustOffsetAndIndex(Info, E, ArgumentInt, SizeOfPointee); + break; + default: + return false; + } + + AtomicPtr.moveInto(Replacement); + } else { + // not float,int,pointer? + return false; + } + + // OP-fetch operation store result, not previous value + if (StoreToResultAfter) + Result = Replacement; + + // store result to atomic's storage + return handleAssignment(Info, E, AtomicStorageLV, AtomicValueTy, Replacement); +} + +static bool EvaluateAtomicCompareExchange(const AtomicExpr *E, APValue &Result, + EvalInfo &Info) { + EvaluateAtomicWeak(E, Info); + + // dereference _Atomic * (atomic value) + LValue AtomicLV; + QualType AtomicTy = E->getValueType(); + if (!EvaluatePointer(E->getPtr(), AtomicLV, Info)) + return false; + + // dereference T * (expected value) + LValue ExpectedLV; + QualType ExpectedTy = E->getVal1()->getType()->getPointeeType(); + if (!EvaluatePointer(E->getVal1(), ExpectedLV, Info)) + return false; + + // get values for atomic and expected + APValue AtomicVal; + APValue ExpectedVal; + + // convert pointer to value + if (!handleLValueToRValueConversion(Info, E->getPtr(), AtomicTy, AtomicLV, + AtomicVal)) + return false; + + if (!handleLValueToRValueConversion(Info, E->getVal1(), ExpectedTy, + ExpectedLV, ExpectedVal)) + return false; + + bool DoExchange = false; + + // compare atomic and friends + if (AtomicTy->isIntegralOrEnumerationType() && + ExpectedTy->isIntegralOrEnumerationType()) { + const APSInt AtomicInt = AtomicVal.getInt(); + const APSInt ExpectedInt = ExpectedVal.getInt(); + if (AtomicInt == ExpectedInt) + DoExchange = true; + + } else if (AtomicTy->isRealFloatingType() && + ExpectedTy->isRealFloatingType()) { + const APFloat AtomicFlt = AtomicVal.getFloat(); + const APFloat ExpectedFlt = ExpectedVal.getFloat(); + if (AtomicFlt == ExpectedFlt) + DoExchange = true; + + } else if (AtomicTy->isPointerType() && ExpectedTy->isPointerType()) { + // get LValue of objects pointed to + LValue LHS; + LHS.setFrom(Info.Ctx, AtomicVal); + + LValue RHS; + RHS.setFrom(Info.Ctx, ExpectedVal); + + if (HasSameBase(LHS, RHS)) { + const CharUnits &LHSOffset = LHS.getLValueOffset(); + const CharUnits &RHSOffset = RHS.getLValueOffset(); + + const unsigned PtrSize = Info.Ctx.getTypeSize(AtomicTy); + assert(PtrSize <= 64 && "Pointer width is larger than expected"); + const uint64_t Mask = ~0ULL >> (64 - PtrSize); + + const uint64_t CompareLHS = LHSOffset.getQuantity() & Mask; + const uint64_t CompareRHS = RHSOffset.getQuantity() & Mask; + + if (CompareLHS == CompareRHS) + DoExchange = true; + + } else { + + // it's implementation-defined to compare distinct literals + // it's not constant-evaluation + if ((IsLiteralLValue(LHS) || IsLiteralLValue(RHS)) && LHS.Base && + RHS.Base) + return false; + + if (IsWeakLValue(LHS) || IsWeakLValue(RHS)) + return false; + + if ((!LHS.Base && !LHS.Offset.isZero()) || + (!RHS.Base && !RHS.Offset.isZero())) + return false; + + if (LHS.Base && LHS.Offset.isZero() && + isOnePastTheEndOfCompleteObject(Info.Ctx, RHS)) + return false; + + if (RHS.Base && RHS.Offset.isZero() && + isOnePastTheEndOfCompleteObject(Info.Ctx, LHS)) + return false; + + if ((RHS.Base && isZeroSized(RHS)) || (LHS.Base && isZeroSized(LHS))) + return false; + + // after all it's a different object + DoExchange = false; + } + + } else { + return false; + } + + APValue Replacement; + if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange) { + // GCC atomic_compare_exchange provides pointer and not value + LValue ReplacementLV; + if (!EvaluatePointer(E->getVal2(), ReplacementLV, Info)) + return false; + + if (!handleLValueToRValueConversion(Info, E->getVal2(), + E->getVal2()->getType(), ReplacementLV, + Replacement)) + return false; + + } else { + if (!Evaluate(Replacement, Info, E->getVal2())) + return false; + } + + if (DoExchange) { + // if values are same do the exchange with replacement value + // but first I must evaluate the replacement value + + // and assign it to atomic + if (!handleAssignment(Info, E, AtomicLV, AtomicTy, Replacement)) + return false; + } + + // to expected pointer I need to put previous value in atomic + if (!handleAssignment(Info, E, ExpectedLV, ExpectedTy, AtomicVal)) + return false; + + // and return boolean if I did the exchange + Result = APValue(Info.Ctx.MakeIntValue(DoExchange, E->getType())); + return true; +} + diff --git a/clang/test/SemaCXX/atomic-constexpr-c11-builtins.cpp b/clang/test/SemaCXX/atomic-constexpr-c11-builtins.cpp new file mode 100644 index 000000000000000..9ff775d37bd52e0 --- /dev/null +++ b/clang/test/SemaCXX/atomic-constexpr-c11-builtins.cpp @@ -0,0 +1,326 @@ +// RUN: %clang_cc1 -std=c++20 %s + +// expected-no-diagnostics + +constexpr int int_min = -2147483648; +constexpr int int_max = 2147483647; + +const int array[2] = {1,2}; +const char small_array[2] = {1,2}; + +template struct identity { + using type = T; +}; + +template using do_not_deduce = typename identity::type; + +// -- SIGNAL and THREAD fence -- +consteval int fence_test(int v) { + // both are no-op in constexpr + __c11_atomic_thread_fence(__ATOMIC_ACQUIRE); + __c11_atomic_signal_fence(__ATOMIC_ACQUIRE); + return v; +} + +static_assert(fence_test(42) == 42); + +// -- LOAD -- + +template consteval T init(T value) { + auto * av = new _Atomic T; + __c11_atomic_init(av, value); + auto result = __c11_atomic_load(av, __ATOMIC_RELAXED); + delete av; + return result; +} + +// integers +static_assert(init(true) == true); +static_assert(init(false) == false); + +static_assert(init(42) == 42); +static_assert(init(-128) == -128); + +static_assert(init(42u) == 42u); +static_assert(init(0xFFFFFFFFu) == 0xFFFFFFFFu); + +// floats +static_assert(init(42.3) == 42.3); +static_assert(init(42.3f) == 42.3f); + +// pointers +static_assert(init(&array[0]) == &array[0]); +static_assert(init(&small_array[1]) == &small_array[1]); + +// -- LOAD -- + +template consteval T load(T value) { + _Atomic(T) av = value; + return __c11_atomic_load(&av, __ATOMIC_RELAXED); +} + +// integers +static_assert(load(true) == true); +static_assert(load(false) == false); + +static_assert(load(42) == 42); +static_assert(load(-128) == -128); + +static_assert(load(42u) == 42u); +static_assert(load(0xFFFFFFFFu) == 0xFFFFFFFFu); + +// floats +static_assert(load(42.3) == 42.3); +static_assert(load(42.3f) == 42.3f); + +// pointers +static_assert(load(&array[0]) == &array[0]); +static_assert(load(&small_array[1]) == &small_array[1]); + +// -- STORE -- + +template consteval T store(T value) { + _Atomic(T) av = T{}; + __c11_atomic_store(&av, value, __ATOMIC_RELAXED); + return __c11_atomic_load(&av, __ATOMIC_RELAXED); +} + +// integers +static_assert(store(true) == true); +static_assert(store(false) == false); + +static_assert(store(42) == 42); +static_assert(store(-128) == -128); + +static_assert(store(42u) == 42u); +static_assert(store(0xFFFFFFFFu) == 0xFFFFFFFFu); + +// floats +static_assert(store(42.3) == 42.3); +static_assert(store(42.3f) == 42.3f); + +// pointers +static_assert(store(&array[0]) == &array[0]); +static_assert(store(&small_array[1]) == &small_array[1]); + +// -- EXCHANGE -- +template struct two_values { + T before; + T after; + constexpr friend bool operator==(two_values, two_values) = default; +}; + +template consteval auto exchange(T value, do_not_deduce replacement) -> two_values { + _Atomic(T) av = T{value}; + T previous = __c11_atomic_exchange(&av, replacement, __ATOMIC_RELAXED); + return two_values{previous, __c11_atomic_load(&av, __ATOMIC_RELAXED)}; +} + +// integers +static_assert(exchange(true,false) == two_values{true, false}); +static_assert(exchange(false,true) == two_values{false, true}); + +static_assert(exchange(10,42) == two_values{10,42}); +static_assert(exchange(14,-128) == two_values{14,-128}); + + +static_assert(exchange(56u,42u) == two_values{56u,42u}); +static_assert(exchange(0xFFu, 0xFFFFFFFFu) == two_values{0xFFu,0xFFFFFFFFu}); + +// floats +static_assert(exchange(42.3,1.2) == two_values{42.3,1.2}); +static_assert(exchange(42.3f,-16.7f) == two_values{42.3f, -16.7f}); + +// pointers +static_assert(exchange(&array[0], &array[1]) == two_values{&array[0],&array[1]}); +static_assert(exchange(&small_array[1], &small_array[0]) == two_values{&small_array[1], &small_array[0]}); + +// -- COMPARE-EXCHANGE -- +template struct comp_result { + bool success; + T output; + T after; + + constexpr comp_result(bool success_, T output_, do_not_deduce after_): success{success_}, output{output_}, after{after_} { } + + constexpr friend bool operator==(comp_result, comp_result) = default; +}; + +template constexpr auto comp_success(T output, do_not_deduce after) { + return comp_result{true, output, after}; +} + +template constexpr auto comp_failure(T after) { + return comp_result{false, after, after}; +} + +template consteval auto compare_exchange_weak(T original, do_not_deduce expected, do_not_deduce replacement) -> comp_result { + _Atomic(T) av = T{original}; + const bool success = __c11_atomic_compare_exchange_weak(&av, &expected, replacement, __ATOMIC_RELAXED, __ATOMIC_RELAXED); + return comp_result{success, expected, __c11_atomic_load(&av, __ATOMIC_RELAXED)}; +} + +// integers +static_assert(compare_exchange_weak(true, true, false) == comp_success(true, false)); +static_assert(compare_exchange_weak(false, false, true) == comp_success(false, true)); +static_assert(compare_exchange_weak(false, true, false) == comp_failure(false)); +static_assert(compare_exchange_weak(true, false, true) == comp_failure(true)); + +static_assert(compare_exchange_weak(10,10,42) == comp_success(10,42)); +static_assert(compare_exchange_weak(14,14,-128) == comp_success(14,-128)); +static_assert(compare_exchange_weak(-10,10,42) == comp_failure(-10)); +static_assert(compare_exchange_weak(-14,14,-128) == comp_failure(-14)); + +static_assert(compare_exchange_weak(56u, 56u,42u) == comp_success(56u,42u)); +static_assert(compare_exchange_weak(0xFFu, 0xFFu, 0xFFFFFFFFu) == comp_success(0xFFu,0xFFFFFFFFu)); +static_assert(compare_exchange_weak(3u, 56u,42u) == comp_failure(3u)); +static_assert(compare_exchange_weak(0xFu, 0xFFu, 0xFFFFFFFFu) == comp_failure(0xFu)); + +// floats +static_assert(compare_exchange_weak(42.3, 42.3,1.2) == comp_success(42.3,1.2)); +static_assert(compare_exchange_weak(42.3f, 42.3f,-16.7f) == comp_success(42.3f, -16.7f)); +static_assert(compare_exchange_weak(0.0, 42.3,1.2) == comp_failure(0.0)); +static_assert(compare_exchange_weak(0.0f, 42.3f,-16.7f) == comp_failure(0.0f)); + +// pointers +static_assert(compare_exchange_weak(&array[0], &array[0], &array[1]) == comp_success(&array[0],&array[1])); +static_assert(compare_exchange_weak(&small_array[1], &small_array[1], &small_array[0]) == comp_success(&small_array[1], &small_array[0])); +static_assert(compare_exchange_weak(&array[1], &array[0], &array[1]) == comp_failure(&array[1])); +static_assert(compare_exchange_weak(&small_array[0], &small_array[1], &small_array[0]) == comp_failure(&small_array[0])); + + +template consteval auto compare_exchange_strong(T original, do_not_deduce expected, do_not_deduce replacement) -> comp_result { + _Atomic(T) av = T{original}; + const bool success = __c11_atomic_compare_exchange_strong(&av, &expected, replacement, __ATOMIC_RELAXED, __ATOMIC_RELAXED); + return comp_result{success, expected, __c11_atomic_load(&av, __ATOMIC_RELAXED)}; +} + +// integers +static_assert(compare_exchange_strong(true, true, false) == comp_success(true, false)); +static_assert(compare_exchange_strong(false, false, true) == comp_success(false, true)); +static_assert(compare_exchange_strong(false, true, false) == comp_failure(false)); +static_assert(compare_exchange_strong(true, false, true) == comp_failure(true)); + +static_assert(compare_exchange_strong(10,10,42) == comp_success(10,42)); +static_assert(compare_exchange_strong(14,14,-128) == comp_success(14,-128)); +static_assert(compare_exchange_strong(-10,10,42) == comp_failure(-10)); +static_assert(compare_exchange_strong(-14,14,-128) == comp_failure(-14)); + +static_assert(compare_exchange_strong(56u, 56u,42u) == comp_success(56u,42u)); +static_assert(compare_exchange_strong(0xFFu, 0xFFu, 0xFFFFFFFFu) == comp_success(0xFFu,0xFFFFFFFFu)); +static_assert(compare_exchange_strong(3u, 56u,42u) == comp_failure(3u)); +static_assert(compare_exchange_strong(0xFu, 0xFFu, 0xFFFFFFFFu) == comp_failure(0xFu)); + +// floats +static_assert(compare_exchange_strong(42.3, 42.3,1.2) == comp_success(42.3,1.2)); +static_assert(compare_exchange_strong(42.3f, 42.3f,-16.7f) == comp_success(42.3f, -16.7f)); +static_assert(compare_exchange_strong(0.0, 42.3,1.2) == comp_failure(0.0)); +static_assert(compare_exchange_strong(0.0f, 42.3f,-16.7f) == comp_failure(0.0f)); + +// pointers +static_assert(compare_exchange_strong(&array[0], &array[0], &array[1]) == comp_success(&array[0],&array[1])); +static_assert(compare_exchange_strong(&small_array[1], &small_array[1], &small_array[0]) == comp_success(&small_array[1], &small_array[0])); +static_assert(compare_exchange_strong(&array[1], &array[0], &array[1]) == comp_failure(&array[1])); +static_assert(compare_exchange_strong(&small_array[0], &small_array[1], &small_array[0]) == comp_failure(&small_array[0])); + + +// --FETCH-OP-- +template consteval auto fetch_add(T original, Y arg) -> two_values { + _Atomic(T) av = T{original}; + const T result = __c11_atomic_fetch_add(&av, arg, __ATOMIC_RELAXED); + return two_values{result, __c11_atomic_load(&av, __ATOMIC_RELAXED)}; +} + +// integers +static_assert(fetch_add(false, 1) == two_values{false, true}); +static_assert(fetch_add(0, 100) == two_values{0, 100}); +static_assert(fetch_add(100, -50) == two_values{100, 50}); + +static_assert(fetch_add(int_max, 1) == two_values{int_max, int_min}); // overflow is defined for atomic +static_assert(fetch_add(int_min, -1) == two_values{int_min, int_max}); + +// floats +static_assert(fetch_add(10.3, 2.1) == two_values{10.3, 12.4}); +static_assert(fetch_add(10.3f, 2.1f) == two_values{10.3f, 12.4f}); + +// pointers +static_assert(fetch_add(&array[0], 1) == two_values{&array[0], &array[1]}); +static_assert(fetch_add(&small_array[0], 1) == two_values{&small_array[0], &small_array[1]}); +static_assert(fetch_add(&array[1], 0) == two_values{&array[1], &array[1]}); +static_assert(fetch_add(&small_array[1], 0) == two_values{&small_array[1], &small_array[1]}); +static_assert(fetch_add(&array[1], -1) == two_values{&array[1], &array[0]}); +static_assert(fetch_add(&small_array[1], -1) == two_values{&small_array[1], &small_array[0]}); + +template consteval auto fetch_sub(T original, Y arg) -> two_values { + _Atomic(T) av = T{original}; + const T result = __c11_atomic_fetch_sub(&av, arg, __ATOMIC_RELAXED); + return two_values{result, __c11_atomic_load(&av, __ATOMIC_RELAXED)}; +} + +// integers +static_assert(fetch_sub(true, 1) == two_values{true, false}); +static_assert(fetch_sub(0, 100) == two_values{0, -100}); +static_assert(fetch_sub(100, -50) == two_values{100, 150}); + +static_assert(fetch_sub(int_min, 1) == two_values{int_min, int_max}); // overflow is defined for atomic +static_assert(fetch_sub(int_max, -1) == two_values{int_max, int_min}); + +// floats +static_assert(fetch_sub(10.3, 2.3) == two_values{10.3, 8.0}); +static_assert(fetch_sub(10.3f, 2.3f) == two_values{10.3f, 8.0f}); + +// pointers +static_assert(fetch_sub(&array[1], 1) == two_values{&array[1], &array[0]}); +static_assert(fetch_sub(&small_array[1], 1) == two_values{&small_array[1], &small_array[0]}); +static_assert(fetch_sub(&array[1], 0) == two_values{&array[1], &array[1]}); +static_assert(fetch_sub(&small_array[1], 0) == two_values{&small_array[1], &small_array[1]}); +static_assert(fetch_sub(&array[0], -1) == two_values{&array[0], &array[1]}); +static_assert(fetch_sub(&small_array[0], -1) == two_values{&small_array[0], &small_array[1]}); + +template consteval auto fetch_and(T original, Y arg) -> two_values { + _Atomic(T) av = T{original}; + const T result = __c11_atomic_fetch_and(&av, arg, __ATOMIC_RELAXED); + return two_values{result, __c11_atomic_load(&av, __ATOMIC_RELAXED)}; +} + +template consteval auto fetch_or(T original, Y arg) -> two_values { + _Atomic(T) av = T{original}; + const T result = __c11_atomic_fetch_or(&av, arg, __ATOMIC_RELAXED); + return two_values{result, __c11_atomic_load(&av, __ATOMIC_RELAXED)}; +} + +template consteval auto fetch_xor(T original, Y arg) -> two_values { + _Atomic(T) av = T{original}; + const T result = __c11_atomic_fetch_xor(&av, arg, __ATOMIC_RELAXED); + return two_values{result, __c11_atomic_load(&av, __ATOMIC_RELAXED)}; +} + +template consteval auto fetch_nand(T original, Y arg) -> two_values { + _Atomic(T) av = T{original}; + const T result = __c11_atomic_fetch_nand(&av, arg, __ATOMIC_RELAXED); + return two_values{result, __c11_atomic_load(&av, __ATOMIC_RELAXED)}; +} + +static_assert(fetch_and(0b1101u, 0b1011u) == two_values{0b1101u, 0b1001u}); +static_assert(fetch_or(0b1101u, 0b1011u) == two_values{0b1101u, 0b1111u}); +static_assert(fetch_xor(0b1101u, 0b1011u) == two_values{0b1101u, 0b0110u}); +static_assert(fetch_nand(0b1001u, 0b1011u) == two_values{0b1001u, 0xFFFFFFF6u}); + +template consteval auto fetch_min(T original, T arg) -> two_values { + _Atomic(T) av = T{original}; + const T result = __c11_atomic_fetch_min(&av, arg, __ATOMIC_RELAXED); + return two_values{result, __c11_atomic_load(&av, __ATOMIC_RELAXED)}; +} + +template consteval auto fetch_max(T original, T arg) -> two_values { + _Atomic(T) av = T{original}; + const T result = __c11_atomic_fetch_max(&av, arg, __ATOMIC_RELAXED); + return two_values{result, __c11_atomic_load(&av, __ATOMIC_RELAXED)}; +} + +static_assert(fetch_max(10, 16) == two_values{10, 16}); +static_assert(fetch_max(16, 10) == two_values{16, 16}); + +static_assert(fetch_min(10, 16) == two_values{10, 10}); +static_assert(fetch_min(16, 10) == two_values{16, 10}); + diff --git a/clang/test/SemaCXX/atomic-constexpr-gcc-builtins.cpp b/clang/test/SemaCXX/atomic-constexpr-gcc-builtins.cpp new file mode 100644 index 000000000000000..a1d23294a2080e0 --- /dev/null +++ b/clang/test/SemaCXX/atomic-constexpr-gcc-builtins.cpp @@ -0,0 +1,504 @@ +// RUN: %clang_cc1 -std=c++20 %s + +// expected-no-diagnostics + +constexpr int int_min = -2147483648; +constexpr int int_max = 2147483647; + +const int array[2] = {1,2}; +const char small_array[2] = {1,2}; + +template struct identity { + using type = T; +}; + +template using do_not_deduce = typename identity::type; + +// -- SIGNAL and THREAD fence -- +consteval int fence_test(int v) { + // both are no-op in constexpr + __atomic_thread_fence(__ATOMIC_ACQUIRE); + __atomic_signal_fence(__ATOMIC_ACQUIRE); + return v; +} + +static_assert(fence_test(42) == 42); + +// -- LOAD -- + +template consteval T load(T value) { + T av = value; + T out{}; + __atomic_load(&av, &out, __ATOMIC_RELAXED); + return out; +} + +// integers +static_assert(load(true) == true); +static_assert(load(false) == false); + +static_assert(load(42) == 42); +static_assert(load(-128) == -128); + +static_assert(load(42u) == 42u); +static_assert(load(0xFFFFFFFFu) == 0xFFFFFFFFu); + +// pointers +static_assert(load(&array[0]) == &array[0]); +static_assert(load(&small_array[1]) == &small_array[1]); + +// -- LOAD-N -- + +template consteval T load_n(T value) { + T av = value; + return __atomic_load_n(&av, __ATOMIC_RELAXED); +} + +// integers +static_assert(load_n(true) == true); +static_assert(load_n(false) == false); + +static_assert(load_n(42) == 42); +static_assert(load_n(-128) == -128); + +static_assert(load_n(42u) == 42u); +static_assert(load_n(0xFFFFFFFFu) == 0xFFFFFFFFu); + +// pointers +static_assert(load_n(&array[0]) == &array[0]); +static_assert(load_n(&small_array[1]) == &small_array[1]); + + +// -- STORE -- + +template consteval T store(T value) { + T av = T{}; + __atomic_store(&av, &value, __ATOMIC_RELAXED); + return __atomic_load_n(&av, __ATOMIC_RELAXED); +} + +// integers +static_assert(store(true) == true); +static_assert(store(false) == false); + +static_assert(store(42) == 42); +static_assert(store(-128) == -128); + +static_assert(store(42u) == 42u); +static_assert(store(0xFFFFFFFFu) == 0xFFFFFFFFu); + +// pointers +static_assert(store(&array[0]) == &array[0]); +static_assert(store(&small_array[1]) == &small_array[1]); + +// -- STORE-N -- + +template consteval T store_n(T value) { + T av = T{}; + __atomic_store_n(&av, value, __ATOMIC_RELAXED); + return __atomic_load_n(&av, __ATOMIC_RELAXED); +} + +// integers +static_assert(store_n(true) == true); +static_assert(store_n(false) == false); + +static_assert(store_n(42) == 42); +static_assert(store_n(-128) == -128); + +static_assert(store_n(42u) == 42u); +static_assert(store_n(0xFFFFFFFFu) == 0xFFFFFFFFu); + +// pointers +static_assert(store_n(&array[0]) == &array[0]); +static_assert(store_n(&small_array[1]) == &small_array[1]); + +// -- EXCHANGE -- +template struct two_values { + T before; + T after; + constexpr friend bool operator==(two_values, two_values) = default; +}; + +template consteval auto exchange(T value, do_not_deduce replacement) -> two_values { + T av = T{value}; + T out{}; + __atomic_exchange(&av, &replacement, &out, __ATOMIC_RELAXED); + return two_values{out, __atomic_load_n(&av, __ATOMIC_RELAXED)}; +} + +// integers +static_assert(exchange(true,false) == two_values{true, false}); +static_assert(exchange(false,true) == two_values{false, true}); + +static_assert(exchange(10,42) == two_values{10,42}); +static_assert(exchange(14,-128) == two_values{14,-128}); + + +static_assert(exchange(56u,42u) == two_values{56u,42u}); +static_assert(exchange(0xFFu, 0xFFFFFFFFu) == two_values{0xFFu,0xFFFFFFFFu}); + +// -- EXCHANGE-N -- +template consteval auto exchange_n(T value, do_not_deduce replacement) -> two_values { + T av = T{value}; + T previous = __atomic_exchange_n(&av, replacement, __ATOMIC_RELAXED); + return two_values{previous, __atomic_load_n(&av, __ATOMIC_RELAXED)}; +} + +// integers +static_assert(exchange_n(true,false) == two_values{true, false}); +static_assert(exchange_n(false,true) == two_values{false, true}); + +static_assert(exchange_n(10,42) == two_values{10,42}); +static_assert(exchange_n(14,-128) == two_values{14,-128}); + + +static_assert(exchange_n(56u,42u) == two_values{56u,42u}); +static_assert(exchange_n(0xFFu, 0xFFFFFFFFu) == two_values{0xFFu,0xFFFFFFFFu}); + +// pointers +static_assert(exchange_n(&array[0], &array[1]) == two_values{&array[0],&array[1]}); +static_assert(exchange_n(&small_array[1], &small_array[0]) == two_values{&small_array[1], &small_array[0]}); + +// -- COMPARE-EXCHANGE -- +template struct comp_result { + bool success; + T output; + T after; + + constexpr comp_result(bool success_, T output_, do_not_deduce after_): success{success_}, output{output_}, after{after_} { } + + constexpr friend bool operator==(comp_result, comp_result) = default; +}; + +template constexpr auto comp_success(T output, do_not_deduce after) { + return comp_result{true, output, after}; +} + +template constexpr auto comp_failure(T after) { + return comp_result{false, after, after}; +} + +template consteval auto compare_exchange_weak(T original, do_not_deduce expected, do_not_deduce replacement) -> comp_result { + T av = T{original}; + const bool success = __atomic_compare_exchange(&av, &expected, &replacement, false, __ATOMIC_RELAXED, __ATOMIC_RELAXED); + return comp_result{success, expected, __atomic_load_n(&av, __ATOMIC_RELAXED)}; +} + +// integers +static_assert(compare_exchange_weak(true, true, false) == comp_success(true, false)); +static_assert(compare_exchange_weak(false, false, true) == comp_success(false, true)); +static_assert(compare_exchange_weak(false, true, false) == comp_failure(false)); +static_assert(compare_exchange_weak(true, false, true) == comp_failure(true)); + +static_assert(compare_exchange_weak(10,10,42) == comp_success(10,42)); +static_assert(compare_exchange_weak(14,14,-128) == comp_success(14,-128)); +static_assert(compare_exchange_weak(-10,10,42) == comp_failure(-10)); +static_assert(compare_exchange_weak(-14,14,-128) == comp_failure(-14)); + +static_assert(compare_exchange_weak(56u, 56u,42u) == comp_success(56u,42u)); +static_assert(compare_exchange_weak(0xFFu, 0xFFu, 0xFFFFFFFFu) == comp_success(0xFFu,0xFFFFFFFFu)); +static_assert(compare_exchange_weak(3u, 56u,42u) == comp_failure(3u)); +static_assert(compare_exchange_weak(0xFu, 0xFFu, 0xFFFFFFFFu) == comp_failure(0xFu)); + +// pointers +static_assert(compare_exchange_weak(&array[0], &array[0], &array[1]) == comp_success(&array[0],&array[1])); +static_assert(compare_exchange_weak(&small_array[1], &small_array[1], &small_array[0]) == comp_success(&small_array[1], &small_array[0])); +static_assert(compare_exchange_weak(&array[1], &array[0], &array[1]) == comp_failure(&array[1])); +static_assert(compare_exchange_weak(&small_array[0], &small_array[1], &small_array[0]) == comp_failure(&small_array[0])); + + +template consteval auto compare_exchange_strong(T original, do_not_deduce expected, do_not_deduce replacement) -> comp_result { + T av = T{original}; + const bool success = __atomic_compare_exchange(&av, &expected, &replacement, true, __ATOMIC_RELAXED, __ATOMIC_RELAXED); + return comp_result{success, expected, __atomic_load_n(&av, __ATOMIC_RELAXED)}; +} + +// integers +static_assert(compare_exchange_strong(true, true, false) == comp_success(true, false)); +static_assert(compare_exchange_strong(false, false, true) == comp_success(false, true)); +static_assert(compare_exchange_strong(false, true, false) == comp_failure(false)); +static_assert(compare_exchange_strong(true, false, true) == comp_failure(true)); + +static_assert(compare_exchange_strong(10,10,42) == comp_success(10,42)); +static_assert(compare_exchange_strong(14,14,-128) == comp_success(14,-128)); +static_assert(compare_exchange_strong(-10,10,42) == comp_failure(-10)); +static_assert(compare_exchange_strong(-14,14,-128) == comp_failure(-14)); + +static_assert(compare_exchange_strong(56u, 56u,42u) == comp_success(56u,42u)); +static_assert(compare_exchange_strong(0xFFu, 0xFFu, 0xFFFFFFFFu) == comp_success(0xFFu,0xFFFFFFFFu)); +static_assert(compare_exchange_strong(3u, 56u,42u) == comp_failure(3u)); +static_assert(compare_exchange_strong(0xFu, 0xFFu, 0xFFFFFFFFu) == comp_failure(0xFu)); + +// pointers +static_assert(compare_exchange_strong(&array[0], &array[0], &array[1]) == comp_success(&array[0],&array[1])); +static_assert(compare_exchange_strong(&small_array[1], &small_array[1], &small_array[0]) == comp_success(&small_array[1], &small_array[0])); +static_assert(compare_exchange_strong(&array[1], &array[0], &array[1]) == comp_failure(&array[1])); +static_assert(compare_exchange_strong(&small_array[0], &small_array[1], &small_array[0]) == comp_failure(&small_array[0])); + +// --COMPARE-EXCHANGE-N-- + +template consteval auto compare_exchange_weak_n(T original, do_not_deduce expected, do_not_deduce replacement) -> comp_result { + T av = T{original}; + const bool success = __atomic_compare_exchange_n(&av, &expected, replacement, false, __ATOMIC_RELAXED, __ATOMIC_RELAXED); + return comp_result{success, expected, __atomic_load_n(&av, __ATOMIC_RELAXED)}; +} + +// integers +static_assert(compare_exchange_weak_n(true, true, false) == comp_success(true, false)); +static_assert(compare_exchange_weak_n(false, false, true) == comp_success(false, true)); +static_assert(compare_exchange_weak_n(false, true, false) == comp_failure(false)); +static_assert(compare_exchange_weak_n(true, false, true) == comp_failure(true)); + +static_assert(compare_exchange_weak_n(10,10,42) == comp_success(10,42)); +static_assert(compare_exchange_weak_n(14,14,-128) == comp_success(14,-128)); +static_assert(compare_exchange_weak_n(-10,10,42) == comp_failure(-10)); +static_assert(compare_exchange_weak_n(-14,14,-128) == comp_failure(-14)); + +static_assert(compare_exchange_weak_n(56u, 56u,42u) == comp_success(56u,42u)); +static_assert(compare_exchange_weak_n(0xFFu, 0xFFu, 0xFFFFFFFFu) == comp_success(0xFFu,0xFFFFFFFFu)); +static_assert(compare_exchange_weak_n(3u, 56u,42u) == comp_failure(3u)); +static_assert(compare_exchange_weak_n(0xFu, 0xFFu, 0xFFFFFFFFu) == comp_failure(0xFu)); + +// pointers +static_assert(compare_exchange_weak_n(&array[0], &array[0], &array[1]) == comp_success(&array[0],&array[1])); +static_assert(compare_exchange_weak_n(&small_array[1], &small_array[1], &small_array[0]) == comp_success(&small_array[1], &small_array[0])); +static_assert(compare_exchange_weak_n(&array[1], &array[0], &array[1]) == comp_failure(&array[1])); +static_assert(compare_exchange_weak_n(&small_array[0], &small_array[1], &small_array[0]) == comp_failure(&small_array[0])); + + +template consteval auto compare_exchange_strong_n(T original, do_not_deduce expected, do_not_deduce replacement) -> comp_result { + T av = T{original}; + const bool success = __atomic_compare_exchange_n(&av, &expected, replacement, true, __ATOMIC_RELAXED, __ATOMIC_RELAXED); + return comp_result{success, expected, __atomic_load_n(&av, __ATOMIC_RELAXED)}; +} + +// integers +static_assert(compare_exchange_strong_n(true, true, false) == comp_success(true, false)); +static_assert(compare_exchange_strong_n(false, false, true) == comp_success(false, true)); +static_assert(compare_exchange_strong_n(false, true, false) == comp_failure(false)); +static_assert(compare_exchange_strong_n(true, false, true) == comp_failure(true)); + +static_assert(compare_exchange_strong_n(10,10,42) == comp_success(10,42)); +static_assert(compare_exchange_strong_n(14,14,-128) == comp_success(14,-128)); +static_assert(compare_exchange_strong_n(-10,10,42) == comp_failure(-10)); +static_assert(compare_exchange_strong_n(-14,14,-128) == comp_failure(-14)); + +static_assert(compare_exchange_strong_n(56u, 56u,42u) == comp_success(56u,42u)); +static_assert(compare_exchange_strong_n(0xFFu, 0xFFu, 0xFFFFFFFFu) == comp_success(0xFFu,0xFFFFFFFFu)); +static_assert(compare_exchange_strong_n(3u, 56u,42u) == comp_failure(3u)); +static_assert(compare_exchange_strong_n(0xFu, 0xFFu, 0xFFFFFFFFu) == comp_failure(0xFu)); + +// pointers +static_assert(compare_exchange_strong_n(&array[0], &array[0], &array[1]) == comp_success(&array[0],&array[1])); +static_assert(compare_exchange_strong_n(&small_array[1], &small_array[1], &small_array[0]) == comp_success(&small_array[1], &small_array[0])); +static_assert(compare_exchange_strong_n(&array[1], &array[0], &array[1]) == comp_failure(&array[1])); +static_assert(compare_exchange_strong_n(&small_array[0], &small_array[1], &small_array[0]) == comp_failure(&small_array[0])); + +// --FETCH-OP-- +template consteval auto fetch_add(T original, Y arg) -> two_values { + T av = T{original}; + const T result = __atomic_fetch_add(&av, arg, __ATOMIC_RELAXED); + return two_values{result, __atomic_load_n(&av, __ATOMIC_RELAXED)}; +} + +template consteval auto fetch_add_ptr(T original, Y arg) -> two_values { + T av = T{original}; + constexpr auto pointee_size = sizeof(*static_cast(nullptr)); + arg *= pointee_size; + const T result = __atomic_fetch_add(&av, arg, __ATOMIC_RELAXED); + return two_values{result, __atomic_load_n(&av, __ATOMIC_RELAXED)}; +} + +// integers +static_assert(fetch_add(false, 1) == two_values{false, true}); +static_assert(fetch_add(0, 100) == two_values{0, 100}); +static_assert(fetch_add(100, -50) == two_values{100, 50}); + +static_assert(fetch_add(int_max, 1) == two_values{int_max, int_min}); // overflow is defined for atomic +static_assert(fetch_add(int_min, -1) == two_values{int_min, int_max}); + +// pointers +static_assert(fetch_add_ptr(&array[0], 1) == two_values{&array[0], &array[1]}); +static_assert(fetch_add_ptr(&small_array[0], 1) == two_values{&small_array[0], &small_array[1]}); +static_assert(fetch_add_ptr(&array[1], 0) == two_values{&array[1], &array[1]}); +static_assert(fetch_add_ptr(&small_array[1], 0) == two_values{&small_array[1], &small_array[1]}); +static_assert(fetch_add_ptr(&array[1], -1) == two_values{&array[1], &array[0]}); +static_assert(fetch_add_ptr(&small_array[1], -1) == two_values{&small_array[1], &small_array[0]}); + +template consteval auto fetch_sub(T original, Y arg) -> two_values { + T av = T{original}; + const T result = __atomic_fetch_sub(&av, arg, __ATOMIC_RELAXED); + return two_values{result, __atomic_load_n(&av, __ATOMIC_RELAXED)}; +} + +template consteval auto fetch_sub_ptr(T original, Y arg) -> two_values { + T av = T{original}; + constexpr auto pointee_size = sizeof(*static_cast(nullptr)); + arg *= pointee_size; + const T result = __atomic_fetch_sub(&av, arg, __ATOMIC_RELAXED); + return two_values{result, __atomic_load_n(&av, __ATOMIC_RELAXED)}; +} + +// integers +static_assert(fetch_sub(true, 1) == two_values{true, false}); +static_assert(fetch_sub(0, 100) == two_values{0, -100}); +static_assert(fetch_sub(100, -50) == two_values{100, 150}); + +static_assert(fetch_sub(int_min, 1) == two_values{int_min, int_max}); // overflow is defined for atomic +static_assert(fetch_sub(int_max, -1) == two_values{int_max, int_min}); + +// pointers +static_assert(fetch_sub_ptr(&array[1], 1) == two_values{&array[1], &array[0]}); +static_assert(fetch_sub_ptr(&small_array[1], 1) == two_values{&small_array[1], &small_array[0]}); +static_assert(fetch_sub_ptr(&array[1], 0) == two_values{&array[1], &array[1]}); +static_assert(fetch_sub_ptr(&small_array[1], 0) == two_values{&small_array[1], &small_array[1]}); +static_assert(fetch_sub_ptr(&array[0], -1) == two_values{&array[0], &array[1]}); +static_assert(fetch_sub_ptr(&small_array[0], -1) == two_values{&small_array[0], &small_array[1]}); + +template consteval auto fetch_and(T original, Y arg) -> two_values { + T av = T{original}; + const T result = __atomic_fetch_and(&av, arg, __ATOMIC_RELAXED); + return two_values{result, __atomic_load_n(&av, __ATOMIC_RELAXED)}; +} + +template consteval auto fetch_or(T original, Y arg) -> two_values { + T av = T{original}; + const T result = __atomic_fetch_or(&av, arg, __ATOMIC_RELAXED); + return two_values{result, __atomic_load_n(&av, __ATOMIC_RELAXED)}; +} + +template consteval auto fetch_xor(T original, Y arg) -> two_values { + T av = T{original}; + const T result = __atomic_fetch_xor(&av, arg, __ATOMIC_RELAXED); + return two_values{result, __atomic_load_n(&av, __ATOMIC_RELAXED)}; +} + +template consteval auto fetch_nand(T original, Y arg) -> two_values { + T av = T{original}; + const T result = __atomic_fetch_nand(&av, arg, __ATOMIC_RELAXED); + return two_values{result, __atomic_load_n(&av, __ATOMIC_RELAXED)}; +} + +static_assert(fetch_and(0b1101u, 0b1011u) == two_values{0b1101u, 0b1001u}); +static_assert(fetch_or(0b1101u, 0b1011u) == two_values{0b1101u, 0b1111u}); +static_assert(fetch_xor(0b1101u, 0b1011u) == two_values{0b1101u, 0b0110u}); +static_assert(fetch_nand(0b1001u, 0b1011u) == two_values{0b1001u, 0xFFFFFFF6u}); + +template consteval auto fetch_min(T original, T arg) -> two_values { + T av = T{original}; + const T result = __atomic_fetch_min(&av, arg, __ATOMIC_RELAXED); + return two_values{result, __atomic_load_n(&av, __ATOMIC_RELAXED)}; +} + +template consteval auto fetch_max(T original, T arg) -> two_values { + T av = T{original}; + const T result = __atomic_fetch_max(&av, arg, __ATOMIC_RELAXED); + return two_values{result, __atomic_load_n(&av, __ATOMIC_RELAXED)}; +} + +static_assert(fetch_max(10, 16) == two_values{10, 16}); +static_assert(fetch_max(16, 10) == two_values{16, 16}); + +static_assert(fetch_min(10, 16) == two_values{10, 10}); +static_assert(fetch_min(16, 10) == two_values{16, 10}); + +// --OP-FETCHP-- +template consteval auto add_fetch(T original, Y arg) -> T { + T av = T{original}; + return __atomic_add_fetch(&av, arg, __ATOMIC_RELAXED); +} + +template consteval auto add_fetch_ptr(T original, Y arg) -> T { + T av = T{original}; + constexpr auto pointee_size = sizeof(*static_cast(nullptr)); + arg *= pointee_size; + return __atomic_add_fetch(&av, arg, __ATOMIC_RELAXED); +} + +// integers +static_assert(add_fetch(false, 1) == true); +static_assert(add_fetch(0, 100) == 100); +static_assert(add_fetch(100, -50) == 50); + +static_assert(add_fetch(int_max, 1) == int_min); // overflow is defined for atomic +static_assert(add_fetch(int_min, -1) == int_max); + +// pointers +static_assert(add_fetch_ptr(&array[0], 1) == &array[1]); +static_assert(add_fetch_ptr(&small_array[0], 1) == &small_array[1]); +static_assert(add_fetch_ptr(&array[1], 0) == &array[1]); +static_assert(add_fetch_ptr(&small_array[1], 0) == &small_array[1]); +static_assert(add_fetch_ptr(&array[1], -1) == &array[0]); +static_assert(add_fetch_ptr(&small_array[1], -1) ==&small_array[0]); + +template consteval auto sub_fetch(T original, Y arg) -> T { + T av = T{original}; + return __atomic_sub_fetch(&av, arg, __ATOMIC_RELAXED); +} + +template consteval auto sub_fetch_ptr(T original, Y arg) -> T { + T av = T{original}; + constexpr auto pointee_size = sizeof(*static_cast(nullptr)); + arg *= pointee_size; + return __atomic_sub_fetch(&av, arg, __ATOMIC_RELAXED); +} + +// integers +static_assert(sub_fetch(true, 1) == false); +static_assert(sub_fetch(0, 100) == -100); +static_assert(sub_fetch(100, -50) == 150); + +static_assert(sub_fetch(int_min, 1) == int_max); // overflow is defined for atomic +static_assert(sub_fetch(int_max, -1) == int_min); + +// pointers +static_assert(sub_fetch_ptr(&array[1], 1) == &array[0]); +static_assert(sub_fetch_ptr(&small_array[1], 1) == &small_array[0]); +static_assert(sub_fetch_ptr(&array[1], 0) == &array[1]); +static_assert(sub_fetch_ptr(&small_array[1], 0) == &small_array[1]); +static_assert(sub_fetch_ptr(&array[0], -1) == &array[1]); +static_assert(sub_fetch_ptr(&small_array[0], -1) == &small_array[1]); + +template consteval auto and_fetch(T original, Y arg) -> T { + T av = T{original}; + return __atomic_and_fetch(&av, arg, __ATOMIC_RELAXED); +} + +template consteval auto or_fetch(T original, Y arg) -> T { + T av = T{original}; + return __atomic_or_fetch(&av, arg, __ATOMIC_RELAXED); +} + +template consteval auto xor_fetch(T original, Y arg) -> T { + T av = T{original}; + return __atomic_xor_fetch(&av, arg, __ATOMIC_RELAXED); +} + +template consteval auto nand_fetch(T original, Y arg) -> T { + T av = T{original}; + return __atomic_nand_fetch(&av, arg, __ATOMIC_RELAXED); +} + +static_assert(and_fetch(0b1101u, 0b1011u) == 0b1001u); +static_assert(or_fetch(0b1101u, 0b1011u) == 0b1111u); +static_assert(xor_fetch(0b1101u, 0b1011u) == 0b0110u); +static_assert(nand_fetch(0b1001u, 0b1011u) == 0xFFFFFFF6u); + +template consteval auto min_fetch(T original, T arg) -> T { + T av = T{original}; + return __atomic_min_fetch(&av, arg, __ATOMIC_RELAXED); +} + +template consteval auto max_fetch(T original, T arg) -> T { + T av = T{original}; + return __atomic_max_fetch(&av, arg, __ATOMIC_RELAXED); +} + +static_assert(max_fetch(10, 16) == 16); +static_assert(max_fetch(16, 10) == 16); + +static_assert(min_fetch(10, 16) == 10); +static_assert(min_fetch(16, 10) == 10); + +