diff --git a/src/core.jl b/src/core.jl index 3d82836..881152b 100644 --- a/src/core.jl +++ b/src/core.jl @@ -48,6 +48,16 @@ for (op, rmwop) in OP_RMW_TABLE first(UnsafeAtomics.modify!(ptr, $op, x, ord)) end +const ATOMIC_INTRINSICS = isdefined(Core.Intrinsics, :atomic_pointerref) + +if VERSION >= v"1.12.0-DEV.161" && Int == Int64 +const MAX_ATOMIC_SIZE = 16 +const MAX_POINTERATOMIC_SIZE = 16 +else +const MAX_ATOMIC_SIZE = 8 +const MAX_POINTERATOMIC_SIZE = 8 +end + # Based on: https://github.com/JuliaLang/julia/blob/v1.6.3/base/atomics.jl for typ in (inttypes..., floattypes...) lt = llvmtypes[typ] @@ -56,35 +66,48 @@ for typ in (inttypes..., floattypes...) for ord in orderings ord in (release, acq_rel) && continue - @eval function UnsafeAtomics.load(x::Ptr{$typ}, ::$(typeof(ord))) - return llvmcall( - $(""" - %ptr = inttoptr i$WORD_SIZE %0 to $lt* - %rv = load atomic $rt %ptr $ord, align $(sizeof(typ)) - ret $lt %rv - """), - $typ, - Tuple{Ptr{$typ}}, - x, - ) + if ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE + @eval function UnsafeAtomics.load(x::Ptr{$typ}, ::$(typeof(ord))) + return Core.Intrinsics.atomic_pointerref(x, base_ordering($ord)) + end + else + @eval function UnsafeAtomics.load(x::Ptr{$typ}, ::$(typeof(ord))) + return llvmcall( + $(""" + %ptr = inttoptr i$WORD_SIZE %0 to $lt* + %rv = load atomic $rt %ptr $ord, align $(sizeof(typ)) + ret $lt %rv + """), + $typ, + Tuple{Ptr{$typ}}, + x, + ) + end end end for ord in orderings ord in (acquire, acq_rel) && continue - @eval function UnsafeAtomics.store!(x::Ptr{$typ}, v::$typ, ::$(typeof(ord))) - return llvmcall( - $(""" - %ptr = inttoptr i$WORD_SIZE %0 to $lt* - store atomic $lt %1, $lt* %ptr $ord, align $(sizeof(typ)) - ret void - """), - Cvoid, - Tuple{Ptr{$typ},$typ}, - x, - v, - ) + if ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE + @eval function UnsafeAtomics.store!(x::Ptr{$typ}, v::$typ, ::$(typeof(ord))) + Core.Intrinsics.atomic_pointerset(x, v, base_ordering($ord)) + return nothing + end + else + @eval function UnsafeAtomics.store!(x::Ptr{$typ}, v::$typ, ::$(typeof(ord))) + return llvmcall( + $(""" + %ptr = inttoptr i$WORD_SIZE %0 to $lt* + store atomic $lt %1, $lt* %ptr $ord, align $(sizeof(typ)) + ret void + """), + Cvoid, + Tuple{Ptr{$typ},$typ}, + x, + v, + ) + end end end @@ -93,37 +116,55 @@ for typ in (inttypes..., floattypes...) typ <: AbstractFloat && break - @eval function UnsafeAtomics.cas!( - x::Ptr{$typ}, - cmp::$typ, - new::$typ, - ::$(typeof(success_ordering)), - ::$(typeof(failure_ordering)), - ) - success = Ref{Int8}() - GC.@preserve success begin - old = llvmcall( - $( - """ - %ptr = inttoptr i$WORD_SIZE %0 to $lt* - %rs = cmpxchg $lt* %ptr, $lt %1, $lt %2 $success_ordering $failure_ordering - %rv = extractvalue { $lt, i1 } %rs, 0 - %s1 = extractvalue { $lt, i1 } %rs, 1 - %s8 = zext i1 %s1 to i8 - %sptr = inttoptr i$WORD_SIZE %3 to i8* - store i8 %s8, i8* %sptr - ret $lt %rv - """ - ), - $typ, - Tuple{Ptr{$typ},$typ,$typ,Ptr{Int8}}, + if ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE + @eval function UnsafeAtomics.cas!( + x::Ptr{$typ}, + cmp::$typ, + new::$typ, + ::$(typeof(success_ordering)), + ::$(typeof(failure_ordering)), + ) + return Core.Intrinsics.atomic_pointerreplace( x, cmp, new, - Ptr{Int8}(pointer_from_objref(success)), + base_ordering($success_ordering), + base_ordering($failure_ordering) ) end - return (old = old, success = !iszero(success[])) + else + @eval function UnsafeAtomics.cas!( + x::Ptr{$typ}, + cmp::$typ, + new::$typ, + ::$(typeof(success_ordering)), + ::$(typeof(failure_ordering)), + ) + success = Ref{Int8}() + GC.@preserve success begin + old = llvmcall( + $( + """ + %ptr = inttoptr i$WORD_SIZE %0 to $lt* + %rs = cmpxchg $lt* %ptr, $lt %1, $lt %2 $success_ordering $failure_ordering + %rv = extractvalue { $lt, i1 } %rs, 0 + %s1 = extractvalue { $lt, i1 } %rs, 1 + %s8 = zext i1 %s1 to i8 + %sptr = inttoptr i$WORD_SIZE %3 to i8* + store i8 %s8, i8* %sptr + ret $lt %rv + """ + ), + $typ, + Tuple{Ptr{$typ},$typ,$typ,Ptr{Int8}}, + x, + cmp, + new, + Ptr{Int8}(pointer_from_objref(success)), + ) + end + return (old = old, success = !iszero(success[])) + end end end @@ -144,24 +185,36 @@ for typ in (inttypes..., floattypes...) end end for ord in orderings - @eval function UnsafeAtomics.modify!( - x::Ptr{$typ}, - ::typeof($op), - v::$typ, - ::$(typeof(ord)), - ) - old = llvmcall( - $(""" - %ptr = inttoptr i$WORD_SIZE %0 to $lt* - %rv = atomicrmw $rmw $lt* %ptr, $lt %1 $ord - ret $lt %rv - """), - $typ, - Tuple{Ptr{$typ},$typ}, - x, - v, + # Enable this code iff https://github.com/JuliaLang/julia/pull/45122 get's merged + if false && ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE + @eval function UnsafeAtomics.modify!( + x::Ptr{$typ}, + op::typeof($op), + v::$typ, + ::$(typeof(ord)), + ) + return Core.Intrinsics.atomic_pointermodify(x, op, v, base_ordering($ord)) + end + else + @eval function UnsafeAtomics.modify!( + x::Ptr{$typ}, + ::typeof($op), + v::$typ, + ::$(typeof(ord)), ) - return old => $op(old, v) + old = llvmcall( + $(""" + %ptr = inttoptr i$WORD_SIZE %0 to $lt* + %rv = atomicrmw $rmw $lt* %ptr, $lt %1 $ord + ret $lt %rv + """), + $typ, + Tuple{Ptr{$typ},$typ}, + x, + v, + ) + return old => $op(old, v) + end end end end diff --git a/src/orderings.jl b/src/orderings.jl index 39cd3cc..37119fd 100644 --- a/src/orderings.jl +++ b/src/orderings.jl @@ -12,7 +12,12 @@ const orderings = (unordered, monotonic, acquire, release, acq_rel, seq_cst) const ConcreteOrdering = Union{map(typeof, orderings)...} llvm_ordering(::LLVMOrdering{name}) where {name} = name + Base.string(o::LLVMOrdering) = String(llvm_ordering(o)) Base.print(io::IO, o::LLVMOrdering) = print(io, string(o)) Base.show(io::IO, o::ConcreteOrdering) = print(io, UnsafeAtomics, '.', llvm_ordering(o)) + +base_ordering(::LLVMOrdering{name}) where {name} = name +base_ordering(::LLVMOrdering{:seq_cst}) = :sequentially_consistent +base_ordering(::LLVMOrdering{:acq_rel}) = :acquire_release \ No newline at end of file