Skip to content

Commit

Permalink
Use atomic_pointer* methods when available (#14)
Browse files Browse the repository at this point in the history 
Co-authored-by: Valentin Churavy <[email protected]>
  • Loading branch information
@jpsamaroo @vchuravy
jpsamaroo and vchuravy authored Nov 28, 2024
1 parent 2eaec48 commit ce7a883
Show file tree
Hide file tree
Showing 2 changed files with 124 additions and 66 deletions.
185 changes: 119 additions & 66 deletions src/core.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,16 @@ for (op, rmwop) in OP_RMW_TABLE
first(UnsafeAtomics.modify!(ptr, $op, x, ord))
end

const ATOMIC_INTRINSICS = isdefined(Core.Intrinsics, :atomic_pointerref)

if VERSION >= v"1.12.0-DEV.161" && Int == Int64
const MAX_ATOMIC_SIZE = 16
const MAX_POINTERATOMIC_SIZE = 16
else
const MAX_ATOMIC_SIZE = 8
const MAX_POINTERATOMIC_SIZE = 8
end

# Based on: https://github.com/JuliaLang/julia/blob/v1.6.3/base/atomics.jl
for typ in (inttypes..., floattypes...)
lt = llvmtypes[typ]
Expand All @@ -56,35 +66,48 @@ for typ in (inttypes..., floattypes...)
for ord in orderings
ord in (release, acq_rel) && continue

@eval function UnsafeAtomics.load(x::Ptr{$typ}, ::$(typeof(ord)))
return llvmcall(
$("""
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
%rv = load atomic $rt %ptr $ord, align $(sizeof(typ))
ret $lt %rv
"""),
$typ,
Tuple{Ptr{$typ}},
x,
)
if ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE
@eval function UnsafeAtomics.load(x::Ptr{$typ}, ::$(typeof(ord)))
return Core.Intrinsics.atomic_pointerref(x, base_ordering($ord))
end
else
@eval function UnsafeAtomics.load(x::Ptr{$typ}, ::$(typeof(ord)))
return llvmcall(
$("""
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
%rv = load atomic $rt %ptr $ord, align $(sizeof(typ))
ret $lt %rv
"""),
$typ,
Tuple{Ptr{$typ}},
x,
)
end
end
end

for ord in orderings
ord in (acquire, acq_rel) && continue

@eval function UnsafeAtomics.store!(x::Ptr{$typ}, v::$typ, ::$(typeof(ord)))
return llvmcall(
$("""
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
store atomic $lt %1, $lt* %ptr $ord, align $(sizeof(typ))
ret void
"""),
Cvoid,
Tuple{Ptr{$typ},$typ},
x,
v,
)
if ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE
@eval function UnsafeAtomics.store!(x::Ptr{$typ}, v::$typ, ::$(typeof(ord)))
Core.Intrinsics.atomic_pointerset(x, v, base_ordering($ord))
return nothing
end
else
@eval function UnsafeAtomics.store!(x::Ptr{$typ}, v::$typ, ::$(typeof(ord)))
return llvmcall(
$("""
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
store atomic $lt %1, $lt* %ptr $ord, align $(sizeof(typ))
ret void
"""),
Cvoid,
Tuple{Ptr{$typ},$typ},
x,
v,
)
end
end
end

Expand All @@ -93,37 +116,55 @@ for typ in (inttypes..., floattypes...)

typ <: AbstractFloat && break

@eval function UnsafeAtomics.cas!(
x::Ptr{$typ},
cmp::$typ,
new::$typ,
::$(typeof(success_ordering)),
::$(typeof(failure_ordering)),
)
success = Ref{Int8}()
GC.@preserve success begin
old = llvmcall(
$(
"""
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
%rs = cmpxchg $lt* %ptr, $lt %1, $lt %2 $success_ordering $failure_ordering
%rv = extractvalue { $lt, i1 } %rs, 0
%s1 = extractvalue { $lt, i1 } %rs, 1
%s8 = zext i1 %s1 to i8
%sptr = inttoptr i$WORD_SIZE %3 to i8*
store i8 %s8, i8* %sptr
ret $lt %rv
"""
),
$typ,
Tuple{Ptr{$typ},$typ,$typ,Ptr{Int8}},
if ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE
@eval function UnsafeAtomics.cas!(
x::Ptr{$typ},
cmp::$typ,
new::$typ,
::$(typeof(success_ordering)),
::$(typeof(failure_ordering)),
)
return Core.Intrinsics.atomic_pointerreplace(
x,
cmp,
new,
Ptr{Int8}(pointer_from_objref(success)),
base_ordering($success_ordering),
base_ordering($failure_ordering)
)
end
return (old = old, success = !iszero(success[]))
else
@eval function UnsafeAtomics.cas!(
x::Ptr{$typ},
cmp::$typ,
new::$typ,
::$(typeof(success_ordering)),
::$(typeof(failure_ordering)),
)
success = Ref{Int8}()
GC.@preserve success begin
old = llvmcall(
$(
"""
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
%rs = cmpxchg $lt* %ptr, $lt %1, $lt %2 $success_ordering $failure_ordering
%rv = extractvalue { $lt, i1 } %rs, 0
%s1 = extractvalue { $lt, i1 } %rs, 1
%s8 = zext i1 %s1 to i8
%sptr = inttoptr i$WORD_SIZE %3 to i8*
store i8 %s8, i8* %sptr
ret $lt %rv
"""
),
$typ,
Tuple{Ptr{$typ},$typ,$typ,Ptr{Int8}},
x,
cmp,
new,
Ptr{Int8}(pointer_from_objref(success)),
)
end
return (old = old, success = !iszero(success[]))
end
end
end

Expand All @@ -144,24 +185,36 @@ for typ in (inttypes..., floattypes...)
end
end
for ord in orderings
@eval function UnsafeAtomics.modify!(
x::Ptr{$typ},
::typeof($op),
v::$typ,
::$(typeof(ord)),
)
old = llvmcall(
$("""
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
%rv = atomicrmw $rmw $lt* %ptr, $lt %1 $ord
ret $lt %rv
"""),
$typ,
Tuple{Ptr{$typ},$typ},
x,
v,
# Enable this code iff https://github.com/JuliaLang/julia/pull/45122 get's merged
if false && ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE
@eval function UnsafeAtomics.modify!(
x::Ptr{$typ},
op::typeof($op),
v::$typ,
::$(typeof(ord)),
)
return Core.Intrinsics.atomic_pointermodify(x, op, v, base_ordering($ord))
end
else
@eval function UnsafeAtomics.modify!(
x::Ptr{$typ},
::typeof($op),
v::$typ,
::$(typeof(ord)),
)
return old => $op(old, v)
old = llvmcall(
$("""
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
%rv = atomicrmw $rmw $lt* %ptr, $lt %1 $ord
ret $lt %rv
"""),
$typ,
Tuple{Ptr{$typ},$typ},
x,
v,
)
return old => $op(old, v)
end
end
end
end
Expand Down
5 changes: 5 additions & 0 deletions src/orderings.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,12 @@ const orderings = (unordered, monotonic, acquire, release, acq_rel, seq_cst)
const ConcreteOrdering = Union{map(typeof, orderings)...}

llvm_ordering(::LLVMOrdering{name}) where {name} = name

Base.string(o::LLVMOrdering) = String(llvm_ordering(o))
Base.print(io::IO, o::LLVMOrdering) = print(io, string(o))

Base.show(io::IO, o::ConcreteOrdering) = print(io, UnsafeAtomics, '.', llvm_ordering(o))

base_ordering(::LLVMOrdering{name}) where {name} = name
base_ordering(::LLVMOrdering{:seq_cst}) = :sequentially_consistent
base_ordering(::LLVMOrdering{:acq_rel}) = :acquire_release

0 comments on commit ce7a883

Please sign in to comment.