Skip to content

Commit

Permalink
AtomicCell: Use atomic-maybe-uninit
Browse files Browse the repository at this point in the history
  • Loading branch information
@taiki-e
taiki-e committed Dec 8, 2024
1 parent 8b0fdfa commit 76b28b7
Show file tree
Hide file tree
Showing 5 changed files with 191 additions and 75 deletions.
4 changes: 4 additions & 0 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,10 @@ jobs:
- rust: nightly
os: ubuntu-latest
target: armv5te-unknown-linux-gnueabi
# Test target without stable inline asm support.
- rust: stable
os: ubuntu-latest
target: sparc64-unknown-linux-gnu
runs-on: ${{ matrix.os }}
timeout-minutes: 60
steps:
Expand Down
3 changes: 2 additions & 1 deletion crossbeam-utils/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,10 @@ default = ["std"]
std = []

# Enable `atomic` module.
atomic = []
atomic = ["atomic-maybe-uninit"]

[dependencies]
atomic-maybe-uninit = { version = "0.3.4", optional = true }

# Enable the use of loom for concurrency testing.
#
Expand Down
10 changes: 6 additions & 4 deletions crossbeam-utils/build.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ include!("build-common.rs");

fn main() {
println!("cargo:rerun-if-changed=no_atomic.rs");
println!("cargo:rustc-check-cfg=cfg(crossbeam_no_atomic,crossbeam_sanitize_thread)");
println!("cargo:rustc-check-cfg=cfg(crossbeam_no_atomic,crossbeam_sanitize_thread,crossbeam_atomic_cell_force_fallback)");

let target = match env::var("TARGET") {
Ok(target) => convert_custom_linux_target(target),
Expand All @@ -39,8 +39,10 @@ fn main() {
}

// `cfg(sanitize = "..")` is not stabilized.
let sanitize = env::var("CARGO_CFG_SANITIZE").unwrap_or_default();
if sanitize.contains("thread") {
println!("cargo:rustc-cfg=crossbeam_sanitize_thread");
if let Ok(sanitize) = env::var("CARGO_CFG_SANITIZE") {
if sanitize.contains("thread") {
println!("cargo:rustc-cfg=crossbeam_sanitize_thread");
}
println!("cargo:rustc-cfg=crossbeam_atomic_cell_force_fallback");
}
}
194 changes: 141 additions & 53 deletions crossbeam-utils/src/atomic/atomic_cell.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,8 @@ impl<T> AtomicCell<T> {
/// # Examples
///
/// ```
/// # // Always use fallback for now on environments that do not support inline assembly.
/// # if cfg!(any(miri, crossbeam_loom, crossbeam_atomic_cell_force_fallback)) { return; }
/// use crossbeam_utils::atomic::AtomicCell;
///
/// // This type is internally represented as `AtomicUsize` so we can just use atomic
Expand Down Expand Up @@ -307,21 +309,37 @@ macro_rules! atomic {
loop {
atomic!(@check, $t, AtomicUnit, $a, $atomic_op);

atomic!(@check, $t, atomic::AtomicU8, $a, $atomic_op);
atomic!(@check, $t, atomic::AtomicU16, $a, $atomic_op);
atomic!(@check, $t, atomic::AtomicU32, $a, $atomic_op);
#[cfg(target_has_atomic = "64")]
atomic!(@check, $t, atomic::AtomicU64, $a, $atomic_op);
// TODO: AtomicU128 is unstable
// atomic!(@check, $t, atomic::AtomicU128, $a, $atomic_op);
// Always use fallback for now on environments that do not support inline assembly.
#[cfg(not(any(
miri,
crossbeam_loom,
crossbeam_atomic_cell_force_fallback,
)))]
atomic_maybe_uninit::cfg_has_atomic_cas! {
atomic_maybe_uninit::cfg_has_atomic_8! {
atomic!(@check, $t, atomic_maybe_uninit::AtomicMaybeUninit<u8>, $a, $atomic_op);
}
atomic_maybe_uninit::cfg_has_atomic_16! {
atomic!(@check, $t, atomic_maybe_uninit::AtomicMaybeUninit<u16>, $a, $atomic_op);
}
atomic_maybe_uninit::cfg_has_atomic_32! {
atomic!(@check, $t, atomic_maybe_uninit::AtomicMaybeUninit<u32>, $a, $atomic_op);
}
atomic_maybe_uninit::cfg_has_atomic_64! {
atomic!(@check, $t, atomic_maybe_uninit::AtomicMaybeUninit<u64>, $a, $atomic_op);
}
atomic_maybe_uninit::cfg_has_atomic_128! {
atomic!(@check, $t, atomic_maybe_uninit::AtomicMaybeUninit<u128>, $a, $atomic_op);
}
}

break $fallback_op;
}
};
}

macro_rules! impl_arithmetic {
($t:ty, fallback, $example:tt) => {
($t:ty, fetch_update, $example:tt) => {
impl AtomicCell<$t> {
/// Increments the current value by `val` and returns the previous value.
///
Expand All @@ -339,11 +357,19 @@ macro_rules! impl_arithmetic {
/// ```
#[inline]
pub fn fetch_add(&self, val: $t) -> $t {
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value = value.wrapping_add(val);
old
atomic! {
$t, _a,
{
self.fetch_update(|old| Some(old.wrapping_add(val))).unwrap()
},
{
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value = value.wrapping_add(val);
old
}
}
}

/// Decrements the current value by `val` and returns the previous value.
Expand All @@ -362,11 +388,19 @@ macro_rules! impl_arithmetic {
/// ```
#[inline]
pub fn fetch_sub(&self, val: $t) -> $t {
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value = value.wrapping_sub(val);
old
atomic! {
$t, _a,
{
self.fetch_update(|old| Some(old.wrapping_sub(val))).unwrap()
},
{
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value = value.wrapping_sub(val);
old
}
}
}

/// Applies bitwise "and" to the current value and returns the previous value.
Expand All @@ -383,11 +417,19 @@ macro_rules! impl_arithmetic {
/// ```
#[inline]
pub fn fetch_and(&self, val: $t) -> $t {
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value &= val;
old
atomic! {
$t, _a,
{
self.fetch_update(|old| Some(old & val)).unwrap()
},
{
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value &= val;
old
}
}
}

/// Applies bitwise "nand" to the current value and returns the previous value.
Expand All @@ -404,11 +446,19 @@ macro_rules! impl_arithmetic {
/// ```
#[inline]
pub fn fetch_nand(&self, val: $t) -> $t {
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value = !(old & val);
old
atomic! {
$t, _a,
{
self.fetch_update(|old| Some(!(old & val))).unwrap()
},
{
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value = !(old & val);
old
}
}
}

/// Applies bitwise "or" to the current value and returns the previous value.
Expand All @@ -425,11 +475,19 @@ macro_rules! impl_arithmetic {
/// ```
#[inline]
pub fn fetch_or(&self, val: $t) -> $t {
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value |= val;
old
atomic! {
$t, _a,
{
self.fetch_update(|old| Some(old | val)).unwrap()
},
{
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value |= val;
old
}
}
}

/// Applies bitwise "xor" to the current value and returns the previous value.
Expand All @@ -446,11 +504,19 @@ macro_rules! impl_arithmetic {
/// ```
#[inline]
pub fn fetch_xor(&self, val: $t) -> $t {
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value ^= val;
old
atomic! {
$t, _a,
{
self.fetch_update(|old| Some(old ^ val)).unwrap()
},
{
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value ^= val;
old
}
}
}

/// Compares and sets the maximum of the current value and `val`,
Expand All @@ -468,11 +534,19 @@ macro_rules! impl_arithmetic {
/// ```
#[inline]
pub fn fetch_max(&self, val: $t) -> $t {
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value = cmp::max(old, val);
old
atomic! {
$t, _a,
{
self.fetch_update(|old| Some(cmp::max(old, val))).unwrap()
},
{
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value = cmp::max(old, val);
old
}
}
}

/// Compares and sets the minimum of the current value and `val`,
Expand All @@ -490,11 +564,19 @@ macro_rules! impl_arithmetic {
/// ```
#[inline]
pub fn fetch_min(&self, val: $t) -> $t {
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value = cmp::min(old, val);
old
atomic! {
$t, _a,
{
self.fetch_update(|old| Some(cmp::min(old, val))).unwrap()
},
{
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value = cmp::min(old, val);
old
}
}
}
}
};
Expand Down Expand Up @@ -754,23 +836,29 @@ impl_arithmetic!(i8, AtomicI8, "let a = AtomicCell::new(7i8);");
impl_arithmetic!(u16, AtomicU16, "let a = AtomicCell::new(7u16);");
impl_arithmetic!(i16, AtomicI16, "let a = AtomicCell::new(7i16);");

#[cfg(target_has_atomic = "32")]
impl_arithmetic!(u32, AtomicU32, "let a = AtomicCell::new(7u32);");
#[cfg(target_has_atomic = "32")]
impl_arithmetic!(i32, AtomicI32, "let a = AtomicCell::new(7i32);");
#[cfg(not(target_has_atomic = "32"))]
impl_arithmetic!(u32, fetch_update, "let a = AtomicCell::new(7u32);");
#[cfg(not(target_has_atomic = "32"))]
impl_arithmetic!(i32, fetch_update, "let a = AtomicCell::new(7i32);");

#[cfg(target_has_atomic = "64")]
impl_arithmetic!(u64, AtomicU64, "let a = AtomicCell::new(7u64);");
#[cfg(target_has_atomic = "64")]
impl_arithmetic!(i64, AtomicI64, "let a = AtomicCell::new(7i64);");
#[cfg(not(target_has_atomic = "64"))]
impl_arithmetic!(u64, fallback, "let a = AtomicCell::new(7u64);");
impl_arithmetic!(u64, fetch_update, "let a = AtomicCell::new(7u64);");
#[cfg(not(target_has_atomic = "64"))]
impl_arithmetic!(i64, fallback, "let a = AtomicCell::new(7i64);");
impl_arithmetic!(i64, fetch_update, "let a = AtomicCell::new(7i64);");

// TODO: AtomicU128 is unstable
// TODO: core::sync::atomic::AtomicU128 is unstable
// impl_arithmetic!(u128, AtomicU128, "let a = AtomicCell::new(7u128);");
// impl_arithmetic!(i128, AtomicI128, "let a = AtomicCell::new(7i128);");
impl_arithmetic!(u128, fallback, "let a = AtomicCell::new(7u128);");
impl_arithmetic!(i128, fallback, "let a = AtomicCell::new(7i128);");
impl_arithmetic!(u128, fetch_update, "let a = AtomicCell::new(7u128);");
impl_arithmetic!(i128, fetch_update, "let a = AtomicCell::new(7i128);");

impl_arithmetic!(usize, AtomicUsize, "let a = AtomicCell::new(7usize);");
impl_arithmetic!(isize, AtomicIsize, "let a = AtomicCell::new(7isize);");
Expand Down
Loading

0 comments on commit 76b28b7

Please sign in to comment.