Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

AtomicCell: Use atomic-maybe-uninit #1015

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

AtomicCell: Use atomic-maybe-uninit #1015

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
1 change: 1 addition & 0 deletions crossbeam-utils/Cargo.toml
View file
Open in desktop
Expand Up @@ -22,6 +22,7 @@ default = ["std"]
std = []

[dependencies]
atomic-maybe-uninit = "0.3"
cfg-if = "1"

# Enable the use of loom for concurrency testing.
Expand Down
3 changes: 3 additions & 0 deletions crossbeam-utils/build.rs
View file
Open in desktop
Expand Up @@ -42,4 +42,7 @@ fn main() {
if sanitize.contains("thread") {
println!("cargo:rustc-cfg=crossbeam_sanitize_thread");
}
if !sanitize.is_empty() {
println!("cargo:rustc-cfg=crossbeam_atomic_cell_force_fallback");
}
}
186 changes: 134 additions & 52 deletions crossbeam-utils/src/atomic/atomic_cell.rs
View file
Open in desktop
Expand Up @@ -97,6 +97,8 @@ impl<T> AtomicCell<T> {
/// # Examples
///
/// ```
/// # // Always use fallback for now on environments that do not support inline assembly.
/// # if cfg!(any(miri, crossbeam_loom, crossbeam_atomic_cell_force_fallback)) { return; }
/// use crossbeam_utils::atomic::AtomicCell;
///
/// // This type is internally represented as `AtomicUsize` so we can just use atomic
Expand Down Expand Up @@ -305,21 +307,37 @@ macro_rules! atomic {
loop {
atomic!(@check, $t, AtomicUnit, $a, $atomic_op);

atomic!(@check, $t, atomic::AtomicU8, $a, $atomic_op);
atomic!(@check, $t, atomic::AtomicU16, $a, $atomic_op);
atomic!(@check, $t, atomic::AtomicU32, $a, $atomic_op);
#[cfg(target_has_atomic = "64")]
atomic!(@check, $t, atomic::AtomicU64, $a, $atomic_op);
// TODO: AtomicU128 is unstable
// atomic!(@check, $t, atomic::AtomicU128, $a, $atomic_op);
// Always use fallback for now on environments that do not support inline assembly.
#[cfg(not(any(
miri,
crossbeam_loom,
crossbeam_atomic_cell_force_fallback,
)))]
{
atomic_maybe_uninit::cfg_has_atomic_8! {
atomic!(@check, $t, atomic_maybe_uninit::AtomicMaybeUninit<u8>, $a, $atomic_op);
}
atomic_maybe_uninit::cfg_has_atomic_16! {
atomic!(@check, $t, atomic_maybe_uninit::AtomicMaybeUninit<u16>, $a, $atomic_op);
}
atomic_maybe_uninit::cfg_has_atomic_32! {
atomic!(@check, $t, atomic_maybe_uninit::AtomicMaybeUninit<u32>, $a, $atomic_op);
}
atomic_maybe_uninit::cfg_has_atomic_64! {
atomic!(@check, $t, atomic_maybe_uninit::AtomicMaybeUninit<u64>, $a, $atomic_op);
}
atomic_maybe_uninit::cfg_has_atomic_128! {
atomic!(@check, $t, atomic_maybe_uninit::AtomicMaybeUninit<u128>, $a, $atomic_op);
}
}

break $fallback_op;
}
};
}

macro_rules! impl_arithmetic {
($t:ty, fallback, $example:tt) => {
($t:ty, fetch_update, $example:tt) => {
impl AtomicCell<$t> {
/// Increments the current value by `val` and returns the previous value.
///
Expand All @@ -337,11 +355,19 @@ macro_rules! impl_arithmetic {
/// ```
#[inline]
pub fn fetch_add(&self, val: $t) -> $t {
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value = value.wrapping_add(val);
old
atomic! {
$t, _a,
{
self.fetch_update(|old| Some(old.wrapping_add(val))).unwrap()
},
{
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value = value.wrapping_add(val);
old
}
}
}

/// Decrements the current value by `val` and returns the previous value.
Expand All @@ -360,11 +386,19 @@ macro_rules! impl_arithmetic {
/// ```
#[inline]
pub fn fetch_sub(&self, val: $t) -> $t {
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value = value.wrapping_sub(val);
old
atomic! {
$t, _a,
{
self.fetch_update(|old| Some(old.wrapping_sub(val))).unwrap()
},
{
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value = value.wrapping_sub(val);
old
}
}
}

/// Applies bitwise "and" to the current value and returns the previous value.
Expand All @@ -381,11 +415,19 @@ macro_rules! impl_arithmetic {
/// ```
#[inline]
pub fn fetch_and(&self, val: $t) -> $t {
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value &= val;
old
atomic! {
$t, _a,
{
self.fetch_update(|old| Some(old & val)).unwrap()
},
{
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value &= val;
old
}
}
}

/// Applies bitwise "nand" to the current value and returns the previous value.
Expand All @@ -402,11 +444,19 @@ macro_rules! impl_arithmetic {
/// ```
#[inline]
pub fn fetch_nand(&self, val: $t) -> $t {
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value = !(old & val);
old
atomic! {
$t, _a,
{
self.fetch_update(|old| Some(!(old & val))).unwrap()
},
{
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value = !(old & val);
old
}
}
}

/// Applies bitwise "or" to the current value and returns the previous value.
Expand All @@ -423,11 +473,19 @@ macro_rules! impl_arithmetic {
/// ```
#[inline]
pub fn fetch_or(&self, val: $t) -> $t {
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value |= val;
old
atomic! {
$t, _a,
{
self.fetch_update(|old| Some(old | val)).unwrap()
},
{
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value |= val;
old
}
}
}

/// Applies bitwise "xor" to the current value and returns the previous value.
Expand All @@ -444,11 +502,19 @@ macro_rules! impl_arithmetic {
/// ```
#[inline]
pub fn fetch_xor(&self, val: $t) -> $t {
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value ^= val;
old
atomic! {
$t, _a,
{
self.fetch_update(|old| Some(old ^ val)).unwrap()
},
{
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value ^= val;
old
}
}
}

/// Compares and sets the maximum of the current value and `val`,
Expand All @@ -466,11 +532,19 @@ macro_rules! impl_arithmetic {
/// ```
#[inline]
pub fn fetch_max(&self, val: $t) -> $t {
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value = cmp::max(old, val);
old
atomic! {
$t, _a,
{
self.fetch_update(|old| Some(cmp::max(old, val))).unwrap()
},
{
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value = cmp::max(old, val);
old
}
}
}

/// Compares and sets the minimum of the current value and `val`,
Expand All @@ -488,11 +562,19 @@ macro_rules! impl_arithmetic {
/// ```
#[inline]
pub fn fetch_min(&self, val: $t) -> $t {
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value = cmp::min(old, val);
old
atomic! {
$t, _a,
{
self.fetch_update(|old| Some(cmp::min(old, val))).unwrap()
},
{
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value = cmp::min(old, val);
old
}
}
}
}
};
Expand Down Expand Up @@ -760,15 +842,15 @@ impl_arithmetic!(u64, AtomicU64, "let a = AtomicCell::new(7u64);");
#[cfg(target_has_atomic = "64")]
impl_arithmetic!(i64, AtomicI64, "let a = AtomicCell::new(7i64);");
#[cfg(not(target_has_atomic = "64"))]
impl_arithmetic!(u64, fallback, "let a = AtomicCell::new(7u64);");
impl_arithmetic!(u64, fetch_update, "let a = AtomicCell::new(7u64);");
#[cfg(not(target_has_atomic = "64"))]
impl_arithmetic!(i64, fallback, "let a = AtomicCell::new(7i64);");
impl_arithmetic!(i64, fetch_update, "let a = AtomicCell::new(7i64);");

// TODO: AtomicU128 is unstable
// impl_arithmetic!(u128, AtomicU128, "let a = AtomicCell::new(7u128);");
// impl_arithmetic!(i128, AtomicI128, "let a = AtomicCell::new(7i128);");
impl_arithmetic!(u128, fallback, "let a = AtomicCell::new(7u128);");
impl_arithmetic!(i128, fallback, "let a = AtomicCell::new(7i128);");
impl_arithmetic!(u128, fetch_update, "let a = AtomicCell::new(7u128);");
impl_arithmetic!(i128, fetch_update, "let a = AtomicCell::new(7i128);");

impl_arithmetic!(usize, AtomicUsize, "let a = AtomicCell::new(7usize);");
impl_arithmetic!(isize, AtomicIsize, "let a = AtomicCell::new(7isize);");
Expand Down
16 changes: 7 additions & 9 deletions crossbeam-utils/src/atomic/mod.rs
View file
Open in desktop
Expand Up @@ -4,7 +4,11 @@
//! * [`AtomicConsume`], for reading from primitive atomic types with "consume" ordering.

#[cfg(target_has_atomic = "ptr")]
// We cannot provide AtomicCell under cfg(crossbeam_loom) because loom's atomic
// types have a different in-memory representation than the underlying type.
// TODO: The latest loom supports fences, so fallback using seqlock may be available.
#[cfg(not(crossbeam_loom))]
atomic_maybe_uninit::cfg_has_atomic_cas! {
cfg_if::cfg_if! {
// Use "wide" sequence lock if the pointer width <= 32 for preventing its counter against wrap
// around.
Expand All @@ -23,15 +27,9 @@ cfg_if::cfg_if! {
}
}

#[cfg(target_has_atomic = "ptr")]
// We cannot provide AtomicCell under cfg(crossbeam_loom) because loom's atomic
// types have a different in-memory representation than the underlying type.
// TODO: The latest loom supports fences, so fallback using seqlock may be available.
#[cfg(not(crossbeam_loom))]
mod atomic_cell;
mod consume;

#[cfg(target_has_atomic = "ptr")]
#[cfg(not(crossbeam_loom))]
pub use self::atomic_cell::AtomicCell;
}

mod consume;
pub use self::consume::AtomicConsume;
23 changes: 17 additions & 6 deletions crossbeam-utils/tests/atomic_cell.rs
View file
Open in desktop
Expand Up @@ -6,6 +6,15 @@ use crossbeam_utils::atomic::AtomicCell;

#[test]
fn is_lock_free() {
// Always use fallback for now on environments that do not support inline assembly.
if cfg!(any(
miri,
crossbeam_loom,
crossbeam_atomic_cell_force_fallback,
)) {
return;
}

struct UsizeWrap(usize);
struct U8Wrap(bool);
struct I16Wrap(i16);
Expand Down Expand Up @@ -44,8 +53,9 @@ fn is_lock_free() {
cfg!(target_has_atomic = "64")
);

// AtomicU128 is unstable
assert!(!AtomicCell::<u128>::is_lock_free());
// TODO
// // AtomicU128 is unstable
// assert!(!AtomicCell::<u128>::is_lock_free());
}

#[test]
Expand Down Expand Up @@ -318,10 +328,11 @@ fn issue_748() {
}

assert_eq!(mem::size_of::<Test>(), 8);
assert_eq!(
AtomicCell::<Test>::is_lock_free(),
cfg!(target_has_atomic = "64")
);
// TODO
// assert_eq!(
// AtomicCell::<Test>::is_lock_free(),
// cfg!(target_has_atomic = "64")
// );
let x = AtomicCell::new(Test::FieldLess);
assert_eq!(x.load(), Test::FieldLess);
}
Expand Down