diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 27e937366..f7a7e96d6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -105,6 +105,17 @@ jobs: - name: clippy run: ./ci/clippy.sh + # Run loom tests. + loom: + name: loom + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - name: Install Rust + run: rustup update stable && rustup default stable + - name: loom + run: ./ci/crossbeam-epoch-loom.sh + # This job doesn't actually test anything, but they're used to tell bors the # build completed, as there is no practical way to detect when a workflow is # successful listening to webhooks only. @@ -120,6 +131,7 @@ jobs: - dependencies - rustfmt - clippy + - loom runs-on: ubuntu-latest steps: - name: Mark the job as a success diff --git a/ci/crossbeam-epoch-loom.sh b/ci/crossbeam-epoch-loom.sh new file mode 100755 index 000000000..8cb393e29 --- /dev/null +++ b/ci/crossbeam-epoch-loom.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +cd "$(dirname "$0")"/../crossbeam-epoch +set -ex + +export RUSTFLAGS="-D warnings --cfg=loom_crossbeam" + +# With MAX_PREEMPTIONS=2 the loom tests (currently) take around 11m. +# If we were to run with =3, they would take several times that, +# which is probably too costly for CI. +env LOOM_MAX_PREEMPTIONS=2 cargo test --test loom --features sanitize --release -- --nocapture diff --git a/crossbeam-epoch/Cargo.toml b/crossbeam-epoch/Cargo.toml index 19538a3c5..a0091612f 100644 --- a/crossbeam-epoch/Cargo.toml +++ b/crossbeam-epoch/Cargo.toml @@ -41,6 +41,13 @@ cfg-if = "1" const_fn = { version = "0.4.4", optional = true } memoffset = "0.6" +# Enable the use of loom for concurrency testing. +# +# This configuration option is outside of the normal semver guarantees: minor +# versions of crossbeam may make breaking changes to it at any time. +[target.'cfg(loom_crossbeam)'.dependencies] +loom = "0.4" + [dependencies.crossbeam-utils] version = "0.8" path = "../crossbeam-utils" diff --git a/crossbeam-epoch/examples/treiber_stack.rs b/crossbeam-epoch/examples/treiber_stack.rs deleted file mode 100644 index a2c3c16ff..000000000 --- a/crossbeam-epoch/examples/treiber_stack.rs +++ /dev/null @@ -1,107 +0,0 @@ -use std::mem::ManuallyDrop; -use std::ptr; -use std::sync::atomic::Ordering::{Acquire, Relaxed, Release}; - -use crossbeam_epoch::{self as epoch, Atomic, Owned}; -use crossbeam_utils::thread::scope; - -/// Treiber's lock-free stack. -/// -/// Usable with any number of producers and consumers. -#[derive(Debug)] -pub struct TreiberStack { - head: Atomic>, -} - -#[derive(Debug)] -struct Node { - data: ManuallyDrop, - next: Atomic>, -} - -impl TreiberStack { - /// Creates a new, empty stack. - pub fn new() -> TreiberStack { - TreiberStack { - head: Atomic::null(), - } - } - - /// Pushes a value on top of the stack. - pub fn push(&self, t: T) { - let mut n = Owned::new(Node { - data: ManuallyDrop::new(t), - next: Atomic::null(), - }); - - let guard = epoch::pin(); - - loop { - let head = self.head.load(Relaxed, &guard); - n.next.store(head, Relaxed); - - match self.head.compare_and_set(head, n, Release, &guard) { - Ok(_) => break, - Err(e) => n = e.new, - } - } - } - - /// Attempts to pop the top element from the stack. - /// - /// Returns `None` if the stack is empty. - pub fn pop(&self) -> Option { - let guard = epoch::pin(); - loop { - let head = self.head.load(Acquire, &guard); - - match unsafe { head.as_ref() } { - Some(h) => { - let next = h.next.load(Relaxed, &guard); - - if self - .head - .compare_and_set(head, next, Relaxed, &guard) - .is_ok() - { - unsafe { - guard.defer_destroy(head); - return Some(ManuallyDrop::into_inner(ptr::read(&(*h).data))); - } - } - } - None => return None, - } - } - } - - /// Returns `true` if the stack is empty. - pub fn is_empty(&self) -> bool { - let guard = epoch::pin(); - self.head.load(Acquire, &guard).is_null() - } -} - -impl Drop for TreiberStack { - fn drop(&mut self) { - while self.pop().is_some() {} - } -} - -fn main() { - let stack = TreiberStack::new(); - - scope(|scope| { - for _ in 0..10 { - scope.spawn(|_| { - for i in 0..10_000 { - stack.push(i); - assert!(stack.pop().is_some()); - } - }); - } - }) - .unwrap(); - - assert!(stack.pop().is_none()); -} diff --git a/crossbeam-epoch/src/atomic.rs b/crossbeam-epoch/src/atomic.rs index fdcf604a3..f01c793a4 100644 --- a/crossbeam-epoch/src/atomic.rs +++ b/crossbeam-epoch/src/atomic.rs @@ -5,10 +5,11 @@ use core::marker::PhantomData; use core::mem::{self, MaybeUninit}; use core::ops::{Deref, DerefMut}; use core::slice; -use core::sync::atomic::{AtomicUsize, Ordering}; +use core::sync::atomic::Ordering; use crate::alloc::alloc; use crate::alloc::boxed::Box; +use crate::primitive::sync::atomic::AtomicUsize; use crate::guard::Guard; use crossbeam_utils::atomic::AtomicConsume; @@ -326,7 +327,7 @@ impl Atomic { /// let a = Atomic::::null(); /// ``` /// - #[cfg_attr(feature = "nightly", const_fn::const_fn)] + #[cfg_attr(all(feature = "nightly", not(loom_crossbeam)), const_fn::const_fn)] pub fn null() -> Atomic { Self { data: AtomicUsize::new(0), @@ -637,7 +638,17 @@ impl Atomic { /// } /// ``` pub unsafe fn into_owned(self) -> Owned { - Owned::from_usize(self.data.into_inner()) + #[cfg(loom_crossbeam)] + { + // FIXME: loom does not yet support into_inner, so we use unsync_load for now, + // which should have the same synchronization properties: + // https://github.com/tokio-rs/loom/issues/117 + Owned::from_usize(self.data.unsync_load()) + } + #[cfg(not(loom_crossbeam))] + { + Owned::from_usize(self.data.into_inner()) + } } } @@ -1357,7 +1368,7 @@ impl Default for Shared<'_, T> { } } -#[cfg(test)] +#[cfg(all(test, not(loom_crossbeam)))] mod tests { use super::Shared; diff --git a/crossbeam-epoch/src/collector.rs b/crossbeam-epoch/src/collector.rs index 8224e1184..8c582f1f1 100644 --- a/crossbeam-epoch/src/collector.rs +++ b/crossbeam-epoch/src/collector.rs @@ -14,7 +14,7 @@ /// ``` use core::fmt; -use crate::alloc::sync::Arc; +use crate::primitive::sync::Arc; use crate::guard::Guard; use crate::internal::{Global, Local}; @@ -109,7 +109,7 @@ impl fmt::Debug for LocalHandle { } } -#[cfg(test)] +#[cfg(all(test, not(loom_crossbeam)))] mod tests { use std::mem; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -151,9 +151,9 @@ mod tests { let a = Owned::new(7).into_shared(guard); guard.defer_destroy(a); - assert!(!(*(*guard.local).bag.get()).is_empty()); + assert!(!(*guard.local).bag.with(|b| (*b).is_empty())); - while !(*(*guard.local).bag.get()).is_empty() { + while !(*guard.local).bag.with(|b| (*b).is_empty()) { guard.flush(); } } @@ -172,7 +172,7 @@ mod tests { let a = Owned::new(7).into_shared(guard); guard.defer_destroy(a); } - assert!(!(*(*guard.local).bag.get()).is_empty()); + assert!(!(*guard.local).bag.with(|b| (*b).is_empty())); } } diff --git a/crossbeam-epoch/src/default.rs b/crossbeam-epoch/src/default.rs index 1deac2114..dd81275c5 100644 --- a/crossbeam-epoch/src/default.rs +++ b/crossbeam-epoch/src/default.rs @@ -5,8 +5,8 @@ //! destructed on thread exit, which in turn unregisters the thread. use crate::collector::{Collector, LocalHandle}; +use crate::primitive::{lazy_static, thread_local}; use crate::guard::Guard; -use lazy_static::lazy_static; lazy_static! { /// The global data for the default garbage collector. @@ -45,7 +45,7 @@ where .unwrap_or_else(|_| f(&COLLECTOR.register())) } -#[cfg(test)] +#[cfg(all(test, not(loom_crossbeam)))] mod tests { use crossbeam_utils::thread; diff --git a/crossbeam-epoch/src/deferred.rs b/crossbeam-epoch/src/deferred.rs index 9f4869b3b..bfad82f75 100644 --- a/crossbeam-epoch/src/deferred.rs +++ b/crossbeam-epoch/src/deferred.rs @@ -79,7 +79,7 @@ impl Deferred { } } -#[cfg(test)] +#[cfg(all(test, not(loom_crossbeam)))] mod tests { use super::Deferred; use std::cell::Cell; diff --git a/crossbeam-epoch/src/epoch.rs b/crossbeam-epoch/src/epoch.rs index 222b5ca62..c24c54a5e 100644 --- a/crossbeam-epoch/src/epoch.rs +++ b/crossbeam-epoch/src/epoch.rs @@ -7,7 +7,8 @@ //! If an object became garbage in some epoch, then we can be sure that after two advancements no //! participant will hold a reference to it. That is the crux of safe memory reclamation. -use core::sync::atomic::{AtomicUsize, Ordering}; +use crate::primitive::sync::atomic::AtomicUsize; +use core::sync::atomic::Ordering; /// An epoch that can be marked as pinned or unpinned. /// diff --git a/crossbeam-epoch/src/internal.rs b/crossbeam-epoch/src/internal.rs index 225d66ac2..78a8a32a0 100644 --- a/crossbeam-epoch/src/internal.rs +++ b/crossbeam-epoch/src/internal.rs @@ -35,10 +35,11 @@ //! Ideally each instance of concurrent data structure may have its own queue that gets fully //! destroyed as soon as the data structure gets dropped. -use core::cell::{Cell, UnsafeCell}; +use crate::primitive::cell::UnsafeCell; +use crate::primitive::sync::atomic; +use core::cell::Cell; use core::mem::{self, ManuallyDrop}; use core::num::Wrapping; -use core::sync::atomic; use core::sync::atomic::Ordering; use core::{fmt, ptr}; @@ -418,7 +419,7 @@ impl Local { /// Returns a reference to the `Collector` in which this `Local` resides. #[inline] pub fn collector(&self) -> &Collector { - unsafe { &**self.collector.get() } + self.collector.with(|c| unsafe { &**c }) } /// Returns `true` if the current participant is pinned. @@ -433,7 +434,7 @@ impl Local { /// /// It should be safe for another thread to execute the given function. pub unsafe fn defer(&self, mut deferred: Deferred, guard: &Guard) { - let bag = &mut *self.bag.get(); + let bag = self.bag.with_mut(|b| &mut *b); while let Err(d) = bag.try_push(deferred) { self.global().push_bag(bag, guard); @@ -442,7 +443,7 @@ impl Local { } pub fn flush(&self, guard: &Guard) { - let bag = unsafe { &mut *self.bag.get() }; + let bag = self.bag.with_mut(|b| unsafe { &mut *b }); if !bag.is_empty() { self.global().push_bag(bag, guard); @@ -583,7 +584,8 @@ impl Local { // Pin and move the local bag into the global queue. It's important that `push_bag` // doesn't defer destruction on any new garbage. let guard = &self.pin(); - self.global().push_bag(&mut *self.bag.get(), guard); + self.global() + .push_bag(self.bag.with_mut(|b| &mut *b), guard); } // Revert the handle count back to zero. self.handle_count.set(0); @@ -592,7 +594,7 @@ impl Local { // Take the reference to the `Global` out of this `Local`. Since we're not protected // by a guard at this time, it's crucial that the reference is read before marking the // `Local` as deleted. - let collector: Collector = ptr::read(&*(*self.collector.get())); + let collector: Collector = ptr::read(self.collector.with(|c| &*(*c))); // Mark this node in the linked list as deleted. self.entry.delete(unprotected()); @@ -623,7 +625,7 @@ impl IsElement for Local { } } -#[cfg(test)] +#[cfg(all(test, not(loom_crossbeam)))] mod tests { use std::sync::atomic::{AtomicUsize, Ordering}; diff --git a/crossbeam-epoch/src/lib.rs b/crossbeam-epoch/src/lib.rs index f64d16cd8..326599cbe 100644 --- a/crossbeam-epoch/src/lib.rs +++ b/crossbeam-epoch/src/lib.rs @@ -64,6 +64,90 @@ use cfg_if::cfg_if; +#[cfg(loom_crossbeam)] +#[allow(unused_imports, dead_code)] +mod primitive { + pub(crate) mod cell { + pub(crate) use loom::cell::UnsafeCell; + } + pub(crate) mod sync { + pub(crate) mod atomic { + use core::sync::atomic::Ordering; + pub(crate) use loom::sync::atomic::AtomicUsize; + pub(crate) fn fence(ord: Ordering) { + if let Ordering::Acquire = ord { + } else { + // FIXME: loom only supports acquire fences at the moment. + // https://github.com/tokio-rs/loom/issues/117 + // let's at least not panic... + // this may generate some false positives (`SeqCst` is stronger than `Acquire` + // for example), and some false negatives (`Relaxed` is weaker than `Acquire`), + // but it's the best we can do for the time being. + } + loom::sync::atomic::fence(Ordering::Acquire) + } + + // FIXME: loom does not support compiler_fence at the moment. + // https://github.com/tokio-rs/loom/issues/117 + // we use fence as a stand-in for compiler_fence for the time being. + // this may miss some races since fence is stronger than compiler_fence, + // but it's the best we can do for the time being. + pub(crate) use self::fence as compiler_fence; + } + pub(crate) use loom::sync::Arc; + } + pub(crate) use loom::lazy_static; + pub(crate) use loom::thread_local; +} +#[cfg(not(loom_crossbeam))] +#[allow(unused_imports, dead_code)] +mod primitive { + #[cfg(any(feature = "alloc", feature = "std"))] + pub(crate) mod cell { + #[derive(Debug)] + #[repr(transparent)] + pub(crate) struct UnsafeCell(::core::cell::UnsafeCell); + + // loom's UnsafeCell has a slightly different API than the standard library UnsafeCell. + // Since we want the rest of the code to be agnostic to whether it's running under loom or + // not, we write this small wrapper that provides the loom-supported API for the standard + // library UnsafeCell. This is also what the loom documentation recommends: + // https://github.com/tokio-rs/loom#handling-loom-api-differences + impl UnsafeCell { + #[inline] + pub(crate) fn new(data: T) -> UnsafeCell { + UnsafeCell(::core::cell::UnsafeCell::new(data)) + } + + #[inline] + pub(crate) fn with(&self, f: impl FnOnce(*const T) -> R) -> R { + f(self.0.get()) + } + + #[inline] + pub(crate) fn with_mut(&self, f: impl FnOnce(*mut T) -> R) -> R { + f(self.0.get()) + } + } + } + #[cfg(any(feature = "alloc", feature = "std"))] + pub(crate) mod sync { + pub(crate) mod atomic { + pub(crate) use core::sync::atomic::compiler_fence; + pub(crate) use core::sync::atomic::fence; + pub(crate) use core::sync::atomic::AtomicUsize; + } + #[cfg_attr(feature = "nightly", cfg(target_has_atomic = "ptr"))] + pub(crate) use alloc::sync::Arc; + } + + #[cfg(feature = "std")] + pub(crate) use std::thread_local; + + #[cfg(feature = "std")] + pub(crate) use lazy_static::lazy_static; +} + #[cfg_attr(feature = "nightly", cfg(target_has_atomic = "ptr"))] cfg_if! { if #[cfg(feature = "alloc")] { diff --git a/crossbeam-epoch/src/sync/list.rs b/crossbeam-epoch/src/sync/list.rs index 656e2a849..4255f8742 100644 --- a/crossbeam-epoch/src/sync/list.rs +++ b/crossbeam-epoch/src/sync/list.rs @@ -295,7 +295,7 @@ impl<'g, T: 'g, C: IsElement> Iterator for Iter<'g, T, C> { } } -#[cfg(test)] +#[cfg(all(test, not(loom_crossbeam)))] mod tests { use super::*; use crate::{Collector, Owned}; diff --git a/crossbeam-epoch/src/sync/queue.rs b/crossbeam-epoch/src/sync/queue.rs index 71ea1bcc0..db9e8de04 100644 --- a/crossbeam-epoch/src/sync/queue.rs +++ b/crossbeam-epoch/src/sync/queue.rs @@ -207,7 +207,7 @@ impl Drop for Queue { } } -#[cfg(test)] +#[cfg(all(test, not(loom_crossbeam)))] mod test { use super::*; use crate::pin; diff --git a/crossbeam-epoch/tests/loom.rs b/crossbeam-epoch/tests/loom.rs new file mode 100644 index 000000000..fa4120f51 --- /dev/null +++ b/crossbeam-epoch/tests/loom.rs @@ -0,0 +1,153 @@ +#![cfg(loom_crossbeam)] + +use crossbeam_epoch as epoch; + +use epoch::*; +use epoch::{Atomic, Owned}; +use loom::sync::atomic::Ordering::{self, Acquire, Relaxed, Release}; +use loom::sync::Arc; +use loom::thread::spawn; +use std::mem::ManuallyDrop; +use std::ptr; + +#[test] +fn it_works() { + loom::model(|| { + let collector = Collector::new(); + let item: Atomic = Atomic::from(Owned::new(String::from("boom"))); + let item2 = item.clone(); + let collector2 = collector.clone(); + let guard = collector.register().pin(); + + let jh = loom::thread::spawn(move || { + let guard = collector2.register().pin(); + guard.defer(move || { + // this isn't really safe, since other threads may still have pointers to the + // value, but in this limited test scenario it's okay, since we know the test won't + // access item after all the pins are released. + let mut item = unsafe { item2.into_owned() }; + // mutate it as a second measure to make sure the assert_eq below would fail + item.retain(|c| c == 'o'); + drop(item); + }); + }); + + let item = item.load(Ordering::SeqCst, &guard); + // we pinned strictly before the call to defer_destroy, + // so item cannot have been dropped yet + assert_eq!(*unsafe { item.deref() }, "boom"); + drop(guard); + + jh.join().unwrap(); + + drop(collector); + }) +} + +#[test] +fn treiber_stack() { + /// Treiber's lock-free stack. + /// + /// Usable with any number of producers and consumers. + #[derive(Debug)] + pub struct TreiberStack { + head: Atomic>, + } + + #[derive(Debug)] + struct Node { + data: ManuallyDrop, + next: Atomic>, + } + + impl TreiberStack { + /// Creates a new, empty stack. + pub fn new() -> TreiberStack { + TreiberStack { + head: Atomic::null(), + } + } + + /// Pushes a value on top of the stack. + pub fn push(&self, t: T) { + let mut n = Owned::new(Node { + data: ManuallyDrop::new(t), + next: Atomic::null(), + }); + + let guard = epoch::pin(); + + loop { + let head = self.head.load(Relaxed, &guard); + n.next.store(head, Relaxed); + + match self.head.compare_and_set(head, n, Release, &guard) { + Ok(_) => break, + Err(e) => n = e.new, + } + } + } + + /// Attempts to pop the top element from the stack. + /// + /// Returns `None` if the stack is empty. + pub fn pop(&self) -> Option { + let guard = epoch::pin(); + loop { + let head = self.head.load(Acquire, &guard); + + match unsafe { head.as_ref() } { + Some(h) => { + let next = h.next.load(Relaxed, &guard); + + if self + .head + .compare_and_set(head, next, Relaxed, &guard) + .is_ok() + { + unsafe { + guard.defer_destroy(head); + return Some(ManuallyDrop::into_inner(ptr::read(&(*h).data))); + } + } + } + None => return None, + } + } + } + + /// Returns `true` if the stack is empty. + pub fn is_empty(&self) -> bool { + let guard = epoch::pin(); + self.head.load(Acquire, &guard).is_null() + } + } + + impl Drop for TreiberStack { + fn drop(&mut self) { + while self.pop().is_some() {} + } + } + + loom::model(|| { + let stack1 = Arc::new(TreiberStack::new()); + let stack2 = Arc::clone(&stack1); + + // use 5 since it's greater than the 4 used for the sanitize feature + let jh = spawn(move || { + for i in 0..5 { + stack2.push(i); + assert!(stack2.pop().is_some()); + } + }); + + for i in 0..5 { + stack1.push(i); + assert!(stack1.pop().is_some()); + } + + jh.join().unwrap(); + assert!(stack1.pop().is_none()); + assert!(stack1.is_empty()); + }); +} diff --git a/crossbeam-utils/Cargo.toml b/crossbeam-utils/Cargo.toml index 3c2360a50..f13d986e2 100644 --- a/crossbeam-utils/Cargo.toml +++ b/crossbeam-utils/Cargo.toml @@ -33,6 +33,13 @@ nightly = [] cfg-if = "1" lazy_static = { version = "1.4.0", optional = true } +# Enable the use of loom for concurrency testing. +# +# This configuration option is outside of the normal semver guarantees: minor +# versions of crossbeam may make breaking changes to it at any time. +[target.'cfg(loom_crossbeam)'.dependencies] +loom = "0.4" + [build-dependencies] autocfg = "1.0.0" diff --git a/crossbeam-utils/src/atomic/atomic_cell.rs b/crossbeam-utils/src/atomic/atomic_cell.rs index f6e164526..3bb06f560 100644 --- a/crossbeam-utils/src/atomic/atomic_cell.rs +++ b/crossbeam-utils/src/atomic/atomic_cell.rs @@ -2,15 +2,19 @@ #![allow(clippy::unit_arg)] #![allow(clippy::let_unit_value)] +use crate::primitive::sync::atomic::{self, AtomicBool}; use core::cell::UnsafeCell; use core::fmt; use core::mem; +use core::sync::atomic::Ordering; + +#[cfg(not(loom_crossbeam))] use core::ptr; -use core::sync::atomic::{self, AtomicBool, Ordering}; #[cfg(feature = "std")] use std::panic::{RefUnwindSafe, UnwindSafe}; +#[cfg(not(loom_crossbeam))] use super::seq_lock::SeqLock; /// A thread-safe mutable memory location. @@ -494,23 +498,23 @@ macro_rules! impl_arithmetic { #[cfg(has_atomic_u8)] impl_arithmetic!(u8, atomic::AtomicU8, "let a = AtomicCell::new(7u8);"); -#[cfg(has_atomic_u8)] +#[cfg(all(has_atomic_u8, not(loom_crossbeam)))] impl_arithmetic!(i8, atomic::AtomicI8, "let a = AtomicCell::new(7i8);"); #[cfg(has_atomic_u16)] impl_arithmetic!(u16, atomic::AtomicU16, "let a = AtomicCell::new(7u16);"); -#[cfg(has_atomic_u16)] +#[cfg(all(has_atomic_u16, not(loom_crossbeam)))] impl_arithmetic!(i16, atomic::AtomicI16, "let a = AtomicCell::new(7i16);"); #[cfg(has_atomic_u32)] impl_arithmetic!(u32, atomic::AtomicU32, "let a = AtomicCell::new(7u32);"); -#[cfg(has_atomic_u32)] +#[cfg(all(has_atomic_u32, not(loom_crossbeam)))] impl_arithmetic!(i32, atomic::AtomicI32, "let a = AtomicCell::new(7i32);"); #[cfg(has_atomic_u64)] impl_arithmetic!(u64, atomic::AtomicU64, "let a = AtomicCell::new(7u64);"); -#[cfg(has_atomic_u64)] +#[cfg(all(has_atomic_u64, not(loom_crossbeam)))] impl_arithmetic!(i64, atomic::AtomicI64, "let a = AtomicCell::new(7i64);"); -#[cfg(has_atomic_u128)] +#[cfg(all(has_atomic_u128, not(loom_crossbeam)))] impl_arithmetic!(u128, atomic::AtomicU128, "let a = AtomicCell::new(7u128);"); -#[cfg(has_atomic_u128)] +#[cfg(all(has_atomic_u128, not(loom_crossbeam)))] impl_arithmetic!(i128, atomic::AtomicI128, "let a = AtomicCell::new(7i128);"); impl_arithmetic!( @@ -518,6 +522,7 @@ impl_arithmetic!( atomic::AtomicUsize, "let a = AtomicCell::new(7usize);" ); +#[cfg(not(loom_crossbeam))] impl_arithmetic!( isize, atomic::AtomicIsize, @@ -626,6 +631,7 @@ const fn can_transmute() -> bool { /// scalability. #[inline] #[must_use] +#[cfg(not(loom_crossbeam))] fn lock(addr: usize) -> &'static SeqLock { // The number of locks is a prime number because we want to make sure `addr % LEN` gets // dispersed across all locks. @@ -812,6 +818,9 @@ macro_rules! atomic { #[cfg(has_atomic_u128)] atomic!(@check, $t, atomic::AtomicU128, $a, $atomic_op); + #[cfg(loom_crossbeam)] + unimplemented!("loom does not support non-atomic atomic ops"); + #[cfg(not(loom_crossbeam))] break $fallback_op; } }; diff --git a/crossbeam-utils/src/atomic/consume.rs b/crossbeam-utils/src/atomic/consume.rs index 584fc34c1..5278504f2 100644 --- a/crossbeam-utils/src/atomic/consume.rs +++ b/crossbeam-utils/src/atomic/consume.rs @@ -1,5 +1,5 @@ #[cfg(any(target_arch = "arm", target_arch = "aarch64"))] -use core::sync::atomic::compiler_fence; +use crate::primitive::sync::atomic::compiler_fence; use core::sync::atomic::Ordering; /// Trait which allows reading from primitive atomic types with "consume" ordering. @@ -53,11 +53,17 @@ macro_rules! impl_atomic { type Val = $val; impl_consume!(); } + #[cfg(loom_crossbeam)] + impl AtomicConsume for ::loom::sync::atomic::$atomic { + type Val = $val; + impl_consume!(); + } }; } impl_atomic!(AtomicBool, bool); impl_atomic!(AtomicUsize, usize); +#[cfg(not(loom_crossbeam))] impl_atomic!(AtomicIsize, isize); #[cfg(has_atomic_u8)] impl_atomic!(AtomicU8, u8); @@ -80,3 +86,9 @@ impl AtomicConsume for ::core::sync::atomic::AtomicPtr { type Val = *mut T; impl_consume!(); } + +#[cfg(loom_crossbeam)] +impl AtomicConsume for ::loom::sync::atomic::AtomicPtr { + type Val = *mut T; + impl_consume!(); +} diff --git a/crossbeam-utils/src/atomic/mod.rs b/crossbeam-utils/src/atomic/mod.rs index 7309c166d..1dc3e7e2e 100644 --- a/crossbeam-utils/src/atomic/mod.rs +++ b/crossbeam-utils/src/atomic/mod.rs @@ -1,7 +1,9 @@ //! Atomic types. +#[cfg(not(loom_crossbeam))] use cfg_if::cfg_if; +#[cfg(not(loom_crossbeam))] cfg_if! { // Use "wide" sequence lock if the pointer width <= 32 for preventing its counter against wrap // around. diff --git a/crossbeam-utils/src/backoff.rs b/crossbeam-utils/src/backoff.rs index ea92d7838..04188e024 100644 --- a/crossbeam-utils/src/backoff.rs +++ b/crossbeam-utils/src/backoff.rs @@ -1,6 +1,6 @@ +use crate::primitive::sync::atomic; use core::cell::Cell; use core::fmt; -use core::sync::atomic; const SPIN_LIMIT: u32 = 6; const YIELD_LIMIT: u32 = 10; diff --git a/crossbeam-utils/src/lib.rs b/crossbeam-utils/src/lib.rs index f2bd460b9..bda9e6e6b 100644 --- a/crossbeam-utils/src/lib.rs +++ b/crossbeam-utils/src/lib.rs @@ -37,6 +37,57 @@ // matches! requires Rust 1.42 #![allow(clippy::match_like_matches_macro)] +#[cfg(loom_crossbeam)] +#[allow(unused_imports)] +mod primitive { + pub(crate) mod sync { + pub(crate) mod atomic { + pub(crate) use loom::sync::atomic::spin_loop_hint; + pub(crate) use loom::sync::atomic::{ + AtomicBool, AtomicU16, AtomicU32, AtomicU64, AtomicU8, AtomicUsize, + }; + + // FIXME: loom does not support compiler_fence at the moment. + // https://github.com/tokio-rs/loom/issues/117 + // we use fence as a stand-in for compiler_fence for the time being. + // this may miss some races since fence is stronger than compiler_fence, + // but it's the best we can do for the time being. + pub(crate) use loom::sync::atomic::fence as compiler_fence; + } + pub(crate) use loom::sync::{Arc, Condvar, Mutex}; + } +} +#[cfg(not(loom_crossbeam))] +#[allow(unused_imports)] +mod primitive { + pub(crate) mod sync { + pub(crate) mod atomic { + pub(crate) use core::sync::atomic::compiler_fence; + pub(crate) use core::sync::atomic::spin_loop_hint; + pub(crate) use core::sync::atomic::{AtomicBool, AtomicIsize, AtomicUsize}; + #[cfg(has_atomic_u16)] + pub(crate) use core::sync::atomic::{AtomicI16, AtomicU16}; + #[cfg(has_atomic_u32)] + pub(crate) use core::sync::atomic::{AtomicI32, AtomicU32}; + #[cfg(has_atomic_u64)] + pub(crate) use core::sync::atomic::{AtomicI64, AtomicU64}; + #[cfg(has_atomic_u8)] + pub(crate) use core::sync::atomic::{AtomicI8, AtomicU8}; + } + + #[cfg(feature = "std")] + pub(crate) use std::sync::{Arc, Condvar, Mutex}; + } +} + +cfg_if! { + if #[cfg(feature = "alloc")] { + extern crate alloc; + } else if #[cfg(feature = "std")] { + extern crate std as alloc; + } +} + #[cfg_attr(feature = "nightly", cfg(target_has_atomic = "ptr"))] pub mod atomic; @@ -51,6 +102,8 @@ use cfg_if::cfg_if; cfg_if! { if #[cfg(feature = "std")] { pub mod sync; + + #[cfg(not(loom_crossbeam))] pub mod thread; } } diff --git a/crossbeam-utils/src/sync/mod.rs b/crossbeam-utils/src/sync/mod.rs index fd400d70e..351940f13 100644 --- a/crossbeam-utils/src/sync/mod.rs +++ b/crossbeam-utils/src/sync/mod.rs @@ -5,9 +5,11 @@ //! * [`WaitGroup`], for synchronizing the beginning or end of some computation. mod parker; +#[cfg(not(loom_crossbeam))] mod sharded_lock; mod wait_group; pub use self::parker::{Parker, Unparker}; +#[cfg(not(loom_crossbeam))] pub use self::sharded_lock::{ShardedLock, ShardedLockReadGuard, ShardedLockWriteGuard}; pub use self::wait_group::WaitGroup; diff --git a/crossbeam-utils/src/sync/parker.rs b/crossbeam-utils/src/sync/parker.rs index bf9d6f347..29cc936bc 100644 --- a/crossbeam-utils/src/sync/parker.rs +++ b/crossbeam-utils/src/sync/parker.rs @@ -1,8 +1,8 @@ +use crate::primitive::sync::atomic::AtomicUsize; +use crate::primitive::sync::{Arc, Condvar, Mutex}; +use core::sync::atomic::Ordering::SeqCst; use std::fmt; use std::marker::PhantomData; -use std::sync::atomic::AtomicUsize; -use std::sync::atomic::Ordering::SeqCst; -use std::sync::{Arc, Condvar, Mutex}; use std::time::{Duration, Instant}; /// A thread parking primitive. diff --git a/crossbeam-utils/src/sync/wait_group.rs b/crossbeam-utils/src/sync/wait_group.rs index cd7af12a3..4206ee42b 100644 --- a/crossbeam-utils/src/sync/wait_group.rs +++ b/crossbeam-utils/src/sync/wait_group.rs @@ -1,8 +1,8 @@ // Necessary for using `Mutex` for conditional variables #![allow(clippy::mutex_atomic)] +use crate::primitive::sync::{Arc, Condvar, Mutex}; use std::fmt; -use std::sync::{Arc, Condvar, Mutex}; /// Enables threads to synchronize the beginning or end of some computation. /// diff --git a/src/lib.rs b/src/lib.rs index 72afa7a48..f9bc86414 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -95,7 +95,10 @@ cfg_if! { pub use crossbeam_channel::select; pub use crossbeam_utils::sync; + + #[cfg(not(loom_crossbeam))] pub use crossbeam_utils::thread; + #[cfg(not(loom_crossbeam))] pub use crossbeam_utils::thread::scope; } }