diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index e884e1b..e313d2b 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -231,8 +231,7 @@ jobs:
       - name: Run miri
         env:
           PROPTEST_CASES: "10"
-          # TODO: Do something with the stacked borrows. Figure out what it means.
-          MIRIFLAGS: "-Zmiri-disable-isolation -Zmiri-disable-stacked-borrows"
+          MIRIFLAGS: "-Zmiri-disable-isolation -Zmiri-permissive-provenance"
         run: cargo miri test --all-features
 
   thread_sanitizer-MacOS:
diff --git a/src/lib.rs b/src/lib.rs
index 7e2e6c2..c9526a3 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -883,7 +883,7 @@ macro_rules! t {
                 const WRITER_CNT: usize = 2;
                 const READER_CNT: usize = 3;
                 #[cfg(miri)]
-                const ITERATIONS: usize = 10;
+                const ITERATIONS: usize = 5;
                 #[cfg(not(miri))]
                 const ITERATIONS: usize = 100;
                 const SEQ: usize = 50;
@@ -1160,6 +1160,9 @@ macro_rules! t {
             #[test]
             /// Make sure the reference count and compare_and_swap works as expected.
             fn cas_ref_cnt() {
+                #[cfg(miri)]
+                const ITERATIONS: usize = 10;
+                #[cfg(not(miri))]
                 const ITERATIONS: usize = 50;
                 let shared = ArcSwap::from(Arc::new(0));
                 for i in 0..ITERATIONS {
@@ -1173,7 +1176,7 @@ macro_rules! t {
                     // Fill up the slots sometimes
                     let fillup = || {
                         if i % 2 == 0 {
-                            Some((0..50).map(|_| shared.load()).collect::<Vec<_>>())
+                            Some((0..ITERATIONS).map(|_| shared.load()).collect::<Vec<_>>())
                         } else {
                             None
                         }
@@ -1272,10 +1275,14 @@ mod tests {
     /// created, but contain full Arcs.
     #[test]
     fn lease_overflow() {
+        #[cfg(miri)]
+        const GUARD_COUNT: usize = 100;
+        #[cfg(not(miri))]
+        const GUARD_COUNT: usize = 1000;
         let a = Arc::new(0);
         let shared = ArcSwap::from(Arc::clone(&a));
         assert_eq!(2, Arc::strong_count(&a));
-        let mut guards = (0..1000).map(|_| shared.load()).collect::<Vec<_>>();
+        let mut guards = (0..GUARD_COUNT).map(|_| shared.load()).collect::<Vec<_>>();
         let count = Arc::strong_count(&a);
         assert!(count > 2);
         let guard = shared.load();
diff --git a/src/ref_cnt.rs b/src/ref_cnt.rs
index 5128f5c..2ea4ec6 100644
--- a/src/ref_cnt.rs
+++ b/src/ref_cnt.rs
@@ -92,7 +92,9 @@ unsafe impl<T> RefCnt for Arc<T> {
         Arc::into_raw(me) as *mut T
     }
     fn as_ptr(me: &Arc<T>) -> *mut T {
-        me as &T as *const T as *mut T
+        // SAFETY: &T cast to *const T will always be aligned, initialised and valid for reads
+        let ptr = Arc::into_raw(unsafe { std::ptr::read(me) });
+        ptr as *mut T
     }
     unsafe fn from_ptr(ptr: *const T) -> Arc<T> {
         Arc::from_raw(ptr)
@@ -105,7 +107,9 @@ unsafe impl<T> RefCnt for Rc<T> {
         Rc::into_raw(me) as *mut T
     }
     fn as_ptr(me: &Rc<T>) -> *mut T {
-        me as &T as *const T as *mut T
+        // SAFETY: &T cast to *const T will always be aligned, initialised and valid for reads
+        let ptr = Rc::into_raw(unsafe { std::ptr::read(me) });
+        ptr as *mut T
     }
     unsafe fn from_ptr(ptr: *const T) -> Rc<T> {
         Rc::from_raw(ptr)