Merge the tests and most of the Tree methods from old sled into sled 1.0

Cargo.toml

@@ -13,6 +13,12 @@ categories = ["database-implementations", "concurrency", "data-structures", "alg
 readme = "README.md"
 exclude = ["benchmarks", "examples", "bindings", "scripts", "experiments"]
 
+[features]
+# initializes allocated memory to 0xa1, writes 0xde to deallocated memory before freeing it
+testing_shred_allocator = []
+# use a counting global allocator that provides the sled::alloc::{allocated, freed, resident, reset} functions
+testing_count_allocator = []
+
 [dependencies]
 bincode = "1.3.3"
 cache-advisor = "1.0.12"
@@ -40,3 +46,12 @@ num-format = "0.4.4"
 rocksdb = "0.21.0"
 rusqlite = "0.29.0"
 old_sled = { version = "0.34", package = "sled" }
+rand = "0.8.5"
+quickcheck = "1.0.3"
+rand_distr = "0.4.3"
+
+[[test]]
+name = "test_crash_recovery"
+path = "tests/test_crash_recovery.rs"
+harness = false
+

examples/bench.rs

@@ -36,9 +36,10 @@ impl Databench for Db {
         sled::Config {
             path: Self::PATH.into(),
             zstd_compression_level: 3,
-            cache_size: 1024 * 1024 * 1024,
+            cache_capacity_bytes: 1024 * 1024 * 1024,
             entry_cache_percent: 20,
             flush_every_ms: Some(200),
+            temporary: false,
         }
         .open()
         .unwrap()

src/alloc.rs

@@ -0,0 +1,81 @@
+#[cfg(any(
+    feature = "testing_shred_allocator",
+    feature = "testing_count_allocator"
+))]
+pub use alloc::*;
+
+// the memshred feature causes all allocated and deallocated
+// memory to be set to a specific non-zero value of 0xa1 for
+// uninitialized allocations and 0xde for deallocated memory,
+// in the hope that it will cause memory errors to surface
+// more quickly.
+
+#[cfg(feature = "testing_shred_allocator")]
+mod alloc {
+    use std::alloc::{Layout, System};
+
+    #[global_allocator]
+    static ALLOCATOR: ShredAllocator = ShredAllocator;
+
+    #[derive(Default, Debug, Clone, Copy)]
+    struct ShredAllocator;
+
+    unsafe impl std::alloc::GlobalAlloc for ShredAllocator {
+        unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
+            let ret = System.alloc(layout);
+            assert_ne!(ret, std::ptr::null_mut());
+            std::ptr::write_bytes(ret, 0xa1, layout.size());
+            ret
+        }
+
+        unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
+            std::ptr::write_bytes(ptr, 0xde, layout.size());
+            System.dealloc(ptr, layout)
+        }
+    }
+}
+
+#[cfg(feature = "testing_shred_allocator")]
+mod alloc {
+    use std::alloc::{Layout, System};
+
+    #[global_allocator]
+    static ALLOCATOR: CountingAllocator = CountingAllocator;
+
+    static ALLOCATED: AtomicUsize = AtomicUsize::new(0);
+    static FREED: AtomicUsize = AtomicUsize::new(0);
+    static RESIDENT: AtomicUsize = AtomicUsize::new(0);
+
+    fn allocated() -> usize {
+        ALLOCATED.swap(0, Ordering::Relaxed)
+    }
+
+    fn freed() -> usize {
+        FREED.swap(0, Ordering::Relaxed)
+    }
+
+    fn resident() -> usize {
+        RESIDENT.load(Ordering::Relaxed)
+    }
+
+    #[derive(Default, Debug, Clone, Copy)]
+    struct CountingAllocator;
+
+    unsafe impl std::alloc::GlobalAlloc for CountingAllocator {
+        unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
+            let ret = System.alloc(layout);
+            assert_ne!(ret, std::ptr::null_mut());
+            ALLOCATED.fetch_add(layout.size(), Ordering::Relaxed);
+            RESIDENT.fetch_add(layout.size(), Ordering::Relaxed);
+            std::ptr::write_bytes(ret, 0xa1, layout.size());
+            ret
+        }
+
+        unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
+            std::ptr::write_bytes(ptr, 0xde, layout.size());
+            FREED.fetch_add(layout.size(), Ordering::Relaxed);
+            RESIDENT.fetch_sub(layout.size(), Ordering::Relaxed);
+            System.dealloc(ptr, layout)
+        }
+    }
+}

src/config.rs

@@ -0,0 +1,66 @@
+use std::path::{Path, PathBuf};
+
+use crate::Db;
+
+#[derive(Debug, Clone)]
+pub struct Config {
+    /// The base directory for storing the database.
+    pub path: PathBuf,
+    /// Cache size in **bytes**. Default is 512mb.
+    pub cache_capacity_bytes: usize,
+    /// The percentage of the cache that is dedicated to the
+    /// scan-resistant entry cache.
+    pub entry_cache_percent: u8,
+    /// Start a background thread that flushes data to disk
+    /// every few milliseconds. Defaults to every 200ms.
+    pub flush_every_ms: Option<usize>,
+    /// The zstd compression level to use when writing data to disk. Defaults to 3.
+    pub zstd_compression_level: i32,
+    /// Attempts to delete all storage files when the last open instance of a Db is dropped.
+    pub temporary: bool,
+}
+
+impl Default for Config {
+    fn default() -> Config {
+        Config {
+            path: "bloodstone.default".into(),
+            flush_every_ms: Some(200),
+            cache_capacity_bytes: 512 * 1024 * 1024,
+            entry_cache_percent: 20,
+            zstd_compression_level: 3,
+            temporary: false,
+        }
+    }
+}
+
+impl Config {
+    /// Returns a default `Config`
+    pub fn new() -> Config {
+        Config::default()
+    }
+
+    /// Set the path of the database (builder).
+    pub fn path<P: AsRef<Path>>(mut self, path: P) -> Config {
+        self.path = path.as_ref().to_path_buf();
+        self
+    }
+
+    builder!(
+        (flush_every_ms, Option<usize>, "Start a background thread that flushes data to disk every few milliseconds. Defaults to every 200ms."),
+        (cache_capacity_bytes, usize, "Cache size in **bytes**. Default is 512mb."),
+        (entry_cache_percent, u8, "The percentage of the cache that is dedicated to the scan-resistant entry cache."),
+        (zstd_compression_level, i32, "The zstd compression level to use when writing data to disk. Defaults to 3."),
+        (temporary, bool, "Attempts to delete all storage files when the last open instance of a Db is dropped.")
+    );
+
+    pub fn open<
+        const INDEX_FANOUT: usize,
+        const LEAF_FANOUT: usize,
+        const EBR_LOCAL_GC_BUFFER_SIZE: usize,
+    >(
+        &self,
+    ) -> std::io::Result<Db<INDEX_FANOUT, LEAF_FANOUT, EBR_LOCAL_GC_BUFFER_SIZE>>
+    {
+        Db::open_with_config(self)
+    }
+}