Multi tree column option (#232)

* Multitree root commit with blocking log write

* fmt

* Implement get_root and get_node

* Initial work on readers

* Working readers

* Working iterator

* Added TreeReader for accessing tree root and nodes

* get_tree returns RwLock reader

* Track ValueTable free entries in memory

* Added ability to claim free entries from ValueTable. Used this to make tree insertion work with commit queue.

* Make ChainGenerator generate trees that share nodes from previous trees

* Tree commits share existing nodes

* Stress test tree pruning. Tree removal (Currently just removes root).

* Implemented tree removal with reference counting for shared nodes

* Empty on shutdown option. This removes all trees and waits for value tables to empty.

* Depth based age histograms for more accurate chain generation. Increased node sharing.

* Prepare for using claim_contiguous_entries

* Append-only mode

* Check RC on dereferencing root

* Correctly use full key or hash

* Safer entry claiming. Deal with tree removal while a commit is being built using that tree by deferring the removal.

* Separate tree operations

* Added various checks for correct usage

* Reference count tables

* fmt

* Remove value table verification of ref counts

* Only create and use ref count table when needed

* Multitree stress fix for appending to existing database

* On restart table data needs to be generated after all log files have been enacted

* In memory ref count cache. Verifies with table.

* Only access ref count table when needed

* Rename

* Remove commented out lines

* madvise

* Ref count overlay reclaiming

* Chunk buffer alignment

* Test fix

* Windows fix

* Fix

* Fix

* Loom RwLock requires Sized

* Debug asserts

* Use INFO_COLUMN as column index

* Avoid changing existing log action values

* Typo

* Fix

* Remove ordered for now

* Remove claim_next_free

* Read lock

* Improved node data packing and unpacking

* Simple ref_count tests

* Remove unused multi tree node compression

* Added allow_direct_node_access column option

---------

Co-authored-by: arkpar <arkady.paronyan@gmail.com>

.gitignore

@@ -15,6 +15,7 @@ Cargo.lock
 *.log
 test_db/
 test_db_stress/
+test_db_multitree_stress/
 
 # vim
 *.swp

admin/Cargo.toml

@@ -11,8 +11,10 @@ env_logger = { version = "0.10.0", default-features = false, features = ["auto-c
 fdlimit = "0.2.1"
 log = "0.4.8"
 parity-db = { path = ".." }
+parking_lot = "0.12.0"
 rand = { version = "0.8.5", features = ["small_rng"] }
 blake2 = "0.10.4"
+siphasher = "0.3.10"
 
 [target.'cfg(not(target_env = "msvc"))'.dependencies]
 jemallocator = "0.5.0"

admin/src/lib.rs

@@ -6,6 +6,7 @@
 use std::path::PathBuf;
 
 mod bench;
+mod multitree_bench;
 
 /// Command line admin client entry point.
 /// Uses default column definition.
@@ -127,6 +128,44 @@ pub fn run() -> Result<(), String> {
 			let db = parity_db::Db::open_or_create(&db_options).unwrap();
 			bench::run_internal(args, db);
 		},
+		SubCommand::MultiTreeStress(bench) => {
+			let args = bench.get_args();
+			// avoid deleting folders by mistake.
+			options.path.push("test_db_multitree_stress");
+			if options.path.exists() && !args.append {
+				std::fs::remove_dir_all(options.path.as_path())
+					.map_err(|e| format!("Error clearing multitree stress db: {e:?}"))?;
+			}
+
+			let mut db_options = options.clone();
+
+			for c in &mut db_options.columns {
+				c.multitree = true;
+				if args.pruning == 0 {
+					c.append_only = true;
+				}
+			}
+
+			if args.compress {
+				for c in &mut db_options.columns {
+					c.compression = parity_db::CompressionType::Lz4;
+				}
+			}
+
+			if db_options.columns.len() < 2 {
+				let info_column: parity_db::ColumnOptions = Default::default();
+				db_options.columns.push(info_column);
+			}
+
+			let info_column = &mut db_options.columns[multitree_bench::INFO_COLUMN as usize];
+			info_column.uniform = false;
+			info_column.multitree = false;
+			info_column.ref_counted = false;
+			info_column.preimage = false;
+
+			let db = parity_db::Db::open_or_create(&db_options).unwrap();
+			multitree_bench::run_internal(args, db)?;
+		},
 	}
 	Ok(())
 }
@@ -187,6 +226,8 @@ pub enum SubCommand {
 	Check(Check),
 	/// Stress tests.
 	Stress(bench::Stress),
+	/// Multitree stress test.
+	MultiTreeStress(multitree_bench::MultiTreeStress),
 }
 
 impl SubCommand {
@@ -207,6 +248,7 @@ impl Cli {
 			SubCommand::Flush(flush) => &flush.shared,
 			SubCommand::Check(check) => &check.shared,
 			SubCommand::Stress(bench) => &bench.shared,
+			SubCommand::MultiTreeStress(bench) => &bench.shared,
 		}
 	}
 }