diff --git a/examples/sync/src/bin/client.rs b/examples/sync/src/bin/client.rs index c4b6b06aea..8eb0ad4090 100644 --- a/examples/sync/src/bin/client.rs +++ b/examples/sync/src/bin/client.rs @@ -10,11 +10,7 @@ use commonware_runtime::{ }; use commonware_storage::qmdb::sync; use commonware_sync::{ - any, crate_version, - databases::{DatabaseType, Syncable}, - immutable, - net::Resolver, - Digest, Error, Key, + any, crate_version, databases::DatabaseType, immutable, net::Resolver, Digest, Error, Key, }; use commonware_utils::DurationExt; use futures::channel::mpsc; diff --git a/examples/sync/src/bin/server.rs b/examples/sync/src/bin/server.rs index 4a49d11833..b5482afcda 100644 --- a/examples/sync/src/bin/server.rs +++ b/examples/sync/src/bin/server.rs @@ -57,8 +57,8 @@ struct Config { /// Server state containing the database and metrics. struct State { - /// The database wrapped in async mutex. - database: RwLock, + /// The database wrapped in async mutex with Option to allow ownership transfers. + database: RwLock>, /// Request counter for metrics. request_counter: Counter, /// Error counter for metrics. @@ -75,7 +75,7 @@ impl State { E: Metrics, { let state = Self { - database: RwLock::new(database), + database: RwLock::new(Some(database)), request_counter: Counter::default(), error_counter: Counter::default(), ops_counter: Counter::default(), @@ -116,11 +116,19 @@ where let new_operations_len = new_operations.len(); // Add operations to database and get the new root let root = { - let mut database = state.database.write().await; - if let Err(err) = DB::add_operations(&mut *database, new_operations).await { - error!(?err, "failed to add operations to database"); + let mut db_opt = state.database.write().await; + let database = db_opt.take().expect("database should exist"); + match database.add_operations(new_operations).await { + Ok(database) => { + let root = database.root(); + *db_opt = Some(database); + root + } + Err(err) => { + error!(?err, "failed to add operations to database"); + return Err(err.into()); + } } - DB::root(&*database) }; state.ops_counter.inc_by(new_operations_len as u64); let root_hex = root @@ -150,7 +158,8 @@ where // Get the current database state let (root, lower_bound, upper_bound) = { - let database = state.database.read().await; + let db_opt = state.database.read().await; + let database = db_opt.as_ref().expect("database should exist"); (database.root(), database.lower_bound(), database.op_count()) }; let response = wire::GetSyncTargetResponse:: { @@ -176,7 +185,8 @@ where state.request_counter.inc(); request.validate()?; - let database = state.database.read().await; + let db_opt = state.database.read().await; + let database = db_opt.as_ref().expect("database should exist"); // Check if we have enough operations let db_size = database.op_count(); @@ -206,7 +216,7 @@ where .historical_proof(request.op_count, request.start_loc, max_ops) .await; - drop(database); + drop(db_opt); let (proof, operations) = result.map_err(|err| { warn!(?err, "failed to generate historical proof"); @@ -354,10 +364,10 @@ where /// Initialize and display database state with initial operations. async fn initialize_database( - database: &mut DB, + database: DB, config: &Config, context: &mut E, -) -> Result<(), Box> +) -> Result> where DB: Syncable, E: RngCore, @@ -370,10 +380,7 @@ where operations_len = initial_ops.len(), "creating initial operations" ); - DB::add_operations(database, initial_ops).await?; - - // Commit the database to ensure operations are persisted - database.commit().await?; + let database = database.add_operations(initial_ops).await?; // Display database state let root = database.root(); @@ -389,14 +396,14 @@ where DB::name() ); - Ok(()) + Ok(database) } /// Run a generic server with the given database. async fn run_helper( mut context: E, config: Config, - mut database: DB, + database: DB, ) -> Result<(), Box> where DB: Syncable + Send + Sync + 'static, @@ -405,7 +412,7 @@ where { info!("starting {} database server", DB::name()); - initialize_database(&mut database, &config, &mut context).await?; + let database = initialize_database(database, &config, &mut context).await?; // Create listener to accept connections let addr = SocketAddr::from((Ipv4Addr::LOCALHOST, config.port)); diff --git a/examples/sync/src/databases/any.rs b/examples/sync/src/databases/any.rs index 3290ff8a6a..fc2bbc8b26 100644 --- a/examples/sync/src/databases/any.rs +++ b/examples/sync/src/databases/any.rs @@ -14,13 +14,14 @@ use commonware_storage::{ }, FixedConfig as Config, }, - store::CleanStore, + operation::Committable, }, }; use commonware_utils::{NZUsize, NZU64}; use std::{future::Future, num::NonZeroU64}; +use tracing::error; -/// Database type alias. +/// Database type alias for the Clean state. pub type Database = Db; /// Operation type alias. @@ -77,32 +78,41 @@ where } async fn add_operations( - database: &mut Self, + self, operations: Vec, - ) -> Result<(), commonware_storage::qmdb::Error> { - for operation in operations { + ) -> Result { + if operations.last().is_none() || !operations.last().unwrap().is_commit() { + // Ignore bad inputs rather than return errors. + error!("operations must end with a commit"); + return Ok(self); + } + let mut db = self.into_mutable(); + let num_ops = operations.len(); + + for (i, operation) in operations.into_iter().enumerate() { match operation { Operation::Update(Update(key, value)) => { - database.update(key, value).await?; + db.update(key, value).await?; } Operation::Delete(key) => { - database.delete(key).await?; + db.delete(key).await?; } Operation::CommitFloor(metadata, _) => { - database.commit(metadata).await?; + let (durable_db, _) = db.commit(metadata).await?; + if i == num_ops - 1 { + // Last operation - return the clean database + return Ok(durable_db.into_merkleized()); + } + // Not the last operation - continue in mutable state + db = durable_db.into_mutable(); } } } - Ok(()) - } - - async fn commit(&mut self) -> Result<(), commonware_storage::qmdb::Error> { - self.commit(None).await?; - Ok(()) + panic!("operations should end with a commit"); } fn root(&self) -> Key { - CleanStore::root(self) + self.root() } fn op_count(&self) -> Location { @@ -119,7 +129,7 @@ where start_loc: Location, max_ops: NonZeroU64, ) -> impl Future, Vec), qmdb::Error>> + Send { - CleanStore::historical_proof(self, op_count, start_loc, max_ops) + self.historical_proof(op_count, start_loc, max_ops) } fn name() -> &'static str { diff --git a/examples/sync/src/databases/immutable.rs b/examples/sync/src/databases/immutable.rs index c90ac70db5..245cccebb3 100644 --- a/examples/sync/src/databases/immutable.rs +++ b/examples/sync/src/databases/immutable.rs @@ -8,13 +8,16 @@ use commonware_storage::{ qmdb::{ self, immutable::{self, Config}, + Durable, Merkleized, }, }; use commonware_utils::{NZUsize, NZU64}; use std::{future::Future, num::NonZeroU64}; +use tracing::error; -/// Database type alias. -pub type Database = immutable::Immutable; +/// Database type alias for the clean (merkleized, durable) state. +pub type Database = + immutable::Immutable, Durable>; /// Operation type alias. pub type Operation = immutable::Operation; @@ -79,24 +82,34 @@ where } async fn add_operations( - database: &mut Self, + self, operations: Vec, - ) -> Result<(), commonware_storage::qmdb::Error> { - for operation in operations { + ) -> Result { + if operations.last().is_none() || !operations.last().unwrap().is_commit() { + // Ignore bad inputs rather than return errors. + error!("operations must end with a commit"); + return Ok(self); + } + let mut db = self.into_mutable(); + let num_ops = operations.len(); + + for (i, operation) in operations.into_iter().enumerate() { match operation { Operation::Set(key, value) => { - database.set(key, value).await?; + db.set(key, value).await?; } Operation::Commit(metadata) => { - database.commit(metadata).await?; + let (durable_db, _) = db.commit(metadata).await?; + if i == num_ops - 1 { + // Last operation - return the clean database + return Ok(durable_db.into_merkleized()); + } + // Not the last operation - continue in mutable state + db = durable_db.into_mutable(); } } } - Ok(()) - } - - async fn commit(&mut self) -> Result<(), commonware_storage::qmdb::Error> { - self.commit(None).await + unreachable!("operations must end with a commit"); } fn root(&self) -> Key { diff --git a/examples/sync/src/databases/mod.rs b/examples/sync/src/databases/mod.rs index cbda68f750..fb1a0e2d55 100644 --- a/examples/sync/src/databases/mod.rs +++ b/examples/sync/src/databases/mod.rs @@ -42,21 +42,20 @@ impl DatabaseType { } /// Helper trait for databases that can be synced. -pub trait Syncable { +pub trait Syncable: Sized { /// The type of operations in the database. type Operation: Operation + Encode + Sync + 'static; /// Create test operations with the given count and seed. + /// The returned operations must end with a commit operation. fn create_test_operations(count: usize, seed: u64) -> Vec; - /// Add operations to the database. + /// Add operations to the database and return the clean database, ignoring any input that + /// doesn't end with a commit operation (since without a commit, we can't return a clean DB). fn add_operations( - database: &mut Self, + self, operations: Vec, - ) -> impl Future>; - - /// Commit pending operations to the database. - fn commit(&mut self) -> impl Future>; + ) -> impl Future>; /// Get the database's root digest. fn root(&self) -> Key; diff --git a/storage/Cargo.toml b/storage/Cargo.toml index 8fea605001..a563a578e8 100644 --- a/storage/Cargo.toml +++ b/storage/Cargo.toml @@ -66,6 +66,7 @@ arbitrary = [ "dep:arbitrary", ] fuzzing = [] +test-traits = [] [lib] bench = false @@ -75,6 +76,7 @@ crate-type = ["rlib", "cdylib"] name = "qmdb" harness = false path = "src/qmdb/benches/bench.rs" +required-features = ["test-traits"] [[bench]] name="archive" diff --git a/storage/fuzz/Cargo.toml b/storage/fuzz/Cargo.toml index 5f670e2bfc..33ce9a53ef 100644 --- a/storage/fuzz/Cargo.toml +++ b/storage/fuzz/Cargo.toml @@ -14,7 +14,7 @@ bytes.workspace = true commonware-codec.workspace = true commonware-cryptography.workspace = true commonware-runtime.workspace = true -commonware-storage = { workspace = true, features = ["std", "fuzzing"] } +commonware-storage = { workspace = true, features = ["std", "fuzzing", "test-traits"] } commonware-utils.workspace = true futures.workspace = true libfuzzer-sys.workspace = true diff --git a/storage/fuzz/fuzz_targets/current_ordered_operations.rs b/storage/fuzz/fuzz_targets/current_ordered_operations.rs index e11770bb35..81bda24787 100644 --- a/storage/fuzz/fuzz_targets/current_ordered_operations.rs +++ b/storage/fuzz/fuzz_targets/current_ordered_operations.rs @@ -1,11 +1,14 @@ #![no_main] use arbitrary::Arbitrary; -use commonware_cryptography::{sha256::Digest, Sha256}; +use commonware_cryptography::{sha256::Digest, Hasher, Sha256}; use commonware_runtime::{buffer::PoolRef, deterministic, Runner}; use commonware_storage::{ - mmr::{hasher::Hasher as _, Location, StandardHasher as Standard}, - qmdb::current::{ordered::fixed::Db as Current, FixedConfig as Config}, + mmr::Location, + qmdb::{ + current::{ordered::fixed::Db as Current, FixedConfig as Config}, + store::MerkleizedStore as _, + }, translator::TwoCap, }; use commonware_utils::{sequence::FixedBytes, NZUsize, NZU64}; @@ -81,7 +84,7 @@ fn fuzz(data: FuzzInput) { let runner = deterministic::Runner::default(); runner.start(|context| async move { - let mut hasher = Standard::::new(); + let mut hasher = Sha256::new(); let cfg = Config { mmr_journal_partition: "fuzz_current_mmr_journal".into(), mmr_metadata_partition: "fuzz_current_mmr_metadata".into(), @@ -98,7 +101,7 @@ fn fuzz(data: FuzzInput) { let mut db = Current::::init(context.clone(), cfg) .await - .expect("Failed to initialize Current database"); + .expect("Failed to initialize Current database").into_mutable(); let mut expected_state: HashMap = HashMap::new(); let mut all_keys = std::collections::HashSet::new(); @@ -112,9 +115,7 @@ fn fuzz(data: FuzzInput) { let v = Value::new(*value); let empty = db.is_empty(); - let mut dirty_db = db.into_dirty(); - dirty_db.update(k, v).await.expect("update should not fail"); - db = dirty_db.merkleize().await.unwrap(); + db.update(k, v).await.expect("update should not fail"); let result = expected_state.insert(*key, *value); all_keys.insert(*key); uncommitted_ops += 1; @@ -130,9 +131,7 @@ fn fuzz(data: FuzzInput) { CurrentOperation::Delete { key } => { let k = Key::new(*key); - let mut dirty_db = db.into_dirty(); - dirty_db.delete(k).await.expect("delete should not fail"); - db = dirty_db.merkleize().await.unwrap(); + db.delete(k).await.expect("delete should not fail"); if expected_state.remove(key).is_some() { all_keys.insert(*key); uncommitted_ops += 1; @@ -180,50 +179,45 @@ fn fuzz(data: FuzzInput) { } CurrentOperation::Commit => { - db.commit(None).await.expect("Commit should not fail"); - last_committed_op_count = db.op_count(); + let (durable_db, _) = db.commit(None).await.expect("Commit should not fail"); + let clean_db = durable_db.into_merkleized().await.expect("into_merkleized should not fail"); + last_committed_op_count = clean_db.op_count(); uncommitted_ops = 0; + db = clean_db.into_mutable(); } CurrentOperation::Prune => { - db.prune(db.inactivity_floor_loc()).await.expect("Prune should not fail"); + let mut merkleized_db = db.into_merkleized().await.expect("into_merkleized should not fail"); + merkleized_db.prune(merkleized_db.inactivity_floor_loc()).await.expect("Prune should not fail"); + db = merkleized_db.into_mutable(); } CurrentOperation::Root => { - if uncommitted_ops > 0 { - db.commit(None).await.expect("Commit before root should not fail"); - last_committed_op_count = db.op_count(); - uncommitted_ops = 0; - } - - let _root = db.root(); + let clean_db = db.into_merkleized().await.expect("into_merkleized should not fail"); + let _root = clean_db.root(); + db = clean_db.into_mutable(); } CurrentOperation::RangeProof { start_loc, max_ops } => { let current_op_count = db.op_count(); if current_op_count > 0 { - if uncommitted_ops > 0 { - db.commit(None).await.expect("Commit before proof should not fail"); - last_committed_op_count = db.op_count(); - uncommitted_ops = 0; - } - - let current_root = db.root(); + let merkleized_db = db.into_merkleized().await.expect("into_merkleized should not fail"); + let current_root = merkleized_db.root(); // Adjust start_loc and max_ops to be within the valid range let start_loc = Location::new(start_loc % *current_op_count).unwrap(); - let oldest_loc = db.inactivity_floor_loc(); + let oldest_loc = merkleized_db.inactivity_floor_loc(); if start_loc >= oldest_loc { - let (proof, ops, chunks) = db - .range_proof(hasher.inner(), start_loc, *max_ops) + let (proof, ops, chunks) = merkleized_db + .range_proof(&mut hasher, start_loc, *max_ops) .await .expect("Range proof should not fail"); assert!( Current::::verify_range_proof( - hasher.inner(), + &mut hasher, &proof, start_loc, &ops, @@ -233,21 +227,22 @@ fn fuzz(data: FuzzInput) { "Range proof verification failed for start_loc={start_loc}, max_ops={max_ops}" ); } + db = merkleized_db.into_mutable(); } } CurrentOperation::ArbitraryProof {start_loc, bad_digests, max_ops, bad_chunks} => { - let mut hasher = Standard::::new(); let current_op_count = db.op_count(); if current_op_count == 0 { continue; } + let merkleized_db = db.into_merkleized().await.expect("into_merkleized should not fail"); let start_loc = Location::new(start_loc % current_op_count.as_u64()).unwrap(); - let root = db.root(); + let root = merkleized_db.root(); - if let Ok((range_proof, ops, chunks)) = db - .range_proof(hasher.inner(), start_loc, *max_ops) + if let Ok((range_proof, ops, chunks)) = merkleized_db + .range_proof(&mut hasher, start_loc, *max_ops) .await { // Try to verify the proof when providing bad proof digests. let bad_digests = bad_digests.iter().map(|d| Digest::from(*d)).collect(); @@ -255,7 +250,7 @@ fn fuzz(data: FuzzInput) { let mut bad_proof = range_proof.clone(); bad_proof.proof.digests = bad_digests; assert!(!Current::::verify_range_proof( - hasher.inner(), + &mut hasher, &bad_proof, start_loc, &ops, @@ -267,7 +262,7 @@ fn fuzz(data: FuzzInput) { // Try to verify the proof when providing bad input chunks. if &chunks != bad_chunks { assert!(!Current::::verify_range_proof( - hasher.inner(), + &mut hasher, &range_proof, start_loc, &ops, @@ -276,24 +271,20 @@ fn fuzz(data: FuzzInput) { ), "proof with bad chunks should not verify"); } } + db = merkleized_db.into_mutable(); } CurrentOperation::KeyValueProof { key } => { let k = Key::new(*key); - if uncommitted_ops > 0 { - db.commit(None).await.expect("Commit before key value proof should not fail"); - last_committed_op_count = db.op_count(); - uncommitted_ops = 0; - } + let merkleized_db = db.into_merkleized().await.expect("into_merkleized should not fail"); + let current_root = merkleized_db.root(); - let current_root = db.root(); - - match db.key_value_proof(hasher.inner(), k.clone()).await { + match merkleized_db.key_value_proof(&mut hasher, k.clone()).await { Ok(proof) => { - let value = db.get(&k).await.expect("get should not fail").expect("key should exist"); + let value = merkleized_db.get(&k).await.expect("get should not fail").expect("key should exist"); let verification_result = Current::::verify_key_value_proof( - hasher.inner(), + &mut hasher, k, value, &proof, @@ -308,23 +299,19 @@ fn fuzz(data: FuzzInput) { panic!("Unexpected error during key value proof generation: {e:?}"); } } + db = merkleized_db.into_mutable(); } CurrentOperation::ExclusionProof { key } => { let k = Key::new(*key); - if uncommitted_ops > 0 { - db.commit(None).await.expect("Commit before exclusion proof should not fail"); - last_committed_op_count = db.op_count(); - uncommitted_ops = 0; - } - - let current_root = db.root(); + let merkleized_db = db.into_merkleized().await.expect("into_merkleized should not fail"); + let current_root = merkleized_db.root(); - match db.exclusion_proof(hasher.inner(), &k).await { + match merkleized_db.exclusion_proof(&mut hasher, &k).await { Ok(proof) => { let verification_result = Current::::verify_exclusion_proof( - hasher.inner(), + &mut hasher, &k, &proof, ¤t_root, @@ -338,17 +325,17 @@ fn fuzz(data: FuzzInput) { panic!("Unexpected error during exclusion proof generation: {e:?}"); } } + db = merkleized_db.into_mutable(); } } } - if uncommitted_ops > 0 { - db.commit(None).await.expect("Final commit should not fail"); - } + let (durable_db, _) = db.commit(None).await.expect("Final commit should not fail"); + let clean_db = durable_db.into_merkleized().await.expect("into_merkleized should not fail"); for key in &all_keys { let k = Key::new(*key); - let result = db.get(&k).await.expect("Final get should not fail"); + let result = clean_db.get(&k).await.expect("Final get should not fail"); match expected_state.get(key) { Some(expected_value) => { @@ -362,6 +349,8 @@ fn fuzz(data: FuzzInput) { } } } + + clean_db.destroy().await.expect("Destroy should not fail"); }); } diff --git a/storage/fuzz/fuzz_targets/current_unordered_operations.rs b/storage/fuzz/fuzz_targets/current_unordered_operations.rs index cf47d1af42..f8220562e7 100644 --- a/storage/fuzz/fuzz_targets/current_unordered_operations.rs +++ b/storage/fuzz/fuzz_targets/current_unordered_operations.rs @@ -1,11 +1,14 @@ #![no_main] use arbitrary::Arbitrary; -use commonware_cryptography::{sha256::Digest, Sha256}; +use commonware_cryptography::{sha256::Digest, Hasher, Sha256}; use commonware_runtime::{buffer::PoolRef, deterministic, Runner}; use commonware_storage::{ - mmr::{hasher::Hasher as _, Location, StandardHasher as Standard}, - qmdb::current::{unordered::fixed::Db as Current, FixedConfig as Config}, + mmr::Location, + qmdb::{ + current::{unordered::fixed::Db as Current, FixedConfig as Config}, + store::MerkleizedStore as _, + }, translator::TwoCap, }; use commonware_utils::{sequence::FixedBytes, NZUsize, NZU64}; @@ -75,7 +78,7 @@ fn fuzz(data: FuzzInput) { let runner = deterministic::Runner::default(); runner.start(|context| async move { - let mut hasher = Standard::::new(); + let mut hasher = Sha256::new(); let cfg = Config { mmr_journal_partition: "fuzz_current_mmr_journal".into(), mmr_metadata_partition: "fuzz_current_mmr_metadata".into(), @@ -92,7 +95,7 @@ fn fuzz(data: FuzzInput) { let mut db = Current::::init(context.clone(), cfg) .await - .expect("Failed to initialize Current database"); + .expect("Failed to initialize Current database").into_mutable(); let mut expected_state: HashMap> = HashMap::new(); let mut all_keys = std::collections::HashSet::new(); @@ -105,9 +108,7 @@ fn fuzz(data: FuzzInput) { let k = Key::new(*key); let v = Value::new(*value); - let mut dirty_db = db.into_dirty(); - dirty_db.update(k, v).await.expect("Update should not fail"); - db = dirty_db.merkleize().await.unwrap(); + db.update(k, v).await.expect("Update should not fail"); expected_state.insert(*key, Some(*value)); all_keys.insert(*key); uncommitted_ops += 1; @@ -118,9 +119,7 @@ fn fuzz(data: FuzzInput) { let k = Key::new(*key); // Check if key exists before deletion let key_existed = db.get(&k).await.expect("Get before delete should not fail").is_some(); - let mut dirty_db = db.into_dirty(); - dirty_db.delete(k).await.expect("Delete should not fail"); - db = dirty_db.merkleize().await.unwrap(); + db.delete(k).await.expect("Delete should not fail"); if key_existed { expected_state.insert(*key, None); all_keys.insert(*key); @@ -158,74 +157,70 @@ fn fuzz(data: FuzzInput) { } CurrentOperation::Commit => { - db.commit(None).await.expect("Commit should not fail"); - last_committed_op_count = db.op_count(); + let (durable_db, _) = db.commit(None).await.expect("Commit should not fail"); + let clean_db = durable_db.into_merkleized().await.expect("into_merkleized should not fail"); + last_committed_op_count = clean_db.op_count(); uncommitted_ops = 0; + db = clean_db.into_mutable(); } CurrentOperation::Prune => { - db.prune(db.inactivity_floor_loc()).await.expect("Prune should not fail"); + let mut merkleized_db = db.into_merkleized().await.expect("into_merkleized should not fail"); + merkleized_db.prune(merkleized_db.inactivity_floor_loc()).await.expect("Prune should not fail"); + db = merkleized_db.into_mutable(); } CurrentOperation::Root => { - if uncommitted_ops > 0 { - db.commit(None).await.expect("Commit before root should not fail"); - last_committed_op_count = db.op_count(); - uncommitted_ops = 0; - } - - let _root = db.root(); + let clean_db = db.into_merkleized().await.expect("into_merkleized should not fail"); + let _root = clean_db.root(); + db = clean_db.into_mutable(); } CurrentOperation::RangeProof { start_loc, max_ops } => { let current_op_count = db.op_count(); + if current_op_count == 0 { + continue; + } - if current_op_count > 0 { - if uncommitted_ops > 0 { - db.commit(None).await.expect("Commit before proof should not fail"); - last_committed_op_count = db.op_count(); - uncommitted_ops = 0; - } - - let current_root = db.root(); - - // Adjust start_loc and max_ops to be within the valid range - let start_loc = Location::new(start_loc % *current_op_count).unwrap(); - - let oldest_loc = db.inactivity_floor_loc(); - if start_loc >= oldest_loc { - let (proof, ops, chunks) = db - .range_proof(hasher.inner(), start_loc, *max_ops) - .await - .expect("Range proof should not fail"); - - assert!( - Current::::verify_range_proof( - hasher.inner(), - &proof, - start_loc, - &ops, - &chunks, - ¤t_root + let merkleized_db = db.into_merkleized().await.expect("into_merkleized should not fail"); + let current_root = merkleized_db.root(); + + // Adjust start_loc and max_ops to be within the valid range + let start_loc = Location::new(start_loc % *current_op_count).unwrap(); + let oldest_loc = merkleized_db.inactivity_floor_loc(); + if start_loc >= oldest_loc { + let (proof, ops, chunks) = merkleized_db + .range_proof(&mut hasher, start_loc, *max_ops) + .await + .expect("Range proof should not fail"); + + assert!( + Current::::verify_range_proof( + &mut hasher, + &proof, + start_loc, + &ops, + &chunks, + ¤t_root ), "Range proof verification failed for start_loc={start_loc}, max_ops={max_ops}" ); - } } + db = merkleized_db.into_mutable(); } CurrentOperation::ArbitraryProof {start_loc, bad_digests, max_ops, bad_chunks} => { - let mut hasher = Standard::::new(); let current_op_count = db.op_count(); if current_op_count == 0 { continue; } + let merkleized_db = db.into_merkleized().await.expect("into_merkleized should not fail"); let start_loc = Location::new(start_loc % current_op_count.as_u64()).unwrap(); - let root = db.root(); + let root = merkleized_db.root(); - if let Ok((range_proof, ops, chunks)) = db - .range_proof(hasher.inner(), start_loc, *max_ops) + if let Ok((range_proof, ops, chunks)) = merkleized_db + .range_proof(&mut hasher, start_loc, *max_ops) .await { // Try to verify the proof when providing bad proof digests. let bad_digests = bad_digests.iter().map(|d| Digest::from(*d)).collect(); @@ -233,7 +228,7 @@ fn fuzz(data: FuzzInput) { let mut bad_proof = range_proof.clone(); bad_proof.proof.digests = bad_digests; assert!(!Current::::verify_range_proof( - hasher.inner(), + &mut hasher, &bad_proof, start_loc, &ops, @@ -245,33 +240,29 @@ fn fuzz(data: FuzzInput) { // Try to verify the proof when providing bad input chunks. if &chunks != bad_chunks { assert!(!Current::::verify_range_proof( - hasher.inner(), - &range_proof, - start_loc, - &ops, - bad_chunks, - &root - ), "proof with bad chunks should not verify"); + &mut hasher, + &range_proof, + start_loc, + &ops, + bad_chunks, + &root + ), "proof with bad chunks should not verify"); } } + db = merkleized_db.into_mutable(); } CurrentOperation::KeyValueProof { key } => { let k = Key::new(*key); - if uncommitted_ops > 0 { - db.commit(None).await.expect("Commit before key value proof should not fail"); - last_committed_op_count = db.op_count(); - uncommitted_ops = 0; - } - - let current_root = db.root(); + let merkleized_db = db.into_merkleized().await.expect("into_merkleized should not fail"); + let current_root = merkleized_db.root(); - match db.key_value_proof(hasher.inner(), k.clone()).await { + match merkleized_db.key_value_proof(&mut hasher, k.clone()).await { Ok(proof) => { - let value = db.get(&k).await.expect("get should not fail").expect("key should exist"); + let value = merkleized_db.get(&k).await.expect("get should not fail").expect("key should exist"); let verification_result = Current::::verify_key_value_proof( - hasher.inner(), + &mut hasher, k, value, &proof, @@ -289,17 +280,17 @@ fn fuzz(data: FuzzInput) { panic!("Unexpected error during key value proof generation: {e:?}"); } } + db = merkleized_db.into_mutable(); } } } - if uncommitted_ops > 0 { - db.commit(None).await.expect("Final commit should not fail"); - } + let (durable_db, _) = db.commit(None).await.expect("Final commit should not fail"); + let clean_db = durable_db.into_merkleized().await.expect("into_merkleized should not fail"); for key in &all_keys { let k = Key::new(*key); - let result = db.get(&k).await.expect("Final get should not fail"); + let result = clean_db.get(&k).await.expect("Final get should not fail"); match expected_state.get(key) { Some(Some(expected_value)) => { @@ -316,6 +307,8 @@ fn fuzz(data: FuzzInput) { } } } + + clean_db.destroy().await.expect("destroy should not fail"); }); } diff --git a/storage/fuzz/fuzz_targets/mmr_journaled.rs b/storage/fuzz/fuzz_targets/mmr_journaled.rs index 31e15661f3..07c5cbbe04 100644 --- a/storage/fuzz/fuzz_targets/mmr_journaled.rs +++ b/storage/fuzz/fuzz_targets/mmr_journaled.rs @@ -1,7 +1,7 @@ #![no_main] use arbitrary::Arbitrary; -use commonware_cryptography::{Hasher, Sha256}; +use commonware_cryptography::Sha256; use commonware_runtime::{buffer::PoolRef, deterministic, Runner}; use commonware_storage::mmr::{ journaled::{CleanMmr, Config, DirtyMmr, Mmr, SyncConfig}, @@ -55,9 +55,6 @@ enum MmrJournaledOperation { GetPrunedToPos, GetOldestRetainedPos, Reinit, - InitFromPinnedNodes { - size: u64, - }, InitSync { lower_bound_seed: u16, upper_bound_seed: u16, @@ -445,41 +442,6 @@ fn fuzz(input: FuzzInput) { MmrState::Clean(new_mmr) } - MmrJournaledOperation::InitFromPinnedNodes { size } => { - let mmr_size = match &mmr { - MmrState::Clean(m) => m.size(), - MmrState::Dirty(m) => m.size(), - }; - - if mmr_size > 0 { - // Ensure limited_size doesn't exceed current MMR size - let size = size.min(*mmr_size); - - // Create a reasonable number of pinned nodes - use a simple heuristic - // For small MMRs, we need fewer pinned nodes; for larger ones, we need more - let estimated_pins = ((size as f64).log2().ceil() as usize).max(1); - - let pinned_nodes: Vec<_> = (0..estimated_pins) - .map(|i| Sha256::hash(&(i as u32).to_be_bytes())) - .collect(); - - if let Ok(new_mmr) = Mmr::init_from_pinned_nodes( - context.clone(), - pinned_nodes.clone(), - size.into(), - test_config("pinned"), - &mut hasher, - ) - .await - { - assert_eq!(new_mmr.size(), size); - assert_eq!(new_mmr.pruned_to_pos(), size); - new_mmr.destroy().await.unwrap(); - } - } - mmr - } - MmrJournaledOperation::InitSync { lower_bound_seed, upper_bound_seed, diff --git a/storage/fuzz/fuzz_targets/qmdb_any_fixed_sync.rs b/storage/fuzz/fuzz_targets/qmdb_any_fixed_sync.rs index c40d3c2ea9..156801031d 100644 --- a/storage/fuzz/fuzz_targets/qmdb_any_fixed_sync.rs +++ b/storage/fuzz/fuzz_targets/qmdb_any_fixed_sync.rs @@ -9,7 +9,6 @@ use commonware_storage::{ unordered::fixed::{Db, Operation as FixedOperation}, FixedConfig as Config, }, - store::CleanStore as _, sync, }, translator::TwoCap, @@ -20,6 +19,7 @@ use std::sync::Arc; type Key = FixedBytes<32>; type Value = FixedBytes<32>; +type FixedDb = Db; const MAX_OPERATIONS: usize = 50; @@ -35,7 +35,7 @@ enum Operation { SyncFull { fetch_batch_size: u64 }, // Failure simulation - SimulateFailure { sync_log: bool }, + SimulateFailure, } impl<'a> Arbitrary<'a> for Operation { @@ -57,10 +57,7 @@ impl<'a> Arbitrary<'a> for Operation { let fetch_batch_size = u.arbitrary()?; Ok(Operation::SyncFull { fetch_batch_size }) } - 6 => { - let sync_log: bool = u.arbitrary()?; - Ok(Operation::SimulateFailure { sync_log }) - } + 6 => Ok(Operation::SimulateFailure {}), 7 => { let key = u.arbitrary()?; Ok(Operation::Delete { key }) @@ -121,10 +118,7 @@ async fn test_sync< let db_config = test_config(test_name); let expected_root = target.root; - let sync_config: sync::engine::Config< - Db, - R, - > = sync::engine::Config { + let sync_config: sync::engine::Config = sync::engine::Config { context, update_rx: None, db_config, @@ -154,10 +148,10 @@ fn fuzz(mut input: FuzzInput) { let runner = deterministic::Runner::default(); runner.start(|context| async move { - let mut db = - Db::<_, Key, Value, Sha256, TwoCap>::init(context.clone(), test_config(TEST_NAME)) - .await - .expect("Failed to init source db"); + let mut db = FixedDb::init(context.clone(), test_config(TEST_NAME)) + .await + .expect("Failed to init source db") + .into_mutable(); let mut sync_id = 0; @@ -188,15 +182,20 @@ fn fuzz(mut input: FuzzInput) { } input.commit_counter += 1; commit_id[..8].copy_from_slice(&input.commit_counter.to_be_bytes()); - db.commit(Some(FixedBytes::new(commit_id))) + let (durable_db, _) = db + .commit(Some(FixedBytes::new(commit_id))) .await .expect("Commit should not fail"); + db = durable_db.into_merkleized().into_mutable(); } Operation::Prune => { - db.prune(db.inactivity_floor_loc()) + let mut merkleized_db = db.into_merkleized(); + merkleized_db + .prune(merkleized_db.inactivity_floor_loc()) .await .expect("Prune should not fail"); + db = merkleized_db.into_mutable(); } Operation::SyncFull { fetch_batch_size } => { @@ -206,16 +205,18 @@ fn fuzz(mut input: FuzzInput) { input.commit_counter += 1; let mut commit_id = [0u8; 32]; commit_id[..8].copy_from_slice(&input.commit_counter.to_be_bytes()); - db.commit(Some(FixedBytes::new(commit_id))) + let (durable_db, _) = db + .commit(Some(FixedBytes::new(commit_id))) .await - .expect("Commit should not fail"); + .expect("commit should not fail"); + let clean_db = durable_db.into_merkleized(); let target = sync::Target { - root: db.root(), - range: db.inactivity_floor_loc()..db.op_count(), + root: clean_db.root(), + range: clean_db.inactivity_floor_loc()..clean_db.op_count(), }; - let wrapped_src = Arc::new(RwLock::new(db)); + let wrapped_src = Arc::new(RwLock::new(clean_db)); let _result = test_sync( context.clone(), wrapped_src.clone(), @@ -226,25 +227,25 @@ fn fuzz(mut input: FuzzInput) { .await; db = Arc::try_unwrap(wrapped_src) .unwrap_or_else(|_| panic!("Failed to unwrap src")) - .into_inner(); + .into_inner() + .into_mutable(); sync_id += 1; } - Operation::SimulateFailure { sync_log } => { - db.simulate_failure(*sync_log) - .await - .expect("Simulate failure should not fail"); + Operation::SimulateFailure => { + // Simulate unclean shutdown by dropping the db without committing + drop(db); - db = Db::<_, Key, Value, Sha256, TwoCap>::init( - context.clone(), - test_config(TEST_NAME), - ) - .await - .expect("Failed to init source db"); + db = FixedDb::init(context.clone(), test_config(TEST_NAME)) + .await + .expect("Failed to init source db") + .into_mutable(); } } } + let db = db.commit(None).await.expect("commit should not fail").0; + let db = db.into_merkleized(); db.destroy().await.expect("Destroy should not fail"); }); } diff --git a/storage/fuzz/fuzz_targets/qmdb_any_variable_sync.rs b/storage/fuzz/fuzz_targets/qmdb_any_variable_sync.rs index ee4b1a68a2..13f4bf1e83 100644 --- a/storage/fuzz/fuzz_targets/qmdb_any_variable_sync.rs +++ b/storage/fuzz/fuzz_targets/qmdb_any_variable_sync.rs @@ -7,7 +7,6 @@ use commonware_storage::{ mmr::{self, hasher::Standard, MAX_LOCATION}, qmdb::{ any::{unordered::variable::Db, VariableConfig as Config}, - store::CleanStore as _, verify_proof, }, translator::TwoCap, @@ -51,9 +50,7 @@ enum Operation { InactivityFloorLoc, OpCount, Root, - SimulateFailure { - sync_log: bool, - }, + SimulateFailure, } impl<'a> Arbitrary<'a> for Operation { @@ -111,10 +108,7 @@ impl<'a> Arbitrary<'a> for Operation { 9 => Ok(Operation::InactivityFloorLoc), 10 => Ok(Operation::OpCount), 11 => Ok(Operation::Root), - 12 | 13 => { - let sync_log: bool = u.arbitrary()?; - Ok(Operation::SimulateFailure { sync_log }) - } + 12 | 13 => Ok(Operation::SimulateFailure {}), _ => unreachable!(), } } @@ -164,43 +158,45 @@ fn fuzz(input: FuzzInput) { test_config("qmdb_any_variable_fuzz_test"), ) .await - .expect("Failed to init source db"); + .expect("Failed to init source db") + .into_mutable(); let mut historical_roots: HashMap< Location, ::Digest, > = HashMap::new(); - let mut has_uncommitted = false; - for op in &input.ops { match op { Operation::Update { key, value_bytes } => { db.update(Key::new(*key), value_bytes.to_vec()) .await .expect("Update should not fail"); - has_uncommitted = true; } Operation::Delete { key } => { db.delete(Key::new(*key)) .await .expect("Delete should not fail"); - has_uncommitted = true; } Operation::Commit { metadata_bytes } => { - db.commit(metadata_bytes.clone()) + let (durable_db, _) = db + .commit(metadata_bytes.clone()) .await .expect("Commit should not fail"); - historical_roots.insert(db.op_count(), db.root()); - has_uncommitted = false; + let clean_db = durable_db.into_merkleized(); + historical_roots.insert(clean_db.op_count(), clean_db.root()); + db = clean_db.into_mutable(); } Operation::Prune => { - db.prune(db.inactivity_floor_loc()) + let mut merkleized_db = db.into_merkleized(); + merkleized_db + .prune(merkleized_db.inactivity_floor_loc()) .await .expect("Prune should not fail"); + db = merkleized_db.into_mutable(); } Operation::Get { key } => { @@ -214,17 +210,19 @@ fn fuzz(input: FuzzInput) { Operation::Proof { start_loc, max_ops } => { let op_count = db.op_count(); let oldest_retained_loc = db.inactivity_floor_loc(); - if op_count > 0 && !has_uncommitted { - if *start_loc < oldest_retained_loc || *start_loc >= *op_count { - continue; - } + if op_count == 0 { + continue; + } + if *start_loc < oldest_retained_loc || *start_loc >= *op_count { + continue; + } - db.sync().await.expect("Sync should not fail"); - if let Ok((proof, log)) = db.proof(*start_loc, *max_ops).await { - let root = db.root(); - assert!(verify_proof(&mut hasher, &proof, *start_loc, &log, &root)); - } + let clean_db = db.into_merkleized(); + if let Ok((proof, log)) = clean_db.proof(*start_loc, *max_ops).await { + let root = clean_db.root(); + assert!(verify_proof(&mut hasher, &proof, *start_loc, &log, &root)); } + db = clean_db.into_mutable(); } Operation::HistoricalProof { @@ -233,25 +231,32 @@ fn fuzz(input: FuzzInput) { max_ops, } => { let op_count = db.op_count(); - if op_count > 0 && !has_uncommitted { - let op_count = Location::new(*size % *op_count).unwrap() + 1; + if op_count == 0 { + continue; + } + let op_count = Location::new(*size % *op_count).unwrap() + 1; - if *start_loc >= op_count || op_count > max_ops.get() { - continue; - } + if *start_loc >= op_count || op_count > max_ops.get() { + continue; + } - if let Ok((proof, log)) = - db.historical_proof(op_count, *start_loc, *max_ops).await - { - if let Some(root) = historical_roots.get(&op_count) { - assert!(verify_proof(&mut hasher, &proof, *start_loc, &log, root)); - } + let clean_db = db.into_merkleized(); + if let Ok((proof, log)) = clean_db + .historical_proof(op_count, *start_loc, *max_ops) + .await + { + if let Some(root) = historical_roots.get(&op_count) { + assert!(verify_proof(&mut hasher, &proof, *start_loc, &log, root)); } } + db = clean_db.into_mutable(); } Operation::Sync => { - db.sync().await.expect("Sync should not fail"); + let (durable_db, _) = db.commit(None).await.expect("commit should not fail"); + let mut clean_db = durable_db.into_merkleized(); + clean_db.sync().await.expect("Sync should not fail"); + db = clean_db.into_mutable(); } Operation::InactivityFloorLoc => { @@ -263,27 +268,28 @@ fn fuzz(input: FuzzInput) { } Operation::Root => { - if !has_uncommitted { - let _ = db.root(); - } + let clean_db = db.into_merkleized(); + let _ = clean_db.root(); + db = clean_db.into_mutable(); } - Operation::SimulateFailure { sync_log } => { - db.simulate_failure(*sync_log) - .await - .expect("Simulate failure should not fail"); + Operation::SimulateFailure => { + // Simulate unclean shutdown by dropping the db without committing + drop(db); - db = Db::<_, Key, Vec, Sha256, TwoCap>::init( + db = Db::<_, Key, Vec, Sha256, TwoCap, _, _>::init( context.clone(), - test_config("src"), + test_config("qmdb_any_variable_fuzz_test"), ) .await - .expect("Failed to init source db"); - has_uncommitted = false; + .expect("Failed to init source db") + .into_mutable(); } } } + let db = db.commit(None).await.expect("commit should not fail").0; + let db = db.into_merkleized(); db.destroy().await.expect("Destroy should not fail"); }); } diff --git a/storage/fuzz/fuzz_targets/qmdb_immutable.rs b/storage/fuzz/fuzz_targets/qmdb_immutable.rs index f491703aed..da42abea36 100644 --- a/storage/fuzz/fuzz_targets/qmdb_immutable.rs +++ b/storage/fuzz/fuzz_targets/qmdb_immutable.rs @@ -118,7 +118,8 @@ fn fuzz(input: FuzzInput) { db_config("fuzz_partition"), ) .await - .unwrap(); + .unwrap() + .into_mutable(); let mut hasher = commonware_storage::mmr::StandardHasher::::new(); let mut keys_set = Vec::new(); @@ -160,21 +161,25 @@ fn fuzz(input: FuzzInput) { None }; - if let Ok(()) = db.commit(metadata).await { - last_commit_loc = Some(db.op_count() - 1); - uncommitted_ops.clear(); - } + let (durable_db, _) = db.commit(metadata).await.unwrap(); + last_commit_loc = Some(durable_db.op_count() - 1); + uncommitted_ops.clear(); + db = durable_db.into_mutable(); } ImmutableOperation::Prune { loc } => { if let Some(commit_loc) = last_commit_loc { let safe_loc = loc % (commit_loc + 1).as_u64(); let safe_loc = Location::new(safe_loc).unwrap(); - if let Ok(()) = db.prune(safe_loc).await { - let oldest = db.oldest_retained_loc(); - set_locations.retain(|(_, l)| *l >= oldest); - keys_set.retain(|(_, l)| *l >= oldest); - } + let mut merkleized_db = db.into_merkleized(); + merkleized_db + .prune(safe_loc) + .await + .expect("prune should not fail"); + let oldest = merkleized_db.oldest_retained_loc(); + set_locations.retain(|(_, l)| *l >= oldest); + keys_set.retain(|(_, l)| *l >= oldest); + db = merkleized_db.into_mutable(); } } @@ -188,11 +193,14 @@ fn fuzz(input: FuzzInput) { let safe_start = Location::new(safe_start).unwrap(); let safe_max_ops = NonZeroU64::new((max_ops % MAX_PROOF_OPS).max(1)).unwrap(); - - if let Ok((proof, ops)) = db.proof(safe_start, safe_max_ops).await { - let root = db.root(); + let merkleized_db = db.into_merkleized(); + if let Ok((proof, ops)) = + merkleized_db.proof(safe_start, safe_max_ops).await + { + let root = merkleized_db.root(); let _ = verify_proof(&mut hasher, &proof, safe_start, &ops, &root); } + db = merkleized_db.into_mutable(); } } @@ -210,11 +218,13 @@ fn fuzz(input: FuzzInput) { let safe_max_ops = NonZeroU64::new((max_ops % MAX_PROOF_OPS).max(1)).unwrap(); - if safe_start >= db.oldest_retained_loc() { - let _ = db + let merkleized_db = db.into_merkleized(); + if safe_start >= merkleized_db.oldest_retained_loc() { + let _ = merkleized_db .historical_proof(safe_size, safe_start, safe_max_ops) .await; } + db = merkleized_db.into_mutable(); } } @@ -231,14 +241,16 @@ fn fuzz(input: FuzzInput) { } ImmutableOperation::Root => { - if uncommitted_ops.is_empty() { - let _ = db.root(); - } + let clean_db = db.into_merkleized(); + let _ = clean_db.root(); + db = clean_db.into_mutable(); } } } - let _ = db.destroy().await; + let (durable_db, _) = db.commit(None).await.unwrap(); + let clean_db = durable_db.into_merkleized(); + clean_db.destroy().await.unwrap(); }); } diff --git a/storage/fuzz/fuzz_targets/qmdb_keyless.rs b/storage/fuzz/fuzz_targets/qmdb_keyless.rs index e254e26b04..66db54d449 100644 --- a/storage/fuzz/fuzz_targets/qmdb_keyless.rs +++ b/storage/fuzz/fuzz_targets/qmdb_keyless.rs @@ -43,10 +43,7 @@ enum Operation { start_offset: u32, max_ops: u16, }, - SimulateFailure { - sync_log: bool, - sync_mmr: bool, - }, + SimulateFailure {}, } impl<'a> Arbitrary<'a> for Operation { @@ -99,11 +96,7 @@ impl<'a> Arbitrary<'a> for Operation { max_ops, }) } - 12 => { - let sync_log: bool = u.arbitrary()?; - let sync_mmr: bool = u.arbitrary()?; - Ok(Operation::SimulateFailure { sync_log, sync_mmr }) - } + 12 => Ok(Operation::SimulateFailure {}), _ => unreachable!(), } } @@ -127,6 +120,8 @@ impl<'a> Arbitrary<'a> for FuzzInput { const PAGE_SIZE: usize = 128; const PAGE_CACHE_SIZE: usize = 8; +type CleanDb = Keyless, Sha256>; + fn test_config(test_name: &str) -> Config<(commonware_codec::RangeCfg, ())> { Config { mmr_journal_partition: format!("{test_name}_mmr"), @@ -148,12 +143,11 @@ fn fuzz(input: FuzzInput) { runner.start(|context| async move { let mut hasher = Standard::::new(); - let mut db = - Keyless::<_, _, Sha256, _>::init(context.clone(), test_config("keyless_fuzz_test")) - .await - .expect("Failed to init keyless db"); + let mut db = CleanDb::init(context.clone(), test_config("keyless_fuzz_test")) + .await + .expect("Failed to init keyless db") + .into_mutable(); - let mut has_uncommitted = false; for op in &input.ops { match op { @@ -161,14 +155,13 @@ fn fuzz(input: FuzzInput) { db.append(value_bytes.clone()) .await .expect("Append should not fail"); - has_uncommitted = true; } Operation::Commit { metadata_bytes } => { - db.commit(metadata_bytes.clone()) + let (durable_db, _) = db.commit(metadata_bytes.clone()) .await .expect("Commit should not fail"); - has_uncommitted = false; + db = durable_db.into_mutable(); } Operation::Get { loc_offset } => { @@ -184,13 +177,18 @@ fn fuzz(input: FuzzInput) { } Operation::Prune => { - db.prune(db.last_commit_loc()) + let mut merkleized_db = db.into_merkleized(); + merkleized_db.prune(merkleized_db.last_commit_loc()) .await .expect("Prune should not fail"); + db = merkleized_db.into_mutable(); } Operation::Sync => { - db.sync().await.expect("Sync should not fail"); + let (durable_db, _) = db.commit(None).await.expect("Commit should not fail"); + let mut clean_db = durable_db.into_merkleized(); + clean_db.sync().await.expect("Sync should not fail"); + db = clean_db.into_mutable(); } Operation::OpCount => { @@ -206,9 +204,9 @@ fn fuzz(input: FuzzInput) { } Operation::Root => { - if !has_uncommitted { - let _ = db.root(); - } + let merkleized_db = db.into_merkleized(); + let _ = merkleized_db.root(); + db = merkleized_db.into_mutable(); } Operation::Proof { @@ -216,18 +214,21 @@ fn fuzz(input: FuzzInput) { max_ops, } => { let op_count = db.op_count(); - if op_count > 0 && !has_uncommitted { - let start_loc = (*start_offset as u64) % op_count.as_u64(); - let max_ops_value = ((*max_ops as u64) % MAX_PROOF_OPS) + 1; - let start_loc = Location::new(start_loc).unwrap(); - let root = db.root(); - if let Ok((proof, ops)) = db.proof(start_loc, NZU64!(max_ops_value)).await { + if op_count == 0 { + continue; + } + let merkleized_db = db.into_merkleized(); + let start_loc = (*start_offset as u64) % op_count.as_u64(); + let max_ops_value = ((*max_ops as u64) % MAX_PROOF_OPS) + 1; + let start_loc = Location::new(start_loc).unwrap(); + let root = merkleized_db.root(); + if let Ok((proof, ops)) = merkleized_db.proof(start_loc, NZU64!(max_ops_value)).await { assert!( verify_proof(&mut hasher, &proof, start_loc, &ops, &root), "Failed to verify proof for start loc{start_loc} with ops {max_ops} ops", ); - } } + db = merkleized_db.into_mutable(); } Operation::HistoricalProof { @@ -235,46 +236,42 @@ fn fuzz(input: FuzzInput) { start_offset, max_ops, } => { - db.sync().await.expect("Sync should not fail"); let op_count = db.op_count(); - if op_count > 0 && !has_uncommitted { - let size = ((*size_offset as u64) % op_count.as_u64()) + 1; - let size = Location::new(size).unwrap(); - let start_loc = (*start_offset as u64) % *size; - let start_loc = Location::new(start_loc).unwrap(); - let max_ops_value = ((*max_ops as u64) % MAX_PROOF_OPS) + 1; - let root = db.root(); - if let Ok((proof, ops)) = db - .historical_proof(op_count, start_loc, NZU64!(max_ops_value)) + if op_count == 0 { + continue; + } + let merkleized_db = db.into_merkleized(); + let size = ((*size_offset as u64) % op_count.as_u64()) + 1; + let size = Location::new(size).unwrap(); + let start_loc = (*start_offset as u64) % *size; + let start_loc = Location::new(start_loc).unwrap(); + let max_ops_value = ((*max_ops as u64) % MAX_PROOF_OPS) + 1; + let root = merkleized_db.root(); + if let Ok((proof, ops)) = merkleized_db + .historical_proof(op_count, start_loc, NZU64!(max_ops_value)) .await { assert!( verify_proof(&mut hasher, &proof, start_loc, &ops, &root), "Failed to verify historical proof for start loc{start_loc} with max ops {max_ops}", ); } - } + db = merkleized_db.into_mutable(); } - Operation::SimulateFailure { - sync_log, - sync_mmr, - } => { - db.simulate_failure(*sync_log, *sync_mmr) - .await - .expect("Simulate failure should not fail"); + Operation::SimulateFailure{} => { + drop(db); - db = Keyless::init( - context.clone(), - test_config("keyless_fuzz_test"), - ) - .await - .expect("Failed to init keyless db"); - has_uncommitted = false; + db = CleanDb::init(context.clone(), test_config("keyless_fuzz_test")) + .await + .expect("Failed to init keyless db") + .into_mutable(); } } } - db.destroy().await.expect("Destroy should not fail"); + let (durable_db, _) = db.commit(None).await.expect("Commit should not fail"); + let clean_db = durable_db.into_merkleized(); + clean_db.destroy().await.expect("Destroy should not fail"); }); } diff --git a/storage/fuzz/fuzz_targets/qmdb_ordered_batching.rs b/storage/fuzz/fuzz_targets/qmdb_ordered_batching.rs index 5e25f77746..b103ca9a12 100644 --- a/storage/fuzz/fuzz_targets/qmdb_ordered_batching.rs +++ b/storage/fuzz/fuzz_targets/qmdb_ordered_batching.rs @@ -19,6 +19,7 @@ type Key = FixedBytes<32>; type Value = FixedBytes<64>; type RawKey = [u8; 32]; type RawValue = [u8; 64]; +type OrderedDb = Db; const MAX_OPS: usize = 25; @@ -55,9 +56,10 @@ fn fuzz(data: FuzzInput) { buffer_pool: PoolRef::new(NZUsize!(PAGE_SIZE), NZUsize!(PAGE_CACHE_SIZE)), }; - let mut db = Db::<_, Key, Value, Sha256, EightCap>::init(context.clone(), cfg.clone()) + let mut db = OrderedDb::init(context.clone(), cfg.clone()) .await - .expect("init qmdb"); + .expect("init qmdb") + .into_mutable(); let mut batch = Some(db.start_batch()); let mut last_commit = None; @@ -99,9 +101,11 @@ fn fuzz(data: FuzzInput) { .await .expect("write batch should not fail"); last_commit = Some(Value::new(*value)); - db.commit(Some(Value::new(*value))) + let (durable_db, _) = db + .commit(Some(Value::new(*value))) .await .expect("commit should not fail"); + db = durable_db.into_merkleized().into_mutable(); // Restore batch for subsequent operations batch = Some(db.start_batch()); @@ -143,7 +147,8 @@ fn fuzz(data: FuzzInput) { db.write_batch(iter) .await .expect("write batch should not fail"); - db.commit(None).await.expect("commit should not fail"); + let (durable_db, _) = db.commit(None).await.expect("commit should not fail"); + let db = durable_db.into_merkleized(); // Comprehensive final verification - check ALL keys ever touched for key in &all_keys { diff --git a/storage/fuzz/fuzz_targets/qmdb_ordered_operations.rs b/storage/fuzz/fuzz_targets/qmdb_ordered_operations.rs index f0e7cf97c0..02e0125c06 100644 --- a/storage/fuzz/fuzz_targets/qmdb_ordered_operations.rs +++ b/storage/fuzz/fuzz_targets/qmdb_ordered_operations.rs @@ -7,7 +7,6 @@ use commonware_storage::{ mmr::{Location, Position, Proof, StandardHasher as Standard}, qmdb::{ any::{ordered::fixed::Db, FixedConfig as Config}, - store::CleanStore as _, verify_proof, }, translator::EightCap, @@ -84,7 +83,7 @@ fn fuzz(data: FuzzInput) { let mut db = Db::<_, Key, Value, Sha256, EightCap>::init(context.clone(), cfg.clone()) .await - .expect("init qmdb"); + .expect("init qmdb").into_mutable(); let mut expected_state: HashMap = HashMap::new(); let mut all_keys: HashSet = HashSet::new(); @@ -132,20 +131,18 @@ fn fuzz(data: FuzzInput) { } QmdbOperation::Commit => { - db.commit(None).await.expect("commit should not fail"); + let (durable_db, _) = db.commit(None).await.expect("commit should not fail"); // After commit, update our last known count since commit may add more operations - last_known_op_count = db.op_count(); + last_known_op_count = durable_db.op_count(); uncommitted_ops = 0; // Reset uncommitted operations counter + db = durable_db.into_mutable(); } QmdbOperation::Root => { - // root requires all operations to be committed - if uncommitted_ops > 0 { - db.commit(None).await.expect("commit should not fail"); - last_known_op_count = db.op_count(); - uncommitted_ops = 0; - } - db.root(); + // root requires merkleization but not commit + let clean_db = db.into_merkleized(); + clean_db.root(); + db = clean_db.into_mutable(); } QmdbOperation::Proof { start_loc, max_ops } => { @@ -153,19 +150,12 @@ fn fuzz(data: FuzzInput) { // Only generate proof if QMDB has operations and valid parameters if actual_op_count > 0 { - // Ensure all operations are committed before generating proof - if uncommitted_ops > 0 { - db.commit(None).await.expect("commit should not fail"); - last_known_op_count = db.op_count(); - uncommitted_ops = 0; - } - - let current_root = db.root(); + let clean_db = db.into_merkleized(); + let current_root = clean_db.root(); // Adjust start_loc to be within valid range // Locations are 0-indexed (first operation is at location 0) let adjusted_start = Location::new(*start_loc % *actual_op_count).unwrap(); - - let (proof, log) = db + let (proof, log) = clean_db .proof(adjusted_start, *max_ops) .await .expect("proof should not fail"); @@ -180,6 +170,7 @@ fn fuzz(data: FuzzInput) { ), "Proof verification failed for start_loc={adjusted_start}, max_ops={max_ops}", ); + db = clean_db.into_mutable(); } } @@ -193,16 +184,11 @@ fn fuzz(data: FuzzInput) { // Only generate proof if QMDB has operations and valid parameters if actual_op_count > 0 { - if uncommitted_ops > 0 { - db.commit(None).await.expect("commit should not fail"); - last_known_op_count = db.op_count(); - uncommitted_ops = 0; - } - - let current_root = db.root(); + let clean_db = db.into_merkleized(); + let current_root = clean_db.root(); let adjusted_start = Location::new(*start_loc % *actual_op_count).unwrap(); - if let Ok(res) = db + if let Ok(res) = clean_db .proof(adjusted_start, *max_ops) .await { let _ = verify_proof( @@ -214,6 +200,7 @@ fn fuzz(data: FuzzInput) { ); } + db = clean_db.into_mutable(); } } @@ -250,7 +237,8 @@ fn fuzz(data: FuzzInput) { // Final commit to ensure all operations are persisted if uncommitted_ops > 0 { - db.commit(None).await.expect("final commit should not fail"); + let (durable_db, _) = db.commit(None).await.expect("final commit should not fail"); + db = durable_db.into_mutable(); } // Comprehensive final verification - check ALL keys ever touched @@ -276,7 +264,8 @@ fn fuzz(data: FuzzInput) { } } - db.destroy().await.expect("destroy should not fail"); + let (durable_db, _) = db.commit(None).await.expect("final commit should not fail"); + durable_db.into_merkleized().destroy().await.expect("destroy should not fail"); expected_state.clear(); all_keys.clear(); }); diff --git a/storage/fuzz/fuzz_targets/qmdb_unordered_operations.rs b/storage/fuzz/fuzz_targets/qmdb_unordered_operations.rs index acade220fb..98ed0bf025 100644 --- a/storage/fuzz/fuzz_targets/qmdb_unordered_operations.rs +++ b/storage/fuzz/fuzz_targets/qmdb_unordered_operations.rs @@ -7,7 +7,6 @@ use commonware_storage::{ mmr::{Location, StandardHasher as Standard}, qmdb::{ any::{unordered::fixed::Db, FixedConfig as Config}, - store::CleanStore as _, verify_proof, }, translator::EightCap, @@ -60,7 +59,7 @@ fn fuzz(data: FuzzInput) { let mut db = Db::<_, Key, Value, Sha256, EightCap>::init(context.clone(), cfg.clone()) .await - .expect("init qmdb"); + .expect("init qmdb").into_mutable(); let mut expected_state: HashMap> = HashMap::new(); let mut all_keys: HashSet = HashSet::new(); @@ -98,56 +97,51 @@ fn fuzz(data: FuzzInput) { } QmdbOperation::Commit => { - db.commit(None).await.expect("commit should not fail"); + let (durable_db, _) = db.commit(None).await.expect("commit should not fail"); // After commit, update our last known count since commit may add more operations - last_known_op_count = db.op_count(); + last_known_op_count = durable_db.op_count(); uncommitted_ops = 0; // Reset uncommitted operations counter + db = durable_db.into_mutable(); } QmdbOperation::Root => { - // root requires all operations to be committed - if uncommitted_ops > 0 { - db.commit(None).await.expect("commit should not fail"); - last_known_op_count = db.op_count(); - uncommitted_ops = 0; - } - db.root(); + // root requires merkleization but not commit + let clean_db = db.into_merkleized(); + clean_db.root(); + db = clean_db.into_mutable(); } QmdbOperation::Proof { start_loc, max_ops } => { let actual_op_count = db.op_count(); + // Only generate proof if proof will have operations. + if actual_op_count == 0 || *max_ops == 0 { + continue; + } - // Only generate proof if QMDB has operations and valid parameters - if actual_op_count > 0 && *max_ops > 0 { - // Ensure all operations are committed before generating proof - if uncommitted_ops > 0 { - db.commit(None).await.expect("commit should not fail"); - last_known_op_count = db.op_count(); - uncommitted_ops = 0; - } + let clean_db = db.into_merkleized(); + + let current_root = clean_db.root(); + // Adjust start_loc to be within valid range + // Locations are 0-indexed (first operation is at location 0) + let adjusted_start = Location::new(*start_loc % *actual_op_count).unwrap(); + let adjusted_max_ops = (*max_ops % 100).max(1); // Ensure at least 1 - let current_root = db.root(); - // Adjust start_loc to be within valid range - // Locations are 0-indexed (first operation is at location 0) - let adjusted_start = Location::new(*start_loc % *actual_op_count).unwrap(); - let adjusted_max_ops = (*max_ops % 100).max(1); // Ensure at least 1 - - let (proof, log) = db - .proof(adjusted_start, NZU64!(adjusted_max_ops)) - .await - .expect("proof should not fail"); - - assert!( - verify_proof( - &mut hasher, - &proof, - adjusted_start, - &log, - ¤t_root + let (proof, log) = clean_db + .proof(adjusted_start, NZU64!(adjusted_max_ops)) + .await + .expect("proof should not fail"); + + assert!( + verify_proof( + &mut hasher, + &proof, + adjusted_start, + &log, + ¤t_root ), - "Proof verification failed for start_loc={adjusted_start}, max_ops={adjusted_max_ops}", - ); - } + "Proof verification failed for start_loc={adjusted_start}, max_ops={adjusted_max_ops}", + ); + db = clean_db.into_mutable(); } QmdbOperation::Get { key } => { @@ -187,7 +181,8 @@ fn fuzz(data: FuzzInput) { // Final commit to ensure all operations are persisted if uncommitted_ops > 0 { - db.commit(None).await.expect("final commit should not fail"); + let (durable_db, _) = db.commit(None).await.expect("final commit should not fail"); + db = durable_db.into_mutable(); } // Comprehensive final verification - check ALL keys ever touched @@ -219,7 +214,8 @@ fn fuzz(data: FuzzInput) { } } - db.destroy().await.expect("destroy should not fail"); + let (durable_db, _) = db.commit(None).await.expect("final commit should not fail"); + durable_db.into_merkleized().destroy().await.expect("destroy should not fail"); expected_state.clear(); all_keys.clear(); }); diff --git a/storage/fuzz/fuzz_targets/store_operations.rs b/storage/fuzz/fuzz_targets/store_operations.rs index c3b234f513..6a1ac79507 100644 --- a/storage/fuzz/fuzz_targets/store_operations.rs +++ b/storage/fuzz/fuzz_targets/store_operations.rs @@ -4,7 +4,7 @@ use arbitrary::Arbitrary; use commonware_cryptography::blake3::Digest; use commonware_runtime::{buffer::PoolRef, deterministic, Runner}; use commonware_storage::{ - qmdb::store::{Config, Store}, + qmdb::store::db::{Config, Db}, translator::TwoCap, }; use commonware_utils::{NZUsize, NZU64}; @@ -14,6 +14,7 @@ const MAX_OPERATIONS: usize = 50; type Key = Digest; type Value = Vec; +type StoreDb = Db; #[derive(Debug)] enum Operation { @@ -104,75 +105,74 @@ fn fuzz(input: FuzzInput) { let runner = deterministic::Runner::default(); runner.start(|context| async move { - let mut store = - Store::<_, Key, Value, TwoCap>::init(context.clone(), test_config("store_fuzz_test")) - .await - .expect("Failed to init store"); + let mut db = StoreDb::init(context.clone(), test_config("store_fuzz_test")) + .await + .expect("Failed to init db") + .into_dirty(); for op in &input.ops { match op { Operation::Update { key, value_bytes } => { - store - .update(Digest(*key), value_bytes.clone()) + db.update(Digest(*key), value_bytes.clone()) .await .expect("Update should not fail"); } Operation::Delete { key } => { - store - .delete(Digest(*key)) + db.delete(Digest(*key)) .await .expect("Delete should not fail"); } Operation::Commit { metadata_bytes } => { - store + let (clean_db, _) = db .commit(metadata_bytes.clone()) .await .expect("Commit should not fail"); + db = clean_db.into_dirty(); } Operation::Get { key } => { - let _ = store.get(&Digest(*key)).await; + let _ = db.get(&Digest(*key)).await; } Operation::GetMetadata => { - let _ = store.get_metadata().await; + let _ = db.get_metadata().await; } Operation::Sync => { - store.sync().await.expect("Sync should not fail"); + let (mut clean_db, _) = db.commit(None).await.expect("Commit should not fail"); + clean_db.sync().await.expect("Sync should not fail"); + db = clean_db.into_dirty(); } Operation::Prune => { - store - .prune(store.inactivity_floor_loc()) + db.prune(db.inactivity_floor_loc()) .await .expect("Prune should not fail"); } Operation::OpCount => { - let _ = store.op_count(); + let _ = db.op_count(); } Operation::InactivityFloorLoc => { - let _ = store.inactivity_floor_loc(); + let _ = db.inactivity_floor_loc(); } Operation::SimulateFailure => { - drop(store); - - store = Store::<_, Key, Value, TwoCap>::init( - context.clone(), - test_config("store_fuzz_test"), - ) - .await - .expect("Failed to init store"); + drop(db); + + db = StoreDb::init(context.clone(), test_config("store_fuzz_test")) + .await + .expect("Failed to init db") + .into_dirty(); } } } - store.destroy().await.expect("Destroy should not fail"); + let (clean_db, _) = db.commit(None).await.expect("Commit should not fail"); + clean_db.destroy().await.expect("Destroy should not fail"); }); } diff --git a/storage/src/journal/authenticated.rs b/storage/src/journal/authenticated.rs index 6b58b82214..174c683c5d 100644 --- a/storage/src/journal/authenticated.rs +++ b/storage/src/journal/authenticated.rs @@ -110,7 +110,7 @@ where impl Journal where E: Storage + Clock + Metrics, - C: MutableContiguous + Persistable, + C: Contiguous + Persistable, H: Hasher, S: State>, { @@ -321,7 +321,7 @@ where impl Journal> where E: Storage + Clock + Metrics, - C: MutableContiguous + Persistable, + C: Contiguous + Persistable, H: Hasher, { /// Destroy the authenticated journal, removing all data from disk. @@ -588,7 +588,7 @@ where impl Persistable for Journal> where E: Storage + Clock + Metrics, - C: MutableContiguous + Persistable, + C: Contiguous + Persistable, H: Hasher, { type Error = JournalError; diff --git a/storage/src/journal/mod.rs b/storage/src/journal/mod.rs index d6c026fc5e..ace260bbcb 100644 --- a/storage/src/journal/mod.rs +++ b/storage/src/journal/mod.rs @@ -14,27 +14,6 @@ pub mod segmented; #[cfg(all(test, feature = "arbitrary"))] mod conformance; -impl crate::qmdb::sync::Journal for contiguous::fixed::Journal -where - E: commonware_runtime::Storage + commonware_runtime::Clock + commonware_runtime::Metrics, - Op: commonware_codec::Codec + commonware_codec::FixedSize, -{ - type Op = Op; - type Error = Error; - - async fn sync(&mut self) -> Result<(), Self::Error> { - Self::sync(self).await - } - - async fn size(&self) -> u64 { - Self::size(self) - } - - async fn append(&mut self, op: Self::Op) -> Result<(), Self::Error> { - Self::append(self, op).await.map(|_| ()) - } -} - /// Errors that can occur when interacting with `Journal`. #[derive(Debug, Error)] pub enum Error { diff --git a/storage/src/mmr/journaled.rs b/storage/src/mmr/journaled.rs index 795e68dd28..91ab56c051 100644 --- a/storage/src/mmr/journaled.rs +++ b/storage/src/mmr/journaled.rs @@ -24,7 +24,7 @@ use crate::{ Error::{self, *}, Proof, }, - qmdb::any::unordered::fixed::sync::{init_journal, init_journal_at_size}, + qmdb::any::unordered::fixed::sync::init_journal, }; use commonware_codec::DecodeExt; use commonware_cryptography::Digest; @@ -223,82 +223,6 @@ impl> Mmr { } impl CleanMmr { - /// Initialize a new journaled MMR from an MMR's size and set of pinned nodes. - /// - /// This creates a journaled MMR that appears to have `mmr_size` elements, all of which - /// are pruned, leaving only the minimal set of `pinned_nodes` required for proof generation. - /// The next element added will be at position `mmr_size`. - /// - /// The returned MMR is functionally equivalent to a journaled MMR that was created, - /// populated, and then pruned up to its size. - /// - /// # Arguments - /// * `context` - Storage context - /// * `pinned_nodes` - Digest values in the order returned by `nodes_to_pin(mmr_size)` - /// * `mmr_size` - The logical size of the MMR (all elements before this are considered pruned) - /// * `config` - Journaled MMR configuration. Any data in the given journal and metadata - /// partitions will be overwritten. - pub async fn init_from_pinned_nodes( - context: E, - pinned_nodes: Vec, - mmr_size: Position, - config: Config, - hasher: &mut impl Hasher, - ) -> Result { - // Destroy any existing journal data - context.remove(&config.journal_partition, None).await.ok(); - context.remove(&config.metadata_partition, None).await.ok(); - - // Create the journal with the desired size - let journal_cfg = JConfig { - partition: config.journal_partition.clone(), - items_per_blob: config.items_per_blob, - buffer_pool: config.buffer_pool.clone(), - write_buffer: config.write_buffer, - }; - let journal = - init_journal_at_size(context.with_label("mmr_journal"), journal_cfg, *mmr_size).await?; - - // Initialize metadata - let metadata_cfg = MConfig { - partition: config.metadata_partition.clone(), - codec_config: ((0..).into(), ()), - }; - let mut metadata = Metadata::init(context.with_label("mmr_metadata"), metadata_cfg).await?; - - // Store the pruning boundary in metadata - let pruning_boundary_key = U64::new(PRUNE_TO_POS_PREFIX, 0); - metadata.put(pruning_boundary_key, mmr_size.to_be_bytes().into()); - - // Store the pinned nodes in metadata - let nodes_to_pin_positions = nodes_to_pin(mmr_size); - for (pos, digest) in nodes_to_pin_positions.zip(pinned_nodes.iter()) { - metadata.put(U64::new(NODE_PREFIX, *pos), digest.to_vec()); - } - - // Sync metadata to disk - metadata.sync().await.map_err(Error::MetadataError)?; - - // Create in-memory MMR in fully pruned state - let mem_mmr = MemMmr::init( - MemConfig { - nodes: vec![], - pruned_to_pos: mmr_size, - pinned_nodes, - }, - hasher, - )?; - - Ok(Self { - mem_mmr, - journal, - journal_size: mmr_size, - metadata, - pruned_to_pos: mmr_size, - pool: config.thread_pool, - }) - } - /// Initialize a new `Mmr` instance. pub async fn init(context: E, hasher: &mut impl Hasher, cfg: Config) -> Result { let journal_cfg = JConfig { @@ -887,7 +811,7 @@ impl DirtyMmr { } } -impl Storage for Mmr> { +impl Storage for CleanMmr { fn size(&self) -> Position { self.size() } @@ -1679,198 +1603,6 @@ mod tests { }); } - #[test_traced] - fn test_journaled_mmr_init_from_pinned_nodes() { - let executor = deterministic::Runner::default(); - executor.start(|context| async move { - let mut hasher = Standard::::new(); - - // Create an in-memory MMR with some elements - let mut original_mmr = Mmr::init( - context.clone(), - &mut hasher, - Config { - journal_partition: "original_journal".into(), - metadata_partition: "original_metadata".into(), - items_per_blob: NZU64!(7), - write_buffer: NZUsize!(1024), - thread_pool: None, - buffer_pool: PoolRef::new(NZUsize!(PAGE_SIZE), NZUsize!(PAGE_CACHE_SIZE)), - }, - ) - .await - .unwrap(); - - // Add some elements and prune to the size of the MMR - const NUM_ELEMENTS: u64 = 1_000; - for i in 0..NUM_ELEMENTS { - original_mmr - .add(&mut hasher, &test_digest(i as usize)) - .await - .unwrap(); - } - original_mmr.sync().await.unwrap(); - let original_size = original_mmr.size(); - original_mmr.prune_to_pos(original_size).await.unwrap(); - - // Get the journal digest - let mut hasher = Standard::::new(); - let original_journal_digest = original_mmr.root(); - - // Get the pinned nodes - let pinned_nodes_map = original_mmr.get_pinned_nodes(); - let pinned_nodes: Vec<_> = nodes_to_pin(original_size) - .map(|pos| pinned_nodes_map[&pos]) - .collect(); - - // Create a journaled MMR from the pinned nodes - let new_mmr_config = Config { - journal_partition: "new_journal".into(), - metadata_partition: "new_metadata".into(), - items_per_blob: NZU64!(7), - write_buffer: NZUsize!(1024), - thread_pool: None, - buffer_pool: PoolRef::new(NZUsize!(PAGE_SIZE), NZUsize!(PAGE_CACHE_SIZE)), - }; - let mut new_mmr = - Mmr::<_, sha256::Digest, Clean>::init_from_pinned_nodes( - context.clone(), - pinned_nodes, - original_size, - new_mmr_config.clone(), - &mut hasher, - ) - .await - .unwrap(); - - // Verify the journaled MMR has the same properties as the original MMR - assert_eq!(new_mmr.size(), original_size); - assert_eq!(new_mmr.pruned_to_pos(), original_size); - assert_eq!(new_mmr.oldest_retained_pos(), None); - // Verify the roots match - let new_journal_digest = new_mmr.root(); - assert_eq!(new_journal_digest, original_journal_digest); - - // Insert a new element into the new journaled MMR and the original MMR - let new_element = test_digest(10); - - let original_mmr_pos = original_mmr.add(&mut hasher, &new_element).await.unwrap(); - assert_eq!(original_mmr_pos, original_size); - - let new_mmr_pos = new_mmr.add(&mut hasher, &new_element).await.unwrap(); - assert_eq!(new_mmr_pos, original_size); // New element is added at the end - - // Verify the roots still match - let original_mmr_root = original_mmr.root(); - let new_mmr_root = new_mmr.root(); - assert_eq!(new_mmr_root, original_mmr_root); - - // Drop and re-open the journaled MMR - new_mmr.sync().await.unwrap(); - drop(new_mmr); - let new_mmr = Mmr::<_, sha256::Digest, Clean>::init( - context.clone(), - &mut hasher, - new_mmr_config, - ) - .await - .unwrap(); - - // Root should be unchanged - let new_mmr_root = new_mmr.root(); - assert_eq!(new_mmr_root, original_mmr_root); - - // Size and other metadata should be unchanged - assert_eq!(new_mmr.size(), original_size + 1); // +1 for element we just added - assert_eq!(new_mmr.pruned_to_pos(), original_size); - assert_eq!(new_mmr.oldest_retained_pos(), Some(original_size)); // Element we just added is the oldest retained - - // Proofs generated from the journaled MMR should be the same as the proofs generated from the original MMR - let proof = new_mmr - .proof(Location::new_unchecked(NUM_ELEMENTS)) - .await - .unwrap(); - let original_proof = original_mmr - .proof(Location::new_unchecked(NUM_ELEMENTS)) - .await - .unwrap(); - assert_eq!(proof.digests, original_proof.digests); - assert_eq!(proof.size, original_proof.size); - - original_mmr.destroy().await.unwrap(); - new_mmr.destroy().await.unwrap(); - }); - } - - #[test_traced] - fn test_journaled_mmr_init_from_pinned_nodes_edge_cases() { - use crate::mmr::mem::CleanMmr as CleanMemMmr; - - let executor = deterministic::Runner::default(); - executor.start(|context| async move { - let mut hasher = Standard::::new(); - - // === TEST 1: Empty MMR (size 0) === - let mut empty_mmr = - Mmr::<_, sha256::Digest, Clean>::init_from_pinned_nodes( - context.clone(), - vec![], // No pinned nodes - Position::new(0), // Size 0 - Config { - journal_partition: "empty_journal".into(), - metadata_partition: "empty_metadata".into(), - items_per_blob: NZU64!(7), - write_buffer: NZUsize!(1024), - thread_pool: None, - buffer_pool: PoolRef::new(NZUsize!(PAGE_SIZE), NZUsize!(PAGE_CACHE_SIZE)), - }, - &mut hasher, - ) - .await - .unwrap(); - - assert_eq!(empty_mmr.size(), 0); - assert_eq!(empty_mmr.pruned_to_pos(), Position::new(0)); - assert_eq!(empty_mmr.oldest_retained_pos(), None); - - // Should be able to add first element at position 0 - let pos = empty_mmr.add(&mut hasher, &test_digest(0)).await.unwrap(); - assert_eq!(pos, 0); - assert_eq!(empty_mmr.size(), 1); - - empty_mmr.destroy().await.unwrap(); - - // === TEST 2: Single element MMR === - let mut single_mem_mmr = CleanMemMmr::new(&mut hasher); - single_mem_mmr.add(&mut hasher, &test_digest(42)); - let single_size = single_mem_mmr.size(); - let single_root = single_mem_mmr.root(); - let single_pinned = single_mem_mmr.node_digests_to_pin(single_size); - - let single_journaled_mmr = - Mmr::<_, sha256::Digest, Clean>::init_from_pinned_nodes( - context.clone(), - single_pinned, - single_size, - Config { - journal_partition: "single_journal".into(), - metadata_partition: "single_metadata".into(), - items_per_blob: NZU64!(7), - write_buffer: NZUsize!(1024), - thread_pool: None, - buffer_pool: PoolRef::new(NZUsize!(PAGE_SIZE), NZUsize!(PAGE_CACHE_SIZE)), - }, - &mut hasher, - ) - .await - .unwrap(); - - assert_eq!(single_journaled_mmr.size(), single_size); - assert_eq!(single_journaled_mmr.root(), *single_root); - - single_journaled_mmr.destroy().await.unwrap(); - }); - } // Test `init_sync` when there is no persisted data. #[test_traced] fn test_journaled_mmr_init_sync_empty() { diff --git a/storage/src/mmr/mem.rs b/storage/src/mmr/mem.rs index d1002a2716..b5766b9301 100644 --- a/storage/src/mmr/mem.rs +++ b/storage/src/mmr/mem.rs @@ -200,6 +200,11 @@ impl> Mmr { /// Return the requested node if it is either retained or present in the pinned_nodes map, and /// panic otherwise. Use `get_node` instead if you require a non-panicking getter. /// + /// # Warning + /// + /// If the requested digest is for an unmerkleized node (only possible in the Dirty state) a + /// dummy digest will be returned. + /// /// # Panics /// /// Panics if the requested node does not exist for any reason such as the node is pruned or diff --git a/storage/src/qmdb/any/db.rs b/storage/src/qmdb/any/db.rs index c5756ec5c4..bec185686f 100644 --- a/storage/src/qmdb/any/db.rs +++ b/storage/src/qmdb/any/db.rs @@ -1,5 +1,6 @@ -//! A shared, generic implementation of the "Any" QMDB. -//! The impl block in this file defines shared functionality across all "Any" QMDB variants. +//! A shared, generic implementation of the _Any_ QMDB. +//! +//! The impl blocks in this file defines shared functionality across all Any QMDB variants. use super::operation::{update::Update, Operation}; use crate::{ @@ -7,17 +8,16 @@ use crate::{ journal::{ authenticated, contiguous::{Contiguous, MutableContiguous}, + Error as JournalError, }, - mmr::{ - mem::{Clean, Dirty, State}, - Location, Proof, - }, + mmr::{Location, Proof}, qmdb::{ any::ValueEncoding, build_snapshot_from_log, operation::{Committable, Operation as OperationTrait}, - store::LogStore, - Error, FloorHelper, + store::{self, LogStore, MerkleizedStore, PrunableStore}, + DurabilityState, Durable, Error, FloorHelper, MerkleizationState, Merkleized, NonDurable, + Unmerkleized, }, AuthenticatedBitMap, Persistable, }; @@ -29,8 +29,7 @@ use core::{num::NonZeroU64, ops::Range}; use tracing::debug; /// Type alias for the authenticated journal used by [Db]. -pub(crate) type AuthenticatedLog>> = - authenticated::Journal; +pub(crate) type AuthenticatedLog> = authenticated::Journal; /// An "Any" QMDB implementation generic over ordered/unordered keys and variable/fixed values. /// Consider using one of the following specialized variants instead, which may be more ergonomic: @@ -44,7 +43,8 @@ pub struct Db< I, H: Hasher, U, - S: State> = Clean>, + M: MerkleizationState> = Merkleized, + D: DurabilityState = Durable, > where C::Item: Codec, { @@ -55,7 +55,7 @@ pub struct Db< /// /// - The log is never pruned beyond the inactivity floor. /// - There is always at least one commit operation in the log. - pub(crate) log: AuthenticatedLog, + pub(crate) log: AuthenticatedLog, /// A location before which all operations are "inactive" (that is, operations before this point /// are over keys that have been updated by some operation at or after this point). @@ -72,18 +72,18 @@ pub struct Db< /// - Only references `Operation::Update`s. pub(crate) snapshot: I, - /// The number of _steps_ to raise the inactivity floor. Each step involves moving exactly one - /// active operation to tip. - pub(crate) steps: u64, - /// The number of active keys in the snapshot. pub(crate) active_keys: usize, + /// Whether the database is in the durable or non-durable state. + pub(crate) durable_state: D, + /// Marker for the update type parameter. pub(crate) _update: core::marker::PhantomData, } -impl Db +// Functionality shared across all DB states, such as most non-mutating operations. +impl Db where E: Storage + Clock + Metrics, K: Array, @@ -92,7 +92,8 @@ where C: Contiguous>, I: UnorderedIndex, H: Hasher, - S: State>, + M: MerkleizationState>, + D: DurabilityState, Operation: Codec, { /// The number of operations that have been applied to this db, including those that have been @@ -128,13 +129,73 @@ where } } -impl Db +// Functionality shared across Merkleized states, such as the ability to prune the log, retrieve the +// state root, and compute proofs. +impl Db, D> +where + E: Storage + Clock + Metrics, + K: Array, + V: ValueEncoding, + U: Update, + C: MutableContiguous> + Persistable, + I: UnorderedIndex, + H: Hasher, + D: DurabilityState, + Operation: Codec, +{ + pub const fn root(&self) -> H::Digest { + self.log.root() + } + + pub async fn proof( + &self, + loc: Location, + max_ops: NonZeroU64, + ) -> Result<(Proof, Vec>), Error> { + self.historical_proof(self.op_count(), loc, max_ops).await + } + + pub async fn historical_proof( + &self, + historical_size: Location, + start_loc: Location, + max_ops: NonZeroU64, + ) -> Result<(Proof, Vec>), Error> { + self.log + .historical_proof(historical_size, start_loc, max_ops) + .await + .map_err(Into::into) + } + + /// Prunes historical operations prior to `prune_loc`. This does not affect the db's root or + /// snapshot. + /// + /// # Errors + /// + /// - Returns [Error::PruneBeyondMinRequired] if `prune_loc` > inactivity floor. + /// - Returns [crate::mmr::Error::LocationOverflow] if `prune_loc` > [crate::mmr::MAX_LOCATION]. + pub async fn prune(&mut self, prune_loc: Location) -> Result<(), Error> { + if prune_loc > self.inactivity_floor_loc { + return Err(Error::PruneBeyondMinRequired( + prune_loc, + self.inactivity_floor_loc, + )); + } + + self.log.prune(prune_loc).await?; + + Ok(()) + } +} + +// Functionality specific to (Merkleized,Durable) state, such as ability to initialize and persist. +impl Db, Durable> where E: Storage + Clock + Metrics, K: Array, V: ValueEncoding, U: Update, - C: MutableContiguous>, + C: MutableContiguous> + Persistable, I: UnorderedIndex, H: Hasher, Operation: Codec, @@ -172,86 +233,76 @@ where inactivity_floor_loc, snapshot: index, last_commit_loc, - steps: 0, active_keys, + durable_state: store::Durable, _update: core::marker::PhantomData, }) } - /// Raises the inactivity floor by exactly one step, moving the first active operation to tip. - /// Raises the floor to the tip if the db is empty. - pub(crate) async fn raise_floor(&mut self) -> Result { - if self.is_empty() { - self.inactivity_floor_loc = self.op_count(); - debug!(tip = ?self.inactivity_floor_loc, "db is empty, raising floor to tip"); - } else { - let steps_to_take = self.steps + 1; - for _ in 0..steps_to_take { - let loc = self.inactivity_floor_loc; - self.inactivity_floor_loc = self.as_floor_helper().raise_floor(loc).await?; - } - } - self.steps = 0; + /// Sync all database state to disk. + pub async fn sync(&mut self) -> Result<(), Error> { + self.log.sync().await.map_err(Into::into) + } - Ok(self.inactivity_floor_loc) + /// Destroy the db, removing all data from disk. + pub async fn destroy(self) -> Result<(), Error> { + self.log.destroy().await.map_err(Into::into) } - /// Same as `raise_floor` but uses the status bitmap to more efficiently find the first active - /// operation above the inactivity floor. - pub(crate) async fn raise_floor_with_bitmap( - &mut self, - status: &mut AuthenticatedBitMap, - ) -> Result { - if self.is_empty() { - self.inactivity_floor_loc = self.op_count(); - debug!(tip = ?self.inactivity_floor_loc, "db is empty, raising floor to tip"); - } else { - let steps_to_take = self.steps + 1; - for _ in 0..steps_to_take { - let loc = self.inactivity_floor_loc; - self.inactivity_floor_loc = self - .as_floor_helper() - .raise_floor_with_bitmap(status, loc) - .await?; - } + /// Convert this database into a mutable state. + pub fn into_mutable(self) -> Db { + Db { + log: self.log.into_dirty(), + inactivity_floor_loc: self.inactivity_floor_loc, + last_commit_loc: self.last_commit_loc, + snapshot: self.snapshot, + active_keys: self.active_keys, + durable_state: NonDurable { steps: 0 }, + _update: core::marker::PhantomData, } - self.steps = 0; - - Ok(self.inactivity_floor_loc) } +} - /// Returns a FloorHelper wrapping the current state of the log. - pub(crate) const fn as_floor_helper( - &mut self, - ) -> FloorHelper<'_, I, AuthenticatedLog> { - FloorHelper { - snapshot: &mut self.snapshot, - log: &mut self.log, +// Functionality specific to (Unmerkleized,Durable) state. +impl Db +where + E: Storage + Clock + Metrics, + K: Array, + V: ValueEncoding, + U: Update, + C: Contiguous>, + I: UnorderedIndex, + H: Hasher, + Operation: Codec, +{ + /// Convert this database into a mutable state. + pub fn into_mutable(self) -> Db { + Db { + log: self.log, + inactivity_floor_loc: self.inactivity_floor_loc, + last_commit_loc: self.last_commit_loc, + snapshot: self.snapshot, + active_keys: self.active_keys, + durable_state: store::NonDurable { steps: 0 }, + _update: core::marker::PhantomData, } } - /// Prunes historical operations prior to `prune_loc`. This does not affect the db's root or - /// snapshot. - /// - /// # Errors - /// - /// - Returns [Error::PruneBeyondMinRequired] if `prune_loc` > inactivity floor. - /// - Returns [crate::mmr::Error::LocationOverflow] if `prune_loc` > [crate::mmr::MAX_LOCATION]. - pub async fn prune(&mut self, prune_loc: Location) -> Result<(), Error> { - if prune_loc > self.inactivity_floor_loc { - return Err(Error::PruneBeyondMinRequired( - prune_loc, - self.inactivity_floor_loc, - )); + pub fn into_merkleized(self) -> Db, Durable> { + Db { + log: self.log.merkleize(), + inactivity_floor_loc: self.inactivity_floor_loc, + last_commit_loc: self.last_commit_loc, + snapshot: self.snapshot, + active_keys: self.active_keys, + durable_state: self.durable_state, + _update: core::marker::PhantomData, } - - self.log.prune(prune_loc).await?; - - Ok(()) } } -impl Db +// Functionality specific to (Unmerkleized,NonDurable) state. +impl Db where E: Storage + Clock + Metrics, K: Array, @@ -262,21 +313,21 @@ where H: Hasher, Operation: Codec, { - /// Convert this database into its dirty counterpart for batched updates. - pub fn into_dirty(self) -> Db { + pub fn into_merkleized(self) -> Db, NonDurable> { Db { - log: self.log.into_dirty(), + log: self.log.merkleize(), inactivity_floor_loc: self.inactivity_floor_loc, last_commit_loc: self.last_commit_loc, snapshot: self.snapshot, - steps: self.steps, active_keys: self.active_keys, + durable_state: self.durable_state, _update: core::marker::PhantomData, } } } -impl Db +// Functionality specific to (Merkleized,NonDurable) state. +impl Db, NonDurable> where E: Storage + Clock + Metrics, K: Array, @@ -287,30 +338,33 @@ where H: Hasher, Operation: Codec, { - /// Merkleize the database and compute the root digest. - pub fn merkleize(self) -> Db> { + /// Convert this database into a mutable state. + pub fn into_mutable(self) -> Db { Db { - log: self.log.merkleize(), + log: self.log.into_dirty(), inactivity_floor_loc: self.inactivity_floor_loc, last_commit_loc: self.last_commit_loc, snapshot: self.snapshot, - steps: self.steps, active_keys: self.active_keys, + durable_state: self.durable_state, _update: core::marker::PhantomData, } } } -impl Db +// Funtionality shared across NonDurable states. +impl Db where E: Storage + Clock + Metrics, K: Array, V: ValueEncoding, U: Update, - C: MutableContiguous> + Persistable, + C: MutableContiguous> + Persistable, I: UnorderedIndex, H: Hasher, + M: MerkleizationState>, Operation: Codec, + AuthenticatedLog: MutableContiguous>, { /// Applies the given commit operation to the log and commits it to disk. Does not raise the /// inactivity floor. @@ -326,21 +380,13 @@ where self.log.commit().await.map_err(Into::into) } - /// Simulate an unclean shutdown by consuming the db. If commit_log is true, the underlying - /// authenticated log will be be committed before consuming. - #[cfg(any(test, feature = "fuzzing"))] - pub async fn simulate_failure(mut self, commit_log: bool) -> Result<(), Error> { - if commit_log { - self.log.commit().await?; - } - - Ok(()) - } - /// Commit any pending operations to the database, ensuring their durability upon return from /// this function. Also raises the inactivity floor according to the schedule. Returns the - /// `(start_loc, end_loc]` location range of committed operations. - pub async fn commit(&mut self, metadata: Option) -> Result, Error> { + /// `[start_loc, end_loc)` location range of committed operations. + pub async fn commit( + mut self, + metadata: Option, + ) -> Result<(Db, Range), Error> { let start_loc = self.last_commit_loc + 1; // Raise the inactivity floor by taking `self.steps` steps, plus 1 to account for the @@ -351,66 +397,118 @@ where self.apply_commit_op(Operation::CommitFloor(metadata, inactivity_floor_loc)) .await?; - Ok(start_loc..self.op_count()) + let range = start_loc..self.op_count(); + + let db = Db { + log: self.log, + inactivity_floor_loc, + last_commit_loc: self.last_commit_loc, + snapshot: self.snapshot, + active_keys: self.active_keys, + durable_state: store::Durable, + _update: core::marker::PhantomData, + }; + + Ok((db, range)) } - /// Sync all database state to disk. - pub async fn sync(&mut self) -> Result<(), Error> { - self.log.sync().await.map_err(Into::into) + /// Raises the inactivity floor by exactly one step, moving the first active operation to tip. + /// Raises the floor to the tip if the db is empty. + pub(crate) async fn raise_floor(&mut self) -> Result { + if self.is_empty() { + self.inactivity_floor_loc = self.op_count(); + debug!(tip = ?self.inactivity_floor_loc, "db is empty, raising floor to tip"); + } else { + let steps_to_take = self.durable_state.steps + 1; + for _ in 0..steps_to_take { + let loc = self.inactivity_floor_loc; + self.inactivity_floor_loc = self.as_floor_helper().raise_floor(loc).await?; + } + } + self.durable_state.steps = 0; + + Ok(self.inactivity_floor_loc) } - /// Destroy the db, removing all data from disk. - pub async fn destroy(self) -> Result<(), Error> { - self.log.destroy().await.map_err(Into::into) + /// Same as `raise_floor` but uses the status bitmap to more efficiently find the first active + /// operation above the inactivity floor. + pub(crate) async fn raise_floor_with_bitmap( + &mut self, + status: &mut AuthenticatedBitMap, + ) -> Result { + if self.is_empty() { + self.inactivity_floor_loc = self.op_count(); + debug!(tip = ?self.inactivity_floor_loc, "db is empty, raising floor to tip"); + } else { + let steps_to_take = self.durable_state.steps + 1; + for _ in 0..steps_to_take { + let loc = self.inactivity_floor_loc; + self.inactivity_floor_loc = self + .as_floor_helper() + .raise_floor_with_bitmap(status, loc) + .await?; + } + } + self.durable_state.steps = 0; + + Ok(self.inactivity_floor_loc) + } + + /// Returns a FloorHelper wrapping the current state of the log. + pub(crate) const fn as_floor_helper( + &mut self, + ) -> FloorHelper<'_, I, AuthenticatedLog> { + FloorHelper { + snapshot: &mut self.snapshot, + log: &mut self.log, + } } } -impl crate::qmdb::store::LogStorePrunable for Db +impl Persistable for Db, Durable> where E: Storage + Clock + Metrics, K: Array, V: ValueEncoding, U: Update, - C: MutableContiguous>, + C: MutableContiguous> + Persistable, I: UnorderedIndex, H: Hasher, Operation: Codec, { - async fn prune(&mut self, prune_loc: Location) -> Result<(), Error> { - self.prune(prune_loc).await + type Error = Error; + + async fn commit(&mut self) -> Result<(), Error> { + // No-op, DB already in recoverable state. + Ok(()) + } + + async fn sync(&mut self) -> Result<(), Error> { + self.sync().await + } + + async fn destroy(self) -> Result<(), Error> { + self.destroy().await } } -impl crate::qmdb::store::CleanStore for Db +impl MerkleizedStore for Db, D> where E: Storage + Clock + Metrics, K: Array, V: ValueEncoding, U: Update, - C: Contiguous>, + C: MutableContiguous> + Persistable, I: UnorderedIndex, H: Hasher, + D: DurabilityState, Operation: Codec, { type Digest = H::Digest; type Operation = Operation; - type Dirty = Db; - - fn into_dirty(self) -> Self::Dirty { - self.into_dirty() - } fn root(&self) -> H::Digest { - self.log.root() - } - - async fn proof( - &self, - start_loc: Location, - max_ops: NonZeroU64, - ) -> Result<(Proof, Vec), Error> { - let size = self.op_count(); - self.historical_proof(size, start_loc, max_ops).await + self.root() } async fn historical_proof( @@ -418,15 +516,13 @@ where historical_size: Location, start_loc: Location, max_ops: NonZeroU64, - ) -> Result<(Proof, Vec), Error> { - self.log - .historical_proof(historical_size, start_loc, max_ops) + ) -> Result<(Proof, Vec>), Error> { + self.historical_proof(historical_size, start_loc, max_ops) .await - .map_err(Into::into) } } -impl LogStore for Db +impl LogStore for Db where E: Storage + Clock + Metrics, K: Array, @@ -435,7 +531,8 @@ where C: Contiguous>, I: UnorderedIndex, H: Hasher, - S: State>, + M: MerkleizationState>, + D: DurabilityState, Operation: Codec, { type Value = V::Value; @@ -457,22 +554,19 @@ where } } -impl crate::qmdb::store::DirtyStore for Db +impl PrunableStore for Db, D> where E: Storage + Clock + Metrics, K: Array, V: ValueEncoding, U: Update, - C: Contiguous>, + C: MutableContiguous> + Persistable, I: UnorderedIndex, H: Hasher, + D: DurabilityState, Operation: Codec, { - type Digest = H::Digest; - type Operation = Operation; - type Clean = Db; - - async fn merkleize(self) -> Result { - Ok(self.merkleize()) + async fn prune(&mut self, prune_loc: Location) -> Result<(), Error> { + self.prune(prune_loc).await } } diff --git a/storage/src/qmdb/any/ext.rs b/storage/src/qmdb/any/ext.rs deleted file mode 100644 index 9fee212697..0000000000 --- a/storage/src/qmdb/any/ext.rs +++ /dev/null @@ -1,198 +0,0 @@ -//! Extension type for simplified usage of Any authenticated databases. -//! -//! The [AnyExt] wrapper provides a traditional mutable key-value store interface over Any -//! databases, automatically handling Clean/Dirty state transitions. This eliminates the need for -//! manual state management while maintaining the performance benefits of deferred merkleization. - -use super::{CleanAny, DirtyAny}; -use crate::{ - kv::{self, Batchable}, - mmr::Location, - qmdb::{ - store::{CleanStore, DirtyStore, LogStore, LogStorePrunable}, - Error, - }, - Persistable, -}; - -/// An extension wrapper for [CleanAny] databases that provides a traditional mutable key-value -/// store interface by internally handling Clean/Dirty state transitions. -pub struct AnyExt { - // Invariant: always Some - inner: Option>, -} - -enum State { - Clean(A), - Dirty(A::Dirty), -} - -impl AnyExt { - /// Create a new wrapper from a Clean Any database. - pub const fn new(db: A) -> Self { - Self { - inner: Some(State::Clean(db)), - } - } - - /// Ensure we're in dirty state, transitioning if necessary. - fn ensure_dirty(&mut self) { - let state = self.inner.take().expect("wrapper should never be empty"); - self.inner = Some(match state { - State::Clean(clean) => State::Dirty(clean.into_dirty()), - State::Dirty(dirty) => State::Dirty(dirty), - }); - } - - /// Merkleize if in dirty state, ensuring we're in clean state. - async fn ensure_clean(&mut self) -> Result<(), Error> { - let state = self.inner.take().expect("wrapper should never be empty"); - self.inner = Some(match state { - State::Clean(clean) => State::Clean(clean), - State::Dirty(dirty) => State::Clean(dirty.merkleize().await?), - }); - Ok(()) - } -} - -impl kv::Gettable for AnyExt -where - A: CleanAny, -{ - type Key = A::Key; - type Value = ::Value; - type Error = Error; - - async fn get(&self, key: &Self::Key) -> Result, Self::Error> { - match self.inner.as_ref().expect("wrapper should never be empty") { - State::Clean(clean) => CleanAny::get(clean, key).await, - State::Dirty(dirty) => DirtyAny::get(dirty, key).await, - } - } -} - -impl kv::Updatable for AnyExt -where - A: CleanAny, -{ - async fn update(&mut self, key: Self::Key, value: Self::Value) -> Result<(), Self::Error> { - self.ensure_dirty(); - match self.inner.as_mut().expect("wrapper should never be empty") { - State::Dirty(dirty) => DirtyAny::update(dirty, key, value).await, - _ => unreachable!("ensure_dirty guarantees Dirty state"), - } - } -} - -impl kv::Deletable for AnyExt -where - A: CleanAny, -{ - async fn delete(&mut self, key: Self::Key) -> Result { - self.ensure_dirty(); - match self.inner.as_mut().expect("wrapper should never be empty") { - State::Dirty(dirty) => DirtyAny::delete(dirty, key).await, - _ => unreachable!("ensure_dirty guarantees Dirty state"), - } - } -} - -impl Persistable for AnyExt -where - A: CleanAny, -{ - type Error = Error; - - async fn commit(&mut self) -> Result<(), Self::Error> { - // Merkleize before commit - self.ensure_clean().await?; - match self.inner.as_mut().expect("wrapper should never be empty") { - State::Clean(clean) => clean.commit(None).await.map(|_| ()), - _ => unreachable!("ensure_clean guarantees Clean state"), - } - } - - async fn sync(&mut self) -> Result<(), Self::Error> { - // Merkleize before sync - self.ensure_clean().await?; - match self.inner.as_mut().expect("wrapper should never be empty") { - State::Clean(clean) => clean.sync().await, - _ => unreachable!("ensure_clean guarantees Clean state"), - } - } - - async fn destroy(mut self) -> Result<(), Self::Error> { - // Merkleize before destroy - self.ensure_clean().await?; - match self.inner.take().expect("wrapper should never be empty") { - State::Clean(clean) => clean.destroy().await, - _ => unreachable!("ensure_clean guarantees Clean state"), - } - } -} - -impl LogStore for AnyExt -where - A: CleanAny, -{ - type Value = ::Value; - - fn is_empty(&self) -> bool { - match self.inner.as_ref().expect("wrapper should never be empty") { - State::Clean(clean) => clean.is_empty(), - State::Dirty(dirty) => dirty.is_empty(), - } - } - - fn op_count(&self) -> Location { - match self.inner.as_ref().expect("wrapper should never be empty") { - State::Clean(clean) => clean.op_count(), - State::Dirty(dirty) => dirty.op_count(), - } - } - - fn inactivity_floor_loc(&self) -> Location { - match self.inner.as_ref().expect("wrapper should never be empty") { - State::Clean(clean) => clean.inactivity_floor_loc(), - State::Dirty(dirty) => dirty.inactivity_floor_loc(), - } - } - - async fn get_metadata(&self) -> Result, Error> { - match self.inner.as_ref().expect("wrapper should never be empty") { - State::Clean(clean) => clean.get_metadata().await, - State::Dirty(dirty) => dirty.get_metadata().await, - } - } -} - -impl LogStorePrunable for AnyExt -where - A: CleanAny, -{ - async fn prune(&mut self, prune_loc: Location) -> Result<(), Error> { - // Merkleize before prune - self.ensure_clean().await?; - match self.inner.as_mut().expect("wrapper should never be empty") { - State::Clean(clean) => clean.prune(prune_loc).await, - _ => unreachable!("ensure_clean guarantees Clean state"), - } - } -} - -impl Batchable for AnyExt -where - A: CleanAny, - ::Dirty: Batchable, -{ - async fn write_batch( - &mut self, - iter: impl Iterator)>, - ) -> Result<(), Error> { - self.ensure_dirty(); - match self.inner.as_mut().expect("wrapper should never be empty") { - State::Dirty(dirty) => dirty.write_batch(iter).await, - _ => unreachable!("ensure_dirty guarantees Dirty state"), - } - } -} diff --git a/storage/src/qmdb/any/mod.rs b/storage/src/qmdb/any/mod.rs index 7414a65b45..ef1b69467c 100644 --- a/storage/src/qmdb/any/mod.rs +++ b/storage/src/qmdb/any/mod.rs @@ -1,97 +1,37 @@ -//! Authenticated databases that provides succinct proofs of _any_ value ever associated with -//! a key. The submodules provide two classes of variants, one specialized for fixed-size values and -//! the other allowing variable-size values. +//! An _Any_ authenticated database provides succinct proofs of any value ever associated with a +//! key. +//! +//! The specific variants provided within this module include: +//! - Unordered: The database does not maintain or require any ordering over the key space. +//! - Fixed-size values +//! - Variable-size values +//! - Ordered: The database maintains a total order over active keys. +//! - Fixed-size values +//! - Variable-size values use crate::{ journal::{ authenticated, contiguous::fixed::{Config as JConfig, Journal}, }, - mmr::{journaled::Config as MmrConfig, mem::Clean, Location}, - qmdb::{ - operation::Committable, - store::{CleanStore, DirtyStore}, - Error, - }, + mmr::journaled::Config as MmrConfig, + qmdb::{operation::Committable, Error, Merkleized}, translator::Translator, }; use commonware_codec::CodecFixed; -use commonware_cryptography::{DigestOf, Hasher}; +use commonware_cryptography::Hasher; use commonware_runtime::{buffer::PoolRef, Clock, Metrics, Storage, ThreadPool}; -use commonware_utils::Array; -use std::{ - future::Future, - num::{NonZeroU64, NonZeroUsize}, - ops::Range, -}; +use std::num::{NonZeroU64, NonZeroUsize}; pub(crate) mod db; mod operation; - +#[cfg(any(test, feature = "test-traits"))] +pub mod states; mod value; pub(crate) use value::{FixedValue, ValueEncoding, VariableValue}; - -mod ext; pub mod ordered; pub mod unordered; -pub use ext::AnyExt; - -/// Extension trait for "Any" QMDBs in a clean (merkleized) state. -pub trait CleanAny: - CleanStore> -{ - /// The key type for this database. - type Key: Array; - - /// Get the value for a given key, or None if it has no value. - fn get(&self, key: &Self::Key) -> impl Future, Error>>; - - /// Commit pending operations to the database, ensuring durability. - /// Returns the location range of committed operations. - fn commit( - &mut self, - metadata: Option, - ) -> impl Future, Error>>; - - /// Sync all database state to disk. - fn sync(&mut self) -> impl Future>; - - /// Prune historical operations prior to `prune_loc`. - fn prune(&mut self, prune_loc: Location) -> impl Future>; - - /// Destroy the db, removing all data from disk. - fn destroy(self) -> impl Future>; -} - -/// Extension trait for "Any" QMDBs in a dirty (deferred merkleization) state. -pub trait DirtyAny: DirtyStore { - /// The key type for this database. - type Key: Array; - - /// Get the value for a given key, or None if it has no value. - fn get(&self, key: &Self::Key) -> impl Future, Error>>; - - /// Update `key` to have value `value`. Subject to rollback until next `commit`. - fn update( - &mut self, - key: Self::Key, - value: Self::Value, - ) -> impl Future>; - - /// Create a new key-value pair. Returns true if created, false if key already existed. - /// Subject to rollback until next `commit`. - fn create( - &mut self, - key: Self::Key, - value: Self::Value, - ) -> impl Future>; - - /// Delete `key` and its value. Returns true if deleted, false if already inactive. - /// Subject to rollback until next `commit`. - fn delete(&mut self, key: Self::Key) -> impl Future>; -} - /// Configuration for an `Any` authenticated db with fixed-size values. #[derive(Clone)] pub struct FixedConfig { @@ -166,8 +106,7 @@ pub struct VariableConfig { pub buffer_pool: PoolRef, } -type AuthenticatedLog>> = - authenticated::Journal, H, S>; +type AuthenticatedLog> = authenticated::Journal, H, S>; /// Initialize the authenticated log from the given config, returning it along with the inactivity /// floor specified by the last commit. @@ -208,7 +147,8 @@ pub(crate) async fn init_fixed_authenticated_log< } #[cfg(test)] -mod test { +// pub(crate) so qmdb/current can use the generic tests. +pub(crate) mod test { use super::*; use crate::{ qmdb::any::{FixedConfig, VariableConfig}, @@ -251,4 +191,52 @@ mod test { buffer_pool: PoolRef::new(NZUsize!(PAGE_SIZE), NZUsize!(PAGE_CACHE_SIZE)), } } + + use crate::{ + kv::Updatable, + qmdb::{ + any::states::{CleanAny, MerkleizedNonDurableAny, MutableAny, UnmerkleizedDurableAny}, + store::MerkleizedStore, + }, + Persistable, + }; + use commonware_cryptography::{sha256::Digest, Sha256}; + + /// Test that merkleization state changes don't reset `steps`. + pub(crate) async fn test_any_db_steps_not_reset(db: D) + where + D: CleanAny + MerkleizedStore, + D::Mutable: Updatable, + { + // Create a db with a couple keys. + let mut db = db.into_mutable(); + + assert!(db + .create(Sha256::fill(1u8), Sha256::fill(2u8)) + .await + .unwrap()); + assert!(db + .create(Sha256::fill(3u8), Sha256::fill(4u8)) + .await + .unwrap()); + let (clean_db, _) = db.commit(None).await.unwrap(); + let mut db = clean_db.into_mutable(); + + // Updating an existing key should make steps non-zero. + db.update(Sha256::fill(1u8), Sha256::fill(5u8)) + .await + .unwrap(); + let steps = db.steps(); + assert_ne!(steps, 0); + + // Steps shouldn't change from merkleization. + let db = db.into_merkleized().await.unwrap(); + let db = db.into_mutable(); + assert_eq!(db.steps(), steps); + + // Cleanup + let (db, _) = db.commit(None).await.unwrap(); + let db = db.into_merkleized().await.unwrap(); + db.destroy().await.unwrap(); + } } diff --git a/storage/src/qmdb/any/ordered/fixed.rs b/storage/src/qmdb/any/ordered/fixed.rs index e9aba5f579..14e10fcc0a 100644 --- a/storage/src/qmdb/any/ordered/fixed.rs +++ b/storage/src/qmdb/any/ordered/fixed.rs @@ -7,17 +7,17 @@ use crate::{ index::ordered::Index, journal::contiguous::fixed::Journal, - mmr::{mem::Clean, Location}, + mmr::Location, qmdb::{ any::{ init_fixed_authenticated_log, ordered, value::FixedEncoding, FixedConfig as Config, FixedValue, }, - Error, + Durable, Error, Merkleized, }, translator::Translator, }; -use commonware_cryptography::{DigestOf, Hasher}; +use commonware_cryptography::Hasher; use commonware_runtime::{Clock, Metrics, Storage}; use commonware_utils::Array; use tracing::warn; @@ -27,11 +27,11 @@ pub type Operation = ordered::Operation>; /// A key-value QMDB based on an authenticated log of operations, supporting authentication of any /// value ever associated with a key. -pub type Db>> = - super::Db>, Index, H, Update, S>; +pub type Db, D = Durable> = + super::Db>, Index, H, Update, S, D>; impl - Db + Db, Durable> { /// Returns a [Db] qmdb initialized from `cfg`. Any uncommitted log operations will be /// discarded and the state of the db will be as of the last committed operation. @@ -74,9 +74,8 @@ mod test { index::Unordered as _, mmr::{Position, StandardHasher as Standard}, qmdb::{ - any::ordered::Update, - store::{batch_tests, CleanStore as _}, - verify_proof, + any::ordered::Update, store::batch_tests, verify_proof, Durable, Merkleized, + NonDurable, Unmerkleized, }, translator::{OneCap, TwoCap}, }; @@ -111,12 +110,15 @@ mod test { } } - /// A type alias for the concrete [Db] type used in these unit tests. - type AnyTest = Db; + /// Type aliases for concrete [Db] types used in these unit tests. + type CleanAnyTest = + Db, Durable>; + type MutableAnyTest = + Db; /// Return an `Any` database initialized with a fixed config. - async fn open_db(context: deterministic::Context) -> AnyTest { - AnyTest::init(context, any_db_config("partition")) + async fn open_db(context: deterministic::Context) -> CleanAnyTest { + CleanAnyTest::init(context, any_db_config("partition")) .await .unwrap() } @@ -141,10 +143,10 @@ mod test { } /// Create a test database with unique partition names - async fn create_test_db(mut context: Context) -> AnyTest { + async fn create_test_db(mut context: Context) -> CleanAnyTest { let seed = context.next_u64(); let config = create_test_config(seed); - AnyTest::init(context, config).await.unwrap() + CleanAnyTest::init(context, config).await.unwrap() } /// Create n random operations. Some portion of the updates are deletes. @@ -172,7 +174,7 @@ mod test { } /// Applies the given operations to the database. - async fn apply_ops(db: &mut AnyTest, ops: Vec>) { + async fn apply_ops(db: &mut MutableAnyTest, ops: Vec>) { for op in ops { match op { Operation::Update(data) => { @@ -181,59 +183,15 @@ mod test { Operation::Delete(key) => { db.delete(key).await.unwrap(); } - Operation::CommitFloor(metadata, _) => { - db.commit(metadata).await.unwrap(); + Operation::CommitFloor(_, _) => { + // CommitFloor consumes self - not supported in this helper. + // Test data from create_test_ops never includes CommitFloor. + panic!("CommitFloor not supported in apply_ops"); } } } } - #[test_traced("WARN")] - fn test_ordered_any_fixed_db_empty() { - let executor = deterministic::Runner::default(); - executor.start(|context| async move { - let mut db = open_db(context.clone()).await; - assert_eq!(db.op_count(), 1); - assert!(db.is_empty()); - assert!(db.get_metadata().await.unwrap().is_none()); - assert!(matches!(db.prune(db.inactivity_floor_loc()).await, Ok(()))); - - // Make sure closing/reopening gets us back to the same state, even after adding an - // uncommitted op, and even without a clean shutdown. - let d1 = Sha256::fill(1u8); - let d2 = Sha256::fill(2u8); - let root = db.root(); - db.update(d1, d2).await.unwrap(); - let mut db = open_db(context.clone()).await; - assert_eq!(db.root(), root); - assert_eq!(db.op_count(), 1); - - // Test calling commit on an empty db. - let metadata = Sha256::fill(3u8); - let range = db.commit(Some(metadata)).await.unwrap(); - assert_eq!(range.start, 1); - assert_eq!(range.end, 2); - assert_eq!(db.op_count(), 2); // floor op added - assert_eq!(db.get_metadata().await.unwrap(), Some(metadata)); - let root = db.root(); - assert!(matches!(db.prune(db.inactivity_floor_loc()).await, Ok(()))); - - // Re-opening the DB without a clean shutdown should still recover the correct state. - let mut db = open_db(context.clone()).await; - assert_eq!(db.op_count(), 2); - assert_eq!(db.get_metadata().await.unwrap(), Some(metadata)); - assert_eq!(db.root(), root); - - // Confirm the inactivity floor doesn't fall endlessly behind with multiple commits. - for _ in 1..100 { - db.commit(None).await.unwrap(); - assert_eq!(db.op_count() - 1, db.inactivity_floor_loc()); - } - - db.destroy().await.unwrap(); - }); - } - #[test_traced("WARN")] // Test the edge case that arises where we're inserting the second key and it precedes the first // key, but shares the same translated key. @@ -242,10 +200,18 @@ mod test { executor.start(|mut context| async move { let seed = context.next_u64(); let config = create_generic_test_config::(seed, OneCap); - let mut db = - Db::, i32, Sha256, OneCap>::init(context.clone(), config) - .await - .unwrap(); + let db = Db::< + Context, + FixedBytes<2>, + i32, + Sha256, + OneCap, + Merkleized, + Durable, + >::init(context.clone(), config) + .await + .unwrap(); + let mut db = db.into_mutable(); let key1 = FixedBytes::<2>::new([1u8, 1u8]); let key2 = FixedBytes::<2>::new([1u8, 3u8]); // Create some keys that will not be added to the snapshot. @@ -255,7 +221,7 @@ mod test { db.update(key1.clone(), 1).await.unwrap(); db.update(key2.clone(), 2).await.unwrap(); - db.commit(None).await.unwrap(); + let (db, _) = db.commit(None).await.unwrap(); assert_eq!(db.get_all(&key1).await.unwrap().unwrap(), (1, key2.clone())); assert_eq!(db.get_all(&key2).await.unwrap().unwrap(), (2, key1.clone())); assert!(db.get_span(&key1).await.unwrap().unwrap().1.next_key == key2.clone()); @@ -264,6 +230,7 @@ mod test { assert!(db.get_span(&middle_key).await.unwrap().unwrap().1.next_key == key2.clone()); assert!(db.get_span(&late_key).await.unwrap().unwrap().1.next_key == key1.clone()); + let mut db = db.into_mutable(); db.delete(key1.clone()).await.unwrap(); assert!(db.get_span(&key1).await.unwrap().unwrap().1.next_key == key2.clone()); assert!(db.get_span(&key2).await.unwrap().unwrap().1.next_key == key2.clone()); @@ -275,13 +242,14 @@ mod test { assert!(db.get_span(&key1).await.unwrap().is_none()); assert!(db.get_span(&key2).await.unwrap().is_none()); - db.commit(None).await.unwrap(); + let (db, _) = db.commit(None).await.unwrap(); assert!(db.is_empty()); // Update the keys in opposite order from earlier. + let mut db = db.into_mutable(); db.update(key2.clone(), 2).await.unwrap(); db.update(key1.clone(), 1).await.unwrap(); - db.commit(None).await.unwrap(); + let (db, _) = db.commit(None).await.unwrap(); assert_eq!(db.get_all(&key1).await.unwrap().unwrap(), (1, key2.clone())); assert_eq!(db.get_all(&key2).await.unwrap().unwrap(), (2, key1.clone())); assert!(db.get_span(&key1).await.unwrap().unwrap().1.next_key == key2.clone()); @@ -291,6 +259,7 @@ mod test { assert!(db.get_span(&late_key).await.unwrap().unwrap().1.next_key == key1.clone()); // Delete the keys in opposite order from earlier. + let mut db = db.into_mutable(); db.delete(key2.clone()).await.unwrap(); assert!(db.get_span(&key1).await.unwrap().unwrap().1.next_key == key1.clone()); assert!(db.get_span(&key2).await.unwrap().unwrap().1.next_key == key1.clone()); @@ -301,7 +270,8 @@ mod test { db.delete(key1.clone()).await.unwrap(); assert!(db.get_span(&key1).await.unwrap().is_none()); assert!(db.get_span(&key2).await.unwrap().is_none()); - db.commit(None).await.unwrap(); + let (db, _) = db.commit(None).await.unwrap(); + let db = db.into_merkleized(); db.destroy().await.unwrap(); }); @@ -315,7 +285,8 @@ mod test { const ELEMENTS: u64 = 1000; executor.start(|context| async move { let mut hasher = Standard::::new(); - let mut db = open_db(context.clone()).await; + let db = open_db(context.clone()).await; + let mut db = db.into_mutable(); let mut map = HashMap::::default(); for i in 0u64..ELEMENTS { @@ -352,7 +323,8 @@ mod test { assert_eq!(db.snapshot.items(), 857); // Test that commit + sync w/ pruning will raise the activity floor. - db.commit(None).await.unwrap(); + let (db, _) = db.commit(None).await.unwrap(); + let mut db = db.into_merkleized(); db.sync().await.unwrap(); db.prune(db.inactivity_floor_loc()).await.unwrap(); assert_eq!(db.op_count(), 4241); @@ -363,7 +335,7 @@ mod test { let root = db.root(); db.sync().await.unwrap(); drop(db); - let mut db = open_db(context.clone()).await; + let db = open_db(context.clone()).await; assert_eq!(root, db.root()); assert_eq!(db.op_count(), 4241); assert_eq!(db.inactivity_floor_loc(), 3383); @@ -390,7 +362,9 @@ mod test { let start_loc = Location::try_from(start_pos).unwrap(); // Raise the inactivity floor via commit and make sure historical inactive operations // are still provable. - db.commit(None).await.unwrap(); + let db = db.into_mutable(); + let (db, _) = db.commit(None).await.unwrap(); + let db = db.into_merkleized(); let root = db.root(); assert!(start_loc < db.inactivity_floor_loc()); @@ -410,7 +384,8 @@ mod test { fn test_ordered_any_fixed_non_empty_db_recovery() { let executor = deterministic::Runner::default(); executor.start(|context| async move { - let mut db = open_db(context.clone()).await; + let db = open_db(context.clone()).await; + let mut db = db.into_mutable(); // Insert 1000 keys then sync. const ELEMENTS: u64 = 1000; @@ -419,19 +394,20 @@ mod test { let v = Sha256::hash(&(i * 1000).to_be_bytes()); db.update(k, v).await.unwrap(); } - db.commit(None).await.unwrap(); + let (db, _) = db.commit(None).await.unwrap(); + let mut db = db.into_merkleized(); db.prune(db.inactivity_floor_loc()).await.unwrap(); let root = db.root(); let op_count = db.op_count(); let inactivity_floor_loc = db.inactivity_floor_loc(); // Reopen DB without clean shutdown and make sure the state is the same. - let mut db = open_db(context.clone()).await; + let db = open_db(context.clone()).await; assert_eq!(db.op_count(), op_count); assert_eq!(db.inactivity_floor_loc(), inactivity_floor_loc); assert_eq!(db.root(), root); - async fn apply_more_ops(db: &mut AnyTest) { + async fn apply_more_ops(db: &mut MutableAnyTest) { for i in 0u64..ELEMENTS { let k = Sha256::hash(&i.to_be_bytes()); let v = Sha256::hash(&((i + 1) * 10000).to_be_bytes()); @@ -439,32 +415,36 @@ mod test { } } - // Insert operations without commit, then simulate failure, syncing nothing. + // Insert operations without commit, then drop without cleanup. + let mut db = db.into_mutable(); apply_more_ops(&mut db).await; - db.simulate_failure(false).await.unwrap(); - let mut db = open_db(context.clone()).await; + drop(db); + let db = open_db(context.clone()).await; assert_eq!(db.op_count(), op_count); assert_eq!(db.inactivity_floor_loc(), inactivity_floor_loc); assert_eq!(db.root(), root); - // Repeat, though this time sync the log. + // Repeat, drop without cleanup again. + let mut db = db.into_mutable(); apply_more_ops(&mut db).await; - db.simulate_failure(true).await.unwrap(); - let mut db = open_db(context.clone()).await; + drop(db); + let db = open_db(context.clone()).await; assert_eq!(db.op_count(), op_count); assert_eq!(db.root(), root); // One last check that re-open without proper shutdown still recovers the correct state. + let mut db = db.into_mutable(); apply_more_ops(&mut db).await; apply_more_ops(&mut db).await; apply_more_ops(&mut db).await; - let mut db = open_db(context.clone()).await; + let db = open_db(context.clone()).await; assert_eq!(db.op_count(), op_count); assert_eq!(db.root(), root); // Apply the ops one last time but fully commit them this time, then clean up. + let mut db = db.into_mutable(); apply_more_ops(&mut db).await; - db.commit(None).await.unwrap(); + let _ = db.commit(None).await.unwrap(); let db = open_db(context.clone()).await; assert!(db.op_count() > op_count); assert_ne!(db.inactivity_floor_loc(), inactivity_floor_loc); @@ -485,11 +465,11 @@ mod test { let root = db.root(); // Reopen DB without clean shutdown and make sure the state is the same. - let mut db = open_db(context.clone()).await; + let db = open_db(context.clone()).await; assert_eq!(db.op_count(), 1); assert_eq!(db.root(), root); - async fn apply_ops(db: &mut AnyTest) { + async fn apply_ops(db: &mut MutableAnyTest) { for i in 0u64..1000 { let k = Sha256::hash(&i.to_be_bytes()); let v = Sha256::hash(&((i + 1) * 10000).to_be_bytes()); @@ -497,31 +477,35 @@ mod test { } } - // Insert operations without commit then simulate failure, syncing nothing. + // Insert operations without commit then drop without cleanup. + let mut db = db.into_mutable(); apply_ops(&mut db).await; - db.simulate_failure(false).await.unwrap(); - let mut db = open_db(context.clone()).await; + drop(db); + let db = open_db(context.clone()).await; assert_eq!(db.op_count(), 1); assert_eq!(db.root(), root); - // Repeat, though this time sync the log. + // Repeat, drop without cleanup again. + let mut db = db.into_mutable(); apply_ops(&mut db).await; - db.simulate_failure(true).await.unwrap(); - let mut db = open_db(context.clone()).await; + drop(db); + let db = open_db(context.clone()).await; assert_eq!(db.op_count(), 1); assert_eq!(db.root(), root); // One last check that re-open without proper shutdown still recovers the correct state. + let mut db = db.into_mutable(); apply_ops(&mut db).await; apply_ops(&mut db).await; apply_ops(&mut db).await; - let mut db = open_db(context.clone()).await; + let db = open_db(context.clone()).await; assert_eq!(db.op_count(), 1); assert_eq!(db.root(), root); // Apply the ops one last time but fully commit them this time, then clean up. + let mut db = db.into_mutable(); apply_ops(&mut db).await; - db.commit(None).await.unwrap(); + let _ = db.commit(None).await.unwrap(); let db = open_db(context.clone()).await; assert!(db.op_count() > 1); assert_ne!(db.root(), root); @@ -536,7 +520,8 @@ mod test { fn test_ordered_any_fixed_db_log_replay() { let executor = deterministic::Runner::default(); executor.start(|context| async move { - let mut db = open_db(context.clone()).await; + let db = open_db(context.clone()).await; + let mut db = db.into_mutable(); // Update the same key many times. const UPDATES: u64 = 100; @@ -545,7 +530,8 @@ mod test { let v = Sha256::hash(&(i * 1000).to_be_bytes()); db.update(k, v).await.unwrap(); } - db.commit(None).await.unwrap(); + let (db, _) = db.commit(None).await.unwrap(); + let db = db.into_merkleized(); let root = db.root(); // Simulate a failed commit and test that the log replay doesn't leave behind old data. @@ -563,7 +549,8 @@ mod test { fn test_ordered_any_fixed_db_multiple_commits_delete_gets_replayed() { let executor = deterministic::Runner::default(); executor.start(|context| async move { - let mut db = open_db(context.clone()).await; + let db = open_db(context.clone()).await; + let mut db = db.into_mutable(); let mut map = HashMap::::default(); const ELEMENTS: u64 = 10; @@ -576,7 +563,8 @@ mod test { db.update(k, v).await.unwrap(); map.insert(k, v); } - db.commit(Some(metadata)).await.unwrap(); + let (new_db, _) = db.commit(Some(metadata)).await.unwrap(); + db = new_db.into_mutable(); } assert_eq!(db.get_metadata().await.unwrap(), Some(metadata)); let k = Sha256::hash(&((ELEMENTS - 1) * 1000 + (ELEMENTS - 1)).to_be_bytes()); @@ -584,11 +572,13 @@ mod test { // Do one last delete operation which will be above the inactivity // floor, to make sure it gets replayed on restart. db.delete(k).await.unwrap(); - db.commit(None).await.unwrap(); + let (db, _) = db.commit(None).await.unwrap(); assert_eq!(db.get_metadata().await.unwrap(), None); assert!(db.get(&k).await.unwrap().is_none()); - // Drop & reopen the db, making sure it has exactly the same state. + // Drop & reopen the db, making sure the re-opened db has exactly the same state. + let (db, _) = db.into_mutable().commit(None).await.unwrap(); + let mut db = db.into_merkleized(); let root = db.root(); db.sync().await.unwrap(); drop(db); @@ -601,14 +591,25 @@ mod test { }); } + // Test that merkleization state changes don't reset `steps`. + #[test_traced("DEBUG")] + fn test_any_ordered_fixed_db_steps_not_reset() { + let executor = deterministic::Runner::default(); + executor.start(|context| async move { + let db = open_db(context).await; + crate::qmdb::any::test::test_any_db_steps_not_reset(db).await; + }); + } + #[test] fn test_ordered_any_fixed_db_historical_proof_basic() { let executor = deterministic::Runner::default(); executor.start(|context| async move { - let mut db = create_test_db(context.clone()).await; + let mut db = create_test_db(context.clone()).await.into_mutable(); let ops = create_test_ops(20); apply_ops(&mut db, ops.clone()).await; - db.commit(None).await.unwrap(); + let (db, _) = db.commit(None).await.unwrap(); + let db = db.into_merkleized(); let mut hasher = Standard::::new(); let root_hash = db.root(); let original_op_count = db.op_count(); @@ -635,8 +636,10 @@ mod test { // Add more operations to the database let more_ops = create_test_ops(5); + let mut db = db.into_mutable(); apply_ops(&mut db, more_ops.clone()).await; - db.commit(None).await.unwrap(); + let (db, _) = db.commit(None).await.unwrap(); + let db = db.into_merkleized(); // Historical proof should remain the same even though database has grown let (historical_proof, historical_ops) = db @@ -667,10 +670,11 @@ mod test { fn test_ordered_any_fixed_db_historical_proof_edge_cases() { let executor = deterministic::Runner::default(); executor.start(|context| async move { - let mut db = create_test_db(context.clone()).await; + let mut db = create_test_db(context.clone()).await.into_mutable(); let ops = create_test_ops(50); apply_ops(&mut db, ops.clone()).await; - db.commit(None).await.unwrap(); + let (db, _) = db.commit(None).await.unwrap(); + let db = db.into_merkleized(); let mut hasher = Standard::::new(); @@ -690,10 +694,10 @@ mod test { assert_eq!(single_ops.len(), 1); // Create historical database with single operation - let mut single_db = create_test_db(context.clone()).await; + let mut single_db = create_test_db(context.clone()).await.into_mutable(); apply_ops(&mut single_db, ops[0..1].to_vec()).await; // Don't commit - this changes the root due to commit operations - single_db.sync().await.unwrap(); + let single_db = single_db.into_merkleized(); let single_root = single_db.root(); assert!(verify_proof( @@ -730,7 +734,7 @@ mod test { ); assert_eq!(min_ops.len(), 3); - single_db.destroy().await.unwrap(); + drop(single_db); db.destroy().await.unwrap(); }); } @@ -739,10 +743,11 @@ mod test { fn test_ordered_any_fixed_db_historical_proof_different_historical_sizes() { let executor = deterministic::Runner::default(); executor.start(|context| async move { - let mut db = create_test_db(context.clone()).await; + let mut db = create_test_db(context.clone()).await.into_mutable(); let ops = create_test_ops(100); apply_ops(&mut db, ops.clone()).await; - db.commit(None).await.unwrap(); + let (db, _) = db.commit(None).await.unwrap(); + let db = db.into_merkleized(); let mut hasher = Standard::::new(); let root = db.root(); @@ -754,12 +759,14 @@ mod test { // Now keep adding operations and make sure we can still generate a historical proof that matches the original. let historical_size = db.op_count(); + let mut db = db.into_mutable(); for _ in 1..10 { let more_ops = create_test_ops(100); apply_ops(&mut db, more_ops).await; - db.commit(None).await.unwrap(); + let (clean_db, _) = db.commit(None).await.unwrap(); + let clean_db = clean_db.into_merkleized(); - let (historical_proof, historical_ops) = db + let (historical_proof, historical_ops) = clean_db .historical_proof(historical_size, start_loc, max_ops) .await .unwrap(); @@ -775,9 +782,12 @@ mod test { &historical_ops, &root )); + + db = clean_db.into_mutable(); } - db.destroy().await.unwrap(); + let (db, _) = db.commit(None).await.unwrap(); + db.into_merkleized().destroy().await.unwrap(); }); } @@ -785,10 +795,11 @@ mod test { fn test_ordered_any_fixed_db_historical_proof_invalid() { let executor = deterministic::Runner::default(); executor.start(|context| async move { - let mut db = create_test_db(context.clone()).await; + let mut db = create_test_db(context.clone()).await.into_mutable(); let ops = create_test_ops(10); apply_ops(&mut db, ops).await; - db.commit(None).await.unwrap(); + let (db, _) = db.commit(None).await.unwrap(); + let db = db.into_merkleized(); let historical_op_count = Location::new_unchecked(5); let historical_mmr_size = Position::try_from(historical_op_count).unwrap(); @@ -904,9 +915,9 @@ mod test { let executor = deterministic::Runner::default(); executor.start(|mut context| async move { async fn insert_random( - db: &mut Db, + mut db: Db, rng: &mut StdRng, - ) { + ) -> Db { let mut keys = BTreeMap::new(); // Insert 1000 random keys into both the db and an ordered map. @@ -916,7 +927,7 @@ mod test { db.update(key, i).await.unwrap(); } - db.commit(None).await.unwrap(); + let (db, _) = db.commit(None).await.unwrap(); // Make sure the db and ordered map agree on contents & key order. let mut iter = keys.iter(); @@ -931,6 +942,7 @@ mod test { } // Delete some random keys and check order agreement again. + let mut db = db.into_mutable(); for _ in 0..500 { let key = keys.keys().choose(rng).cloned().unwrap(); keys.remove(&key); @@ -957,6 +969,7 @@ mod test { assert_eq!(keys.len(), 0); assert!(db.is_empty()); assert_eq!(db.get_span(&Digest::random(&mut *rng)).await.unwrap(), None); + db } let mut rng = StdRng::seed_from_u64(context.next_u64()); @@ -964,24 +977,32 @@ mod test { // Use a OneCap to ensure many collisions. let config = create_generic_test_config::(seed, OneCap); - let mut db = Db::::init(context.clone(), config) - .await - .unwrap(); - insert_random(&mut db, &mut rng).await; - db.destroy().await.unwrap(); + let db = Db::, Durable>::init( + context.clone(), + config, + ) + .await + .unwrap(); + let db = insert_random(db.into_mutable(), &mut rng).await; + let (db, _) = db.commit(None).await.unwrap(); + db.into_merkleized().destroy().await.unwrap(); // Repeat test with TwoCap to test low/no collisions. let config = create_generic_test_config::(seed, TwoCap); - let mut db = Db::::init(context.clone(), config) - .await - .unwrap(); - insert_random(&mut db, &mut rng).await; - db.destroy().await.unwrap(); + let db = Db::, Durable>::init( + context.clone(), + config, + ) + .await + .unwrap(); + let db = insert_random(db.into_mutable(), &mut rng).await; + let (db, _) = db.commit(None).await.unwrap(); + db.into_merkleized().destroy().await.unwrap(); }); } #[test_traced("DEBUG")] - fn test_batch() { - batch_tests::test_batch(|ctx| async move { create_test_db(ctx).await }); + fn test_any_ordered_fixed_batch() { + batch_tests::test_batch(|ctx| async move { create_test_db(ctx).await.into_mutable() }); } } diff --git a/storage/src/qmdb/any/ordered/mod.rs b/storage/src/qmdb/any/ordered/mod.rs index 278e69b60c..e6e61b388b 100644 --- a/storage/src/qmdb/any/ordered/mod.rs +++ b/storage/src/qmdb/any/ordered/mod.rs @@ -1,22 +1,23 @@ use crate::{ index::{Cursor as _, Ordered as Index}, - journal::{ - contiguous::{Contiguous, MutableContiguous}, - Error as JournalError, - }, + journal::contiguous::{Contiguous, MutableContiguous}, kv::{self, Batchable}, - mmr::{ - mem::{Dirty, State}, - Location, - }, + mmr::Location, qmdb::{ any::{ db::{AuthenticatedLog, Db}, - CleanAny, DirtyAny, ValueEncoding, + ValueEncoding, }, delete_known_loc, operation::Operation as OperationTrait, - update_known_loc, Error, + update_known_loc, DurabilityState, Error, MerkleizationState, NonDurable, Unmerkleized, + }, +}; +#[cfg(any(test, feature = "test-traits"))] +use crate::{ + qmdb::{ + any::states::{CleanAny, MerkleizedNonDurableAny, MutableAny, UnmerkleizedDurableAny}, + Durable, Merkleized, }, Persistable, }; @@ -24,6 +25,7 @@ use commonware_codec::Codec; use commonware_cryptography::{DigestOf, Hasher}; use commonware_runtime::{Clock, Metrics, Storage}; use commonware_utils::Array; +#[cfg(any(test, feature = "test-traits"))] use core::ops::Range; use futures::{ future::try_join_all, @@ -59,13 +61,14 @@ impl< C: Contiguous>, I: Index, H: Hasher, - S: State>, - > Db, S> + M: MerkleizationState>, + D: DurabilityState, + > Db, M, D> where Operation: Codec, { async fn get_update_op( - log: &AuthenticatedLog, + log: &AuthenticatedLog, loc: Location, ) -> Result, Error> { match log.read(loc).await? { @@ -264,8 +267,7 @@ impl< C: MutableContiguous>, I: Index, H: Hasher, - S: State>, - > Db, S> + > Db, Unmerkleized, NonDurable> where Operation: Codec, { @@ -431,7 +433,7 @@ where // For either a new key or an update of existing key, we inactivate exactly one previous // operation. A new key inactivates a previous span, and an update of existing key // inactivates a previous value. - self.steps += 1; + self.durable_state.steps += 1; Ok(()) } @@ -480,7 +482,7 @@ where }; // Creating a new key involves inactivating a previous span, requiring we increment `steps`. - self.steps += 1; + self.durable_state.steps += 1; Ok(true) } @@ -535,7 +537,7 @@ where self.active_keys -= 1; let op = Operation::Delete(key.clone()); self.log.append(op).await?; - self.steps += 1; + self.durable_state.steps += 1; if self.is_empty() { // This was the last key in the DB so there is no span to update. @@ -563,7 +565,7 @@ where next_key, }); self.log.append(op).await?; - self.steps += 1; + self.durable_state.steps += 1; Ok(()) } @@ -661,7 +663,7 @@ where // Each delete reduces the active key count by one and inactivates that key. self.active_keys -= 1; - self.steps += 1; + self.durable_state.steps += 1; } } @@ -721,7 +723,7 @@ where callback(true, Some(loc)); // Each update of an existing key inactivates its previous location. - self.steps += 1; + self.durable_state.steps += 1; already_updated.insert(key); } @@ -763,7 +765,7 @@ where callback(true, Some(*prev_loc)); // Each key whose next-key value is updated inactivates its previous location. - self.steps += 1; + self.durable_state.steps += 1; } if possible_next.is_empty() || possible_previous.is_empty() { @@ -790,39 +792,13 @@ where callback(true, Some(*prev_loc)); // Each key whose next-key value is updated inactivates its previous location. - self.steps += 1; + self.durable_state.steps += 1; } Ok(()) } } -impl< - E: Storage + Clock + Metrics, - K: Array, - V: ValueEncoding, - C: MutableContiguous> + Persistable, - I: Index, - H: Hasher, - > Persistable for Db> -where - Operation: Codec, -{ - type Error = Error; - - async fn commit(&mut self) -> Result<(), Error> { - self.commit(None).await.map(|_| ()) - } - - async fn sync(&mut self) -> Result<(), Error> { - self.sync().await - } - - async fn destroy(self) -> Result<(), Error> { - self.destroy().await - } -} - impl< E: Storage + Clock + Metrics, K: Array, @@ -830,7 +806,9 @@ impl< C: Contiguous>, I: Index, H: Hasher, - > kv::Gettable for Db> + M: MerkleizationState>, + D: DurabilityState, + > kv::Gettable for Db, M, D> where Operation: Codec, { @@ -850,7 +828,7 @@ impl< C: MutableContiguous>, I: Index, H: Hasher, - > kv::Updatable for Db> + > kv::Updatable for Db, Unmerkleized, NonDurable> where Operation: Codec, { @@ -866,7 +844,7 @@ impl< C: MutableContiguous>, I: Index, H: Hasher, - > kv::Deletable for Db> + > kv::Deletable for Db, Unmerkleized, NonDurable> where Operation: Codec, { @@ -875,70 +853,6 @@ where } } -impl< - E: Storage + Clock + Metrics, - K: Array, - V: ValueEncoding, - C: MutableContiguous> + Persistable, - I: Index, - H: Hasher, - > CleanAny for Db> -where - Operation: Codec, -{ - type Key = K; - - async fn get(&self, key: &Self::Key) -> Result, Error> { - self.get(key).await - } - - async fn commit(&mut self, metadata: Option) -> Result, Error> { - self.commit(metadata).await - } - - async fn sync(&mut self) -> Result<(), Error> { - self.sync().await - } - - async fn prune(&mut self, prune_loc: Location) -> Result<(), Error> { - self.prune(prune_loc).await - } - - async fn destroy(self) -> Result<(), Error> { - self.destroy().await - } -} - -impl< - E: Storage + Clock + Metrics, - K: Array, - V: ValueEncoding, - C: MutableContiguous>, - I: Index, - H: Hasher, - > DirtyAny for Db, Dirty> -where - Operation: Codec, -{ - type Key = K; - - async fn get(&self, key: &Self::Key) -> Result, Error> { - self.get(key).await - } - - async fn update(&mut self, key: Self::Key, value: Self::Value) -> Result<(), Error> { - self.update(key, value).await - } - - async fn create(&mut self, key: Self::Key, value: Self::Value) -> Result { - self.create(key, value).await - } - - async fn delete(&mut self, key: Self::Key) -> Result { - self.delete(key).await - } -} - /// Returns the next key to `key` within `possible_next`. The result will "cycle around" to the /// first key if `key` is the last key. /// @@ -977,7 +891,7 @@ fn find_prev_key<'a, K: Ord, V>(key: &K, possible_previous: &'a BTreeMap) .expect("possible_previous should not be empty") } -impl Batchable for Db> +impl Batchable for Db, Unmerkleized, NonDurable> where E: Storage + Clock + Metrics, K: Array, @@ -995,14 +909,117 @@ where } } +#[cfg(any(test, feature = "test-traits"))] +impl CleanAny for Db, Merkleized, Durable> +where + E: Storage + Clock + Metrics, + K: Array, + V: ValueEncoding, + C: MutableContiguous> + Persistable, + I: Index, + H: Hasher, + Operation: Codec, +{ + type Mutable = Db, Unmerkleized, NonDurable>; + + fn into_mutable(self) -> Self::Mutable { + self.into_mutable() + } +} + +#[cfg(any(test, feature = "test-traits"))] +impl UnmerkleizedDurableAny + for Db, Unmerkleized, Durable> +where + E: Storage + Clock + Metrics, + K: Array, + V: ValueEncoding, + C: MutableContiguous> + Persistable, + I: Index, + H: Hasher, + Operation: Codec, +{ + type Digest = H::Digest; + type Operation = Operation; + type Mutable = Db, Unmerkleized, NonDurable>; + type Merkleized = Db, Merkleized, Durable>; + + fn into_mutable(self) -> Self::Mutable { + self.into_mutable() + } + + async fn into_merkleized(self) -> Result { + Ok(self.into_merkleized()) + } +} + +#[cfg(any(test, feature = "test-traits"))] +impl MerkleizedNonDurableAny + for Db, Merkleized, NonDurable> +where + E: Storage + Clock + Metrics, + K: Array, + V: ValueEncoding, + C: MutableContiguous> + Persistable, + I: Index, + H: Hasher, + Operation: Codec, +{ + type Mutable = Db, Unmerkleized, NonDurable>; + type Durable = Db, Merkleized, Durable>; + + async fn commit( + self, + metadata: Option, + ) -> Result<(Self::Durable, Range), Error> { + self.commit(metadata).await + } + + fn into_mutable(self) -> Self::Mutable { + self.into_mutable() + } +} + +#[cfg(any(test, feature = "test-traits"))] +impl MutableAny for Db, Unmerkleized, NonDurable> +where + E: Storage + Clock + Metrics, + K: Array, + V: ValueEncoding, + C: MutableContiguous> + Persistable, + I: Index, + H: Hasher, + Operation: Codec, +{ + type Digest = H::Digest; + type Operation = Operation; + type Durable = Db, Unmerkleized, Durable>; + type Merkleized = Db, Merkleized, NonDurable>; + + async fn commit( + self, + metadata: Option, + ) -> Result<(Self::Durable, Range), Error> { + self.commit(metadata).await + } + + async fn into_merkleized(self) -> Result { + Ok(self.into_merkleized()) + } + + fn steps(&self) -> u64 { + self.durable_state.steps + } +} + #[cfg(test)] mod test { use super::*; use crate::{ - kv::{Deletable as _, Updatable as _}, + kv::{Deletable as _, Gettable as _, Updatable as _}, qmdb::{ any::test::{fixed_db_config, variable_db_config}, - store::{DirtyStore as _, LogStore as _}, + store::{LogStore as _, MerkleizedStore}, }, translator::TwoCap, }; @@ -1022,6 +1039,20 @@ mod test { /// A type alias for the concrete [variable::Db] type used in these unit tests. type VariableDb = variable::Db, Digest, Sha256, TwoCap>; + /// A type alias for a variable db with Digest keys (for generic tests). + type DigestVariableDb = variable::Db; + + /// Helper trait for testing Any databases that cycle through all four states. + trait TestableAnyDb: + CleanAny> + MerkleizedStore + { + } + + impl TestableAnyDb for T where + T: CleanAny> + MerkleizedStore + { + } + /// Return an `Any` database initialized with a fixed config. async fn open_fixed_db(context: Context) -> FixedDb { FixedDb::init(context, fixed_db_config("partition")) @@ -1036,13 +1067,18 @@ mod test { .unwrap() } - async fn test_ordered_any_db_empty( + /// Return a variable db with Digest keys for generic tests. + async fn open_digest_variable_db(context: Context) -> DigestVariableDb { + DigestVariableDb::init(context, variable_db_config("digest_partition")) + .await + .unwrap() + } + + async fn test_ordered_any_db_empty>( context: Context, mut db: D, reopen_db: impl Fn(Context) -> Pin + Send>>, - ) where - D: CleanAny, Value = Digest, Digest = Digest>, - { + ) { assert_eq!(db.op_count(), 1); assert!(db.get_metadata().await.unwrap().is_none()); assert!(matches!(db.prune(db.inactivity_floor_loc()).await, Ok(()))); @@ -1052,15 +1088,17 @@ mod test { let d1 = FixedBytes::from([1u8; 4]); let d2 = Sha256::fill(2u8); let root = db.root(); - let mut db = db.into_dirty(); + let mut db = db.into_mutable(); db.update(d1, d2).await.unwrap(); - let mut db = reopen_db(context.clone()).await; + let db = reopen_db(context.clone()).await; assert_eq!(db.root(), root); assert_eq!(db.op_count(), 1); // Test calling commit on an empty db. let metadata = Sha256::fill(3u8); - let range = db.commit(Some(metadata)).await.unwrap(); + let db = db.into_mutable(); + let (db, range) = db.commit(Some(metadata)).await.unwrap(); + let mut db = db.into_merkleized().await.unwrap(); assert_eq!(range.start, Location::new_unchecked(1)); assert_eq!(range.end, Location::new_unchecked(2)); assert_eq!(db.op_count(), 2); // floor op added @@ -1069,18 +1107,30 @@ mod test { assert!(matches!(db.prune(db.inactivity_floor_loc()).await, Ok(()))); // Re-opening the DB without a clean shutdown should still recover the correct state. - let mut db = reopen_db(context.clone()).await; + let db = reopen_db(context.clone()).await; assert_eq!(db.op_count(), 2); assert_eq!(db.get_metadata().await.unwrap(), Some(metadata)); assert_eq!(db.root(), root); // Confirm the inactivity floor doesn't fall endlessly behind with multiple commits. + let mut db = db.into_mutable(); for _ in 1..100 { - db.commit(None).await.unwrap(); - assert_eq!(db.op_count() - 1, db.inactivity_floor_loc()); + let (durable_db, _) = db.commit(None).await.unwrap(); + let clean_db = durable_db.into_merkleized().await.unwrap(); + assert_eq!(clean_db.op_count() - 1, clean_db.inactivity_floor_loc()); + db = clean_db.into_mutable(); } - db.destroy().await.unwrap(); + db.commit(None) + .await + .unwrap() + .0 + .into_merkleized() + .await + .unwrap() + .destroy() + .await + .unwrap(); } #[test_traced("WARN")] @@ -1101,13 +1151,11 @@ mod test { }); } - async fn test_ordered_any_db_basic( + async fn test_ordered_any_db_basic>( context: Context, db: D, reopen_db: impl Fn(Context) -> Pin + Send>>, - ) where - D: CleanAny, Value = Digest, Digest = Digest>, - { + ) { // Build a db with 2 keys and make sure updates and deletions of those keys work as // expected. let key1 = FixedBytes::from([1u8; 4]); @@ -1118,7 +1166,7 @@ mod test { assert!(db.get(&key1).await.unwrap().is_none()); assert!(db.get(&key2).await.unwrap().is_none()); - let mut db = db.into_dirty(); + let mut db = db.into_mutable(); assert!(db.create(key1.clone(), val1).await.unwrap()); assert_eq!(db.get(&key1).await.unwrap().unwrap(), val1); assert!(db.get(&key2).await.unwrap().is_none()); @@ -1141,9 +1189,8 @@ mod test { // 2 new keys (4 ops), 2 updates (2 ops), 1 deletion (2 ops) + 1 initial commit = 9 ops assert_eq!(db.op_count(), 9); assert_eq!(db.inactivity_floor_loc(), Location::new_unchecked(0)); - let mut db = db.merkleize().await.unwrap(); - db.commit(None).await.unwrap(); - let mut db = db.into_dirty(); + let (durable_db, _) = db.commit(None).await.unwrap(); + let mut db = durable_db.into_merkleized().await.unwrap().into_mutable(); // Make sure create won't modify active keys. assert!(!db.create(key1.clone(), val1).await.unwrap()); @@ -1155,18 +1202,21 @@ mod test { assert!(db.get(&key1).await.unwrap().is_none()); assert!(db.get(&key2).await.unwrap().is_none()); - let mut db = db.merkleize().await.unwrap(); - db.commit(None).await.unwrap(); - let root = db.root(); + let db = db + .commit(None) + .await + .unwrap() + .0 + .into_merkleized() + .await + .unwrap(); // Multiple deletions of the same key should be a no-op. let prev_op_count = db.op_count(); - let mut db = db.into_dirty(); + let mut db = db.into_mutable(); + // Note: commit always adds a floor op, so op_count will increase by 1 after commit. assert!(!db.delete(key1.clone()).await.unwrap()); assert_eq!(db.op_count(), prev_op_count); - let db = db.merkleize().await.unwrap(); - assert_eq!(db.root(), root); - let mut db = db.into_dirty(); // Deletions of non-existent keys should be a no-op. let key3 = FixedBytes::from([6u8; 4]); @@ -1174,14 +1224,20 @@ mod test { assert_eq!(db.op_count(), prev_op_count); // Make sure closing/reopening gets us back to the same state. - let mut db = db.merkleize().await.unwrap(); - db.commit(None).await.unwrap(); + let db = db + .commit(None) + .await + .unwrap() + .0 + .into_merkleized() + .await + .unwrap(); let op_count = db.op_count(); let root = db.root(); let db = reopen_db(context.clone()).await; assert_eq!(db.op_count(), op_count); assert_eq!(db.root(), root); - let mut db = db.into_dirty(); + let mut db = db.into_mutable(); // Re-activate the keys by updating them. db.update(key1.clone(), val1).await.unwrap(); @@ -1190,20 +1246,34 @@ mod test { db.update(key2.clone(), val1).await.unwrap(); db.update(key1.clone(), val2).await.unwrap(); - let mut db = db.merkleize().await.unwrap(); - db.commit(None).await.unwrap(); + let db = db + .commit(None) + .await + .unwrap() + .0 + .into_merkleized() + .await + .unwrap(); // Confirm close/reopen gets us back to the same state. let op_count = db.op_count(); let root = db.root(); - let mut db = reopen_db(context.clone()).await; + let db = reopen_db(context.clone()).await; assert_eq!(db.root(), root); assert_eq!(db.op_count(), op_count); // Commit will raise the inactivity floor, which won't affect state but will affect the // root. - db.commit(None).await.unwrap(); + let db = db.into_mutable(); + let mut db = db + .commit(None) + .await + .unwrap() + .0 + .into_merkleized() + .await + .unwrap(); assert!(db.root() != root); @@ -1233,12 +1303,19 @@ mod test { }); } + // Test that merkleization state changes don't reset `steps`. + #[test_traced("DEBUG")] + fn test_any_ordered_variable_db_steps_not_reset() { + let executor = Runner::default(); + executor.start(|context| async move { + let db = open_digest_variable_db(context).await; + crate::qmdb::any::test::test_any_db_steps_not_reset(db).await; + }); + } + /// Builds a db with colliding keys to make sure the "cycle around when there are translated /// key collisions" edge case is exercised. - async fn test_ordered_any_update_collision_edge_case(db: D) - where - D: CleanAny, Value = Digest, Digest = Digest>, - { + async fn test_ordered_any_update_collision_edge_case>(db: D) { // This DB uses a TwoCap so we use equivalent two byte prefixes for each key to ensure // collisions. let key1 = FixedBytes::from([0xFFu8, 0xFFu8, 5u8, 5u8]); @@ -1247,7 +1324,7 @@ mod test { let key3 = FixedBytes::from([0xFFu8, 0xFFu8, 0u8, 0u8]); let val = Sha256::fill(1u8); - let mut db = db.into_dirty(); + let mut db = db.into_mutable(); db.update(key1.clone(), val).await.unwrap(); db.update(key2.clone(), val).await.unwrap(); db.update(key3.clone(), val).await.unwrap(); @@ -1256,8 +1333,8 @@ mod test { assert_eq!(db.get(&key2).await.unwrap().unwrap(), val); assert_eq!(db.get(&key3).await.unwrap().unwrap(), val); - let db = db.merkleize().await.unwrap(); - db.destroy().await.unwrap(); + let db = db.commit(None).await.unwrap().0; + db.into_merkleized().await.unwrap().destroy().await.unwrap(); } #[test_traced("WARN")] @@ -1284,7 +1361,7 @@ mod test { fn test_ordered_any_update_batch_create_between_collisions() { let executor = Runner::default(); executor.start(|context| async move { - let mut db = open_variable_db(context.clone()).await; + let mut db = open_variable_db(context.clone()).await.into_mutable(); // This DB uses a TwoCap so we use equivalent two byte prefixes for each key to ensure // collisions. @@ -1295,17 +1372,18 @@ mod test { db.update(key1.clone(), val).await.unwrap(); db.update(key3.clone(), val).await.unwrap(); - db.commit(None).await.unwrap(); + let (db, _) = db.commit(None).await.unwrap(); assert_eq!(db.get(&key1).await.unwrap().unwrap(), val); assert!(db.get(&key2).await.unwrap().is_none()); assert_eq!(db.get(&key3).await.unwrap().unwrap(), val); // Batch-insert the middle key. + let mut db = db.into_mutable(); let mut batch = db.start_batch(); batch.update(key2.clone(), val).await.unwrap(); db.write_batch(batch.into_iter()).await.unwrap(); - db.commit(None).await.unwrap(); + let (db, _) = db.commit(None).await.unwrap(); assert_eq!(db.get(&key1).await.unwrap().unwrap(), val); assert_eq!(db.get(&key2).await.unwrap().unwrap(), val); @@ -1318,7 +1396,8 @@ mod test { let span3 = db.get_span(&key3).await.unwrap().unwrap(); assert_eq!(span3.1.next_key, key1); - db.destroy().await.unwrap(); + let db = db.into_mutable().commit(None).await.unwrap().0; + db.into_merkleized().destroy().await.unwrap(); }); } @@ -1329,20 +1408,20 @@ mod test { fn test_ordered_any_batch_create_with_cycling_next_key() { let executor = Runner::default(); executor.start(|context| async move { - let mut db = open_fixed_db(context.clone()).await; + let mut db = open_fixed_db(context.clone()).await.into_mutable(); let mid_key = FixedBytes::from([0xAAu8; 4]); let val = Sha256::fill(1u8); - db.create(mid_key.clone(), val).await.unwrap(); - db.commit(None).await.unwrap(); + let (db, _) = db.commit(None).await.unwrap(); // Batch-insert a preceeding non-translated-colliding key. let preceeding_key = FixedBytes::from([0x55u8; 4]); + let mut db = db.into_mutable(); let mut batch = db.start_batch(); assert!(batch.create(preceeding_key.clone(), val).await.unwrap()); db.write_batch(batch.into_iter()).await.unwrap(); - db.commit(None).await.unwrap(); + let (db, _) = db.commit(None).await.unwrap(); assert_eq!(db.get(&preceeding_key).await.unwrap().unwrap(), val); assert_eq!(db.get(&mid_key).await.unwrap().unwrap(), val); @@ -1352,7 +1431,8 @@ mod test { let span2 = db.get_span(&mid_key).await.unwrap().unwrap(); assert_eq!(span2.1.next_key, preceeding_key); - db.destroy().await.unwrap(); + let db = db.into_mutable().commit(None).await.unwrap().0; + db.into_merkleized().destroy().await.unwrap(); }); } @@ -1362,7 +1442,7 @@ mod test { fn test_ordered_any_batch_delete_middle_key() { let executor = Runner::default(); executor.start(|context| async move { - let mut db = open_fixed_db(context.clone()).await; + let mut db = open_fixed_db(context.clone()).await.into_mutable(); let key_a = FixedBytes::from([0x11u8; 4]); let key_b = FixedBytes::from([0x22u8; 4]); @@ -1373,7 +1453,7 @@ mod test { db.create(key_a.clone(), val).await.unwrap(); db.create(key_b.clone(), val).await.unwrap(); db.create(key_c.clone(), val).await.unwrap(); - db.commit(None).await.unwrap(); + let mut db = db.commit(None).await.unwrap().0.into_mutable(); // Verify initial spans let span_a = db.get_span(&key_a).await.unwrap().unwrap(); @@ -1387,7 +1467,7 @@ mod test { let mut batch = db.start_batch(); batch.delete(key_b.clone()).await.unwrap(); db.write_batch(batch.into_iter()).await.unwrap(); - db.commit(None).await.unwrap(); + let db = db.commit(None).await.unwrap().0.into_merkleized(); // Verify B is deleted assert!(db.get(&key_b).await.unwrap().is_none()); @@ -1418,10 +1498,10 @@ mod test { let val3 = Sha256::fill(3u8); // Delete the previous key of a newly created key. - let mut db = open_variable_db(context.clone()).await; + let mut db = open_variable_db(context.clone()).await.into_mutable(); db.update(key1.clone(), val1).await.unwrap(); db.update(key3.clone(), val3).await.unwrap(); - db.commit(None).await.unwrap(); + let mut db = db.commit(None).await.unwrap().0.into_mutable(); let mut batch = db.start_batch(); batch.delete(key1.clone()).await.unwrap(); @@ -1435,13 +1515,14 @@ mod test { assert_eq!(span2.1.next_key, key3); let span3 = db.get_span(&key3).await.unwrap().unwrap(); assert_eq!(span3.1.next_key, key2); - db.destroy().await.unwrap(); + let db = db.commit(None).await.unwrap().0; + db.into_merkleized().destroy().await.unwrap(); // Create a key that becomes the previous key of a concurrently deleted key. - let mut db = open_variable_db(context.clone()).await; + let mut db = open_variable_db(context.clone()).await.into_mutable(); db.update(key1.clone(), val1).await.unwrap(); db.update(key3.clone(), val3).await.unwrap(); - db.commit(None).await.unwrap(); + let mut db = db.commit(None).await.unwrap().0.into_mutable(); let mut batch = db.start_batch(); batch.create(key2.clone(), val2).await.unwrap(); @@ -1455,7 +1536,8 @@ mod test { assert_eq!(span1.1.next_key, key2); let span2 = db.get_span(&key2).await.unwrap().unwrap(); assert_eq!(span2.1.next_key, key1); - db.destroy().await.unwrap(); + let db = db.commit(None).await.unwrap().0; + db.into_merkleized().destroy().await.unwrap(); }); } @@ -1463,14 +1545,14 @@ mod test { fn test_ordered_any_stream_range() { let executor = Runner::default(); executor.start(|context| async move { - let mut db = open_fixed_db(context.clone()).await; + let mut db = open_fixed_db(context.clone()).await.into_mutable(); let key1 = FixedBytes::from([0x10u8, 0x00, 0x00, 0x05]); let val = Sha256::fill(1u8); // Test the single-bucket case. db.create(key1.clone(), val).await.unwrap(); - db.commit(None).await.unwrap(); + let db = db.commit(None).await.unwrap().0; // Start key is in the DB. { @@ -1514,10 +1596,11 @@ mod test { let key2_2 = FixedBytes::from([0x20u8, 0x00, 0x00, 0x11]); let key3 = FixedBytes::from([0x30u8, 0x00, 0x00, 0x05]); + let mut db = db.into_mutable(); db.create(key2_1.clone(), val).await.unwrap(); db.create(key2_2.clone(), val).await.unwrap(); db.create(key3.clone(), val).await.unwrap(); - db.commit(None).await.unwrap(); + let db = db.commit(None).await.unwrap().0; // Start key is in the DB. { @@ -1573,7 +1656,8 @@ mod test { assert!(stream.next().await.is_none()); } - db.destroy().await.unwrap(); + let db = db.into_mutable().commit(None).await.unwrap().0; + db.into_merkleized().destroy().await.unwrap(); }); } } diff --git a/storage/src/qmdb/any/ordered/variable.rs b/storage/src/qmdb/any/ordered/variable.rs index 96f38e78e8..8de9282066 100644 --- a/storage/src/qmdb/any/ordered/variable.rs +++ b/storage/src/qmdb/any/ordered/variable.rs @@ -11,16 +11,16 @@ use crate::{ authenticated, contiguous::variable::{Config as JournalConfig, Journal}, }, - mmr::{journaled::Config as MmrConfig, mem::Clean, Location}, + mmr::{journaled::Config as MmrConfig, Location}, qmdb::{ any::{ordered, value::VariableEncoding, VariableConfig, VariableValue}, operation::Committable as _, - Error, + Durable, Error, Merkleized, }, translator::Translator, }; use commonware_codec::Read; -use commonware_cryptography::{DigestOf, Hasher}; +use commonware_cryptography::Hasher; use commonware_runtime::{Clock, Metrics, Storage}; use commonware_utils::Array; use tracing::warn; @@ -30,11 +30,11 @@ pub type Operation = ordered::Operation>; /// A key-value QMDB based on an authenticated log of operations, supporting authentication of any /// value ever associated with a key. -pub type Db>> = - super::Db>, Index, H, Update, S>; +pub type Db, D = Durable> = + super::Db>, Index, H, Update, S, D>; impl - Db + Db, Durable> { /// Returns a [Db] QMDB initialized from `cfg`. Any uncommitted log operations will be /// discarded and the state of the db will be as of the last committed operation. diff --git a/storage/src/qmdb/any/states.rs b/storage/src/qmdb/any/states.rs new file mode 100644 index 0000000000..cfa8d9dccc --- /dev/null +++ b/storage/src/qmdb/any/states.rs @@ -0,0 +1,144 @@ +//! Traits representing the 4 possible states of an Any database. These are used to share test and +//! benchmark code across the variants. + +use crate::{ + kv::{Batchable, Deletable, Gettable}, + mmr::Location, + qmdb::{ + store::{LogStore, MerkleizedStore, PrunableStore}, + Error, + }, + Persistable, +}; +use commonware_codec::Codec; +use commonware_cryptography::Digest; +use commonware_utils::Array; +use std::{future::Future, ops::Range}; + +/// Trait for the (Merkleized,Durable) state. +/// +/// This state allows authentication (root, proofs), pruning, and persistence operations +/// (sync/close/destroy). Use `into_mutable` to transition to the (Unmerkleized,Non-durable) state. +pub trait CleanAny: + MerkleizedStore + + PrunableStore + + Persistable + + Gettable::Value, Error = Error> +{ + /// The mutable state type (Unmerkleized,Non-durable). + type Mutable: MutableAny< + Key = Self::Key, + Digest = ::Digest, + Operation = ::Operation, + // Cycle constraint for path: into_merkleized() then commit() + Merkleized: MerkleizedNonDurableAny, + // Cycle constraint for path: commit() then into_merkleized() or into_mutable() + Durable: UnmerkleizedDurableAny, + > + LogStore::Value>; + + /// Convert this database into the mutable (Unmerkleized, Non-durable) state. + fn into_mutable(self) -> Self::Mutable; +} + +/// Trait for the (Unmerkleized,Durable) state. +/// +/// Use `into_mutable` to transition to the (Unmerkleized,NonDurable) state, or `into_merkleized` to +/// transition to the (Merkleized,Durable) state. +pub trait UnmerkleizedDurableAny: + LogStore + Gettable::Value, Error = Error> +{ + /// The digest type used by Merkleized states in this database's state machine. + type Digest: Digest; + + /// The operation type used by Merkleized states in this database's state machine. + type Operation: Codec; + + /// The mutable state type (Unmerkleized,NonDurable). + type Mutable: MutableAny + + LogStore::Value>; + + /// The provable state type (Merkleized,Durable). + type Merkleized: CleanAny + + MerkleizedStore< + Value = ::Value, + Digest = Self::Digest, + Operation = Self::Operation, + >; + + /// Convert this database into the mutable state. + fn into_mutable(self) -> Self::Mutable; + + /// Convert this database into the provable (Merkleized,Durable) state. + fn into_merkleized(self) -> impl Future>; +} + +/// Trait for the (Merkleized,NonDurable) state. +/// +/// This state allows authentication (root, proofs) and pruning. Use `commit` to transition to the +/// Merkleized, Durable state. +pub trait MerkleizedNonDurableAny: + MerkleizedStore + + PrunableStore + + Gettable::Value, Error = Error> +{ + /// The mutable state type (Unmerkleized,NonDurable). + type Mutable: MutableAny; + + /// The durable state type (Merkleized,Durable). + type Durable: CleanAny + + MerkleizedStore< + Value = ::Value, + Digest = ::Digest, + Operation = ::Operation, + >; + + /// Commit any pending operations to the database, ensuring their durability. Returns the + /// durable state and the location range of committed operations. + fn commit( + self, + metadata: Option<::Value>, + ) -> impl Future), Error>>; + + /// Convert this database into the mutable (Unmerkleized, NonDurable) state. + fn into_mutable(self) -> Self::Mutable; +} + +/// Trait for the (Unmerkleized,NonDurable) state. +/// +/// This is the only state that allows mutations (create/update/delete). Use `commit` to transition +/// to the Unmerkleized, Durable state, or `into_merkleized` to transition to the Merkleized, +/// NonDurable state. +pub trait MutableAny: + LogStore + Deletable::Value, Error = Error> + Batchable +{ + /// The digest type used by Merkleized states in this database's state machine. + type Digest: Digest; + + /// The operation type used by Merkleized states in this database's state machine. + type Operation: Codec; + + /// The durable state type (Unmerkleized,Durable). + type Durable: UnmerkleizedDurableAny + + LogStore::Value>; + + /// The provable state type (Merkleized,NonDurable). + type Merkleized: MerkleizedNonDurableAny + + MerkleizedStore< + Value = ::Value, + Digest = Self::Digest, + Operation = Self::Operation, + >; + + /// Commit any pending operations to the database, ensuring their durability. Returns the + /// durable state and the location range of committed operations. + fn commit( + self, + metadata: Option<::Value>, + ) -> impl Future), Error>>; + + /// Convert this database into the provable (Merkleized, Non-durable) state. + fn into_merkleized(self) -> impl Future>; + + /// Returns the number of steps to raise the inactivity floor on the next commit. + fn steps(&self) -> u64; +} diff --git a/storage/src/qmdb/any/unordered/fixed/mod.rs b/storage/src/qmdb/any/unordered/fixed/mod.rs index b9e8e03d73..33e7c05291 100644 --- a/storage/src/qmdb/any/unordered/fixed/mod.rs +++ b/storage/src/qmdb/any/unordered/fixed/mod.rs @@ -3,17 +3,17 @@ use crate::{ index::unordered::Index, journal::contiguous::fixed::Journal, - mmr::{mem::Clean, Location}, + mmr::Location, qmdb::{ any::{ init_fixed_authenticated_log, unordered, value::FixedEncoding, FixedConfig as Config, FixedValue, }, - Error, + Durable, Error, Merkleized, }, translator::Translator, }; -use commonware_cryptography::{DigestOf, Hasher}; +use commonware_cryptography::Hasher; use commonware_runtime::{Clock, Metrics, Storage}; use commonware_utils::Array; use tracing::warn; @@ -25,11 +25,11 @@ pub type Operation = unordered::Operation>; /// A key-value QMDB based on an authenticated log of operations, supporting authentication of any /// value ever associated with a key. -pub type Db>> = - super::Db>, Index, H, Update, S>; +pub type Db, D = Durable> = + super::Db>, Index, H, Update, S, D>; impl - Db + Db, Durable> { /// Returns a [Db] QMDB initialized from `cfg`. Uncommitted log operations will be /// discarded and the state of the db will be as of the last committed operation. @@ -79,8 +79,8 @@ pub(super) mod test { mmr::{Position, StandardHasher}, qmdb::{ any::unordered::{fixed::Operation, Update}, - store::{batch_tests, CleanStore as _}, - verify_proof, + store::batch_tests, + verify_proof, NonDurable, Unmerkleized, }, translator::TwoCap, }; @@ -115,7 +115,10 @@ pub(super) mod test { } /// A type alias for the concrete [Db] type used in these unit tests. - pub(crate) type AnyTest = Db; + pub(crate) type AnyTest = + Db, Durable>; + pub(crate) type DirtyAnyTest = + Db; #[inline] fn to_digest(i: u64) -> Digest { @@ -171,7 +174,7 @@ pub(super) mod test { } /// Applies the given operations to the database. - pub(crate) async fn apply_ops(db: &mut AnyTest, ops: Vec>) { + pub(crate) async fn apply_ops(db: &mut DirtyAnyTest, ops: Vec>) { for op in ops { match op { Operation::Update(Update(key, value)) => { @@ -180,8 +183,8 @@ pub(super) mod test { Operation::Delete(key) => { db.delete(key).await.unwrap(); } - Operation::CommitFloor(metadata, _) => { - db.commit(metadata).await.unwrap(); + Operation::CommitFloor(_, _) => { + panic!("CommitFloor not supported in apply_ops"); } } } @@ -242,7 +245,7 @@ pub(super) mod test { fn test_any_fixed_db_log_replay() { let executor = deterministic::Runner::default(); executor.start(|context| async move { - let mut db = open_db(context.clone()).await; + let mut db = open_db(context.clone()).await.into_mutable(); // Update the same key many times. const UPDATES: u64 = 100; @@ -251,7 +254,7 @@ pub(super) mod test { let v = Sha256::hash(&(i * 1000).to_be_bytes()); db.update(k, v).await.unwrap(); } - db.commit(None).await.unwrap(); + let db = db.commit(None).await.unwrap().0.into_merkleized(); let root = db.root(); // Simulate a failed commit and test that the log replay doesn't leave behind old data. @@ -280,14 +283,24 @@ pub(super) mod test { }); } + // Test that merkleization state changes don't reset `steps`. + #[test_traced("DEBUG")] + fn test_any_unordered_fixed_db_steps_not_reset() { + let executor = deterministic::Runner::default(); + executor.start(|context| async move { + let db = open_db(context).await; + crate::qmdb::any::test::test_any_db_steps_not_reset(db).await; + }); + } + #[test] fn test_any_fixed_db_historical_proof_basic() { let executor = deterministic::Runner::default(); executor.start(|context| async move { - let mut db = create_test_db(context.clone()).await; + let mut db = create_test_db(context.clone()).await.into_mutable(); let ops = create_test_ops(20); apply_ops(&mut db, ops.clone()).await; - db.commit(None).await.unwrap(); + let db = db.commit(None).await.unwrap().0.into_merkleized(); let root_hash = db.root(); let original_op_count = db.op_count(); @@ -314,9 +327,10 @@ pub(super) mod test { )); // Add more operations to the database + let mut db = db.into_mutable(); let more_ops = create_test_ops(5); apply_ops(&mut db, more_ops.clone()).await; - db.commit(None).await.unwrap(); + let db = db.commit(None).await.unwrap().0.into_merkleized(); // Historical proof should remain the same even though database has grown let (historical_proof, historical_ops) = db @@ -355,10 +369,10 @@ pub(super) mod test { fn test_any_fixed_db_historical_proof_edge_cases() { let executor = deterministic::Runner::default(); executor.start(|context| async move { - let mut db = create_test_db(context.clone()).await; + let mut db = create_test_db(context.clone()).await.into_mutable(); let ops = create_test_ops(50); apply_ops(&mut db, ops.clone()).await; - db.commit(None).await.unwrap(); + let db = db.commit(None).await.unwrap().0.into_merkleized(); let mut hasher = StandardHasher::::new(); @@ -377,11 +391,10 @@ pub(super) mod test { ); assert_eq!(single_ops.len(), 1); - // Create historical database with single operation - let mut single_db = create_test_db(context.clone()).await; + // Create historical database with single operation without committing it. + let mut single_db = create_test_db(context.clone()).await.into_mutable(); apply_ops(&mut single_db, ops[0..1].to_vec()).await; - // Don't commit - this changes the root due to commit operations - single_db.sync().await.unwrap(); + let single_db = single_db.into_merkleized(); let single_root = single_db.root(); assert!(verify_proof( @@ -420,7 +433,10 @@ pub(super) mod test { assert_eq!(min_ops.len(), 3); assert_eq!(min_ops, ops[0..3]); + // Can't destroy the db unless it's durable, so we need to commit first. + let (single_db, _) = single_db.commit(None).await.unwrap(); single_db.destroy().await.unwrap(); + db.destroy().await.unwrap(); }); } @@ -429,10 +445,10 @@ pub(super) mod test { fn test_any_fixed_db_historical_proof_different_historical_sizes() { let executor = deterministic::Runner::default(); executor.start(|context| async move { - let mut db = create_test_db(context.clone()).await; + let mut db = create_test_db(context.clone()).await.into_mutable(); let ops = create_test_ops(100); apply_ops(&mut db, ops.clone()).await; - db.commit(None).await.unwrap(); + let db = db.commit(None).await.unwrap().0.into_merkleized(); let mut hasher = StandardHasher::::new(); @@ -449,10 +465,9 @@ pub(super) mod test { assert_eq!(historical_proof.size, Position::try_from(end_loc).unwrap()); // Create reference database at the given historical size - let mut ref_db = create_test_db(context.clone()).await; + let mut ref_db = create_test_db(context.clone()).await.into_mutable(); apply_ops(&mut ref_db, ops[0..(*end_loc - 1) as usize].to_vec()).await; - // Sync to process dirty nodes but don't commit - commit changes the root due to commit operations - ref_db.sync().await.unwrap(); + let ref_db = ref_db.into_merkleized(); let (ref_proof, ref_ops) = ref_db.proof(start_loc, max_ops).await.unwrap(); assert_eq!(ref_proof.size, historical_proof.size); @@ -472,8 +487,9 @@ pub(super) mod test { start_loc, &historical_ops, &ref_root - ),); + )); + let (ref_db, _) = ref_db.commit(None).await.unwrap(); ref_db.destroy().await.unwrap(); } @@ -485,10 +501,10 @@ pub(super) mod test { fn test_any_fixed_db_historical_proof_invalid() { let executor = deterministic::Runner::default(); executor.start(|context| async move { - let mut db = create_test_db(context.clone()).await; + let mut db = create_test_db(context.clone()).await.into_mutable(); let ops = create_test_ops(10); apply_ops(&mut db, ops).await; - db.commit(None).await.unwrap(); + let db = db.commit(None).await.unwrap().0.into_merkleized(); let historical_op_count = Location::new_unchecked(5); let historical_mmr_size = Position::try_from(historical_op_count).unwrap(); @@ -598,8 +614,8 @@ pub(super) mod test { } #[test_traced("DEBUG")] - fn test_batch() { - batch_tests::test_batch(|ctx| async move { create_test_db(ctx).await }); + fn test_any_unordered_fixed_batch() { + batch_tests::test_batch(|ctx| async move { create_test_db(ctx).await.into_mutable() }); } // FromSyncTestable implementation for from_sync_result tests diff --git a/storage/src/qmdb/any/unordered/fixed/sync.rs b/storage/src/qmdb/any/unordered/fixed/sync.rs index 222f69add3..d3462c390c 100644 --- a/storage/src/qmdb/any/unordered/fixed/sync.rs +++ b/storage/src/qmdb/any/unordered/fixed/sync.rs @@ -6,7 +6,7 @@ use crate::{ journal::{authenticated, contiguous::fixed}, mmr::{mem::Clean, Location, Position, StandardHasher}, // TODO(https://github.com/commonwarexyz/monorepo/issues/1873): support any::fixed::ordered - qmdb::{self, any::FixedValue}, + qmdb::{self, any::FixedValue, Durable, Merkleized}, translator::Translator, }; use commonware_codec::CodecFixed; @@ -19,7 +19,7 @@ use prometheus_client::metrics::{counter::Counter, gauge::Gauge}; use std::{collections::BTreeMap, marker::PhantomData, ops::Range}; use tracing::debug; -impl qmdb::sync::Database for Db +impl qmdb::sync::Database for Db, Durable> where E: Storage + Clock + Metrics, K: Array, @@ -316,8 +316,10 @@ mod tests { AnyTest::init(ctx, config).await.unwrap() } - async fn apply_ops(db: &mut Self::Db, ops: Vec>) { - apply_ops(db, ops).await + async fn apply_ops(db: Self::Db, ops: Vec>) -> Self::Db { + let mut db = db.into_mutable(); + apply_ops(&mut db, ops).await; + db.commit(None::).await.unwrap().0.into_merkleized() } } diff --git a/storage/src/qmdb/any/unordered/mod.rs b/storage/src/qmdb/any/unordered/mod.rs index 40c908315c..84e46fc2a8 100644 --- a/storage/src/qmdb/any/unordered/mod.rs +++ b/storage/src/qmdb/any/unordered/mod.rs @@ -1,29 +1,28 @@ use crate::{ - journal::{ - contiguous::{Contiguous, MutableContiguous}, - Error as JournalError, - }, + index::Unordered as Index, + journal::contiguous::{Contiguous, MutableContiguous}, kv::{self, Batchable}, - mmr::{ - mem::{Dirty, State}, - Location, - }, + mmr::Location, qmdb::{ any::{ db::{AuthenticatedLog, Db}, - CleanAny, DirtyAny, ValueEncoding, + ValueEncoding, }, build_snapshot_from_log, create_key, delete_key, delete_known_loc, operation::{Committable as _, Operation as OperationTrait}, - update_key, update_known_loc, Error, Index, + update_key, update_known_loc, DurabilityState, Durable, Error, MerkleizationState, + Merkleized, NonDurable, Unmerkleized, }, +}; +#[cfg(any(test, feature = "test-traits"))] +use crate::{ + qmdb::any::states::{CleanAny, MerkleizedNonDurableAny, MutableAny, UnmerkleizedDurableAny}, Persistable, }; use commonware_codec::Codec; use commonware_cryptography::{DigestOf, Hasher}; use commonware_runtime::{Clock, Metrics, Storage}; use commonware_utils::Array; -use core::ops::Range; use futures::future::try_join_all; use std::collections::BTreeMap; @@ -42,8 +41,9 @@ impl< C: Contiguous>, I: Index, H: Hasher, - S: State>, - > Db, S> + M: MerkleizationState>, + D: DurabilityState, + > Db, M, D> where Operation: Codec, { @@ -83,8 +83,7 @@ impl< C: MutableContiguous>, I: Index, H: Hasher, - S: State>, - > Db, S> + > Db, Unmerkleized, NonDurable> where Operation: Codec, { @@ -95,7 +94,7 @@ where return Ok(None); }; self.log.append(Operation::Delete(key)).await?; - self.steps += 1; + self.durable_state.steps += 1; self.active_keys -= 1; Ok(Some(loc)) @@ -115,7 +114,7 @@ where .append(Operation::Update(Update(key, value))) .await?; if res.is_some() { - self.steps += 1; + self.durable_state.steps += 1; } else { self.active_keys += 1; } @@ -214,7 +213,7 @@ where callback(false, Some(old_loc)); self.active_keys -= 1; } - self.steps += 1; + self.durable_state.steps += 1; } // Process the creates. @@ -241,7 +240,7 @@ impl< C: MutableContiguous>, I: Index, H: Hasher, - > Db> + > Db, Merkleized, Durable> where Operation: Codec, { @@ -263,7 +262,7 @@ where inactivity_floor_loc, snapshot, last_commit_loc, - steps: 0, + durable_state: Durable {}, active_keys, _update: core::marker::PhantomData, }) @@ -277,7 +276,9 @@ impl< C: Contiguous>, I: Index, H: Hasher, - > kv::Gettable for Db> + M: MerkleizationState>, + D: DurabilityState, + > kv::Gettable for Db, M, D> where Operation: Codec, { @@ -297,7 +298,7 @@ impl< C: MutableContiguous>, I: Index, H: Hasher, - > kv::Updatable for Db> + > kv::Updatable for Db, Unmerkleized, NonDurable> where Operation: Codec, { @@ -313,7 +314,7 @@ impl< C: MutableContiguous>, I: Index, H: Hasher, - > kv::Deletable for Db> + > kv::Deletable for Db, Unmerkleized, NonDurable> where Operation: Codec, { @@ -322,111 +323,124 @@ where } } -impl< - E: Storage + Clock + Metrics, - K: Array, - V: ValueEncoding, - C: MutableContiguous> + Persistable, - I: Index, - H: Hasher, - > Persistable for Db> +impl Batchable for Db, Unmerkleized, NonDurable> where + E: Storage + Clock + Metrics, + K: Array, + V: ValueEncoding, + C: MutableContiguous>, + I: Index, + H: Hasher, Operation: Codec, { - type Error = Error; - - async fn commit(&mut self) -> Result<(), Error> { - self.commit(None).await.map(|_| ()) - } - - async fn sync(&mut self) -> Result<(), Error> { - self.sync().await - } - - async fn destroy(self) -> Result<(), Error> { - self.destroy().await + async fn write_batch( + &mut self, + iter: impl Iterator)>, + ) -> Result<(), Error> { + self.write_batch_with_callback(iter, |_, _| {}).await } } -impl< - E: Storage + Clock + Metrics, - K: Array, - V: ValueEncoding, - C: MutableContiguous> + Persistable, - I: Index, - H: Hasher, - > CleanAny for Db> +#[cfg(any(test, feature = "test-traits"))] +impl CleanAny for Db, Merkleized, Durable> where + E: Storage + Clock + Metrics, + K: Array, + V: ValueEncoding, + C: MutableContiguous> + Persistable, + I: Index, + H: Hasher, Operation: Codec, { - type Key = K; - - async fn get(&self, key: &Self::Key) -> Result, Error> { - self.get(key).await - } + type Mutable = Db, Unmerkleized, NonDurable>; - async fn commit(&mut self, metadata: Option) -> Result, Error> { - self.commit(metadata).await + fn into_mutable(self) -> Self::Mutable { + self.into_mutable() } +} - async fn sync(&mut self) -> Result<(), Error> { - self.sync().await - } +#[cfg(any(test, feature = "test-traits"))] +impl UnmerkleizedDurableAny + for Db, Unmerkleized, Durable> +where + E: Storage + Clock + Metrics, + K: Array, + V: ValueEncoding, + C: MutableContiguous> + Persistable, + I: Index, + H: Hasher, + Operation: Codec, +{ + type Digest = H::Digest; + type Operation = Operation; + type Mutable = Db, Unmerkleized, NonDurable>; + type Merkleized = Db, Merkleized, Durable>; - async fn prune(&mut self, prune_loc: Location) -> Result<(), Error> { - self.prune(prune_loc).await + fn into_mutable(self) -> Self::Mutable { + self.into_mutable() } - async fn destroy(self) -> Result<(), Error> { - self.destroy().await + async fn into_merkleized(self) -> Result { + Ok(self.into_merkleized()) } } -impl< - E: Storage + Clock + Metrics, - K: Array, - V: ValueEncoding, - C: MutableContiguous>, - I: Index, - H: Hasher, - > DirtyAny for Db, Dirty> +#[cfg(any(test, feature = "test-traits"))] +impl MerkleizedNonDurableAny + for Db, Merkleized, NonDurable> where + E: Storage + Clock + Metrics, + K: Array, + V: ValueEncoding, + C: MutableContiguous> + Persistable, + I: Index, + H: Hasher, Operation: Codec, { - type Key = K; - - async fn get(&self, key: &Self::Key) -> Result, Error> { - self.get(key).await - } - - async fn update(&mut self, key: Self::Key, value: Self::Value) -> Result<(), Error> { - self.update(key, value).await - } + type Mutable = Db, Unmerkleized, NonDurable>; + type Durable = Db, Merkleized, Durable>; - async fn create(&mut self, key: Self::Key, value: Self::Value) -> Result { - self.create(key, value).await + async fn commit( + self, + metadata: Option, + ) -> Result<(Self::Durable, core::ops::Range), Error> { + self.commit(metadata).await } - async fn delete(&mut self, key: Self::Key) -> Result { - self.delete(key).await + fn into_mutable(self) -> Self::Mutable { + self.into_mutable() } } -impl Batchable for Db> +#[cfg(any(test, feature = "test-traits"))] +impl MutableAny for Db, Unmerkleized, NonDurable> where E: Storage + Clock + Metrics, K: Array, V: ValueEncoding, - C: MutableContiguous>, + C: MutableContiguous> + Persistable, I: Index, H: Hasher, Operation: Codec, { - async fn write_batch( - &mut self, - iter: impl Iterator)>, - ) -> Result<(), Error> { - self.write_batch_with_callback(iter, |_, _| {}).await + type Digest = H::Digest; + type Operation = Operation; + type Durable = Db, Unmerkleized, Durable>; + type Merkleized = Db, Merkleized, NonDurable>; + + async fn commit( + self, + metadata: Option, + ) -> Result<(Self::Durable, core::ops::Range), Error> { + self.commit(metadata).await + } + + async fn into_merkleized(self) -> Result { + Ok(self.into_merkleized()) + } + + fn steps(&self) -> u64 { + self.durable_state.steps } } @@ -435,15 +449,15 @@ where pub(super) mod test { use super::*; use crate::{ + kv::{Deletable as _, Gettable as _, Updatable as _}, mmr::StandardHasher, qmdb::{ any::test::{fixed_db_config, variable_db_config}, - store::{DirtyStore as _, LogStore as _}, + store::{LogStore, MerkleizedStore}, verify_proof, }, translator::TwoCap, }; - use commonware_codec::Encode; use commonware_cryptography::{sha256::Digest, Sha256}; use commonware_macros::test_traced; use commonware_runtime::{ @@ -455,10 +469,22 @@ pub(super) mod test { use std::collections::HashMap; /// A type alias for the concrete [fixed::Db] type used in these unit tests. - type FixedDb = fixed::Db; + type FixedDb = fixed::Db, Durable>; /// A type alias for the concrete [variable::Db] type used in these unit tests. - type VariableDb = variable::Db; + type VariableDb = + variable::Db, Durable>; + + /// Helper trait for testing Any databases that cycle through all four states. + pub(crate) trait TestableAnyDb: + CleanAny + MerkleizedStore + { + } + + impl TestableAnyDb for T where + T: CleanAny + MerkleizedStore + { + } /// Return an `Any` database initialized with a fixed config. pub(crate) async fn open_fixed_db(context: Context) -> FixedDb { @@ -474,13 +500,11 @@ pub(super) mod test { .unwrap() } - async fn test_any_db_empty( + async fn test_any_db_empty>( context: Context, mut db: D, reopen_db: impl Fn(Context) -> Pin + Send>>, - ) where - D: CleanAny, - { + ) { assert_eq!(db.op_count(), 1); assert!(matches!(db.prune(db.inactivity_floor_loc()).await, Ok(()))); assert!(db.get_metadata().await.unwrap().is_none()); @@ -491,15 +515,24 @@ pub(super) mod test { // Make sure closing/reopening gets us back to the same state, even after adding an // uncommitted op, and even without a clean shutdown. - let mut db = db.into_dirty(); + let mut db = db.into_mutable(); db.update(k1, v1).await.unwrap(); - let mut db = reopen_db(context.clone()).await; + drop(db); + let db = reopen_db(context.clone()).await; assert_eq!(db.op_count(), 1); assert_eq!(db.root(), empty_root); // Test calling commit on an empty db. let metadata = Sha256::fill(3u8); - let range = db.commit(Some(metadata)).await.unwrap(); + let db = db.into_mutable(); + // into_merkleized() -> MerkleizedNonDurable, then commit() -> MerkleizedDurable + let (mut db, range) = db + .into_merkleized() + .await + .unwrap() + .commit(Some(metadata)) + .await + .unwrap(); assert_eq!(range.start, 1); assert_eq!(range.end, 2); assert_eq!(db.op_count(), 2); // another commit op added @@ -508,6 +541,7 @@ pub(super) mod test { assert!(matches!(db.prune(db.inactivity_floor_loc()).await, Ok(()))); // Re-opening the DB without a clean shutdown should still recover the correct state. + drop(db); let db = reopen_db(context.clone()).await; assert_eq!(db.op_count(), 2); assert_eq!(db.get_metadata().await.unwrap(), Some(metadata)); @@ -515,21 +549,32 @@ pub(super) mod test { // Confirm the inactivity floor doesn't fall endlessly behind with multiple commits on a // non-empty db. - let mut db = db.into_dirty(); + let mut db = db.into_mutable(); db.update(k1, v1).await.unwrap(); for _ in 1..100 { - let mut clean_db = db.merkleize().await.unwrap(); - clean_db.commit(None).await.unwrap(); - db = clean_db.into_dirty(); + let (clean_db, _) = db + .into_merkleized() + .await + .unwrap() + .commit(None) + .await + .unwrap(); // Distance should equal 3 after the second commit, with inactivity_floor // referencing the previous commit operation. + assert!(clean_db.op_count() - clean_db.inactivity_floor_loc() <= 3); + db = clean_db.into_mutable(); assert!(db.op_count() - db.inactivity_floor_loc() <= 3); } // Confirm the inactivity floor is raised to tip when the db becomes empty. db.delete(k1).await.unwrap(); - let mut db = db.merkleize().await.unwrap(); - db.commit(None).await.unwrap(); + let (db, _) = db + .into_merkleized() + .await + .unwrap() + .commit(None) + .await + .unwrap(); assert!(db.is_empty()); assert_eq!(db.op_count() - 1, db.inactivity_floor_loc()); @@ -554,19 +599,20 @@ pub(super) mod test { }); } - /// Shared test: build a db with mixed updates/deletes, verify state, proofs, reopen. - pub(crate) async fn test_any_db_build_and_authenticate( + pub(crate) async fn test_any_db_build_and_authenticate< + D: TestableAnyDb, + V: Codec + Clone + Eq + std::hash::Hash + std::fmt::Debug, + >( context: Context, db: D, reopen_db: impl Fn(Context) -> Pin + Send>>, make_value: impl Fn(u64) -> V, ) where - D: CleanAny, - V: Clone + Eq + std::hash::Hash + std::fmt::Debug + Codec, + ::Operation: Codec, { const ELEMENTS: u64 = 1000; - let mut db = db.into_dirty(); + let mut db = db.into_mutable(); let mut map = HashMap::::default(); for i in 0u64..ELEMENTS { let k = Sha256::hash(&i.to_be_bytes()); @@ -600,8 +646,8 @@ pub(super) mod test { assert_eq!(db.inactivity_floor_loc(), Location::new_unchecked(0)); // Commit + sync with pruning raises inactivity floor. - let mut db = db.merkleize().await.unwrap(); - db.commit(None).await.unwrap(); + let (db, _) = db.commit(None).await.unwrap(); + let mut db = db.into_merkleized().await.unwrap(); db.sync().await.unwrap(); db.prune(db.inactivity_floor_loc()).await.unwrap(); assert_eq!(db.op_count(), Location::new_unchecked(1957)); @@ -640,14 +686,12 @@ pub(super) mod test { } /// Test basic CRUD and commit behavior. - pub(crate) async fn test_any_db_basic( + pub(crate) async fn test_any_db_basic>( context: Context, db: D, reopen_db: impl Fn(Context) -> Pin + Send>>, - ) where - D: CleanAny, - { - let mut db = db.into_dirty(); + ) { + let mut db = db.into_mutable(); // Build a db with 2 keys and make sure updates and deletions of those keys work as // expected. @@ -679,9 +723,8 @@ pub(super) mod test { assert_eq!(db.op_count(), 6); // 4 updates, 1 deletion + initial commit. assert_eq!(db.inactivity_floor_loc(), Location::new_unchecked(0)); - let mut db = db.merkleize().await.unwrap(); - db.commit(None).await.unwrap(); - let mut db = db.into_dirty(); + let (db, _) = db.commit(None).await.unwrap(); + let mut db = db.into_mutable(); // Make sure create won't modify active keys. assert!(!db.create(d1, v1).await.unwrap()); @@ -699,9 +742,8 @@ pub(super) mod test { assert_eq!(db.op_count(), 12); // 2 new delete ops. assert_eq!(db.inactivity_floor_loc(), Location::new_unchecked(7)); - let mut db = db.merkleize().await.unwrap(); - db.commit(None).await.unwrap(); - let mut db = db.into_dirty(); + let (db, _) = db.commit(None).await.unwrap(); + let mut db = db.into_mutable(); assert_eq!(db.inactivity_floor_loc(), Location::new_unchecked(12)); assert_eq!(db.op_count(), 13); // only commit should remain. @@ -715,15 +757,21 @@ pub(super) mod test { assert_eq!(db.op_count(), 13); // Make sure closing/reopening gets us back to the same state. - let mut db = db.merkleize().await.unwrap(); - db.commit(None).await.unwrap(); + let db = db + .commit(None) + .await + .unwrap() + .0 + .into_merkleized() + .await + .unwrap(); assert_eq!(db.op_count(), 14); let root = db.root(); drop(db); let db = reopen_db(context.clone()).await; assert_eq!(db.op_count(), 14); assert_eq!(db.root(), root); - let mut db = db.into_dirty(); + let mut db = db.into_mutable(); // Re-activate the keys by updating them. db.update(d1, v1).await.unwrap(); @@ -733,20 +781,34 @@ pub(super) mod test { db.update(d1, v2).await.unwrap(); // Make sure last_commit is updated by changing the metadata back to None. - let mut db = db.merkleize().await.unwrap(); - db.commit(None).await.unwrap(); + let db = db + .commit(None) + .await + .unwrap() + .0 + .into_merkleized() + .await + .unwrap(); // Confirm close/reopen gets us back to the same state. assert_eq!(db.op_count(), 23); let root = db.root(); - let mut db = reopen_db(context.clone()).await; + let db = reopen_db(context.clone()).await; assert_eq!(db.root(), root); assert_eq!(db.op_count(), 23); // Commit will raise the inactivity floor, which won't affect state but will affect the // root. - db.commit(None).await.unwrap(); + let db = db.into_mutable(); + let mut db = db + .commit(None) + .await + .unwrap() + .0 + .into_merkleized() + .await + .unwrap(); assert!(db.root() != root); @@ -777,25 +839,28 @@ pub(super) mod test { } /// Test recovery on non-empty db. - pub(crate) async fn test_any_db_non_empty_recovery( + pub(crate) async fn test_any_db_non_empty_recovery, V: Clone>( context: Context, db: D, reopen_db: impl Fn(Context) -> Pin + Send>>, make_value: impl Fn(u64) -> V, - ) where - D: CleanAny, - V: Clone, - { + ) { const ELEMENTS: u64 = 1000; - let mut db = db.into_dirty(); + let mut db = db.into_mutable(); for i in 0u64..ELEMENTS { let k = Sha256::hash(&i.to_be_bytes()); let v = make_value(i * 1000); db.update(k, v).await.unwrap(); } - let mut db = db.merkleize().await.unwrap(); - db.commit(None).await.unwrap(); + let mut db = db + .commit(None) + .await + .unwrap() + .0 + .into_merkleized() + .await + .unwrap(); db.prune(db.inactivity_floor_loc()).await.unwrap(); let root = db.root(); let op_count = db.op_count(); @@ -806,7 +871,7 @@ pub(super) mod test { assert_eq!(db.inactivity_floor_loc(), inactivity_floor_loc); assert_eq!(db.root(), root); - let mut db = db.into_dirty(); + let mut db = db.into_mutable(); for i in 0u64..ELEMENTS { let k = Sha256::hash(&i.to_be_bytes()); let v = make_value((i + 1) * 10000); @@ -817,7 +882,7 @@ pub(super) mod test { assert_eq!(db.inactivity_floor_loc(), inactivity_floor_loc); assert_eq!(db.root(), root); - let mut dirty = db.into_dirty(); + let mut dirty = db.into_mutable(); for i in 0u64..ELEMENTS { let k = Sha256::hash(&i.to_be_bytes()); let v = make_value((i + 1) * 10000); @@ -827,7 +892,7 @@ pub(super) mod test { assert_eq!(db.op_count(), op_count); assert_eq!(db.root(), root); - let mut db = db.into_dirty(); + let mut db = db.into_mutable(); for _ in 0..3 { for i in 0u64..ELEMENTS { let k = Sha256::hash(&i.to_be_bytes()); @@ -839,14 +904,13 @@ pub(super) mod test { assert_eq!(db.op_count(), op_count); assert_eq!(db.root(), root); - let mut db = db.into_dirty(); + let mut db = db.into_mutable(); for i in 0u64..ELEMENTS { let k = Sha256::hash(&i.to_be_bytes()); let v = make_value((i + 1) * 10000); db.update(k, v).await.unwrap(); } - let mut db = db.merkleize().await.unwrap(); - db.commit(None).await.unwrap(); + let _ = db.commit(None).await.unwrap(); let db = reopen_db(context.clone()).await; assert!(db.op_count() > op_count); assert_ne!(db.inactivity_floor_loc(), inactivity_floor_loc); @@ -856,22 +920,19 @@ pub(super) mod test { } /// Test recovery on empty db. - pub(crate) async fn test_any_db_empty_recovery( + pub(crate) async fn test_any_db_empty_recovery, V: Clone>( context: Context, db: D, reopen_db: impl Fn(Context) -> Pin + Send>>, make_value: impl Fn(u64) -> V, - ) where - D: CleanAny, - V: Clone, - { + ) { let root = db.root(); let db = reopen_db(context.clone()).await; assert_eq!(db.op_count(), 1); assert_eq!(db.root(), root); - let mut db = db.into_dirty(); + let mut db = db.into_mutable(); for i in 0u64..1000 { let k = Sha256::hash(&i.to_be_bytes()); let v = make_value((i + 1) * 10000); @@ -881,17 +942,18 @@ pub(super) mod test { assert_eq!(db.op_count(), 1); assert_eq!(db.root(), root); - let mut db = db.into_dirty(); + let mut db = db.into_mutable(); for i in 0u64..1000 { let k = Sha256::hash(&i.to_be_bytes()); let v = make_value((i + 1) * 10000); db.update(k, v).await.unwrap(); } + drop(db); let db = reopen_db(context.clone()).await; assert_eq!(db.op_count(), 1); assert_eq!(db.root(), root); - let mut db = db.into_dirty(); + let mut db = db.into_mutable(); for _ in 0..3 { for i in 0u64..1000 { let k = Sha256::hash(&i.to_be_bytes()); @@ -899,18 +961,26 @@ pub(super) mod test { db.update(k, v).await.unwrap(); } } + drop(db); let db = reopen_db(context.clone()).await; assert_eq!(db.op_count(), 1); assert_eq!(db.root(), root); - let mut db = db.into_dirty(); + let mut db = db.into_mutable(); for i in 0u64..1000 { let k = Sha256::hash(&i.to_be_bytes()); let v = make_value((i + 1) * 10000); db.update(k, v).await.unwrap(); } - let mut db = db.merkleize().await.unwrap(); - db.commit(None).await.unwrap(); + let db = db + .commit(None) + .await + .unwrap() + .0 + .into_merkleized() + .await + .unwrap(); + drop(db); let db = reopen_db(context.clone()).await; assert!(db.op_count() > 1); assert_ne!(db.root(), root); @@ -919,19 +989,18 @@ pub(super) mod test { } /// Test making multiple commits, one of which deletes a key from a previous commit. - pub(crate) async fn test_any_db_multiple_commits_delete_replayed( + pub(crate) async fn test_any_db_multiple_commits_delete_replayed, V>( context: Context, db: D, reopen_db: impl Fn(Context) -> Pin + Send>>, make_value: impl Fn(u64) -> V, ) where - D: CleanAny, V: Clone + Eq + std::fmt::Debug, { let mut map = HashMap::::default(); const ELEMENTS: u64 = 10; let metadata_value = make_value(42); - let mut db = db.into_dirty(); + let mut db = db.into_mutable(); let key_at = |j: u64, i: u64| Sha256::hash(&(j * 1000 + i).to_be_bytes()); for j in 0u64..ELEMENTS { for i in 0u64..ELEMENTS { @@ -940,16 +1009,15 @@ pub(super) mod test { db.update(k, v.clone()).await.unwrap(); map.insert(k, v); } - let mut clean_db = db.merkleize().await.unwrap(); - clean_db.commit(Some(metadata_value.clone())).await.unwrap(); - db = clean_db.into_dirty(); + let (clean_db, _) = db.commit(Some(metadata_value.clone())).await.unwrap(); + db = clean_db.into_merkleized().await.unwrap().into_mutable(); } assert_eq!(db.get_metadata().await.unwrap(), Some(metadata_value)); let k = key_at(ELEMENTS - 1, ELEMENTS - 1); db.delete(k).await.unwrap(); - let mut db = db.merkleize().await.unwrap(); - db.commit(None).await.unwrap(); + let (db, _) = db.commit(None).await.unwrap(); + let db = db.into_merkleized().await.unwrap(); assert_eq!(db.get_metadata().await.unwrap(), None); assert!(db.get(&k).await.unwrap().is_none()); diff --git a/storage/src/qmdb/any/unordered/sync_tests.rs b/storage/src/qmdb/any/unordered/sync_tests.rs index 3746669769..1b60d07312 100644 --- a/storage/src/qmdb/any/unordered/sync_tests.rs +++ b/storage/src/qmdb/any/unordered/sync_tests.rs @@ -5,12 +5,13 @@ use crate::{ journal::contiguous::Contiguous, + kv::Gettable, mmr::{Location, Position}, qmdb::{ self, - any::CleanAny, + any::states::CleanAny, operation::Operation as OperationTrait, - store::{CleanStore, LogStore as _}, + store::{LogStore as _, MerkleizedStore, PrunableStore}, sync::{ self, engine::{Config, NextStep}, @@ -18,6 +19,7 @@ use crate::{ Engine, Target, }, }, + Persistable, }; use commonware_codec::Encode; use commonware_cryptography::sha256::Digest; @@ -73,10 +75,11 @@ pub(crate) trait FromSyncTestable: qmdb::sync::Database { /// Harness for sync tests. pub(crate) trait SyncTestHarness: Sized + 'static { - /// The database type being tested. + /// The database type being tested (Clean state: Merkleized + Durable). type Db: qmdb::sync::Database - + CleanAny - + CleanStore; + + CleanAny + + MerkleizedStore + + Gettable; /// Create a config with unique partition names fn config(suffix: &str) -> ConfigOf; @@ -96,11 +99,11 @@ pub(crate) trait SyncTestHarness: Sized + 'static { config: ConfigOf, ) -> impl std::future::Future + Send; - /// Apply operations to a database + /// Apply operations to a database and commit (returns to Clean state) fn apply_ops( - db: &mut Self::Db, + db: Self::Db, ops: Vec>, - ) -> impl std::future::Future + Send; + ) -> impl std::future::Future + Send; } /// Test that invalid bounds are rejected @@ -186,8 +189,8 @@ where // Create and populate target database let mut target_db = H::init_db(context.clone()).await; let target_ops = H::create_ops(target_db_ops); - H::apply_ops(&mut target_db, target_ops).await; - target_db.commit(None).await.unwrap(); + target_db = H::apply_ops(target_db, target_ops).await; + // commit already done in apply_ops target_db .prune(target_db.inactivity_floor_loc()) .await @@ -259,8 +262,8 @@ where let target_ops = H::create_ops(target_db_ops); // Apply all but the last operation - H::apply_ops(&mut target_db, target_ops[0..target_db_ops - 1].to_vec()).await; - target_db.commit(None).await.unwrap(); + target_db = H::apply_ops(target_db, target_ops[0..target_db_ops - 1].to_vec()).await; + // commit already done in apply_ops let upper_bound = target_db.op_count(); let root = target_db.root(); @@ -269,8 +272,8 @@ where // Add another operation after the sync range let final_op = target_ops[target_db_ops - 1].clone(); let final_key = final_op.key().cloned(); // Store the key before applying - H::apply_ops(&mut target_db, vec![final_op]).await; - target_db.commit(None).await.unwrap(); + target_db = H::apply_ops(target_db, vec![final_op]).await; + // commit already done in apply_ops // Sync to the original root (before final_op was added) let db_config = H::config(&context.next_u64().to_string()); @@ -326,17 +329,17 @@ where H::init_db_with_config(context.clone(), H::clone_config(&sync_db_config)).await; // Apply the same operations to both databases - H::apply_ops(&mut target_db, original_ops_data.clone()).await; - H::apply_ops(&mut sync_db, original_ops_data.clone()).await; - target_db.commit(None).await.unwrap(); - sync_db.commit(None).await.unwrap(); + target_db = H::apply_ops(target_db, original_ops_data.clone()).await; + sync_db = H::apply_ops(sync_db, original_ops_data.clone()).await; + // commit already done in apply_ops + // commit already done in apply_ops drop(sync_db); // Add more operations and commit the target database let more_ops = H::create_ops(1); - H::apply_ops(&mut target_db, more_ops.clone()).await; - target_db.commit(None).await.unwrap(); + target_db = H::apply_ops(target_db, more_ops.clone()).await; + // commit already done in apply_ops let root = target_db.root(); let lower_bound = target_db.inactivity_floor_loc(); @@ -416,10 +419,10 @@ where H::init_db_with_config(context.clone(), H::clone_config(&sync_config)).await; // Apply the same operations to both databases - H::apply_ops(&mut target_db, target_ops.clone()).await; - H::apply_ops(&mut sync_db, target_ops.clone()).await; - target_db.commit(None).await.unwrap(); - sync_db.commit(None).await.unwrap(); + target_db = H::apply_ops(target_db, target_ops.clone()).await; + sync_db = H::apply_ops(sync_db, target_ops.clone()).await; + // commit already done in apply_ops + // commit already done in apply_ops target_db .prune(target_db.inactivity_floor_loc()) @@ -488,8 +491,8 @@ where // Create and populate target database let mut target_db = H::init_db(context.clone()).await; let target_ops = H::create_ops(50); - H::apply_ops(&mut target_db, target_ops).await; - target_db.commit(None).await.unwrap(); + target_db = H::apply_ops(target_db, target_ops).await; + // commit already done in apply_ops // Capture initial target state let initial_lower_bound = target_db.inactivity_floor_loc(); @@ -553,8 +556,8 @@ where // Create and populate target database let mut target_db = H::init_db(context.clone()).await; let target_ops = H::create_ops(50); - H::apply_ops(&mut target_db, target_ops).await; - target_db.commit(None).await.unwrap(); + target_db = H::apply_ops(target_db, target_ops).await; + // commit already done in apply_ops // Capture initial target state let initial_lower_bound = target_db.inactivity_floor_loc(); @@ -617,8 +620,8 @@ where // Create and populate target database let mut target_db = H::init_db(context.clone()).await; let target_ops = H::create_ops(100); - H::apply_ops(&mut target_db, target_ops).await; - target_db.commit(None).await.unwrap(); + target_db = H::apply_ops(target_db, target_ops).await; + // commit already done in apply_ops // Capture initial target state let initial_lower_bound = target_db.inactivity_floor_loc(); @@ -628,8 +631,8 @@ where // Apply more operations to the target database let additional_ops = H::create_ops(1); let new_root = { - H::apply_ops(&mut target_db, additional_ops).await; - target_db.commit(None).await.unwrap(); + target_db = H::apply_ops(target_db, additional_ops).await; + // commit already done in apply_ops // Capture new target state let new_lower_bound = target_db.inactivity_floor_loc(); @@ -698,8 +701,8 @@ where // Create and populate target database let mut target_db = H::init_db(context.clone()).await; let target_ops = H::create_ops(50); - H::apply_ops(&mut target_db, target_ops).await; - target_db.commit(None).await.unwrap(); + target_db = H::apply_ops(target_db, target_ops).await; + // commit already done in apply_ops // Capture initial target state let initial_lower_bound = target_db.inactivity_floor_loc(); @@ -760,8 +763,8 @@ where // Create and populate target database let mut target_db = H::init_db(context.clone()).await; let target_ops = H::create_ops(10); - H::apply_ops(&mut target_db, target_ops).await; - target_db.commit(None).await.unwrap(); + target_db = H::apply_ops(target_db, target_ops).await; + // commit already done in apply_ops // Capture target state let lower_bound = target_db.inactivity_floor_loc(); @@ -819,7 +822,7 @@ pub(crate) fn test_target_update_during_sync( initial_ops: usize, additional_ops: usize, ) where - Arc>>: Resolver, Digest = Digest>, + Arc>>>: Resolver, Digest = Digest>, OpOf: Encode + Clone, JournalOf: Contiguous, { @@ -828,16 +831,16 @@ pub(crate) fn test_target_update_during_sync( // Create and populate target database with initial operations let mut target_db = H::init_db(context.clone()).await; let target_ops = H::create_ops(initial_ops); - H::apply_ops(&mut target_db, target_ops).await; - target_db.commit(None).await.unwrap(); + target_db = H::apply_ops(target_db, target_ops).await; + // commit already done in apply_ops // Capture initial target state let initial_lower_bound = target_db.inactivity_floor_loc(); let initial_upper_bound = target_db.op_count(); let initial_root = target_db.root(); - // Wrap target database for shared mutable access - let target_db = Arc::new(RwLock::new(target_db)); + // Wrap target database for shared mutable access (using Option so we can take ownership) + let target_db = Arc::new(RwLock::new(Some(target_db))); // Create client with initial target and small batch size let (mut update_sender, update_receiver) = mpsc::channel(1); @@ -873,14 +876,15 @@ pub(crate) fn test_target_update_during_sync( // Modify the target database by adding more operations let additional_ops_data = H::create_ops(additional_ops); let new_root = { - let mut db = target_db.write().await; - H::apply_ops(&mut db, additional_ops_data).await; - db.commit(None).await.unwrap(); + let mut db_guard = target_db.write().await; + let db = db_guard.take().unwrap(); + let db = H::apply_ops(db, additional_ops_data).await; // Capture new target state let new_lower_bound = db.inactivity_floor_loc(); let new_upper_bound = db.op_count(); let new_root = db.root(); + *db_guard = Some(db); // Send target update with new target update_sender @@ -903,7 +907,7 @@ pub(crate) fn test_target_update_during_sync( // Verify the target database matches the synced database let target_db = Arc::try_unwrap(target_db).map_or_else( |_| panic!("Failed to unwrap Arc - still has references"), - |rw_lock| rw_lock.into_inner(), + |rw_lock| rw_lock.into_inner().expect("db should be present"), ); { assert_eq!(synced_db.op_count(), target_db.op_count()); @@ -931,8 +935,8 @@ where // Create and populate a simple target database let mut target_db = H::init_db(context.clone()).await; let target_ops = H::create_ops(10); - H::apply_ops(&mut target_db, target_ops).await; - target_db.commit(None).await.unwrap(); + target_db = H::apply_ops(target_db, target_ops).await; + // commit already done in apply_ops // Capture target state let target_root = target_db.root(); @@ -1001,8 +1005,8 @@ where let db_config = H::config(&context.next_u64().to_string()); let mut db = H::init_db_with_config(context.clone(), H::clone_config(&db_config)).await; let ops = H::create_ops(100); - H::apply_ops(&mut db, ops).await; - db.commit(None).await.unwrap(); + db = H::apply_ops(db, ops).await; + // commit already done in apply_ops let sync_lower_bound = db.inactivity_floor_loc(); let sync_upper_bound = db.op_count(); @@ -1054,14 +1058,14 @@ where let mut sync_db = H::init_db_with_config(context.clone(), H::clone_config(&sync_db_config)).await; let original_ops = H::create_ops(NUM_OPS); - H::apply_ops(&mut target_db, original_ops.clone()).await; - target_db.commit(None).await.unwrap(); + target_db = H::apply_ops(target_db, original_ops.clone()).await; + // commit already done in apply_ops target_db .prune(target_db.inactivity_floor_loc()) .await .unwrap(); - H::apply_ops(&mut sync_db, original_ops.clone()).await; - sync_db.commit(None).await.unwrap(); + sync_db = H::apply_ops(sync_db, original_ops.clone()).await; + // commit already done in apply_ops sync_db.prune(sync_db.inactivity_floor_loc()).await.unwrap(); let sync_db_original_size = sync_db.op_count(); @@ -1074,8 +1078,8 @@ where // Add more operations to the target db let more_ops = H::create_ops(NUM_ADDITIONAL_OPS); - H::apply_ops(&mut target_db, more_ops).await; - target_db.commit(None).await.unwrap(); + target_db = H::apply_ops(target_db, more_ops).await; + // commit already done in apply_ops // Capture target db state for comparison let target_db_op_count = target_db.op_count(); @@ -1128,8 +1132,8 @@ where // Create and populate a source database let mut source_db = H::init_db(context.clone()).await; let ops = H::create_ops(NUM_OPS); - H::apply_ops(&mut source_db, ops).await; - source_db.commit(None).await.unwrap(); + source_db = H::apply_ops(source_db, ops).await; + // commit already done in apply_ops source_db .prune(source_db.inactivity_floor_loc()) .await @@ -1214,8 +1218,7 @@ where // Test that we can perform operations on the synced database let ops = H::create_ops(10); - H::apply_ops(&mut synced_db, ops).await; - synced_db.commit(None).await.unwrap(); + synced_db = H::apply_ops(synced_db, ops).await; // Verify the operations worked assert!(synced_db.op_count() > Location::new_unchecked(1)); diff --git a/storage/src/qmdb/any/unordered/variable/mod.rs b/storage/src/qmdb/any/unordered/variable/mod.rs index ed3148a5d4..471088a318 100644 --- a/storage/src/qmdb/any/unordered/variable/mod.rs +++ b/storage/src/qmdb/any/unordered/variable/mod.rs @@ -12,16 +12,16 @@ use crate::{ authenticated, contiguous::variable::{Config as JournalConfig, Journal}, }, - mmr::{journaled::Config as MmrConfig, mem::Clean, Location}, + mmr::{journaled::Config as MmrConfig, Location}, qmdb::{ any::{unordered, value::VariableEncoding, VariableConfig, VariableValue}, operation::Committable as _, - Error, + Durable, Error, Merkleized, }, translator::Translator, }; use commonware_codec::Read; -use commonware_cryptography::{DigestOf, Hasher}; +use commonware_cryptography::Hasher; use commonware_runtime::{Clock, Metrics, Storage}; use commonware_utils::Array; use tracing::warn; @@ -31,11 +31,11 @@ pub type Operation = unordered::Operation>; /// A key-value QMDB based on an authenticated log of operations, supporting authentication of any /// value ever associated with a key. -pub type Db>> = - super::Db>, Index, H, Update, S>; +pub type Db, D = Durable> = + super::Db>, Index, H, Update, S, D>; impl - Db + Db, Durable> { /// Returns a [Db] QMDB initialized from `cfg`. Uncommitted log operations will be /// discarded and the state of the db will be as of the last committed operation. @@ -91,11 +91,7 @@ impl, Sha256, TwoCap>; + type AnyTest = + Db, Sha256, TwoCap, Merkleized, Durable>; /// Deterministic byte vector generator for variable-value tests. fn to_bytes(i: u64) -> Vec { @@ -160,7 +157,7 @@ pub(super) mod test { pub fn test_any_variable_db_log_replay() { let executor = deterministic::Runner::default(); executor.start(|context| async move { - let mut db = open_db(context.clone()).await; + let mut db = open_db(context.clone()).await.into_mutable(); // Update the same key many times. const UPDATES: u64 = 100; @@ -169,7 +166,7 @@ pub(super) mod test { let v = to_bytes(i); db.update(k, v).await.unwrap(); } - db.commit(None).await.unwrap(); + let db = db.commit(None).await.unwrap().0.into_merkleized(); let root = db.root(); // Simulate a failed commit and test that the log replay doesn't leave behind old data. @@ -204,8 +201,9 @@ pub(super) mod test { // Build a db with 1000 keys, some of which we update and some of which we delete. const ELEMENTS: u64 = 1000; executor.start(|context| async move { - let mut db = open_db(context.clone()).await; + let db = open_db(context.clone()).await; let root = db.root(); + let mut db = db.into_mutable(); for i in 0u64..ELEMENTS { let k = Sha256::hash(&i.to_be_bytes()); @@ -214,20 +212,22 @@ pub(super) mod test { } // Simulate a failure and test that we rollback to the previous root. - db.simulate_failure(false).await.unwrap(); - let mut db = open_db(context.clone()).await; + drop(db); + let db = open_db(context.clone()).await; assert_eq!(root, db.root()); // re-apply the updates and commit them this time. + let mut db = db.into_mutable(); for i in 0u64..ELEMENTS { let k = Sha256::hash(&i.to_be_bytes()); let v = vec![(i % 255) as u8; ((i % 13) + 7) as usize]; db.update(k, v.clone()).await.unwrap(); } - db.commit(None).await.unwrap(); + let db = db.commit(None).await.unwrap().0.into_merkleized(); let root = db.root(); // Update every 3rd key + let mut db = db.into_mutable(); for i in 0u64..ELEMENTS { if i % 3 != 0 { continue; @@ -238,11 +238,12 @@ pub(super) mod test { } // Simulate a failure and test that we rollback to the previous root. - db.simulate_failure(false).await.unwrap(); - let mut db = open_db(context.clone()).await; + drop(db); + let db = open_db(context.clone()).await; assert_eq!(root, db.root()); // Re-apply updates for every 3rd key and commit them this time. + let mut db = db.into_mutable(); for i in 0u64..ELEMENTS { if i % 3 != 0 { continue; @@ -251,10 +252,11 @@ pub(super) mod test { let v = vec![((i + 1) % 255) as u8; ((i % 13) + 8) as usize]; db.update(k, v.clone()).await.unwrap(); } - db.commit(None).await.unwrap(); + let db = db.commit(None).await.unwrap().0.into_merkleized(); let root = db.root(); // Delete every 7th key + let mut db = db.into_mutable(); for i in 0u64..ELEMENTS { if i % 7 != 1 { continue; @@ -264,11 +266,12 @@ pub(super) mod test { } // Simulate a failure and test that we rollback to the previous root. - db.simulate_failure(false).await.unwrap(); - let mut db = open_db(context.clone()).await; + drop(db); + let db = open_db(context.clone()).await; assert_eq!(root, db.root()); // Re-delete every 7th key and commit this time. + let mut db = db.into_mutable(); for i in 0u64..ELEMENTS { if i % 7 != 1 { continue; @@ -276,7 +279,7 @@ pub(super) mod test { let k = Sha256::hash(&i.to_be_bytes()); db.delete(k).await.unwrap(); } - db.commit(None).await.unwrap(); + let mut db = db.commit(None).await.unwrap().0.into_merkleized(); let root = db.root(); assert_eq!(db.op_count(), 1961); @@ -312,73 +315,17 @@ pub(super) mod test { /// Test that various types of unclean shutdown while updating a non-empty DB recover to the /// empty DB on re-open. #[test_traced("WARN")] - fn test_any_variable_non_empty_db_recovery() { + fn test_any_fixed_non_empty_db_recovery() { let executor = deterministic::Runner::default(); executor.start(|context| async move { - let mut db = open_db(context.clone()).await; - - // Insert 1000 keys then sync. - for i in 0u64..1000 { - let k = Sha256::hash(&i.to_be_bytes()); - let v = vec![(i % 255) as u8; ((i % 13) + 7) as usize]; - db.update(k, v).await.unwrap(); - } - db.commit(None).await.unwrap(); - db.prune(db.inactivity_floor_loc()).await.unwrap(); - let root = db.root(); - let op_count = db.op_count(); - let inactivity_floor_loc = db.inactivity_floor_loc(); - - // Reopen DB without clean shutdown and make sure the state is the same. - let mut db = open_db(context.clone()).await; - assert_eq!(db.op_count(), op_count); - assert_eq!(db.inactivity_floor_loc, inactivity_floor_loc); - assert_eq!(db.root(), root); - - async fn apply_more_ops( - db: &mut Db, Sha256, TwoCap>, - ) { - for i in 0u64..1000 { - let k = Sha256::hash(&i.to_be_bytes()); - let v = vec![(i % 255) as u8; ((i % 13) + 8) as usize]; - db.update(k, v).await.unwrap(); - } - } - - // Insert operations without commit, then simulate failure, syncing nothing. - apply_more_ops(&mut db).await; - db.simulate_failure(false).await.unwrap(); - let mut db = open_db(context.clone()).await; - assert_eq!(db.op_count(), op_count); - assert_eq!(db.inactivity_floor_loc, inactivity_floor_loc); - assert_eq!(db.root(), root); - - // Repeat, though this time sync the log. - apply_more_ops(&mut db).await; - db.simulate_failure(true).await.unwrap(); - let mut db = open_db(context.clone()).await; - assert_eq!(db.op_count(), op_count); - assert_eq!(db.inactivity_floor_loc(), inactivity_floor_loc); - assert_eq!(db.root(), root); - - // One last check that re-open without proper shutdown still recovers the correct state. - apply_more_ops(&mut db).await; - apply_more_ops(&mut db).await; - apply_more_ops(&mut db).await; - let mut db = open_db(context.clone()).await; - assert_eq!(db.op_count(), op_count); - assert_eq!(db.inactivity_floor_loc(), inactivity_floor_loc); - assert_eq!(db.root(), root); - - // Apply the ops one last time but fully commit them this time, then clean up. - apply_more_ops(&mut db).await; - db.commit(None).await.unwrap(); let db = open_db(context.clone()).await; - assert!(db.op_count() > op_count); - assert_ne!(db.inactivity_floor_loc(), inactivity_floor_loc); - assert_ne!(db.root(), root); - - db.destroy().await.unwrap(); + crate::qmdb::any::unordered::test::test_any_db_non_empty_recovery( + context, + db, + |ctx| Box::pin(open_db(ctx)), + to_bytes, + ) + .await; }); } @@ -388,70 +335,23 @@ pub(super) mod test { fn test_any_variable_empty_db_recovery() { let executor = deterministic::Runner::default(); executor.start(|context| async move { - // Initialize an empty db. - let db = open_db(context.clone()).await; - let root = db.root(); - - // Reopen DB without clean shutdown and make sure the state is the same. - let mut db = open_db(context.clone()).await; - assert_eq!(db.op_count(), 1); - assert_eq!(db.root(), root); - - async fn apply_ops( - db: &mut Db, Sha256, TwoCap>, - ) { - for i in 0u64..1000 { - let k = Sha256::hash(&i.to_be_bytes()); - let v = vec![(i % 255) as u8; ((i % 13) + 8) as usize]; - db.update(k, v).await.unwrap(); - } - } - - // Insert operations without commit then simulate failure. - apply_ops(&mut db).await; - db.simulate_failure(false).await.unwrap(); - let mut db = open_db(context.clone()).await; - assert_eq!(db.op_count(), 1); - assert_eq!(db.root(), root); - - // Insert another 1000 keys then simulate failure after syncing the log. - apply_ops(&mut db).await; - db.simulate_failure(true).await.unwrap(); - let mut db = open_db(context.clone()).await; - assert_eq!(db.op_count(), 1); - assert_eq!(db.root(), root); - - // Insert another 1000 keys then simulate failure (sync only the mmr). - apply_ops(&mut db).await; - db.simulate_failure(false).await.unwrap(); - let mut db = open_db(context.clone()).await; - assert_eq!(db.op_count(), 1); - assert_eq!(db.root(), root); - - // One last check that re-open without proper shutdown still recovers the correct state. - apply_ops(&mut db).await; - apply_ops(&mut db).await; - apply_ops(&mut db).await; - let mut db = open_db(context.clone()).await; - assert_eq!(db.op_count(), 1); - assert_eq!(db.root(), root); - - // Apply the ops one last time but fully commit them this time, then clean up. - apply_ops(&mut db).await; - db.commit(None).await.unwrap(); let db = open_db(context.clone()).await; - assert!(db.op_count() > 1); - assert_ne!(db.root(), root); - - db.destroy().await.unwrap(); + crate::qmdb::any::unordered::test::test_any_db_empty_recovery( + context, + db, + |ctx| Box::pin(open_db(ctx)), + to_bytes, + ) + .await; }); } #[test_traced] - fn test_variable_db_prune_beyond_inactivity_floor() { + fn test_any_variable_db_prune_beyond_inactivity_floor() { let executor = deterministic::Runner::default(); executor.start(|mut context| async move { - let mut db = open_db(context.clone()).await; + let db = open_db(context.clone()).await; + let mut db = db.into_mutable(); // Add some operations let key1 = Digest::random(&mut context); @@ -461,13 +361,14 @@ pub(super) mod test { db.update(key1, vec![10]).await.unwrap(); db.update(key2, vec![20]).await.unwrap(); db.update(key3, vec![30]).await.unwrap(); - db.commit(None).await.unwrap(); + let (db, _) = db.commit(None).await.unwrap(); // inactivity_floor should be at some location < op_count let inactivity_floor = db.inactivity_floor_loc(); let beyond_floor = Location::new_unchecked(*inactivity_floor + 1); // Try to prune beyond the inactivity floor + let mut db = db.into_merkleized(); let result = db.prune(beyond_floor).await; assert!( matches!(result, Err(Error::PruneBeyondMinRequired(loc, floor)) @@ -479,11 +380,21 @@ pub(super) mod test { } #[test_traced("DEBUG")] - fn test_batch() { + fn test_any_unordered_variable_batch() { batch_tests::test_batch(|mut ctx| async move { let seed = ctx.next_u64(); let cfg = db_config(&format!("batch_{seed}")); - AnyTest::init(ctx, cfg).await.unwrap() + AnyTest::init(ctx, cfg).await.unwrap().into_mutable() + }); + } + + // Test that merkleization state changes don't reset `steps`. + #[test_traced("DEBUG")] + fn test_any_unordered_variable_db_steps_not_reset() { + let executor = deterministic::Runner::default(); + executor.start(|context| async move { + let db = crate::qmdb::any::unordered::test::open_variable_db(context).await; + crate::qmdb::any::test::test_any_db_steps_not_reset(db).await; }); } diff --git a/storage/src/qmdb/any/unordered/variable/sync.rs b/storage/src/qmdb/any/unordered/variable/sync.rs index ddf856ca7d..a5f9ced5b9 100644 --- a/storage/src/qmdb/any/unordered/variable/sync.rs +++ b/storage/src/qmdb/any/unordered/variable/sync.rs @@ -5,7 +5,7 @@ use crate::{ index::unordered::Index, journal::{authenticated, contiguous::variable}, mmr::{mem::Clean, Location, Position, StandardHasher}, - qmdb::{self, any::VariableValue}, + qmdb::{self, any::VariableValue, Durable, Merkleized}, translator::Translator, }; use commonware_codec::Read; @@ -14,7 +14,7 @@ use commonware_runtime::{Clock, Metrics, Storage}; use commonware_utils::Array; use std::ops::Range; -impl qmdb::sync::Database for Db +impl qmdb::sync::Database for Db, Durable> where E: Storage + Clock + Metrics, K: Array, @@ -124,7 +124,10 @@ where mod tests { use super::*; use crate::{ - qmdb::any::unordered::{sync_tests::SyncTestHarness, Update}, + qmdb::{ + any::unordered::{sync_tests::SyncTestHarness, Update}, + NonDurable, Unmerkleized, + }, translator::TwoCap, }; use commonware_cryptography::{sha256::Digest, Sha256}; @@ -162,7 +165,8 @@ mod tests { } /// Type alias for tests - type AnyTest = Db, Sha256, TwoCap>; + type AnyTest = + Db, Sha256, TwoCap, Merkleized, Durable>; fn test_value(i: u64) -> Vec { let len = ((i % 13) + 7) as usize; @@ -194,8 +198,14 @@ mod tests { ops } + type DirtyAnyTest = + Db, Sha256, TwoCap, Unmerkleized, NonDurable>; + /// Applies the given operations to the database. - async fn apply_ops(db: &mut AnyTest, ops: Vec>>) { + async fn apply_ops_inner( + mut db: DirtyAnyTest, + ops: Vec>>, + ) -> DirtyAnyTest { for op in ops { match op { Operation::Update(Update(key, value)) => { @@ -205,10 +215,11 @@ mod tests { db.delete(key).await.unwrap(); } Operation::CommitFloor(metadata, _) => { - db.commit(metadata).await.unwrap(); + db = db.commit(metadata).await.unwrap().0.into_mutable(); } } } + db } /// Harness for sync tests. @@ -237,8 +248,14 @@ mod tests { AnyTest::init(ctx, config).await.unwrap() } - async fn apply_ops(db: &mut Self::Db, ops: Vec>>) { - apply_ops(db, ops).await + async fn apply_ops(db: Self::Db, ops: Vec>>) -> Self::Db { + apply_ops_inner(db.into_mutable(), ops) + .await + .commit(None) + .await + .unwrap() + .0 + .into_merkleized() } } diff --git a/storage/src/qmdb/benches/fixed/generate.rs b/storage/src/qmdb/benches/fixed/generate.rs index d11cc6e4de..1972df869a 100644 --- a/storage/src/qmdb/benches/fixed/generate.rs +++ b/storage/src/qmdb/benches/fixed/generate.rs @@ -4,17 +4,16 @@ use crate::fixed::{ gen_random_kv, gen_random_kv_batched, get_any_ordered_fixed, get_any_ordered_variable, get_any_unordered_fixed, get_any_unordered_variable, get_current_ordered_fixed, - get_current_unordered_fixed, get_store, Variant, VARIANTS, + get_current_unordered_fixed, Digest, Variant, VARIANTS, }; -use commonware_cryptography::{Hasher, Sha256}; use commonware_runtime::{ benchmarks::{context, tokio}, tokio::{Config, Context}, }; -use commonware_storage::{ - kv::Batchable, - qmdb::{any::AnyExt, store::LogStorePrunable, Error}, - Persistable, +use commonware_storage::qmdb::{ + any::states::{CleanAny, MutableAny, UnmerkleizedDurableAny}, + store::LogStore, + Error, }; use criterion::{criterion_group, Criterion}; use std::time::{Duration, Instant}; @@ -71,18 +70,6 @@ fn bench_fixed_generate(c: &mut Criterion) { .await .unwrap() } - Variant::Store => { - let db = get_store(ctx.clone()).await; - test_db( - db, - use_batch, - elements, - operations, - commit_frequency, - ) - .await - .unwrap() - } Variant::AnyUnorderedVariable => { let db = get_any_unordered_variable(ctx.clone()).await; test_db( @@ -109,7 +96,6 @@ fn bench_fixed_generate(c: &mut Criterion) { } Variant::CurrentUnorderedFixed => { let db = get_current_unordered_fixed(ctx.clone()).await; - let db = AnyExt::new(db); test_db( db, use_batch, @@ -122,7 +108,6 @@ fn bench_fixed_generate(c: &mut Criterion) { } Variant::CurrentOrderedFixed => { let db = get_current_ordered_fixed(ctx.clone()).await; - let db = AnyExt::new(db); test_db( db, use_batch, @@ -146,28 +131,41 @@ fn bench_fixed_generate(c: &mut Criterion) { } } -async fn test_db( - db: A, +/// Test the database generation and cleanup. +/// +/// Takes a clean database, converts to mutable, generates data, then prunes and destroys. +async fn test_db( + db: C, use_batch: bool, elements: u64, operations: u64, commit_frequency: u32, -) -> Result +) -> Result where - A: Batchable::Digest, Value = ::Digest> - + Persistable - + LogStorePrunable, + C: CleanAny, + C::Mutable: MutableAny + LogStore, + ::Durable: + UnmerkleizedDurableAny, { let start = Instant::now(); - let mut db = if use_batch { - gen_random_kv_batched(db, elements, operations, Some(commit_frequency)).await + + // Convert clean → mutable + let mutable = db.into_mutable(); + + // Generate random operations, returns in durable state + let durable = if use_batch { + gen_random_kv_batched(mutable, elements, operations, Some(commit_frequency)).await } else { - gen_random_kv(db, elements, operations, Some(commit_frequency)).await + gen_random_kv(mutable, elements, operations, Some(commit_frequency)).await }; - db.commit().await?; - db.prune(db.inactivity_floor_loc()).await?; + + // Convert durable → provable (clean) for pruning + let mut clean = durable.into_merkleized().await?; + clean.prune(clean.inactivity_floor_loc()).await?; + clean.sync().await?; + let res = start.elapsed(); - db.destroy().await?; // don't time destroy + clean.destroy().await?; // don't time destroy Ok(res) } diff --git a/storage/src/qmdb/benches/fixed/init.rs b/storage/src/qmdb/benches/fixed/init.rs index b4fd6dbbe3..13fb9f2895 100644 --- a/storage/src/qmdb/benches/fixed/init.rs +++ b/storage/src/qmdb/benches/fixed/init.rs @@ -4,8 +4,8 @@ use crate::fixed::{ any_cfg, current_cfg, gen_random_kv, get_any_ordered_fixed, get_any_ordered_variable, get_any_unordered_fixed, get_any_unordered_variable, get_current_ordered_fixed, - get_current_unordered_fixed, get_store, store_cfg, variable_any_cfg, OCurrentDb, OFixedDb, - OVAnyDb, StoreDb, UCurrentDb, UFixedDb, UVAnyDb, Variant, THREADS, VARIANTS, + get_current_unordered_fixed, variable_any_cfg, Digest, OCurrentDb, OFixedDb, OVAnyDb, + UCurrentDb, UFixedDb, UVAnyDb, Variant, THREADS, VARIANTS, }; use commonware_runtime::{ benchmarks::{context, tokio}, @@ -13,8 +13,8 @@ use commonware_runtime::{ Runner as _, }; use commonware_storage::qmdb::{ - any::AnyExt, - store::{LogStore, LogStorePrunable}, + any::states::{CleanAny, MutableAny, UnmerkleizedDurableAny}, + store::LogStore, }; use criterion::{criterion_group, Criterion}; use std::time::Instant; @@ -33,70 +33,61 @@ cfg_if::cfg_if! { } } +/// Helper function to setup a database with random data, prune, and close it. +async fn setup_db(db: C, elements: u64, operations: u64) +where + C: CleanAny, + C::Mutable: MutableAny + LogStore, + ::Durable: + UnmerkleizedDurableAny, +{ + let mutable = db.into_mutable(); + let durable = gen_random_kv(mutable, elements, operations, Some(COMMIT_FREQUENCY)).await; + let mut clean = durable.into_merkleized().await.unwrap(); + clean.prune(clean.inactivity_floor_loc()).await.unwrap(); + clean.sync().await.unwrap(); + drop(clean); +} + /// Benchmark the initialization of a large randomly generated any db. fn bench_fixed_init(c: &mut Criterion) { let cfg = Config::default(); for elements in ELEMENTS { for operations in OPERATIONS { for variant in VARIANTS { + // Setup phase: create and populate the database let runner = Runner::new(cfg.clone()); runner.start(|ctx| async move { match variant { - Variant::Store => { - let db = get_store(ctx.clone()).await; - let mut db = - gen_random_kv(db, elements, operations, Some(COMMIT_FREQUENCY)) - .await; - db.prune(db.inactivity_floor_loc()).await.unwrap(); - } Variant::AnyUnorderedFixed => { let db = get_any_unordered_fixed(ctx.clone()).await; - let mut db = - gen_random_kv(db, elements, operations, Some(COMMIT_FREQUENCY)) - .await; - db.prune(db.inactivity_floor_loc()).await.unwrap(); + setup_db(db, elements, operations).await; } Variant::AnyOrderedFixed => { let db = get_any_ordered_fixed(ctx.clone()).await; - let mut db = - gen_random_kv(db, elements, operations, Some(COMMIT_FREQUENCY)) - .await; - db.prune(db.inactivity_floor_loc()).await.unwrap(); + setup_db(db, elements, operations).await; } Variant::CurrentUnorderedFixed => { let db = get_current_unordered_fixed(ctx.clone()).await; - let db = AnyExt::new(db); - let mut db = - gen_random_kv(db, elements, operations, Some(COMMIT_FREQUENCY)) - .await; - db.prune(db.inactivity_floor_loc()).await.unwrap(); + setup_db(db, elements, operations).await; } Variant::CurrentOrderedFixed => { let db = get_current_ordered_fixed(ctx.clone()).await; - let db = AnyExt::new(db); - let mut db = - gen_random_kv(db, elements, operations, Some(COMMIT_FREQUENCY)) - .await; - db.prune(db.inactivity_floor_loc()).await.unwrap(); + setup_db(db, elements, operations).await; } Variant::AnyUnorderedVariable => { let db = get_any_unordered_variable(ctx.clone()).await; - let mut db = - gen_random_kv(db, elements, operations, Some(COMMIT_FREQUENCY)) - .await; - db.prune(db.inactivity_floor_loc()).await.unwrap(); + setup_db(db, elements, operations).await; } Variant::AnyOrderedVariable => { let db = get_any_ordered_variable(ctx.clone()).await; - let mut db = - gen_random_kv(db, elements, operations, Some(COMMIT_FREQUENCY)) - .await; - db.prune(db.inactivity_floor_loc()).await.unwrap(); + setup_db(db, elements, operations).await; } } }); - let runner = tokio::Runner::new(cfg.clone()); + // Benchmark phase: measure initialization time + let runner = tokio::Runner::new(cfg.clone()); c.bench_function( &format!( "{}/variant={}, elements={} operations={}", @@ -113,16 +104,9 @@ fn bench_fixed_init(c: &mut Criterion) { let any_cfg = any_cfg(pool.clone()); let current_cfg = current_cfg(pool.clone()); let variable_any_cfg = variable_any_cfg(pool); - let store_cfg = store_cfg(); let start = Instant::now(); for _ in 0..iters { match variant { - Variant::Store => { - let db = StoreDb::init(ctx.clone(), store_cfg.clone()) - .await - .unwrap(); - assert_ne!(db.op_count(), 0); - } Variant::AnyUnorderedFixed => { let db = UFixedDb::init(ctx.clone(), any_cfg.clone()) .await @@ -168,14 +152,10 @@ fn bench_fixed_init(c: &mut Criterion) { }, ); + // Cleanup phase: destroy the database let runner = Runner::new(cfg.clone()); runner.start(|ctx| async move { - // Clean up the database after the benchmark. match variant { - Variant::Store => { - let db = get_store(ctx.clone()).await; - db.destroy().await.unwrap(); - } Variant::AnyUnorderedFixed => { let db = get_any_unordered_fixed(ctx.clone()).await; db.destroy().await.unwrap(); diff --git a/storage/src/qmdb/benches/fixed/mod.rs b/storage/src/qmdb/benches/fixed/mod.rs index 9627befd35..2c7a3aae12 100644 --- a/storage/src/qmdb/benches/fixed/mod.rs +++ b/storage/src/qmdb/benches/fixed/mod.rs @@ -7,10 +7,11 @@ use commonware_cryptography::{Hasher, Sha256}; use commonware_runtime::{buffer::PoolRef, create_pool, tokio::Context, ThreadPool}; use commonware_storage::{ - kv::{Batchable, Deletable, Updatable as _}, + kv::{Deletable as _, Updatable as _}, qmdb::{ any::{ ordered::{fixed::Db as OFixed, variable::Db as OVariable}, + states::{MutableAny, UnmerkleizedDurableAny}, unordered::{fixed::Db as UFixed, variable::Db as UVariable}, FixedConfig as AConfig, VariableConfig as VariableAnyConfig, }, @@ -18,11 +19,9 @@ use commonware_storage::{ ordered::fixed::Db as OCurrent, unordered::fixed::Db as UCurrent, FixedConfig as CConfig, }, - store::{Config as SConfig, Store}, - Error, + store::LogStore, }, translator::EightCap, - Persistable, }; use commonware_utils::{NZUsize, NZU64}; use rand::{rngs::StdRng, RngCore, SeedableRng}; @@ -31,9 +30,10 @@ use std::num::{NonZeroU64, NonZeroUsize}; pub mod generate; pub mod init; +pub type Digest = ::Digest; + #[derive(Debug, Clone, Copy)] enum Variant { - Store, AnyUnorderedFixed, AnyOrderedFixed, AnyUnorderedVariable, @@ -45,7 +45,6 @@ enum Variant { impl Variant { pub const fn name(&self) -> &'static str { match self { - Self::Store => "store", Self::AnyUnorderedFixed => "any::unordered::fixed", Self::AnyOrderedFixed => "any::ordered::fixed", Self::AnyUnorderedVariable => "any::unordered::variable", @@ -56,8 +55,7 @@ impl Variant { } } -const VARIANTS: [Variant; 7] = [ - Variant::Store, +const VARIANTS: [Variant; 6] = [ Variant::AnyUnorderedFixed, Variant::AnyOrderedFixed, Variant::AnyUnorderedVariable, @@ -89,31 +87,14 @@ const DELETE_FREQUENCY: u32 = 10; /// Default write buffer size. const WRITE_BUFFER_SIZE: NonZeroUsize = NZUsize!(1024); -type UFixedDb = - UFixed::Digest, ::Digest, Sha256, EightCap>; -type OFixedDb = - OFixed::Digest, ::Digest, Sha256, EightCap>; -type UCurrentDb = UCurrent< - Context, - ::Digest, - ::Digest, - Sha256, - EightCap, - CHUNK_SIZE, ->; -type OCurrentDb = OCurrent< - Context, - ::Digest, - ::Digest, - Sha256, - EightCap, - CHUNK_SIZE, ->; -type StoreDb = Store::Digest, ::Digest, EightCap>; -type UVAnyDb = - UVariable::Digest, ::Digest, Sha256, EightCap>; -type OVAnyDb = - OVariable::Digest, ::Digest, Sha256, EightCap>; +/// Clean (Merkleized, Durable) Db type aliases for Any databases. +type UFixedDb = UFixed; +type OFixedDb = OFixed; +type UVAnyDb = UVariable; +type OVAnyDb = OVariable; + +type UCurrentDb = UCurrent; +type OCurrentDb = OCurrent; /// Configuration for any QMDB. fn any_cfg(pool: ThreadPool) -> AConfig { @@ -148,18 +129,6 @@ fn current_cfg(pool: ThreadPool) -> CConfig { } } -fn store_cfg() -> SConfig { - SConfig:: { - log_partition: format!("journal_{PARTITION_SUFFIX}"), - log_write_buffer: WRITE_BUFFER_SIZE, - log_compression: None, - log_codec_config: (), - log_items_per_section: ITEMS_PER_BLOB, - translator: EightCap, - buffer_pool: PoolRef::new(PAGE_SIZE, PAGE_CACHE_SIZE), - } -} - fn variable_any_cfg(pool: ThreadPool) -> VariableAnyConfig { VariableAnyConfig:: { mmr_journal_partition: format!("journal_{PARTITION_SUFFIX}"), @@ -177,22 +146,32 @@ fn variable_any_cfg(pool: ThreadPool) -> VariableAnyConfig { } } -/// Get an unordered any QMDB instance. +/// Get an unordered fixed Any QMDB instance in clean state. async fn get_any_unordered_fixed(ctx: Context) -> UFixedDb { let pool = create_pool(ctx.clone(), THREADS).unwrap(); let any_cfg = any_cfg(pool); - UFixed::<_, _, _, Sha256, EightCap>::init(ctx, any_cfg) - .await - .unwrap() + UFixedDb::init(ctx, any_cfg).await.unwrap() } -/// Get an ordered any QMDB instance. +/// Get an ordered fixed Any QMDB instance in clean state. async fn get_any_ordered_fixed(ctx: Context) -> OFixedDb { let pool = create_pool(ctx.clone(), THREADS).unwrap(); let any_cfg = any_cfg(pool); - OFixed::<_, _, _, Sha256, EightCap>::init(ctx, any_cfg) - .await - .unwrap() + OFixedDb::init(ctx, any_cfg).await.unwrap() +} + +/// Get an unordered variable Any QMDB instance in clean state. +async fn get_any_unordered_variable(ctx: Context) -> UVAnyDb { + let pool = create_pool(ctx.clone(), THREADS).unwrap(); + let variable_any_cfg = variable_any_cfg(pool); + UVAnyDb::init(ctx, variable_any_cfg).await.unwrap() +} + +/// Get an ordered variable Any QMDB instance in clean state. +async fn get_any_ordered_variable(ctx: Context) -> OVAnyDb { + let pool = create_pool(ctx.clone(), THREADS).unwrap(); + let variable_any_cfg = variable_any_cfg(pool); + OVAnyDb::init(ctx, variable_any_cfg).await.unwrap() } /// Get an unordered current QMDB instance. @@ -213,40 +192,22 @@ async fn get_current_ordered_fixed(ctx: Context) -> OCurrentDb { .unwrap() } -async fn get_store(ctx: Context) -> StoreDb { - let store_cfg = store_cfg(); - Store::init(ctx, store_cfg).await.unwrap() -} - -async fn get_any_unordered_variable(ctx: Context) -> UVAnyDb { - let pool = create_pool(ctx.clone(), THREADS).unwrap(); - let variable_any_cfg = variable_any_cfg(pool); - UVariable::init(ctx, variable_any_cfg).await.unwrap() -} - -async fn get_any_ordered_variable(ctx: Context) -> OVAnyDb { - let pool = create_pool(ctx.clone(), THREADS).unwrap(); - let variable_any_cfg = variable_any_cfg(pool); - OVariable::init(ctx, variable_any_cfg).await.unwrap() -} - /// Generate a large db with random data. The function seeds the db with exactly `num_elements` /// elements by inserting them in order, each with a new random value. Then, it performs /// `num_operations` over these elements, each selected uniformly at random for each operation. The /// database is committed after every `commit_frequency` operations (if Some), or at the end (if /// None). -async fn gen_random_kv( - mut db: A, +/// +/// Takes a mutable database and returns it in durable state after final commit. +async fn gen_random_kv( + mut db: M, num_elements: u64, num_operations: u64, commit_frequency: Option, -) -> A +) -> M::Durable where - A: Deletable< - Key = ::Digest, - Value = ::Digest, - Error = Error, - > + Persistable, + M: MutableAny + LogStore, + M::Durable: UnmerkleizedDurableAny, { // Insert a random value for every possible element into the db. let mut rng = StdRng::seed_from_u64(42); @@ -267,24 +228,25 @@ where db.update(rand_key, v).await.unwrap(); if let Some(freq) = commit_frequency { if rng.next_u32() % freq == 0 { - db.commit().await.unwrap(); + let (durable, _) = db.commit(None).await.unwrap(); + db = durable.into_mutable(); } } } - db.commit().await.unwrap(); - db + let (durable, _) = db.commit(None).await.unwrap(); + durable } -async fn gen_random_kv_batched( - mut db: A, +async fn gen_random_kv_batched( + mut db: M, num_elements: u64, num_operations: u64, commit_frequency: Option, -) -> A +) -> M::Durable where - A: Batchable::Digest, Value = ::Digest> - + Persistable, + M: MutableAny + LogStore, + M::Durable: UnmerkleizedDurableAny, { let mut rng = StdRng::seed_from_u64(42); let mut batch = db.start_batch(); @@ -292,32 +254,42 @@ where for i in 0u64..num_elements { let k = Sha256::hash(&i.to_be_bytes()); let v = Sha256::hash(&rng.next_u32().to_be_bytes()); - batch.update(k, v).await.unwrap(); + batch.update(k, v).await.expect("update shouldn't fail"); } let iter = batch.into_iter(); - db.write_batch(iter).await.unwrap(); + db.write_batch(iter) + .await + .expect("write_batch shouldn't fail"); batch = db.start_batch(); for _ in 0u64..num_operations { let rand_key = Sha256::hash(&(rng.next_u64() % num_elements).to_be_bytes()); if rng.next_u32() % DELETE_FREQUENCY == 0 { - batch.delete(rand_key).await.unwrap(); + batch.delete(rand_key).await.expect("delete shouldn't fail"); continue; } let v = Sha256::hash(&rng.next_u32().to_be_bytes()); - batch.update(rand_key, v).await.unwrap(); + batch + .update(rand_key, v) + .await + .expect("update shouldn't fail"); if let Some(freq) = commit_frequency { if rng.next_u32() % freq == 0 { let iter = batch.into_iter(); - db.write_batch(iter).await.unwrap(); - db.commit().await.unwrap(); + db.write_batch(iter) + .await + .expect("write_batch shouldn't fail"); + let (durable, _) = db.commit(None).await.expect("commit shouldn't fail"); + db = durable.into_mutable(); batch = db.start_batch(); } } } let iter = batch.into_iter(); - db.write_batch(iter).await.unwrap(); - db.commit().await.unwrap(); - db + db.write_batch(iter) + .await + .expect("write_batch shouldn't fail"); + let (durable, _) = db.commit(None).await.expect("commit shouldn't fail"); + durable } diff --git a/storage/src/qmdb/benches/keyless_generate.rs b/storage/src/qmdb/benches/keyless_generate.rs index a32e6a86ec..6a95c194ba 100644 --- a/storage/src/qmdb/benches/keyless_generate.rs +++ b/storage/src/qmdb/benches/keyless_generate.rs @@ -1,4 +1,4 @@ -use commonware_cryptography::{Hasher, Sha256}; +use commonware_cryptography::Sha256; use commonware_runtime::{ benchmarks::{context, tokio}, buffer::PoolRef, @@ -6,9 +6,9 @@ use commonware_runtime::{ tokio::{Config, Context}, ThreadPool, }; -use commonware_storage::{ - mmr::mem::Clean, - qmdb::keyless::{Config as KConfig, Keyless}, +use commonware_storage::qmdb::{ + keyless::{Config as KConfig, Keyless}, + NonDurable, Unmerkleized, }; use commonware_utils::{NZUsize, NZU64}; use criterion::{criterion_group, Criterion}; @@ -49,12 +49,21 @@ fn keyless_cfg(pool: ThreadPool) -> KConfig<(commonware_codec::RangeCfg, } } +/// Clean (Merkleized, Durable) db type alias for Keyless. +type KeylessDb = Keyless, Sha256>; + +/// Mutable (Unmerkleized, NonDurable) type alias for Keyless. +type KeylessMutable = Keyless, Sha256, Unmerkleized, NonDurable>; + /// Generate a keyless db by appending `num_operations` random values in total. The database is /// committed after every `COMMIT_FREQUENCY` operations. async fn gen_random_keyless(ctx: Context, num_operations: u64) -> KeylessDb { let pool = create_pool(ctx.clone(), THREADS).unwrap(); let keyless_cfg = keyless_cfg(pool); - let mut db = Keyless::init(ctx, keyless_cfg).await.unwrap(); + let clean = KeylessDb::init(ctx, keyless_cfg).await.unwrap(); + + // Convert to mutable state for operations. + let mut db: KeylessMutable = clean.into_mutable(); // Randomly append. let mut rng = StdRng::seed_from_u64(42); @@ -62,17 +71,17 @@ async fn gen_random_keyless(ctx: Context, num_operations: u64) -> KeylessDb { let v = vec![(rng.next_u32() % 255) as u8; ((rng.next_u32() % 300) + 10) as usize]; db.append(v).await.unwrap(); if rng.next_u32() % COMMIT_FREQUENCY == 0 { - db.commit(None).await.unwrap(); + let (durable, _) = db.commit(None).await.unwrap(); + db = durable.into_mutable(); } } - db.commit(None).await.unwrap(); - db.sync().await.unwrap(); + let (durable, _) = db.commit(None).await.unwrap(); + let mut clean = durable.into_merkleized(); + clean.sync().await.unwrap(); - db + clean } -type KeylessDb = Keyless, Sha256, Clean<::Digest>>; - /// Benchmark the generation of a large randomly generated keyless db. fn bench_keyless_generate(c: &mut Criterion) { let cfg = Config::default(); diff --git a/storage/src/qmdb/benches/variable/generate.rs b/storage/src/qmdb/benches/variable/generate.rs index 9d0972d79a..6da5c65b9e 100644 --- a/storage/src/qmdb/benches/variable/generate.rs +++ b/storage/src/qmdb/benches/variable/generate.rs @@ -2,18 +2,17 @@ //! that supports variable-size values. use crate::variable::{ - gen_random_kv, gen_random_kv_batched, get_any_ordered, get_any_unordered, get_store, Variant, + gen_random_kv, gen_random_kv_batched, get_any_ordered, get_any_unordered, Digest, Variant, VARIANTS, }; -use commonware_cryptography::{Hasher, Sha256}; use commonware_runtime::{ benchmarks::{context, tokio}, tokio::{Config, Context}, }; -use commonware_storage::{ - kv::{Batchable, Deletable}, - qmdb::{store::LogStorePrunable, Error}, - Persistable, +use commonware_storage::qmdb::{ + any::states::{CleanAny, MutableAny, UnmerkleizedDurableAny}, + store::LogStore, + Error, }; use criterion::{criterion_group, Criterion}; use std::time::{Duration, Instant}; @@ -47,18 +46,6 @@ fn bench_variable_generate(c: &mut Criterion) { let commit_frequency = (operations / COMMITS_PER_ITERATION) as u32; let elapsed = match variant { - Variant::Store => { - let db = get_store(ctx.clone()).await; - test_db( - db, - use_batch, - elements, - operations, - commit_frequency, - ) - .await - .unwrap() - } Variant::AnyUnordered => { let db = get_any_unordered(ctx.clone()).await; test_db( @@ -96,27 +83,42 @@ fn bench_variable_generate(c: &mut Criterion) { } } -async fn test_db( - db: A, +/// Test the database generation and cleanup. +/// +/// Takes a clean database, converts to mutable, generates data, then prunes and destroys. +async fn test_db( + db: C, use_batch: bool, elements: u64, operations: u64, commit_frequency: u32, ) -> Result where - A: Batchable::Digest, Value = Vec> - + Persistable - + Deletable - + LogStorePrunable, + C: CleanAny, + C::Mutable: MutableAny + LogStore>, + ::Durable: + UnmerkleizedDurableAny, { let start = Instant::now(); - let db = if use_batch { - gen_random_kv_batched(db, elements, operations, commit_frequency).await + + // Convert clean → mutable + let mutable = db.into_mutable(); + + // Generate random operations, returns in durable state + let durable = if use_batch { + gen_random_kv_batched(mutable, elements, operations, commit_frequency).await } else { - gen_random_kv(db, elements, operations, commit_frequency).await + gen_random_kv(mutable, elements, operations, commit_frequency).await }; + + // Convert durable → provable (clean) for pruning + let mut clean = durable.into_merkleized().await?; + clean.prune(clean.inactivity_floor_loc()).await?; + clean.sync().await?; + let elapsed = start.elapsed(); - db.destroy().await?; + clean.destroy().await?; // don't time destroy + Ok(elapsed) } diff --git a/storage/src/qmdb/benches/variable/init.rs b/storage/src/qmdb/benches/variable/init.rs index 97b9b61a5a..477bfddd30 100644 --- a/storage/src/qmdb/benches/variable/init.rs +++ b/storage/src/qmdb/benches/variable/init.rs @@ -2,13 +2,18 @@ //! database with variable-sized values. use crate::variable::{ - gen_random_kv, get_any_ordered, get_any_unordered, get_store, Variant, VARIANTS, + any_cfg, gen_random_kv, get_any_ordered, get_any_unordered, Digest, OVariableDb, UVariableDb, + Variant, THREADS, VARIANTS, }; use commonware_runtime::{ benchmarks::{context, tokio}, tokio::{Config, Runner}, Runner as _, }; +use commonware_storage::qmdb::{ + any::states::{CleanAny, MutableAny, UnmerkleizedDurableAny}, + store::LogStore, +}; use criterion::{criterion_group, Criterion}; use std::time::Instant; @@ -26,37 +31,45 @@ cfg_if::cfg_if! { } } +/// Helper function to setup a database with random data, prune, and close it. +async fn setup_db(db: C, elements: u64, operations: u64) +where + C: CleanAny, + C::Mutable: MutableAny + LogStore>, + ::Durable: + UnmerkleizedDurableAny, +{ + let mutable = db.into_mutable(); + let durable = gen_random_kv(mutable, elements, operations, COMMIT_FREQUENCY).await; + let mut clean = durable.into_merkleized().await.unwrap(); + clean.prune(clean.inactivity_floor_loc()).await.unwrap(); + clean.sync().await.unwrap(); + drop(clean); +} + /// Benchmark the initialization of a large randomly generated any db. fn bench_variable_init(c: &mut Criterion) { let cfg = Config::default(); for elements in ELEMENTS { for operations in OPERATIONS { for variant in VARIANTS { + // Setup phase: create and populate the database let runner = Runner::new(cfg.clone()); runner.start(|ctx| async move { match variant { - Variant::Store => { - let db = get_store(ctx.clone()).await; - let mut db = - gen_random_kv(db, elements, operations, COMMIT_FREQUENCY).await; - db.prune(db.inactivity_floor_loc()).await.unwrap(); - } Variant::AnyUnordered => { let db = get_any_unordered(ctx.clone()).await; - let mut db = - gen_random_kv(db, elements, operations, COMMIT_FREQUENCY).await; - db.prune(db.inactivity_floor_loc()).await.unwrap(); + setup_db(db, elements, operations).await; } Variant::AnyOrdered => { let db = get_any_ordered(ctx.clone()).await; - let mut db = - gen_random_kv(db, elements, operations, COMMIT_FREQUENCY).await; - db.prune(db.inactivity_floor_loc()).await.unwrap(); + setup_db(db, elements, operations).await; } } }); - let runner = tokio::Runner::new(cfg.clone()); + // Benchmark phase: measure initialization time + let runner = tokio::Runner::new(cfg.clone()); c.bench_function( &format!( "{}/variant={} elements={} operations={}", @@ -71,16 +84,22 @@ fn bench_variable_init(c: &mut Criterion) { let start = Instant::now(); for _ in 0..iters { match variant { - Variant::Store => { - let db = get_store(ctx.clone()).await; - assert_ne!(db.op_count(), 0); - } Variant::AnyUnordered => { - let db = get_any_unordered(ctx.clone()).await; + let pool = + commonware_runtime::create_pool(ctx.clone(), THREADS) + .unwrap(); + let db = UVariableDb::init(ctx.clone(), any_cfg(pool)) + .await + .unwrap(); assert_ne!(db.op_count(), 0); } Variant::AnyOrdered => { - let db = get_any_ordered(ctx.clone()).await; + let pool = + commonware_runtime::create_pool(ctx.clone(), THREADS) + .unwrap(); + let db = OVariableDb::init(ctx.clone(), any_cfg(pool)) + .await + .unwrap(); assert_ne!(db.op_count(), 0); } } @@ -91,14 +110,10 @@ fn bench_variable_init(c: &mut Criterion) { }, ); + // Cleanup phase: destroy the database let runner = Runner::new(cfg.clone()); runner.start(|ctx| async move { - // Clean up the databases after the benchmark. match variant { - Variant::Store => { - let db = get_store(ctx).await; - db.destroy().await.unwrap(); - } Variant::AnyUnordered => { let db = get_any_unordered(ctx).await; db.destroy().await.unwrap(); diff --git a/storage/src/qmdb/benches/variable/mod.rs b/storage/src/qmdb/benches/variable/mod.rs index 82e152422b..d9db9a1328 100644 --- a/storage/src/qmdb/benches/variable/mod.rs +++ b/storage/src/qmdb/benches/variable/mod.rs @@ -3,17 +3,17 @@ use commonware_cryptography::{Hasher, Sha256}; use commonware_runtime::{buffer::PoolRef, create_pool, tokio::Context, ThreadPool}; use commonware_storage::{ - kv::{Batchable, Deletable, Updatable as _}, + kv::{Deletable as _, Updatable as _}, qmdb::{ any::{ - ordered::variable::Db as OVariable, unordered::variable::Db as UVariable, + ordered::variable::Db as OVariable, + states::{MutableAny, UnmerkleizedDurableAny}, + unordered::variable::Db as UVariable, VariableConfig as AConfig, }, - store::{Config as SConfig, LogStorePrunable, Store}, - Error, + store::LogStore, }, translator::EightCap, - Persistable, }; use commonware_utils::{NZUsize, NZU64}; use rand::{rngs::StdRng, RngCore, SeedableRng}; @@ -22,9 +22,10 @@ use std::num::{NonZeroU64, NonZeroUsize}; pub mod generate; pub mod init; +pub type Digest = ::Digest; + #[derive(Debug, Clone, Copy)] enum Variant { - Store, AnyUnordered, AnyOrdered, } @@ -32,14 +33,13 @@ enum Variant { impl Variant { pub const fn name(&self) -> &'static str { match self { - Self::Store => "store", Self::AnyUnordered => "any_unordered", Self::AnyOrdered => "any_ordered", } } } -const VARIANTS: [Variant; 3] = [Variant::Store, Variant::AnyUnordered, Variant::AnyOrdered]; +const VARIANTS: [Variant; 2] = [Variant::AnyUnordered, Variant::AnyOrdered]; const ITEMS_PER_BLOB: NonZeroU64 = NZU64!(50_000); const PARTITION_SUFFIX: &str = "any_variable_bench_partition"; @@ -60,21 +60,9 @@ const DELETE_FREQUENCY: u32 = 10; /// Default write buffer size. const WRITE_BUFFER_SIZE: NonZeroUsize = NZUsize!(1024); -type StoreDb = Store::Digest, Vec, EightCap>; -type UVariableDb = UVariable::Digest, Vec, Sha256, EightCap>; -type OVariableDb = OVariable::Digest, Vec, Sha256, EightCap>; - -fn store_cfg() -> SConfig, ())> { - SConfig::, ())> { - log_partition: format!("journal_{PARTITION_SUFFIX}"), - log_write_buffer: WRITE_BUFFER_SIZE, - log_compression: None, - log_codec_config: ((0..=10000).into(), ()), - log_items_per_section: ITEMS_PER_BLOB, - translator: EightCap, - buffer_pool: PoolRef::new(PAGE_SIZE, PAGE_CACHE_SIZE), - } -} +/// Clean (Merkleized, Durable) db type aliases. +type UVariableDb = UVariable, Sha256, EightCap>; +type OVariableDb = OVariable, Sha256, EightCap>; fn any_cfg(pool: ThreadPool) -> AConfig, ())> { AConfig::, ())> { @@ -93,21 +81,16 @@ fn any_cfg(pool: ThreadPool) -> AConfig StoreDb { - let store_cfg = store_cfg(); - Store::init(ctx, store_cfg).await.unwrap() -} - async fn get_any_unordered(ctx: Context) -> UVariableDb { let pool = create_pool(ctx.clone(), THREADS).unwrap(); let any_cfg = any_cfg(pool); - UVariable::init(ctx, any_cfg).await.unwrap() + UVariableDb::init(ctx, any_cfg).await.unwrap() } async fn get_any_ordered(ctx: Context) -> OVariableDb { let pool = create_pool(ctx.clone(), THREADS).unwrap(); let any_cfg = any_cfg(pool); - OVariable::init(ctx, any_cfg).await.unwrap() + OVariableDb::init(ctx, any_cfg).await.unwrap() } /// Generate a large db with random data. The function seeds the db with exactly `num_elements` @@ -115,54 +98,53 @@ async fn get_any_ordered(ctx: Context) -> OVariableDb { /// `num_operations` over these elements, each selected uniformly at random for each operation. The /// ratio of updates to deletes is configured with `DELETE_FREQUENCY`. The database is committed /// after every `commit_frequency` operations. -async fn gen_random_kv( - mut db: A, +/// +/// Takes a mutable database and returns it in durable state after final commit. +async fn gen_random_kv( + mut db: M, num_elements: u64, num_operations: u64, commit_frequency: u32, -) -> A +) -> M::Durable where - A: Deletable::Digest, Value = Vec, Error = Error> - + Persistable - + LogStorePrunable, + M: MutableAny + LogStore>, + M::Durable: UnmerkleizedDurableAny, { // Insert a random value for every possible element into the db. let mut rng = StdRng::seed_from_u64(42); for i in 0u64..num_elements { let k = Sha256::hash(&i.to_be_bytes()); let v = vec![(rng.next_u32() % 255) as u8; ((rng.next_u32() % 16) + 24) as usize]; - db.update(k, v).await.unwrap(); + assert!(db.update(k, v).await.is_ok()); } // Randomly update / delete them + randomly commit. for _ in 0u64..num_operations { let rand_key = Sha256::hash(&(rng.next_u64() % num_elements).to_be_bytes()); if rng.next_u32() % DELETE_FREQUENCY == 0 { - db.delete(rand_key).await.unwrap(); + assert!(db.delete(rand_key).await.is_ok()); continue; } let v = vec![(rng.next_u32() % 255) as u8; ((rng.next_u32() % 24) + 20) as usize]; - db.update(rand_key, v).await.unwrap(); + assert!(db.update(rand_key, v).await.is_ok()); if rng.next_u32() % commit_frequency == 0 { - db.commit().await.unwrap(); + let (durable, _) = db.commit(None).await.unwrap(); + db = durable.into_mutable(); } } - db.commit().await.unwrap(); - db.prune(db.inactivity_floor_loc()).await.unwrap(); - - db + let (durable, _) = db.commit(None).await.unwrap(); + durable } -async fn gen_random_kv_batched( - mut db: A, +async fn gen_random_kv_batched( + mut db: M, num_elements: u64, num_operations: u64, commit_frequency: u32, -) -> A +) -> M::Durable where - A: Batchable::Digest, Value = Vec> - + Persistable - + LogStorePrunable, + M: MutableAny + LogStore>, + M::Durable: UnmerkleizedDurableAny, { let mut rng = StdRng::seed_from_u64(42); let mut batch = db.start_batch(); @@ -170,32 +152,33 @@ where for i in 0u64..num_elements { let k = Sha256::hash(&i.to_be_bytes()); let v = vec![(rng.next_u32() % 255) as u8; ((rng.next_u32() % 16) + 24) as usize]; - batch.update(k, v).await.unwrap(); + assert!(batch.update(k, v).await.is_ok()); } let iter = batch.into_iter(); - db.write_batch(iter).await.unwrap(); + assert!(db.write_batch(iter).await.is_ok()); batch = db.start_batch(); for _ in 0u64..num_operations { let rand_key = Sha256::hash(&(rng.next_u64() % num_elements).to_be_bytes()); if rng.next_u32() % DELETE_FREQUENCY == 0 { - batch.delete(rand_key).await.unwrap(); + assert!(batch.delete(rand_key).await.is_ok()); continue; } let v = vec![(rng.next_u32() % 255) as u8; ((rng.next_u32() % 24) + 20) as usize]; - batch.update(rand_key, v).await.unwrap(); + assert!(batch.update(rand_key, v).await.is_ok()); if rng.next_u32() % commit_frequency == 0 { let iter = batch.into_iter(); - db.write_batch(iter).await.unwrap(); - db.commit().await.unwrap(); + assert!(db.write_batch(iter).await.is_ok()); + let (durable, _) = db.commit(None).await.unwrap(); + db = durable.into_mutable(); batch = db.start_batch(); } } let iter = batch.into_iter(); - db.write_batch(iter).await.unwrap(); - db.commit().await.unwrap(); - db.prune(db.inactivity_floor_loc()).await.unwrap(); - - db + db.write_batch(iter) + .await + .expect("write_batch shouldn't fail"); + let (durable, _) = db.commit(None).await.expect("commit shouldn't fail"); + durable } diff --git a/storage/src/qmdb/current/ordered/fixed.rs b/storage/src/qmdb/current/ordered/fixed.rs index 15fad6482e..8484a8c05c 100644 --- a/storage/src/qmdb/current/ordered/fixed.rs +++ b/storage/src/qmdb/current/ordered/fixed.rs @@ -7,18 +7,18 @@ //! //! See [Db] for the main database type and [ExclusionProof] for proving key inactivity. +#[cfg(any(test, feature = "test-traits"))] +use crate::qmdb::any::states::{ + CleanAny, MerkleizedNonDurableAny, MutableAny, UnmerkleizedDurableAny, +}; use crate::{ - bitmap::{CleanBitMap, DirtyBitMap}, + bitmap::CleanBitMap, kv::{self, Batchable}, - mmr::{ - grafting::Storage as GraftingStorage, - mem::{Clean, Dirty, State}, - Location, Proof, StandardHasher, - }, + mmr::{grafting::Storage as GraftingStorage, Location, Proof, StandardHasher}, qmdb::{ any::{ ordered::fixed::{Db as AnyDb, Operation, Update}, - CleanAny, DirtyAny, FixedValue, + FixedValue, }, current::{ merkleize_grafted_bitmap, @@ -26,11 +26,12 @@ use crate::{ proof::{OperationProof, RangeProof}, root, FixedConfig as Config, }, - store::{CleanStore, DirtyStore, LogStore}, - Error, + store, + store::{LogStore, MerkleizedStore, PrunableStore}, + DurabilityState, Durable, Error, MerkleizationState, Merkleized, NonDurable, Unmerkleized, }, translator::Translator, - AuthenticatedBitMap as BitMap, + AuthenticatedBitMap as BitMap, Persistable, }; use commonware_codec::FixedSize; use commonware_cryptography::{Digest, DigestOf, Hasher}; @@ -53,15 +54,16 @@ pub struct Db< H: Hasher, T: Translator, const N: usize, - S: State> = Clean>, + M: MerkleizationState> = Merkleized, + D: DurabilityState = Durable, > { /// An authenticated database that provides the ability to prove whether a key ever had a /// specific value. - any: AnyDb, + any: AnyDb, /// The bitmap over the activity status of each operation. Supports augmenting [Db] proofs in /// order to further prove whether a key _currently_ has a specific value. - status: BitMap, + status: BitMap, context: E, @@ -78,6 +80,7 @@ pub struct KeyValueProof { pub next_key: K, } +// Functionality shared across all DB states, such as most non-mutating operations. impl< E: RStorage + Clock + Metrics, K: Array, @@ -85,8 +88,9 @@ impl< H: Hasher, T: Translator, const N: usize, - S: State>, - > Db + M: MerkleizationState>, + D: DurabilityState, + > Db { /// The number of operations that have been applied to this db, including those that have been /// pruned and those that are not yet committed. @@ -172,7 +176,11 @@ impl< // The provided `key` is in the DB if it matches the start of the span. return false; } - if !AnyDb::::span_contains(&data.key, &data.next_key, key) { + if !AnyDb::, Durable>::span_contains( + &data.key, + &data.next_key, + key, + ) { // If the key is not within the span, then this proof cannot prove its // exclusion. return false; @@ -194,8 +202,24 @@ impl< op_proof.verify(hasher, Self::grafting_height(), op, root) } + + /// Return true if the given sequence of `ops` were applied starting at location `start_loc` in + /// the log with the provided root. + pub fn verify_range_proof( + hasher: &mut H, + proof: &RangeProof, + start_loc: Location, + ops: &[Operation], + chunks: &[[u8; N]], + root: &H::Digest, + ) -> bool { + let height = Self::grafting_height(); + + proof.verify(hasher, height, start_loc, ops, chunks, root) + } } +// Functionality for the Clean state. impl< E: RStorage + Clock + Metrics, K: Array, @@ -203,7 +227,7 @@ impl< H: Hasher, T: Translator, const N: usize, - > Db + > Db, Durable> { /// Initializes a [Db] from the given `config`. Leverages parallel Merkleization to initialize /// the bitmap MMR if a thread pool is provided. @@ -270,6 +294,53 @@ impl< self.cached_root.expect("Clean state must have cached root") } + /// Sync all database state to disk. + pub async fn sync(&mut self) -> Result<(), Error> { + self.any.sync().await?; + + // Write the bitmap pruning boundary to disk so that next startup doesn't have to + // re-Merkleize the inactive portion up to the inactivity floor. + self.status + .write_pruned( + self.context.with_label("bitmap"), + &self.bitmap_metadata_partition, + ) + .await + .map_err(Into::into) + } + + /// Destroy the db, removing all data from disk. + pub async fn destroy(self) -> Result<(), Error> { + // Clean up bitmap metadata partition. + CleanBitMap::::destroy(self.context, &self.bitmap_metadata_partition).await?; + + // Clean up Any components (MMR and log). + self.any.destroy().await + } + + /// Transition into the mutable state. + pub fn into_mutable(self) -> Db { + Db { + any: self.any.into_mutable(), + status: self.status.into_dirty(), + context: self.context, + bitmap_metadata_partition: self.bitmap_metadata_partition, + cached_root: None, + } + } +} + +// Functionality for any Merkleized state (both Durable and NonDurable). +impl< + E: RStorage + Clock + Metrics, + K: Array, + V: FixedValue, + H: Hasher, + T: Translator, + const N: usize, + D: store::State, + > Db, D> +{ /// Returns a proof that the specified range of operations are part of the database, along with /// the operations from the range. A truncated range (from hitting the max) can be detected by /// looking at the length of the returned operations vector. Also returns the bitmap chunks @@ -297,21 +368,6 @@ impl< .await } - /// Return true if the given sequence of `ops` were applied starting at location `start_loc` in - /// the log with the provided root. - pub fn verify_range_proof( - hasher: &mut H, - proof: &RangeProof, - start_loc: Location, - ops: &[Operation], - chunks: &[[u8; N]], - root: &H::Digest, - ) -> bool { - let height = Self::grafting_height(); - - proof.verify(hasher, height, start_loc, ops, chunks, root) - } - /// Generate and return a proof of the current value of `key`, along with the other /// [KeyValueProof] required to verify the proof. Returns KeyNotFound error if the key is not /// currently assigned any value. @@ -384,106 +440,6 @@ impl< }) } - /// Destroy the db, removing all data from disk. - pub async fn destroy(self) -> Result<(), Error> { - // Clean up bitmap metadata partition. - CleanBitMap::::destroy(self.context, &self.bitmap_metadata_partition).await?; - - // Clean up Any components (MMR and log). - self.any.destroy().await - } - - #[cfg(test)] - /// Simulate a crash that prevents any data from being written to disk, which involves simply - /// consuming the db before it can be cleanly closed. - fn simulate_commit_failure_before_any_writes(self) { - // Don't successfully complete any of the commit operations. - } - - #[cfg(test)] - /// Simulate a crash that happens during commit and prevents the any db from being pruned of - /// inactive operations, and bitmap state from being written/pruned. - async fn simulate_commit_failure_after_any_db_commit(mut self) -> Result<(), Error> { - // Only successfully complete the log write part of the commit process. - let _ = self.commit_to_log(None).await?; - Ok(()) - } - - /// Helper that performs the commit operations up to and including writing to the log, - /// but does not merkleize the bitmap or prune. Used for simulating partial commit failures - /// in tests, and as the first phase of the full commit operation. - /// - /// Returns the dirty bitmap that needs to be merkleized and pruned. - async fn commit_to_log( - &mut self, - metadata: Option, - ) -> Result, Error> { - let empty_status = CleanBitMap::::new(&mut self.any.log.hasher, None); - let mut status = std::mem::replace(&mut self.status, empty_status).into_dirty(); - - // Inactivate the current commit operation. - status.set_bit(*self.any.last_commit_loc, false); - - // Raise the inactivity floor by taking `self.steps` steps, plus 1 to account for the - // previous commit becoming inactive. - let inactivity_floor_loc = self.any.raise_floor_with_bitmap(&mut status).await?; - - // Append the commit operation with the new floor and tag it as active in the bitmap. - status.push(true); - let commit_op = Operation::CommitFloor(metadata, inactivity_floor_loc); - - self.any.apply_commit_op(commit_op).await?; - - Ok(status) - } - - /// Commit any pending operations to the database, ensuring their durability upon return from - /// this function. Also raises the inactivity floor according to the schedule. Returns the - /// `(start_loc, end_loc]` location range of committed operations. - pub async fn commit(&mut self, metadata: Option) -> Result, Error> { - let start_loc = self.any.last_commit_loc + 1; - - // Commit to log (recovery is ensured after this returns) - let status = self.commit_to_log(metadata).await?; - - // Merkleize the new bitmap entries. - let height = Self::grafting_height(); - self.status = - merkleize_grafted_bitmap(&mut self.any.log.hasher, status, &self.any.log.mmr, height) - .await?; - - // Prune bits that are no longer needed because they precede the inactivity floor. - self.status.prune_to_bit(*self.any.inactivity_floor_loc())?; - - // Refresh cached root after commit - self.cached_root = Some( - root( - &mut self.any.log.hasher, - height, - &self.status, - &self.any.log.mmr, - ) - .await?, - ); - - Ok(start_loc..self.op_count()) - } - - /// Sync all database state to disk. - pub async fn sync(&mut self) -> Result<(), Error> { - self.any.sync().await?; - - // Write the bitmap pruning boundary to disk so that next startup doesn't have to - // re-Merkleize the inactive portion up to the inactivity floor. - self.status - .write_pruned( - self.context.with_label("bitmap"), - &self.bitmap_metadata_partition, - ) - .await - .map_err(Into::into) - } - /// Prune historical operations prior to `prune_loc`. This does not affect the db's root /// or current snapshot. pub async fn prune(&mut self, prune_loc: Location) -> Result<(), Error> { @@ -499,19 +455,9 @@ impl< self.any.prune(prune_loc).await } - - /// Convert this clean database into its dirty counterpart for performing mutations. - pub fn into_dirty(self) -> Db { - Db { - any: self.any.into_dirty(), - status: self.status.into_dirty(), - context: self.context, - bitmap_metadata_partition: self.bitmap_metadata_partition, - cached_root: None, - } - } } +// Functionality for the Mutable state. impl< E: RStorage + Clock + Metrics, K: Array, @@ -519,7 +465,7 @@ impl< H: Hasher, T: Translator, const N: usize, - > Db + > Db { /// Updates `key` to have value `value`. The operation is reflected in the snapshot, but will be /// subject to rollback until the next successful `commit`. @@ -566,24 +512,90 @@ impl< Ok(r) } - /// Merkleize the bitmap and convert this dirty database into its clean counterpart. - /// This computes the Merkle tree over any new bitmap entries but does NOT persist - /// changes to storage. Use `commit()` for durable state transitions. - pub async fn merkleize(self) -> Result>>, Error> { - // First merkleize the any to get a Clean MMR - let clean_any = self.any.merkleize(); + /// Commit any pending operations to the database, ensuring their durability upon return from + /// this function. Also raises the inactivity floor according to the schedule. Returns the + /// `[start_loc, end_loc)` location range of committed operations. + async fn apply_commit_op(&mut self, metadata: Option) -> Result, Error> { + let start_loc = self.any.last_commit_loc + 1; - // Now use the clean MMR for bitmap merkleization + // Inactivate the current commit operation. + self.status.set_bit(*self.any.last_commit_loc, false); + + // Raise the inactivity floor by taking `self.steps` steps, plus 1 to account for the + // previous commit becoming inactive. + let inactivity_floor_loc = self.any.raise_floor_with_bitmap(&mut self.status).await?; + + // Append the commit operation with the new floor and tag it as active in the bitmap. + self.status.push(true); + let commit_op = Operation::CommitFloor(metadata, inactivity_floor_loc); + + self.any.apply_commit_op(commit_op).await?; + + Ok(start_loc..self.op_count()) + } + + /// Commit any pending operations to the database, ensuring their durability upon return. + /// This transitions to the Durable state without merkleizing. Returns the committed database + /// and the `[start_loc, end_loc)` range of committed operations. + pub async fn commit( + mut self, + metadata: Option, + ) -> Result<(Db, Range), Error> { + let range = self.apply_commit_op(metadata).await?; + + // Transition to Durable state without merkleizing + let any = AnyDb { + log: self.any.log, + inactivity_floor_loc: self.any.inactivity_floor_loc, + last_commit_loc: self.any.last_commit_loc, + snapshot: self.any.snapshot, + durable_state: store::Durable, + active_keys: self.any.active_keys, + _update: core::marker::PhantomData, + }; + + Ok(( + Db { + any, + status: self.status, + context: self.context, + bitmap_metadata_partition: self.bitmap_metadata_partition, + cached_root: None, // Not merkleized yet + }, + range, + )) + } + + /// Merkleize the database and transition to the provable state without committing. + /// This enables proof generation while keeping the database in the non-durable state. + pub async fn into_merkleized( + self, + ) -> Result, NonDurable>, Error> { + // Merkleize the any db's log + let any = AnyDb { + log: self.any.log.merkleize(), + inactivity_floor_loc: self.any.inactivity_floor_loc, + last_commit_loc: self.any.last_commit_loc, + snapshot: self.any.snapshot, + durable_state: self.any.durable_state, + active_keys: self.any.active_keys, + _update: core::marker::PhantomData, + }; + + // Merkleize the bitmap using the clean MMR let mut hasher = StandardHasher::::new(); - let height = Self::grafting_height(); - let status = - merkleize_grafted_bitmap(&mut hasher, self.status, &clean_any.log.mmr, height).await?; + let height = Db::, NonDurable>::grafting_height(); + let mut status = + merkleize_grafted_bitmap(&mut hasher, self.status, &any.log.mmr, height).await?; + + // Prune the bitmap of no-longer-necessary bits. + status.prune_to_bit(*any.inactivity_floor_loc)?; // Compute and cache the root - let cached_root = Some(root(&mut hasher, height, &status, &clean_any.log.mmr).await?); + let cached_root = Some(root(&mut hasher, height, &status, &any.log.mmr).await?); Ok(Db { - any: clean_any, + any, status, context: self.context, bitmap_metadata_partition: self.bitmap_metadata_partition, @@ -592,6 +604,7 @@ impl< } } +// Functionality for (Merkleized, NonDurable) state. impl< E: RStorage + Clock + Metrics, K: Array, @@ -599,13 +612,37 @@ impl< H: Hasher, T: Translator, const N: usize, - > crate::qmdb::store::LogStorePrunable for Db + > Db, NonDurable> { - async fn prune(&mut self, prune_loc: Location) -> Result<(), Error> { - self.prune(prune_loc).await + /// Transition into the mutable state. + pub fn into_mutable(self) -> Db { + Db { + any: self.any.into_mutable(), + status: self.status.into_dirty(), + context: self.context, + bitmap_metadata_partition: self.bitmap_metadata_partition, + cached_root: None, + } + } + + /// Commit any pending operations to the database, ensuring their durability upon return. + /// Returns the committed database and the range of committed operations. + pub async fn commit( + self, + metadata: Option, + ) -> Result< + ( + Db, Durable>, + Range, + ), + Error, + > { + let (durable, range) = self.into_mutable().commit(metadata).await?; + Ok((durable.into_merkleized().await?, range)) } } +// Functionality for (Unmerkleized, Durable) state. impl< E: RStorage + Clock + Metrics, K: Array, @@ -613,8 +650,67 @@ impl< H: Hasher, T: Translator, const N: usize, - S: State>, - > LogStore for Db + > Db +{ + /// Merkleize the database, transitioning to the provable state. + pub async fn into_merkleized( + self, + ) -> Result, Durable>, Error> { + // Merkleize the any db's log + let any = AnyDb { + log: self.any.log.merkleize(), + inactivity_floor_loc: self.any.inactivity_floor_loc, + last_commit_loc: self.any.last_commit_loc, + snapshot: self.any.snapshot, + durable_state: self.any.durable_state, + active_keys: self.any.active_keys, + _update: core::marker::PhantomData, + }; + + // Merkleize the bitmap using the clean MMR + let mut hasher = StandardHasher::::new(); + let height = Db::, Durable>::grafting_height(); + let mut status = + merkleize_grafted_bitmap(&mut hasher, self.status, &any.log.mmr, height).await?; + + // Prune the bitmap of no-longer-necessary bits. + status.prune_to_bit(*any.inactivity_floor_loc)?; + + // Compute and cache the root + let cached_root = Some(root(&mut hasher, height, &status, &any.log.mmr).await?); + + Ok(Db { + any, + status, + context: self.context, + bitmap_metadata_partition: self.bitmap_metadata_partition, + cached_root, + }) + } + + /// Transition into the mutable state. + pub fn into_mutable(self) -> Db { + Db { + any: self.any.into_mutable(), + status: self.status, + context: self.context, + bitmap_metadata_partition: self.bitmap_metadata_partition, + cached_root: None, + } + } +} + +// LogStore implementation for all states. +impl< + E: RStorage + Clock + Metrics, + K: Array, + V: FixedValue, + H: Hasher, + T: Translator, + const N: usize, + M: MerkleizationState>, + D: DurabilityState, + > LogStore for Db { type Value = V; @@ -635,6 +731,7 @@ impl< } } +// Store implementation for all states impl< E: RStorage + Clock + Metrics, K: Array, @@ -642,8 +739,9 @@ impl< H: Hasher, T: Translator, const N: usize, - S: State>, - > kv::Gettable for Db + M: MerkleizationState>, + D: DurabilityState, + > kv::Gettable for Db { type Key = K; type Value = V; @@ -654,6 +752,7 @@ impl< } } +// StoreMut for (Unmerkleized, NonDurable) (aka mutable) state impl< E: RStorage + Clock + Metrics, K: Array, @@ -661,13 +760,14 @@ impl< H: Hasher, T: Translator, const N: usize, - > kv::Updatable for Db + > kv::Updatable for Db { async fn update(&mut self, key: Self::Key, value: Self::Value) -> Result<(), Self::Error> { self.update(key, value).await } } +// StoreDeletable for (Unmerkleized, NonDurable) (aka mutable) state impl< E: RStorage + Clock + Metrics, K: Array, @@ -675,13 +775,42 @@ impl< H: Hasher, T: Translator, const N: usize, - > kv::Deletable for Db + > kv::Deletable for Db { async fn delete(&mut self, key: Self::Key) -> Result { self.delete(key).await } } +// Batchable for (Unmerkleized, NonDurable) (aka mutable) state +impl Batchable for Db +where + E: RStorage + Clock + Metrics, + K: Array, + V: FixedValue, + T: Translator, + H: Hasher, +{ + async fn write_batch( + &mut self, + iter: impl Iterator)>, + ) -> Result<(), Error> { + let status = &mut self.status; + self.any + .write_batch_with_callback(iter, move |append: bool, loc: Option| { + status.push(append); + if let Some(loc) = loc { + status.set_bit(*loc, false); + } + }) + .await + } +} + +// MerkleizedStore for Merkleized states (both Durable and NonDurable) +// TODO(https://github.com/commonwarexyz/monorepo/issues/2560): This is broken -- it's computing +// proofs only over the any db mmr not the grafted mmr, so they won't validate against the grafted +// root. impl< E: RStorage + Clock + Metrics, K: Array, @@ -689,22 +818,15 @@ impl< H: Hasher, T: Translator, const N: usize, - > CleanStore for Db>> + D: store::State, + > MerkleizedStore for Db, D> { type Digest = H::Digest; type Operation = Operation; - type Dirty = Db; fn root(&self) -> Self::Digest { - self.root() - } - - async fn proof( - &self, - start_loc: Location, - max_ops: NonZeroU64, - ) -> Result<(Proof, Vec), Error> { - self.any.proof(start_loc, max_ops).await + self.cached_root + .expect("Merkleized state must have cached root") } async fn historical_proof( @@ -717,36 +839,52 @@ impl< .historical_proof(historical_size, start_loc, max_ops) .await } +} - fn into_dirty(self) -> Self::Dirty { - self.into_dirty() +// PrunableStore for Merkleized states (both Durable and NonDurable) +impl< + E: RStorage + Clock + Metrics, + K: Array, + V: FixedValue, + H: Hasher, + T: Translator, + const N: usize, + D: DurabilityState, + > PrunableStore for Db, D> +{ + async fn prune(&mut self, prune_loc: Location) -> Result<(), Error> { + self.prune(prune_loc).await } } -impl Batchable for Db -where - E: RStorage + Clock + Metrics, - K: Array, - V: FixedValue, - T: Translator, - H: Hasher, +// Persistable for Clean state +impl< + E: RStorage + Clock + Metrics, + K: Array, + V: FixedValue, + H: Hasher, + T: Translator, + const N: usize, + > Persistable for Db, Durable> { - async fn write_batch( - &mut self, - iter: impl Iterator)>, - ) -> Result<(), Error> { - let status = &mut self.status; - self.any - .write_batch_with_callback(iter, move |append: bool, loc: Option| { - status.push(append); - if let Some(loc) = loc { - status.set_bit(*loc, false); - } - }) - .await + type Error = Error; + + async fn commit(&mut self) -> Result<(), Self::Error> { + // No-op, DB already recoverable. + Ok(()) + } + + async fn sync(&mut self) -> Result<(), Self::Error> { + self.sync().await + } + + async fn destroy(self) -> Result<(), Self::Error> { + self.destroy().await } } +// CleanAny trait implementation +#[cfg(any(test, feature = "test-traits"))] impl< E: RStorage + Clock + Metrics, K: Array, @@ -754,17 +892,17 @@ impl< H: Hasher, T: Translator, const N: usize, - > DirtyStore for Db + > CleanAny for Db, Durable> { - type Digest = H::Digest; - type Operation = Operation; - type Clean = Db>>; + type Mutable = Db; - async fn merkleize(self) -> Result { - self.merkleize().await + fn into_mutable(self) -> Self::Mutable { + self.into_mutable() } } +// UnmerkleizedDurableAny trait implementation +#[cfg(any(test, feature = "test-traits"))] impl< E: RStorage + Clock + Metrics, K: Array, @@ -772,31 +910,47 @@ impl< H: Hasher, T: Translator, const N: usize, - > CleanAny for Db>> + > UnmerkleizedDurableAny for Db { - type Key = K; + type Digest = H::Digest; + type Operation = Operation; + type Mutable = Db; + type Merkleized = Db, Durable>; - async fn get(&self, key: &Self::Key) -> Result, Error> { - self.get(key).await + fn into_mutable(self) -> Self::Mutable { + self.into_mutable() } - async fn commit(&mut self, metadata: Option) -> Result, Error> { - self.commit(metadata).await + async fn into_merkleized(self) -> Result { + self.into_merkleized().await } +} - async fn sync(&mut self) -> Result<(), Error> { - self.sync().await - } +// MerkleizedNonDurableAny trait implementation +#[cfg(any(test, feature = "test-traits"))] +impl< + E: RStorage + Clock + Metrics, + K: Array, + V: FixedValue, + H: Hasher, + T: Translator, + const N: usize, + > MerkleizedNonDurableAny for Db, NonDurable> +{ + type Mutable = Db; + type Durable = Db, Durable>; - async fn prune(&mut self, prune_loc: Location) -> Result<(), Error> { - self.prune(prune_loc).await + async fn commit(self, metadata: Option) -> Result<(Self::Durable, Range), Error> { + self.commit(metadata).await } - async fn destroy(self) -> Result<(), Error> { - self.destroy().await + fn into_mutable(self) -> Self::Mutable { + self.into_mutable() } } +// MutableAny trait implementation +#[cfg(any(test, feature = "test-traits"))] impl< E: RStorage + Clock + Metrics, K: Array, @@ -804,24 +958,23 @@ impl< H: Hasher, T: Translator, const N: usize, - > DirtyAny for Db + > MutableAny for Db { - type Key = K; - - async fn get(&self, key: &Self::Key) -> Result, Error> { - self.get(key).await - } + type Digest = H::Digest; + type Operation = Operation; + type Merkleized = Db, NonDurable>; + type Durable = Db; - async fn update(&mut self, key: Self::Key, value: Self::Value) -> Result<(), Error> { - self.update(key, value).await + async fn commit(self, metadata: Option) -> Result<(Self::Durable, Range), Error> { + self.commit(metadata).await } - async fn create(&mut self, key: Self::Key, value: Self::Value) -> Result { - self.create(key, value).await + async fn into_merkleized(self) -> Result { + self.into_merkleized().await } - async fn delete(&mut self, key: Self::Key) -> Result { - self.delete(key).await + fn steps(&self) -> u64 { + self.any.durable_state.steps } } @@ -829,9 +982,7 @@ impl< pub mod test { use super::*; use crate::{ - index::Unordered as _, - mmr::hasher::Hasher as _, - qmdb::{any::AnyExt, store::batch_tests}, + index::Unordered as _, mmr::hasher::Hasher as _, qmdb::store::batch_tests, translator::OneCap, }; use commonware_cryptography::{sha256::Digest, Sha256}; @@ -861,11 +1012,13 @@ pub mod test { } } - /// A type alias for the concrete [Db] type used in these unit tests. - type CleanCurrentTest = Db; + /// A type alias for the concrete [Db] type used in these unit tests (Merkleized, Durable state). + type CleanCurrentTest = + Db, Durable>; - /// A type alias for the Dirty variant of CurrentTest. - type DirtyCurrentTest = Db; + /// A type alias for the Mutable variant of CurrentTest (Unmerkleized, NonDurable state). + type MutableCurrentTest = + Db; /// Return an [Db] database initialized with a fixed config. async fn open_db(context: deterministic::Context, partition_prefix: &str) -> CleanCurrentTest { @@ -893,11 +1046,11 @@ pub mod test { // Add one key. let k1 = Sha256::hash(&0u64.to_be_bytes()); let v1 = Sha256::hash(&10u64.to_be_bytes()); - let mut db = db.into_dirty(); + let mut db = db.into_mutable(); assert!(db.create(k1, v1).await.unwrap()); assert_eq!(db.get(&k1).await.unwrap().unwrap(), v1); - let mut db = db.merkleize().await.unwrap(); - db.commit(None).await.unwrap(); + let (db, _) = db.commit(None).await.unwrap(); + let db = db.into_merkleized().await.unwrap(); assert_eq!(db.op_count(), 4); // 1 update, 1 commit, 1 move + 1 initial commit. assert!(db.get_metadata().await.unwrap().is_none()); let root1 = db.root(); @@ -909,15 +1062,15 @@ pub mod test { assert_eq!(db.root(), root1); // Create of same key should fail. - let mut db = db.into_dirty(); + let mut db = db.into_mutable(); assert!(!db.create(k1, v1).await.unwrap()); // Delete that one key. assert!(db.delete(k1).await.unwrap()); let metadata = Sha256::hash(&1u64.to_be_bytes()); - let mut db = db.merkleize().await.unwrap(); - db.commit(Some(metadata)).await.unwrap(); + let (db, _) = db.commit(Some(metadata)).await.unwrap(); + let db = db.into_merkleized().await.unwrap(); assert_eq!(db.op_count(), 6); // 1 update, 2 commits, 1 move, 1 delete. assert_eq!(db.get_metadata().await.unwrap().unwrap(), metadata); assert_eq!(db.inactivity_floor_loc(), Location::new_unchecked(5)); @@ -931,9 +1084,10 @@ pub mod test { assert_eq!(db.root(), root2); // Repeated delete of same key should fail. - let mut db = db.into_dirty(); + let mut db = db.into_mutable(); assert!(!db.delete(k1).await.unwrap()); - let db = db.merkleize().await.unwrap(); + let (db, _) = db.commit(None).await.unwrap(); + let db = db.into_merkleized().await.unwrap(); // Confirm all activity bits except the last are false. for i in 0..*db.op_count() - 1 { @@ -952,7 +1106,7 @@ pub mod test { // confirm that the end state of the db matches that of an identically updated hashmap. const ELEMENTS: u64 = 1000; executor.start(|context| async move { - let mut db = open_db(context.clone(), "build_big").await.into_dirty(); + let mut db = open_db(context.clone(), "build_big").await.into_mutable(); let mut map = HashMap::::default(); for i in 0u64..ELEMENTS { @@ -988,8 +1142,8 @@ pub mod test { assert_eq!(db.any.snapshot.items(), 857); // Test that commit + sync w/ pruning will raise the activity floor. - let mut db = db.merkleize().await.unwrap(); - db.commit(None).await.unwrap(); + let (db, _) = db.commit(None).await.unwrap(); + let mut db = db.into_merkleized().await.unwrap(); db.sync().await.unwrap(); db.prune(db.inactivity_floor_loc()).await.unwrap(); assert_eq!(db.op_count(), 4241); @@ -1020,6 +1174,16 @@ pub mod test { }); } + // Test that merkleization state changes don't reset `steps`. + #[test_traced("DEBUG")] + fn test_current_ordered_fixed_db_steps_not_reset() { + let executor = deterministic::Runner::default(); + executor.start(|context| async move { + let db = open_db(context, "steps_test").await; + crate::qmdb::any::test::test_any_db_steps_not_reset(db).await; + }); + } + /// Build a tiny database and make sure we can't convince the verifier that some old value of a /// key is active. We specifically test over the partial chunk case, since these bits are yet to /// be committed to the underlying MMR. @@ -1029,14 +1193,14 @@ pub mod test { executor.start(|context| async move { let mut hasher = StandardHasher::::new(); let partition = "build_small"; - let mut db = open_db(context.clone(), partition).await.into_dirty(); + let mut db = open_db(context.clone(), partition).await.into_mutable(); // Add one key. let k = Sha256::fill(0x01); let v1 = Sha256::fill(0xA1); db.update(k, v1).await.unwrap(); - let mut db = db.merkleize().await.unwrap(); - db.commit(None).await.unwrap(); + let (db, _) = db.commit(None).await.unwrap(); + let db = db.into_merkleized().await.unwrap(); let (_, op_loc) = db.any.get_with_loc(&k).await.unwrap().unwrap(); let proof = db.key_value_proof(hasher.inner(), k).await.unwrap(); @@ -1072,10 +1236,10 @@ pub mod test { )); // Update the key to a new value (v2), which inactivates the previous operation. - let mut db = db.into_dirty(); + let mut db = db.into_mutable(); db.update(k, v2).await.unwrap(); - let mut db = db.merkleize().await.unwrap(); - db.commit(None).await.unwrap(); + let (db, _) = db.commit(None).await.unwrap(); + let db = db.into_merkleized().await.unwrap(); let root = db.root(); // New value should not be verifiable against the old proof. @@ -1189,13 +1353,13 @@ pub mod test { } /// Apply random operations to the given db, committing them (randomly & at the end) only if - /// `commit_changes` is true. + /// `commit_changes` is true. Returns a mutable db; callers should commit if needed. async fn apply_random_ops( num_elements: u64, commit_changes: bool, rng_seed: u64, - mut db: DirtyCurrentTest, - ) -> Result { + mut db: MutableCurrentTest, + ) -> Result { // Log the seed with high visibility to make failures reproducible. warn!("rng_seed={}", rng_seed); let mut rng = StdRng::seed_from_u64(rng_seed); @@ -1218,18 +1382,17 @@ pub mod test { db.update(rand_key, v).await.unwrap(); if commit_changes && rng.next_u32() % 20 == 0 { // Commit every ~20 updates. - let mut clean_db = db.merkleize().await?; - clean_db.commit(None).await?; - db = clean_db.into_dirty(); + let (durable_db, _) = db.commit(None).await?; + let clean_db = durable_db.into_merkleized().await?; + db = clean_db.into_mutable(); } } if commit_changes { - let mut clean_db = db.merkleize().await?; - clean_db.commit(None).await?; - Ok(clean_db) - } else { - db.merkleize().await + let (durable_db, _) = db.commit(None).await?; + let clean_db = durable_db.into_merkleized().await?; + db = clean_db.into_mutable(); } + Ok(db) } #[test_traced("DEBUG")] @@ -1256,9 +1419,11 @@ pub mod test { &root, )); - let db = apply_random_ops(200, true, context.next_u64(), db.into_dirty()) + let db = apply_random_ops(200, true, context.next_u64(), db.into_mutable()) .await .unwrap(); + let (db, _) = db.commit(None).await.unwrap(); + let db = db.into_merkleized().await.unwrap(); let root = db.root(); // Make sure size-constrained batches of operations are provable from the oldest @@ -1307,10 +1472,12 @@ pub mod test { executor.start(|mut context| async move { let partition = "range_proofs"; let mut hasher = StandardHasher::::new(); - let db = open_db(context.clone(), partition).await.into_dirty(); + let db = open_db(context.clone(), partition).await.into_mutable(); let db = apply_random_ops(500, true, context.next_u64(), db) .await .unwrap(); + let (db, _) = db.commit(None).await.unwrap(); + let db = db.into_merkleized().await.unwrap(); let root = db.root(); // Confirm bad keys produce the expected error. @@ -1395,10 +1562,12 @@ pub mod test { executor.start(|mut context| async move { let partition = "build_random"; let rng_seed = context.next_u64(); - let db = open_db(context.clone(), partition).await.into_dirty(); - let mut db = apply_random_ops(ELEMENTS, true, rng_seed, db) + let db = open_db(context.clone(), partition).await.into_mutable(); + let db = apply_random_ops(ELEMENTS, true, rng_seed, db) .await .unwrap(); + let (db, _) = db.commit(None).await.unwrap(); + let mut db = db.into_merkleized().await.unwrap(); db.sync().await.unwrap(); // Drop and reopen the db @@ -1428,11 +1597,12 @@ pub mod test { let mut old_val = Sha256::fill(0x00); for i in 1u8..=255 { let v = Sha256::fill(i); - let mut dirty_db = db.into_dirty(); + let mut dirty_db = db.into_mutable(); dirty_db.update(k, v).await.unwrap(); assert_eq!(dirty_db.get(&k).await.unwrap().unwrap(), v); - db = dirty_db.merkleize().await.unwrap(); - db.commit(None).await.unwrap(); + let (dirty_db, _) = dirty_db.commit(None).await.unwrap(); + let clean_db = dirty_db.into_merkleized().await.unwrap(); + db = clean_db; let root = db.root(); // Create a proof for the current value of k. @@ -1470,37 +1640,41 @@ pub mod test { executor.start(|mut context| async move { let partition = "build_random_fail_commit"; let rng_seed = context.next_u64(); - let db = open_db(context.clone(), partition).await.into_dirty(); - let mut db = apply_random_ops(ELEMENTS, true, rng_seed, db) + let db = open_db(context.clone(), partition).await.into_mutable(); + let db = apply_random_ops(ELEMENTS, true, rng_seed, db) .await .unwrap(); + let (db, _) = db.commit(None).await.unwrap(); + let mut db = db.into_merkleized().await.unwrap(); let committed_root = db.root(); let committed_op_count = db.op_count(); let committed_inactivity_floor = db.any.inactivity_floor_loc(); db.prune(committed_inactivity_floor).await.unwrap(); // Perform more random operations without committing any of them. - let db = apply_random_ops(ELEMENTS, false, rng_seed + 1, db.into_dirty()) + let db = apply_random_ops(ELEMENTS, false, rng_seed + 1, db.into_mutable()) .await .unwrap(); // SCENARIO #1: Simulate a crash that happens before any writes. Upon reopening, the // state of the DB should be as of the last commit. - db.simulate_commit_failure_before_any_writes(); + drop(db); let db = open_db(context.clone(), partition).await; assert_eq!(db.root(), committed_root); assert_eq!(db.op_count(), committed_op_count); // Re-apply the exact same uncommitted operations. - let db = apply_random_ops(ELEMENTS, false, rng_seed + 1, db.into_dirty()) + let db = apply_random_ops(ELEMENTS, false, rng_seed + 1, db.into_mutable()) .await .unwrap(); // SCENARIO #2: Simulate a crash that happens after the any db has been committed, but - // before the state of the pruned bitmap can be written to disk. - db.simulate_commit_failure_after_any_db_commit() - .await - .unwrap(); + // before the state of the pruned bitmap can be written to disk (i.e., before + // into_merkleized is called). We do this by committing and then dropping the durable + // db without calling close or into_merkleized. + let (durable_db, _) = db.commit(None).await.unwrap(); + let committed_op_count = durable_db.op_count(); + drop(durable_db); // We should be able to recover, so the root should differ from the previous commit, and // the op count should be greater than before. @@ -1510,17 +1684,21 @@ pub mod test { // To confirm the second committed hash is correct we'll re-build the DB in a new // partition, but without any failures. They should have the exact same state. let fresh_partition = "build_random_fail_commit_fresh"; - let db = open_db(context.clone(), fresh_partition).await.into_dirty(); + let db = open_db(context.clone(), fresh_partition) + .await + .into_mutable(); let db = apply_random_ops(ELEMENTS, true, rng_seed, db) .await .unwrap(); - let db = apply_random_ops(ELEMENTS, false, rng_seed + 1, db.into_dirty()) + let (db, _) = db.commit(None).await.unwrap(); + let db = apply_random_ops(ELEMENTS, false, rng_seed + 1, db.into_mutable()) .await .unwrap(); - let mut db = db.into_dirty().merkleize().await.unwrap(); - db.commit(None).await.unwrap(); + let (db, _) = db.commit(None).await.unwrap(); + let mut db = db.into_merkleized().await.unwrap(); db.prune(db.any.inactivity_floor_loc()).await.unwrap(); // State from scenario #2 should match that of a successful commit. + assert_eq!(db.op_count(), committed_op_count); assert_eq!(db.root(), scenario_2_root); db.destroy().await.unwrap(); @@ -1540,11 +1718,11 @@ pub mod test { CleanCurrentTest::init(context.clone(), db_config_no_pruning.clone()) .await .unwrap() - .into_dirty(); + .into_mutable(); let mut db_pruning = CleanCurrentTest::init(context.clone(), db_config_pruning.clone()) .await .unwrap() - .into_dirty(); + .into_mutable(); // Apply identical operations to both databases, but only prune one. const NUM_OPERATIONS: u64 = 1000; @@ -1557,24 +1735,24 @@ pub mod test { // Commit periodically if i % 50 == 49 { - let mut clean_no_pruning = db_no_pruning.merkleize().await.unwrap(); - clean_no_pruning.commit(None).await.unwrap(); - let mut clean_pruning = db_pruning.merkleize().await.unwrap(); - clean_pruning.commit(None).await.unwrap(); + let (durable_no_pruning, _) = db_no_pruning.commit(None).await.unwrap(); + let clean_no_pruning = durable_no_pruning.into_merkleized().await.unwrap(); + let (durable_pruning, _) = db_pruning.commit(None).await.unwrap(); + let mut clean_pruning = durable_pruning.into_merkleized().await.unwrap(); clean_pruning .prune(clean_no_pruning.any.inactivity_floor_loc()) .await .unwrap(); - db_no_pruning = clean_no_pruning.into_dirty(); - db_pruning = clean_pruning.into_dirty(); + db_no_pruning = clean_no_pruning.into_mutable(); + db_pruning = clean_pruning.into_mutable(); } } // Final commit - let mut db_no_pruning = db_no_pruning.merkleize().await.unwrap(); - db_no_pruning.commit(None).await.unwrap(); - let mut db_pruning = db_pruning.merkleize().await.unwrap(); - db_pruning.commit(None).await.unwrap(); + let (db_no_pruning, _) = db_no_pruning.commit(None).await.unwrap(); + let db_no_pruning = db_no_pruning.into_merkleized().await.unwrap(); + let (db_pruning, _) = db_pruning.commit(None).await.unwrap(); + let db_pruning = db_pruning.into_merkleized().await.unwrap(); // Get roots from both databases let root_no_pruning = db_no_pruning.root(); @@ -1637,10 +1815,10 @@ pub mod test { // Add `key_exists_1` and test exclusion proving over the single-key database case. let v1 = Sha256::fill(0xA1); - let mut db = db.into_dirty(); + let mut db = db.into_mutable(); db.update(key_exists_1, v1).await.unwrap(); - let mut db = db.merkleize().await.unwrap(); - db.commit(None).await.unwrap(); + let (db, _) = db.commit(None).await.unwrap(); + let db = db.into_merkleized().await.unwrap(); let root = db.root(); // We shouldn't be able to generate an exclusion proof for a key already in the db. @@ -1687,10 +1865,10 @@ pub mod test { let key_exists_2 = Sha256::fill(0x30); let v2 = Sha256::fill(0xB2); - let mut db = db.into_dirty(); + let mut db = db.into_mutable(); db.update(key_exists_2, v2).await.unwrap(); - let mut db = db.merkleize().await.unwrap(); - db.commit(None).await.unwrap(); + let (db, _) = db.commit(None).await.unwrap(); + let db = db.into_merkleized().await.unwrap(); let root = db.root(); // Use a lesser/greater key that has a translated-key conflict based @@ -1780,12 +1958,12 @@ pub mod test { // Make the DB empty again by deleting the keys and check the empty case // again. - let mut db = db.into_dirty(); + let mut db = db.into_mutable(); db.delete(key_exists_1).await.unwrap(); db.delete(key_exists_2).await.unwrap(); - let mut db = db.merkleize().await.unwrap(); + let (db, _) = db.commit(None).await.unwrap(); + let mut db = db.into_merkleized().await.unwrap(); db.sync().await.unwrap(); - db.commit(None).await.unwrap(); let root = db.root(); // This root should be different than the empty root from earlier since the DB now has a // non-zero number of operations. @@ -1831,7 +2009,7 @@ pub mod test { batch_tests::test_batch(|mut ctx| async move { let seed = ctx.next_u64(); let partition = format!("current_ordered_batch_{seed}"); - AnyExt::new(open_db(ctx, &partition).await) + open_db(ctx, &partition).await.into_mutable() }); } } diff --git a/storage/src/qmdb/current/unordered/fixed.rs b/storage/src/qmdb/current/unordered/fixed.rs index 8e0d15f4be..5ddee2788d 100644 --- a/storage/src/qmdb/current/unordered/fixed.rs +++ b/storage/src/qmdb/current/unordered/fixed.rs @@ -6,31 +6,32 @@ //! //! See [Db] for the main database type. +#[cfg(any(test, feature = "test-traits"))] +use crate::qmdb::any::states::{ + CleanAny, MerkleizedNonDurableAny, MutableAny, UnmerkleizedDurableAny, +}; use crate::{ - bitmap::{CleanBitMap, DirtyBitMap}, + bitmap::CleanBitMap, kv::{self, Batchable}, - mmr::{ - mem::{Clean, Dirty, State}, - Location, Proof, StandardHasher, - }, + mmr::{Location, Proof, StandardHasher}, qmdb::{ any::{ unordered::{ fixed::{Db as AnyDb, Operation}, Update, }, - CleanAny, DirtyAny, FixedValue, + FixedValue, }, current::{ merkleize_grafted_bitmap, proof::{OperationProof, RangeProof}, root, FixedConfig as Config, }, - store::{CleanStore, DirtyStore, LogStore}, - Error, + store::{self, LogStore, MerkleizedStore, PrunableStore}, + DurabilityState, Durable, Error, MerkleizationState, Merkleized, NonDurable, Unmerkleized, }, translator::Translator, - AuthenticatedBitMap as BitMap, + AuthenticatedBitMap as BitMap, Persistable, }; use commonware_codec::FixedSize; use commonware_cryptography::{DigestOf, Hasher}; @@ -55,15 +56,16 @@ pub struct Db< H: Hasher, T: Translator, const N: usize, - S: State> = Clean>, + M: MerkleizationState> = Merkleized, + D: DurabilityState = Durable, > { /// An authenticated database that provides the ability to prove whether a key ever had a /// specific value. - any: AnyDb, + any: AnyDb, /// The bitmap over the activity status of each operation. Supports augmenting [Db] proofs in /// order to further prove whether a key _currently_ has a specific value. - status: BitMap, + status: BitMap, context: E, @@ -73,6 +75,7 @@ pub struct Db< cached_root: Option, } +// Functionality shared across all DB states, such as most non-mutating operations. impl< E: RStorage + Clock + Metrics, K: Array, @@ -80,8 +83,9 @@ impl< H: Hasher, T: Translator, const N: usize, - S: State>, - > Db + M: MerkleizationState>, + D: DurabilityState, + > Db { /// The number of operations that have been applied to this db, including those that have been /// pruned and those that are not yet committed. @@ -105,6 +109,11 @@ impl< self.any.get_metadata().await } + /// Whether the db currently has no active keys. + pub const fn is_empty(&self) -> bool { + self.any.is_empty() + } + /// Get the level of the base MMR into which we are grafting. /// /// This value is log2 of the chunk size in bits. Since we assume the chunk size is a power of @@ -143,6 +152,7 @@ impl< } } +// Functionality for the Clean state. impl< E: RStorage + Clock + Metrics, K: Array, @@ -150,7 +160,7 @@ impl< H: Hasher, T: Translator, const N: usize, - > Db + > Db, Durable> { /// Initializes a [Db] authenticated database from the given `config`. Leverages parallel /// Merkleization to initialize the bitmap MMR if a thread pool is provided. @@ -217,6 +227,69 @@ impl< self.cached_root.expect("Clean state must have cached root") } + /// Sync all database state to disk. + pub async fn sync(&mut self) -> Result<(), Error> { + self.any.sync().await?; + + // Write the bitmap pruning boundary to disk so that next startup doesn't have to + // re-Merkleize the inactive portion up to the inactivity floor. + self.status + .write_pruned( + self.context.with_label("bitmap"), + &self.bitmap_metadata_partition, + ) + .await + .map_err(Into::into) + } + + /// Destroy the db, removing all data from disk. + pub async fn destroy(self) -> Result<(), Error> { + // Clean up bitmap metadata partition. + CleanBitMap::::destroy(self.context, &self.bitmap_metadata_partition).await?; + + // Clean up Any components (MMR and log). + self.any.destroy().await + } + + /// Transition into the mutable state. + pub fn into_mutable(self) -> Db { + Db { + any: self.any.into_mutable(), + status: self.status.into_dirty(), + context: self.context, + bitmap_metadata_partition: self.bitmap_metadata_partition, + cached_root: None, + } + } +} + +// Functionality for any Merkleized state (both Durable and NonDurable). +impl< + E: RStorage + Clock + Metrics, + K: Array, + V: FixedValue, + H: Hasher, + T: Translator, + const N: usize, + D: store::State, + > Db, D> +{ + /// Prune historical operations prior to `prune_loc`. This does not affect the db's root + /// or current snapshot. + pub async fn prune(&mut self, prune_loc: Location) -> Result<(), Error> { + // Write the pruned portion of the bitmap to disk *first* to ensure recovery in case of + // failure during pruning. If we don't do this, we may not be able to recover the bitmap + // because it may require replaying of pruned operations. + self.status + .write_pruned( + self.context.with_label("bitmap"), + &self.bitmap_metadata_partition, + ) + .await?; + + self.any.prune(prune_loc).await + } + /// Returns a proof that the specified range of operations are part of the database, along with /// the operations from the range. A truncated range (from hitting the max) can be detected by /// looking at the length of the returned operations vector. Also returns the bitmap chunks @@ -265,127 +338,9 @@ impl< OperationProof::::new(hasher, &self.status, height, mmr, loc).await } - - #[cfg(test)] - /// Simulate a crash that prevents any data from being written to disk, which involves simply - /// consuming the db before it can be cleanly closed. - fn simulate_commit_failure_before_any_writes(self) { - // Don't successfully complete any of the commit operations. - } - - #[cfg(test)] - /// Simulate a crash that happens during commit and prevents the any db from being pruned of - /// inactive operations, and bitmap state from being written/pruned. - async fn simulate_commit_failure_after_any_db_commit(mut self) -> Result<(), Error> { - // Only successfully complete the log write part of the commit process. - let _ = self.commit_to_log(None).await?; - Ok(()) - } - - /// Helper that performs the commit operations up to and including writing to the log, - /// but does not merkleize the bitmap or prune. Used for simulating partial commit failures - /// in tests, and as the first phase of the full commit operation. - /// - /// Returns the dirty bitmap that needs to be merkleized and pruned. - async fn commit_to_log( - &mut self, - metadata: Option, - ) -> Result, Error> { - let empty_status = CleanBitMap::::new(&mut self.any.log.hasher, None); - let mut status = std::mem::replace(&mut self.status, empty_status).into_dirty(); - - // Inactivate the current commit operation. - status.set_bit(*self.any.last_commit_loc, false); - - // Raise the inactivity floor by taking `self.steps` steps, plus 1 to account for the - // previous commit becoming inactive. - let inactivity_floor_loc = self.any.raise_floor_with_bitmap(&mut status).await?; - - // Append the commit operation with the new floor and tag it as active in the bitmap. - status.push(true); - let commit_op = Operation::CommitFloor(metadata, inactivity_floor_loc); - - self.any.apply_commit_op(commit_op).await?; - - Ok(status) - } - - /// Commit any pending operations to the database, ensuring their durability upon return from - /// this function. Also raises the inactivity floor according to the schedule. Returns the - /// `(start_loc, end_loc]` location range of committed operations. - pub async fn commit(&mut self, metadata: Option) -> Result, Error> { - let start_loc = self.any.last_commit_loc + 1; - - // Phase 1: Commit to log (recovery is ensured after this returns) - let status = self.commit_to_log(metadata).await?; - - // Phase 2: Merkleize the new bitmap entries. - let mmr = &self.any.log.mmr; - let height = Self::grafting_height(); - self.status = - merkleize_grafted_bitmap(&mut self.any.log.hasher, status, mmr, height).await?; - - // Phase 3: Prune bits that are no longer needed because they precede the inactivity floor. - self.status.prune_to_bit(*self.any.inactivity_floor_loc())?; - - // Phase 4: Refresh cached root after commit - self.cached_root = Some(root(&mut self.any.log.hasher, height, &self.status, mmr).await?); - - Ok(start_loc..self.op_count()) - } - - /// Sync all database state to disk. - pub async fn sync(&mut self) -> Result<(), Error> { - self.any.sync().await?; - - // Write the bitmap pruning boundary to disk so that next startup doesn't have to - // re-Merkleize the inactive portion up to the inactivity floor. - self.status - .write_pruned( - self.context.with_label("bitmap"), - &self.bitmap_metadata_partition, - ) - .await - .map_err(Into::into) - } - - /// Prune historical operations prior to `prune_loc`. This does not affect the db's root - /// or current snapshot. - pub async fn prune(&mut self, prune_loc: Location) -> Result<(), Error> { - // Write the pruned portion of the bitmap to disk *first* to ensure recovery in case of - // failure during pruning. If we don't do this, we may not be able to recover the bitmap - // because it may require replaying of pruned operations. - self.status - .write_pruned( - self.context.with_label("bitmap"), - &self.bitmap_metadata_partition, - ) - .await?; - - self.any.prune(prune_loc).await - } - - /// Destroy the db, removing all data from disk. - pub async fn destroy(self) -> Result<(), Error> { - // Clean up bitmap metadata partition. - CleanBitMap::::destroy(self.context, &self.bitmap_metadata_partition).await?; - - // Clean up Any components (MMR and log). - self.any.destroy().await - } - - /// Convert this clean database into its dirty counterpart for performing mutations. - pub fn into_dirty(self) -> Db { - Db { - any: self.any.into_dirty(), - status: self.status.into_dirty(), - context: self.context, - bitmap_metadata_partition: self.bitmap_metadata_partition, - cached_root: None, - } - } } +// Functionality for the Mutable state. impl< E: RStorage + Clock + Metrics, K: Array, @@ -393,7 +348,7 @@ impl< H: Hasher, T: Translator, const N: usize, - > Db + > Db { /// Updates `key` to have value `value`. The operation is reflected in the snapshot, but will be /// subject to rollback until the next successful `commit`. @@ -432,24 +387,90 @@ impl< Ok(true) } - /// Merkleize the bitmap and convert this dirty database into its clean counterpart. - /// This computes the Merkle tree over any new bitmap entries but does NOT persist - /// changes to storage. Use `commit()` for durable state transitions. - pub async fn merkleize(self) -> Result>>, Error> { - // First merkleize the any to get a Clean MMR - let clean_any = self.any.merkleize(); + /// Commit any pending operations to the database, ensuring their durability upon return from + /// this function. Also raises the inactivity floor according to the schedule. Returns the + /// `[start_loc, end_loc)` location range of committed operations. + async fn apply_commit_op(&mut self, metadata: Option) -> Result, Error> { + let start_loc = self.any.last_commit_loc + 1; - // Now use the clean MMR for bitmap merkleization + // Inactivate the current commit operation. + self.status.set_bit(*self.any.last_commit_loc, false); + + // Raise the inactivity floor by taking `self.steps` steps, plus 1 to account for the + // previous commit becoming inactive. + let inactivity_floor_loc = self.any.raise_floor_with_bitmap(&mut self.status).await?; + + // Append the commit operation with the new floor and tag it as active in the bitmap. + self.status.push(true); + let commit_op = Operation::CommitFloor(metadata, inactivity_floor_loc); + + self.any.apply_commit_op(commit_op).await?; + + Ok(start_loc..self.op_count()) + } + + /// Commit any pending operations to the database, ensuring their durability upon return. + /// This transitions to the Durable state without merkleizing. Returns the committed database + /// and the `[start_loc, end_loc)` range of committed operations. + pub async fn commit( + mut self, + metadata: Option, + ) -> Result<(Db, Range), Error> { + let range = self.apply_commit_op(metadata).await?; + + // Transition to Durable state without merkleizing + let any = AnyDb { + log: self.any.log, + inactivity_floor_loc: self.any.inactivity_floor_loc, + last_commit_loc: self.any.last_commit_loc, + snapshot: self.any.snapshot, + durable_state: store::Durable, + active_keys: self.any.active_keys, + _update: core::marker::PhantomData, + }; + + Ok(( + Db { + any, + status: self.status, + context: self.context, + bitmap_metadata_partition: self.bitmap_metadata_partition, + cached_root: None, // Not merkleized yet + }, + range, + )) + } + + /// Merkleize the database and transition to the provable state without committing. + /// This enables proof generation while keeping the database in the non-durable state. + pub async fn into_merkleized( + self, + ) -> Result, NonDurable>, Error> { + // Merkleize the any db's log + let any = AnyDb { + log: self.any.log.merkleize(), + inactivity_floor_loc: self.any.inactivity_floor_loc, + last_commit_loc: self.any.last_commit_loc, + snapshot: self.any.snapshot, + durable_state: self.any.durable_state, + active_keys: self.any.active_keys, + _update: core::marker::PhantomData, + }; + + // Merkleize the bitmap using the clean MMR let mut hasher = StandardHasher::::new(); - let height = Self::grafting_height(); - let status = - merkleize_grafted_bitmap(&mut hasher, self.status, &clean_any.log.mmr, height).await?; + let height = Db::, NonDurable>::grafting_height(); + let mut status = + merkleize_grafted_bitmap(&mut hasher, self.status, &any.log.mmr, height).await?; + + // Prune the bitmap of no-longer-necessary bits. + status.prune_to_bit(*any.inactivity_floor_loc)?; // Compute and cache the root - let cached_root = Some(root(&mut hasher, height, &status, &clean_any.log.mmr).await?); + let cached_root = Some(root(&mut hasher, height, &status, &any.log.mmr).await?); Ok(Db { - any: clean_any, + any, status, context: self.context, bitmap_metadata_partition: self.bitmap_metadata_partition, @@ -458,6 +479,7 @@ impl< } } +// Functionality for (Merkleized, NonDurable) state. impl< E: RStorage + Clock + Metrics, K: Array, @@ -465,13 +487,95 @@ impl< H: Hasher, T: Translator, const N: usize, - > crate::qmdb::store::LogStorePrunable for Db + > Db, NonDurable> { - async fn prune(&mut self, prune_loc: Location) -> Result<(), Error> { - self.prune(prune_loc).await + /// Transition into the mutable state. + pub fn into_mutable(self) -> Db { + Db { + any: self.any.into_mutable(), + status: self.status.into_dirty(), + context: self.context, + bitmap_metadata_partition: self.bitmap_metadata_partition, + cached_root: None, + } + } + + /// Commit any pending operations to the database, ensuring their durability upon return. + /// Returns the committed database and the range of committed operations. + pub async fn commit( + self, + metadata: Option, + ) -> Result< + ( + Db, Durable>, + Range, + ), + Error, + > { + let (durable, range) = self.into_mutable().commit(metadata).await?; + Ok((durable.into_merkleized().await?, range)) + } +} + +// Functionality for (Unmerkleized, Durable) state. +impl< + E: RStorage + Clock + Metrics, + K: Array, + V: FixedValue, + H: Hasher, + T: Translator, + const N: usize, + > Db +{ + /// Merkleize the database and transition to the provable state. + pub async fn into_merkleized( + self, + ) -> Result, Durable>, Error> { + // Merkleize the any db's log + let any = AnyDb { + log: self.any.log.merkleize(), + inactivity_floor_loc: self.any.inactivity_floor_loc, + last_commit_loc: self.any.last_commit_loc, + snapshot: self.any.snapshot, + durable_state: self.any.durable_state, + active_keys: self.any.active_keys, + _update: core::marker::PhantomData, + }; + + // Merkleize the bitmap using the clean MMR + let mut hasher = StandardHasher::::new(); + let height = Db::, Durable>::grafting_height(); + let mut status = + merkleize_grafted_bitmap(&mut hasher, self.status, &any.log.mmr, height).await?; + + // Prune the bitmap of no-longer-necessary bits. + status.prune_to_bit(*any.inactivity_floor_loc)?; + + // Compute and cache the root + let cached_root = Some(root(&mut hasher, height, &status, &any.log.mmr).await?); + + Ok(Db { + any, + status, + context: self.context, + bitmap_metadata_partition: self.bitmap_metadata_partition, + cached_root, + }) + } + + /// Transition into the mutable state. + pub fn into_mutable(self) -> Db { + Db { + any: self.any.into_mutable(), + status: self.status, + context: self.context, + bitmap_metadata_partition: self.bitmap_metadata_partition, + cached_root: None, + } } } +// LogStore implementation for all states. impl< E: RStorage + Clock + Metrics, K: Array, @@ -479,8 +583,9 @@ impl< H: Hasher, T: Translator, const N: usize, - S: State>, - > LogStore for Db + M: MerkleizationState>, + D: DurabilityState, + > LogStore for Db { type Value = V; @@ -501,6 +606,7 @@ impl< } } +// Store implementation for all states impl< E: RStorage + Clock + Metrics, K: Array, @@ -508,8 +614,9 @@ impl< H: Hasher, T: Translator, const N: usize, - S: State>, - > kv::Gettable for Db + M: MerkleizationState>, + D: DurabilityState, + > kv::Gettable for Db { type Key = K; type Value = V; @@ -520,6 +627,7 @@ impl< } } +// StoreMut for (Unmerkleized,NonDurable) (aka mutable) state impl< E: RStorage + Clock + Metrics, K: Array, @@ -527,13 +635,14 @@ impl< H: Hasher, T: Translator, const N: usize, - > kv::Updatable for Db + > kv::Updatable for Db { async fn update(&mut self, key: Self::Key, value: Self::Value) -> Result<(), Self::Error> { self.update(key, value).await } } +// StoreDeletable for (Unmerkleized,NonDurable) (aka mutable) state impl< E: RStorage + Clock + Metrics, K: Array, @@ -541,13 +650,42 @@ impl< H: Hasher, T: Translator, const N: usize, - > kv::Deletable for Db + > kv::Deletable for Db { async fn delete(&mut self, key: Self::Key) -> Result { self.delete(key).await } } +// Batchable for (Unmerkleized,NonDurable) (aka mutable) state +impl Batchable for Db +where + E: RStorage + Clock + Metrics, + K: Array, + V: FixedValue, + T: Translator, + H: Hasher, +{ + async fn write_batch( + &mut self, + iter: impl Iterator)>, + ) -> Result<(), Error> { + let status = &mut self.status; + self.any + .write_batch_with_callback(iter, move |append: bool, loc: Option| { + status.push(append); + if let Some(loc) = loc { + status.set_bit(*loc, false); + } + }) + .await + } +} + +// MerkleizedStore for Merkleized states (both Durable and NonDurable) +// TODO(https://github.com/commonwarexyz/monorepo/issues/2560): This is broken -- it's computing +// proofs only over the any db mmr not the grafted mmr, so they won't validate against the grafted +// root. impl< E: RStorage + Clock + Metrics, K: Array, @@ -555,22 +693,15 @@ impl< H: Hasher, T: Translator, const N: usize, - > CleanStore for Db>> + D: store::State, + > MerkleizedStore for Db, D> { type Digest = H::Digest; type Operation = Operation; - type Dirty = Db; fn root(&self) -> Self::Digest { - self.root() - } - - async fn proof( - &self, - start_loc: Location, - max_ops: NonZeroU64, - ) -> Result<(Proof, Vec), Error> { - self.any.proof(start_loc, max_ops).await + self.cached_root + .expect("Merkleized state must have cached root") } async fn historical_proof( @@ -583,36 +714,52 @@ impl< .historical_proof(historical_size, start_loc, max_ops) .await } +} - fn into_dirty(self) -> Self::Dirty { - self.into_dirty() +// PrunableStore for both Merkleized states. +impl< + E: RStorage + Clock + Metrics, + K: Array, + V: FixedValue, + H: Hasher, + T: Translator, + const N: usize, + D: DurabilityState, + > PrunableStore for Db, D> +{ + async fn prune(&mut self, prune_loc: Location) -> Result<(), Error> { + self.prune(prune_loc).await } } -impl Batchable for Db -where - E: RStorage + Clock + Metrics, - K: Array, - V: FixedValue, - T: Translator, - H: Hasher, +// Persistable for Clean state +impl< + E: RStorage + Clock + Metrics, + K: Array, + V: FixedValue, + H: Hasher, + T: Translator, + const N: usize, + > Persistable for Db, Durable> { - async fn write_batch( - &mut self, - iter: impl Iterator)>, - ) -> Result<(), Error> { - let status = &mut self.status; - self.any - .write_batch_with_callback(iter, move |append: bool, loc: Option| { - status.push(append); - if let Some(loc) = loc { - status.set_bit(*loc, false); - } - }) - .await + type Error = Error; + + async fn commit(&mut self) -> Result<(), Self::Error> { + // No-op, DB already recoverable. + Ok(()) + } + + async fn sync(&mut self) -> Result<(), Self::Error> { + self.sync().await + } + + async fn destroy(self) -> Result<(), Self::Error> { + self.destroy().await } } +// CleanAny implementation +#[cfg(any(test, feature = "test-traits"))] impl< E: RStorage + Clock + Metrics, K: Array, @@ -620,17 +767,17 @@ impl< H: Hasher, T: Translator, const N: usize, - > DirtyStore for Db + > CleanAny for Db, Durable> { - type Digest = H::Digest; - type Operation = Operation; - type Clean = Db>>; + type Mutable = Db; - async fn merkleize(self) -> Result { - self.merkleize().await + fn into_mutable(self) -> Self::Mutable { + self.into_mutable() } } +// UnmerkleizedDurableAny implementation +#[cfg(any(test, feature = "test-traits"))] impl< E: RStorage + Clock + Metrics, K: Array, @@ -638,31 +785,53 @@ impl< H: Hasher, T: Translator, const N: usize, - > CleanAny for Db>> + > UnmerkleizedDurableAny for Db { - type Key = K; + type Digest = H::Digest; + type Operation = Operation; + type Mutable = Db; + type Merkleized = Db, Durable>; - async fn get(&self, key: &Self::Key) -> Result, Error> { - self.get(key).await + fn into_mutable(self) -> Self::Mutable { + Db { + any: self.any.into_mutable(), + status: self.status, + context: self.context, + bitmap_metadata_partition: self.bitmap_metadata_partition, + cached_root: None, + } } - async fn commit(&mut self, metadata: Option) -> Result, Error> { - self.commit(metadata).await + async fn into_merkleized(self) -> Result { + self.into_merkleized().await } +} - async fn sync(&mut self) -> Result<(), Error> { - self.sync().await - } +// MerkleizedNonDurableAny implementation +#[cfg(any(test, feature = "test-traits"))] +impl< + E: RStorage + Clock + Metrics, + K: Array, + V: FixedValue, + H: Hasher, + T: Translator, + const N: usize, + > MerkleizedNonDurableAny for Db, NonDurable> +{ + type Mutable = Db; + type Durable = Db, Durable>; - async fn prune(&mut self, prune_loc: Location) -> Result<(), Error> { - self.prune(prune_loc).await + async fn commit(self, metadata: Option) -> Result<(Self::Durable, Range), Error> { + self.commit(metadata).await } - async fn destroy(self) -> Result<(), Error> { - self.destroy().await + fn into_mutable(self) -> Self::Mutable { + self.into_mutable() } } +// MutableAny implementation +#[cfg(any(test, feature = "test-traits"))] impl< E: RStorage + Clock + Metrics, K: Array, @@ -670,24 +839,23 @@ impl< H: Hasher, T: Translator, const N: usize, - > DirtyAny for Db + > MutableAny for Db { - type Key = K; - - async fn get(&self, key: &Self::Key) -> Result, Error> { - self.get(key).await - } + type Digest = H::Digest; + type Operation = Operation; + type Durable = Db; + type Merkleized = Db, NonDurable>; - async fn update(&mut self, key: Self::Key, value: Self::Value) -> Result<(), Error> { - self.update(key, value).await + async fn commit(self, metadata: Option) -> Result<(Self::Durable, Range), Error> { + self.commit(metadata).await } - async fn create(&mut self, key: Self::Key, value: Self::Value) -> Result { - self.create(key, value).await + async fn into_merkleized(self) -> Result { + self.into_merkleized().await } - async fn delete(&mut self, key: Self::Key) -> Result { - self.delete(key).await + fn steps(&self) -> u64 { + self.any.durable_state.steps } } @@ -695,9 +863,7 @@ impl< pub mod test { use super::*; use crate::{ - index::Unordered as _, - mmr::hasher::Hasher as _, - qmdb::{any::AnyExt, store::batch_tests}, + index::Unordered as _, mmr::hasher::Hasher as _, qmdb::store::batch_tests, translator::TwoCap, }; use commonware_cryptography::{sha256::Digest, Sha256}; @@ -727,11 +893,12 @@ pub mod test { } } - /// A type alias for the concrete [Db] type used in these unit tests. + /// A type alias for the concrete [Db] type used in these unit tests (Merkleized, Durable). type CleanCurrentTest = Db; - /// A type alias for the Dirty variant of CurrentTest. - type DirtyCurrentTest = Db; + /// A type alias for the Dirty (Unmerkleized, NonDurable) variant of CurrentTest. + type DirtyCurrentTest = + Db; /// Return an [Db] database initialized with a fixed config. async fn open_db(context: deterministic::Context, partition_prefix: &str) -> CleanCurrentTest { @@ -757,15 +924,15 @@ pub mod test { assert_eq!(db.root(), root0); // Add one key. + let mut db = db.into_mutable(); let k1 = Sha256::hash(&0u64.to_be_bytes()); let v1 = Sha256::hash(&10u64.to_be_bytes()); - let mut db = db.into_dirty(); assert!(db.create(k1, v1).await.unwrap()); assert_eq!(db.get(&k1).await.unwrap().unwrap(), v1); - let mut db = db.merkleize().await.unwrap(); - let range = db.commit(None).await.unwrap(); - assert_eq!(range.start, 1); - assert_eq!(range.end, 4); + let (db, range) = db.commit(None).await.unwrap(); + let db = db.into_merkleized().await.unwrap(); + assert_eq!(*range.start, 1); + assert_eq!(*range.end, 4); assert!(db.get_metadata().await.unwrap().is_none()); assert_eq!(db.op_count(), 4); // 1 update, 1 commit, 1 move + 1 initial commit. let root1 = db.root(); @@ -777,33 +944,38 @@ pub mod test { assert_eq!(db.root(), root1); // Create of same key should fail. - let mut db = db.into_dirty(); + let mut db = db.into_mutable(); assert!(!db.create(k1, v1).await.unwrap()); // Delete that one key. assert!(db.delete(k1).await.unwrap()); let metadata = Sha256::hash(&1u64.to_be_bytes()); - let mut db = db.merkleize().await.unwrap(); - let range = db.commit(Some(metadata)).await.unwrap(); - assert_eq!(range.start, 4); - assert_eq!(range.end, 6); - + let (db, range) = db.commit(Some(metadata)).await.unwrap(); + let db = db.into_merkleized().await.unwrap(); + assert_eq!(*range.start, 4); + assert_eq!(*range.end, 6); assert_eq!(db.op_count(), 6); // 1 update, 2 commits, 1 move, 1 delete. assert_eq!(db.get_metadata().await.unwrap().unwrap(), metadata); let root2 = db.root(); // Repeated delete of same key should fail. - let mut db = db.into_dirty(); + let mut db = db.into_mutable(); assert!(!db.delete(k1).await.unwrap()); - let mut db = db.merkleize().await.unwrap(); + let (db, _) = db.commit(None).await.unwrap(); + let mut db = db.into_merkleized().await.unwrap(); db.sync().await.unwrap(); + // Commit adds a commit even for no-op, so op_count increases and root changes. + assert_eq!(db.op_count(), 7); + let root3 = db.root(); + assert!(root3 != root2); // Confirm re-open preserves state. drop(db); let db = open_db(context.clone(), partition).await; - assert_eq!(db.op_count(), 6); // 1 update, 2 commits, 1 move, 1 delete + 1 initial commit. - assert_eq!(db.get_metadata().await.unwrap().unwrap(), metadata); - assert_eq!(db.root(), root2); + assert_eq!(db.op_count(), 7); + // Last commit had no metadata (passed None to commit). + assert!(db.get_metadata().await.unwrap().is_none()); + assert_eq!(db.root(), root3); // Confirm all activity bits are false except for the last commit. for i in 0..*db.op_count() - 1 { @@ -811,6 +983,17 @@ pub mod test { } assert!(db.status.get_bit(*db.op_count() - 1)); + // Test that we can do a non-durable root. + let mut db = db.into_mutable(); + db.update(k1, v1).await.unwrap(); + let db = db.into_merkleized().await.unwrap(); + assert_ne!(db.root(), root3); + + // Test that we can do a merkleized commit. + let (db, _) = db.commit(None).await.unwrap(); + assert!(db.get_metadata().await.unwrap().is_none()); + assert_eq!(db.op_count(), 10); + db.destroy().await.unwrap(); }); } @@ -822,7 +1005,7 @@ pub mod test { // confirm that the end state of the db matches that of an identically updated hashmap. const ELEMENTS: u64 = 1000; executor.start(|context| async move { - let mut db = open_db(context.clone(), "build_big").await.into_dirty(); + let mut db = open_db(context.clone(), "build_big").await.into_mutable(); let mut map = HashMap::::default(); for i in 0u64..ELEMENTS { @@ -859,8 +1042,8 @@ pub mod test { assert_eq!(db.any.snapshot.items(), 857); // Test that commit + sync w/ pruning will raise the activity floor. - let mut db = db.merkleize().await.unwrap(); - db.commit(None).await.unwrap(); + let (db, _) = db.commit(None).await.unwrap(); + let mut db = db.into_merkleized().await.unwrap(); db.sync().await.unwrap(); db.prune(db.inactivity_floor_loc()).await.unwrap(); assert_eq!(db.op_count(), 1957); @@ -892,6 +1075,16 @@ pub mod test { }); } + // Test that merkleization state changes don't reset `steps`. + #[test_traced("DEBUG")] + fn test_current_unordered_fixed_db_steps_not_reset() { + let executor = deterministic::Runner::default(); + executor.start(|context| async move { + let db = open_db(context, "steps_test").await; + crate::qmdb::any::test::test_any_db_steps_not_reset(db).await; + }); + } + /// Build a tiny database and make sure we can't convince the verifier that some old value of a /// key is active. We specifically test over the partial chunk case, since these bits are yet to /// be committed to the underlying MMR. @@ -901,14 +1094,14 @@ pub mod test { executor.start(|context| async move { let mut hasher = StandardHasher::::new(); let partition = "build_small"; - let mut db = open_db(context.clone(), partition).await.into_dirty(); + let mut db = open_db(context.clone(), partition).await.into_mutable(); // Add one key. let k = Sha256::fill(0x01); let v1 = Sha256::fill(0xA1); db.update(k, v1).await.unwrap(); - let mut db = db.merkleize().await.unwrap(); - db.commit(None).await.unwrap(); + let (db, _) = db.commit(None).await.unwrap(); + let db = db.into_merkleized().await.unwrap(); let (_, op_loc) = db.any.get_with_loc(&k).await.unwrap().unwrap(); let proof = db.key_value_proof(hasher.inner(), k).await.unwrap(); @@ -934,10 +1127,10 @@ pub mod test { )); // Update the key to a new value (v2), which inactivates the previous operation. - let mut db = db.into_dirty(); + let mut db = db.into_mutable(); db.update(k, v2).await.unwrap(); - let mut db = db.merkleize().await.unwrap(); - db.commit(None).await.unwrap(); + let (db, _) = db.commit(None).await.unwrap(); + let db = db.into_merkleized().await.unwrap(); let root = db.root(); // New value should not be verifiable against the old proof. @@ -1046,13 +1239,13 @@ pub mod test { } /// Apply random operations to the given db, committing them (randomly & at the end) only if - /// `commit_changes` is true. + /// `commit_changes` is true. Returns a dirty db; callers should commit if needed. async fn apply_random_ops( num_elements: u64, commit_changes: bool, rng_seed: u64, mut db: DirtyCurrentTest, - ) -> Result { + ) -> Result { // Log the seed with high visibility to make failures reproducible. warn!("rng_seed={}", rng_seed); let mut rng = StdRng::seed_from_u64(rng_seed); @@ -1075,18 +1268,17 @@ pub mod test { db.update(rand_key, v).await.unwrap(); if commit_changes && rng.next_u32() % 20 == 0 { // Commit every ~20 updates. - let mut clean_db = db.merkleize().await?; - clean_db.commit(None).await?; - db = clean_db.into_dirty(); + let (durable_db, _) = db.commit(None).await?; + let clean_db = durable_db.into_merkleized().await?; + db = clean_db.into_mutable(); } } if commit_changes { - let mut clean_db = db.merkleize().await?; - clean_db.commit(None).await?; - Ok(clean_db) - } else { - db.merkleize().await + let (durable_db, _) = db.commit(None).await?; + let clean_db = durable_db.into_merkleized().await?; + db = clean_db.into_mutable(); } + Ok(db) } #[test_traced("DEBUG")] @@ -1113,9 +1305,11 @@ pub mod test { &root, )); - let db = apply_random_ops(200, true, context.next_u64(), db.into_dirty()) + let db = apply_random_ops(200, true, context.next_u64(), db.into_mutable()) .await .unwrap(); + let (db, _) = db.commit(None).await.unwrap(); + let db = db.into_merkleized().await.unwrap(); let root = db.root(); // Make sure size-constrained batches of operations are provable from the oldest @@ -1164,10 +1358,12 @@ pub mod test { executor.start(|mut context| async move { let partition = "range_proofs"; let mut hasher = StandardHasher::::new(); - let db = open_db(context.clone(), partition).await.into_dirty(); + let db = open_db(context.clone(), partition).await.into_mutable(); let db = apply_random_ops(500, true, context.next_u64(), db) .await .unwrap(); + let (db, _) = db.commit(None).await.unwrap(); + let db = db.into_merkleized().await.unwrap(); let root = db.root(); // Confirm bad keys produce the expected error. @@ -1244,10 +1440,12 @@ pub mod test { executor.start(|mut context| async move { let partition = "build_random"; let rng_seed = context.next_u64(); - let db = open_db(context.clone(), partition).await.into_dirty(); - let mut db = apply_random_ops(ELEMENTS, true, rng_seed, db) + let db = open_db(context.clone(), partition).await.into_mutable(); + let db = apply_random_ops(ELEMENTS, true, rng_seed, db) .await .unwrap(); + let (db, _) = db.commit(None).await.unwrap(); + let mut db = db.into_merkleized().await.unwrap(); db.sync().await.unwrap(); // Drop and reopen the db @@ -1278,11 +1476,11 @@ pub mod test { let mut old_val = Sha256::fill(0x00); for i in 1u8..=255 { let v = Sha256::fill(i); - let mut dirty_db = db.into_dirty(); + let mut dirty_db = db.into_mutable(); dirty_db.update(k, v).await.unwrap(); assert_eq!(dirty_db.get(&k).await.unwrap().unwrap(), v); - db = dirty_db.merkleize().await.unwrap(); - db.commit(None).await.unwrap(); + let (durable_db, _) = dirty_db.commit(None).await.unwrap(); + db = durable_db.into_merkleized().await.unwrap(); let root = db.root(); // Create a proof for the current value of k. @@ -1320,37 +1518,41 @@ pub mod test { executor.start(|mut context| async move { let partition = "build_random_fail_commit"; let rng_seed = context.next_u64(); - let db = open_db(context.clone(), partition).await.into_dirty(); - let mut db = apply_random_ops(ELEMENTS, true, rng_seed, db) + let db = open_db(context.clone(), partition).await.into_mutable(); + let db = apply_random_ops(ELEMENTS, true, rng_seed, db) .await .unwrap(); + let (db, _) = db.commit(None).await.unwrap(); + let mut db = db.into_merkleized().await.unwrap(); let committed_root = db.root(); let committed_op_count = db.op_count(); let committed_inactivity_floor = db.any.inactivity_floor_loc(); db.prune(committed_inactivity_floor).await.unwrap(); // Perform more random operations without committing any of them. - let db = apply_random_ops(ELEMENTS, false, rng_seed + 1, db.into_dirty()) + let db = apply_random_ops(ELEMENTS, false, rng_seed + 1, db.into_mutable()) .await .unwrap(); // SCENARIO #1: Simulate a crash that happens before any writes. Upon reopening, the // state of the DB should be as of the last commit. - db.simulate_commit_failure_before_any_writes(); + drop(db); let db = open_db(context.clone(), partition).await; assert_eq!(db.root(), committed_root); assert_eq!(db.op_count(), committed_op_count); // Re-apply the exact same uncommitted operations. - let db = apply_random_ops(ELEMENTS, false, rng_seed + 1, db.into_dirty()) + let db = apply_random_ops(ELEMENTS, false, rng_seed + 1, db.into_mutable()) .await .unwrap(); // SCENARIO #2: Simulate a crash that happens after the any db has been committed, but - // before the state of the pruned bitmap can be written to disk. - db.simulate_commit_failure_after_any_db_commit() - .await - .unwrap(); + // before the state of the pruned bitmap can be written to disk (i.e., before + // into_merkleized is called). We do this by committing and then dropping the durable + // db without calling close or into_merkleized. + let (durable_db, _) = db.commit(None).await.unwrap(); + let committed_op_count = durable_db.op_count(); + drop(durable_db); // We should be able to recover, so the root should differ from the previous commit, and // the op count should be greater than before. @@ -1360,17 +1562,21 @@ pub mod test { // To confirm the second committed hash is correct we'll re-build the DB in a new // partition, but without any failures. They should have the exact same state. let fresh_partition = "build_random_fail_commit_fresh"; - let db = open_db(context.clone(), fresh_partition).await.into_dirty(); + let db = open_db(context.clone(), fresh_partition) + .await + .into_mutable(); let db = apply_random_ops(ELEMENTS, true, rng_seed, db) .await .unwrap(); - let db = apply_random_ops(ELEMENTS, false, rng_seed + 1, db.into_dirty()) + let (db, _) = db.commit(None).await.unwrap(); + let db = apply_random_ops(ELEMENTS, false, rng_seed + 1, db.into_mutable()) .await .unwrap(); - let mut db = db.into_dirty().merkleize().await.unwrap(); - db.commit(None).await.unwrap(); + let (db, _) = db.commit(None).await.unwrap(); + let mut db = db.into_merkleized().await.unwrap(); db.prune(db.any.inactivity_floor_loc()).await.unwrap(); // State from scenario #2 should match that of a successful commit. + assert_eq!(db.op_count(), committed_op_count); assert_eq!(db.root(), scenario_2_root); db.destroy().await.unwrap(); @@ -1390,11 +1596,11 @@ pub mod test { CleanCurrentTest::init(context.clone(), db_config_no_pruning.clone()) .await .unwrap() - .into_dirty(); + .into_mutable(); let mut db_pruning = CleanCurrentTest::init(context.clone(), db_config_pruning.clone()) .await .unwrap() - .into_dirty(); + .into_mutable(); // Apply identical operations to both databases, but only prune one. const NUM_OPERATIONS: u64 = 1000; @@ -1407,24 +1613,24 @@ pub mod test { // Commit periodically if i % 50 == 49 { - let mut clean_no_pruning = db_no_pruning.merkleize().await.unwrap(); - clean_no_pruning.commit(None).await.unwrap(); - let mut clean_pruning = db_pruning.merkleize().await.unwrap(); - clean_pruning.commit(None).await.unwrap(); + let (db_1, _) = db_no_pruning.commit(None).await.unwrap(); + let clean_no_pruning = db_1.into_merkleized().await.unwrap(); + let (db_2, _) = db_pruning.commit(None).await.unwrap(); + let mut clean_pruning = db_2.into_merkleized().await.unwrap(); clean_pruning .prune(clean_no_pruning.any.inactivity_floor_loc()) .await .unwrap(); - db_no_pruning = clean_no_pruning.into_dirty(); - db_pruning = clean_pruning.into_dirty(); + db_no_pruning = clean_no_pruning.into_mutable(); + db_pruning = clean_pruning.into_mutable(); } } // Final commit - let mut db_no_pruning = db_no_pruning.merkleize().await.unwrap(); - db_no_pruning.commit(None).await.unwrap(); - let mut db_pruning = db_pruning.merkleize().await.unwrap(); - db_pruning.commit(None).await.unwrap(); + let (db_1, _) = db_no_pruning.commit(None).await.unwrap(); + let db_no_pruning = db_1.into_merkleized().await.unwrap(); + let (db_2, _) = db_pruning.commit(None).await.unwrap(); + let db_pruning = db_2.into_merkleized().await.unwrap(); // Get roots from both databases let root_no_pruning = db_no_pruning.root(); @@ -1466,7 +1672,7 @@ pub mod test { batch_tests::test_batch(|mut ctx| async move { let seed = ctx.next_u64(); let prefix = format!("current_unordered_batch_{seed}"); - AnyExt::new(open_db(ctx, &prefix).await) + open_db(ctx, &prefix).await.into_mutable() }); } } diff --git a/storage/src/qmdb/immutable/mod.rs b/storage/src/qmdb/immutable/mod.rs index ef22afd3b6..a2af6f5a12 100644 --- a/storage/src/qmdb/immutable/mod.rs +++ b/storage/src/qmdb/immutable/mod.rs @@ -10,17 +10,22 @@ use crate::{ kv, mmr::{ journaled::{Config as MmrConfig, Mmr}, - mem::{Clean, Dirty, State}, Location, Position, Proof, StandardHasher as Standard, }, - qmdb::{any::VariableValue, build_snapshot_from_log, Error}, + qmdb::{ + any::VariableValue, build_snapshot_from_log, DurabilityState, Durable, Error, + MerkleizationState, Merkleized, NonDurable, Unmerkleized, + }, translator::Translator, }; use commonware_codec::Read; use commonware_cryptography::{DigestOf, Hasher as CHasher}; use commonware_runtime::{buffer::PoolRef, Clock, Metrics, Storage as RStorage, ThreadPool}; use commonware_utils::Array; -use std::num::{NonZeroU64, NonZeroUsize}; +use std::{ + num::{NonZeroU64, NonZeroUsize}, + ops::Range, +}; use tracing::warn; mod operation; @@ -79,10 +84,11 @@ pub struct Immutable< V: VariableValue, H: CHasher, T: Translator, - S: State> = Clean>, + M: MerkleizationState> = Merkleized, + D: DurabilityState = Durable, > { /// Authenticated journal of operations. - journal: Journal, + journal: Journal, /// A map from each active key to the location of the operation that set its value. /// @@ -93,16 +99,21 @@ pub struct Immutable< /// The location of the last commit operation. last_commit_loc: Location, + + /// Marker for the durability state. + _durable: core::marker::PhantomData, } +// Functionality shared across all DB states. impl< E: RStorage + Clock + Metrics, K: Array, V: VariableValue, H: CHasher, T: Translator, - S: State>, - > Immutable + M: MerkleizationState>, + D: DurabilityState, + > Immutable { /// Return the oldest location that remains retrievable. pub fn oldest_retained_loc(&self) -> Location { @@ -162,18 +173,80 @@ impl< Ok(metadata) } +} - /// Update the operations MMR with the given operation, and append the operation to the log. The - /// `commit` method must be called to make any applied operation persistent & recoverable. - pub(super) async fn apply_op(&mut self, op: Operation) -> Result<(), Error> { - self.journal.append(op).await?; +// Functionality shared across Merkleized states. +impl< + E: RStorage + Clock + Metrics, + K: Array, + V: VariableValue, + H: CHasher, + T: Translator, + D: DurabilityState, + > Immutable, D> +{ + /// Return the root of the db. + pub const fn root(&self) -> H::Digest { + self.journal.root() + } + + /// Generate and return: + /// 1. a proof of all operations applied to the db in the range starting at (and including) + /// location `start_loc`, and ending at the first of either: + /// - the last operation performed, or + /// - the operation `max_ops` from the start. + /// 2. the operations corresponding to the leaves in this range. + pub async fn proof( + &self, + start_index: Location, + max_ops: NonZeroU64, + ) -> Result<(Proof, Vec>), Error> { + let op_count = self.op_count(); + self.historical_proof(op_count, start_index, max_ops).await + } + + /// Analogous to proof but with respect to the state of the database when it had `op_count` + /// operations. + /// + /// # Errors + /// + /// Returns [crate::mmr::Error::LocationOverflow] if `op_count` or `start_loc` > + /// [crate::mmr::MAX_LOCATION]. + /// Returns [crate::mmr::Error::RangeOutOfBounds] if `op_count` > number of operations, or + /// if `start_loc` >= `op_count`. + /// Returns [`Error::OperationPruned`] if `start_loc` has been pruned. + pub async fn historical_proof( + &self, + op_count: Location, + start_loc: Location, + max_ops: NonZeroU64, + ) -> Result<(Proof, Vec>), Error> { + Ok(self + .journal + .historical_proof(op_count, start_loc, max_ops) + .await?) + } + + /// Prune historical operations prior to `prune_loc`. This does not affect the db's root or + /// current snapshot. + /// + /// # Errors + /// + /// - Returns [Error::PruneBeyondMinRequired] if `prune_loc` > inactivity floor. + /// - Returns [crate::mmr::Error::LocationOverflow] if `prune_loc` > [crate::mmr::MAX_LOCATION]. + pub async fn prune(&mut self, loc: Location) -> Result<(), Error> { + if loc > self.last_commit_loc { + return Err(Error::PruneBeyondMinRequired(loc, self.last_commit_loc)); + } + self.journal.prune(loc).await?; Ok(()) } } +// Functionality specific to (Merkleized, Durable) state. impl - Immutable> + Immutable, Durable> { /// Returns an [Immutable] qmdb initialized from `cfg`. Any uncommitted log operations will be /// discarded and the state of the db will be as of the last committed operation. @@ -227,6 +300,7 @@ impl>>::from_components( + let journal = Journal::<_, _, _, _, Merkleized>::from_components( mmr, cfg.log, hasher, @@ -286,24 +360,84 @@ impl inactivity floor. - /// - Returns [crate::mmr::Error::LocationOverflow] if `prune_loc` > [crate::mmr::MAX_LOCATION]. - pub async fn prune(&mut self, loc: Location) -> Result<(), Error> { - if loc > self.last_commit_loc { - return Err(Error::PruneBeyondMinRequired(loc, self.last_commit_loc)); + /// Sync all database state to disk. While this isn't necessary to ensure durability of + /// committed operations, periodic invocation may reduce memory usage and the time required to + /// recover the database on restart. + pub async fn sync(&mut self) -> Result<(), Error> { + Ok(self.journal.sync().await?) + } + + /// Destroy the db, removing all data from disk. + pub async fn destroy(self) -> Result<(), Error> { + Ok(self.journal.destroy().await?) + } + + /// Convert this database into a mutable state for batched updates. + pub fn into_mutable(self) -> Immutable { + Immutable { + journal: self.journal.into_dirty(), + snapshot: self.snapshot, + last_commit_loc: self.last_commit_loc, + _durable: core::marker::PhantomData, } - self.journal.prune(loc).await?; + } +} + +// Functionality specific to (Unmerkleized, Durable) state. +impl + Immutable +{ + /// Convert this database into a mutable state for batched updates. + pub fn into_mutable(self) -> Immutable { + Immutable { + journal: self.journal, + snapshot: self.snapshot, + last_commit_loc: self.last_commit_loc, + _durable: core::marker::PhantomData, + } + } + + /// Convert to merkleized state. + pub fn into_merkleized(self) -> Immutable, Durable> { + Immutable { + journal: self.journal.merkleize(), + snapshot: self.snapshot, + last_commit_loc: self.last_commit_loc, + _durable: core::marker::PhantomData, + } + } +} + +// Functionality specific to (Merkleized, NonDurable) state. +impl + Immutable, NonDurable> +{ + /// Convert this database into a mutable state for batched updates. + pub fn into_mutable(self) -> Immutable { + Immutable { + journal: self.journal.into_dirty(), + snapshot: self.snapshot, + last_commit_loc: self.last_commit_loc, + _durable: core::marker::PhantomData, + } + } +} + +// Functionality specific to (Unmerkleized, NonDurable) state - the mutable state. +impl + Immutable +{ + /// Update the operations MMR with the given operation, and append the operation to the log. The + /// `commit` method must be called to make any applied operation persistent & recoverable. + pub(super) async fn apply_op(&mut self, op: Operation) -> Result<(), Error> { + self.journal.append(op).await?; Ok(()) } @@ -324,133 +458,54 @@ impl H::Digest { - self.journal.root() - } - - /// Generate and return: - /// 1. a proof of all operations applied to the db in the range starting at (and including) - /// location `start_loc`, and ending at the first of either: - /// - the last operation performed, or - /// - the operation `max_ops` from the start. - /// 2. the operations corresponding to the leaves in this range. - pub async fn proof( - &self, - start_index: Location, - max_ops: NonZeroU64, - ) -> Result<(Proof, Vec>), Error> { - let op_count = self.op_count(); - self.historical_proof(op_count, start_index, max_ops).await - } - - /// Analogous to proof but with respect to the state of the database when it had `op_count` - /// operations. - /// - /// # Errors - /// - /// Returns [crate::mmr::Error::LocationOverflow] if `op_count` or `start_loc` > - /// [crate::mmr::MAX_LOCATION]. - /// Returns [crate::mmr::Error::RangeOutOfBounds] if `op_count` > number of operations, or - /// if `start_loc` >= `op_count`. - /// Returns [`Error::OperationPruned`] if `start_loc` has been pruned. - pub async fn historical_proof( - &self, - op_count: Location, - start_loc: Location, - max_ops: NonZeroU64, - ) -> Result<(Proof, Vec>), Error> { - Ok(self - .journal - .historical_proof(op_count, start_loc, max_ops) - .await?) - } - /// Commit any pending operations to the database, ensuring their durability upon return from /// this function. Caller can associate an arbitrary `metadata` value with the commit. - /// - /// Failures after commit (but before `sync` or `close`) may still require reprocessing to - /// recover the database on restart. - pub async fn commit(&mut self, metadata: Option) -> Result<(), Error> { + /// Returns the committed database and the range of committed locations. + pub async fn commit( + mut self, + metadata: Option, + ) -> Result< + ( + Immutable, + Range, + ), + Error, + > { let loc = self.journal.append(Operation::Commit(metadata)).await?; self.journal.commit().await?; self.last_commit_loc = loc; + let range = loc..self.op_count(); - Ok(()) - } - - /// Sync all database state to disk. While this isn't necessary to ensure durability of - /// committed operations, periodic invocation may reduce memory usage and the time required to - /// recover the database on restart. - pub(super) async fn sync(&mut self) -> Result<(), Error> { - Ok(self.journal.sync().await?) - } - - /// Destroy the db, removing all data from disk. - pub async fn destroy(self) -> Result<(), Error> { - Ok(self.journal.destroy().await?) - } - - /// Convert this database into its dirty counterpart for batched updates. - pub fn into_dirty(self) -> Immutable { - Immutable { - journal: self.journal.into_dirty(), + let db = Immutable { + journal: self.journal, snapshot: self.snapshot, last_commit_loc: self.last_commit_loc, - } - } - - /// Simulate a failed commit that successfully writes the log to the commit point, but without - /// fully committing the MMR's cached elements to trigger MMR node recovery on reopening. - #[cfg(test)] - pub async fn simulate_failed_commit_mmr(mut self, write_limit: usize) -> Result<(), Error> - where - V: Default, - { - self.apply_op(Operation::Commit(None)).await?; - self.journal.journal.sync().await?; - self.journal.mmr.simulate_partial_sync(write_limit).await?; - - Ok(()) - } - - /// Simulate a failed commit that successfully writes the MMR to the commit point, but without - /// fully committing the log, requiring rollback of the MMR and log upon reopening. - #[cfg(test)] - pub async fn simulate_failed_commit_log(mut self) -> Result<(), Error> - where - V: Default, - { - self.apply_op(Operation::Commit(None)).await?; - let log_size = self.journal.journal.size(); - - self.journal.mmr.sync().await?; - - // Rewind the operation log over the commit op to force rollback to the previous commit. - if log_size > 0 { - self.journal.journal.rewind(log_size - 1).await?; - } - self.journal.journal.sync().await?; + _durable: core::marker::PhantomData, + }; - Ok(()) + Ok((db, range)) } -} -impl - Immutable -{ - /// Merkleize the database and compute the root digest. - pub fn merkleize(self) -> Immutable> { + /// Convert to merkleized state without committing (for read-only merkle operations). + pub fn into_merkleized(self) -> Immutable, NonDurable> { Immutable { journal: self.journal.merkleize(), snapshot: self.snapshot, last_commit_loc: self.last_commit_loc, + _durable: core::marker::PhantomData, } } } -impl - kv::Gettable for Immutable> +impl< + E: RStorage + Clock + Metrics, + K: Array, + V: VariableValue, + H: CHasher, + T: Translator, + M: MerkleizationState>, + D: DurabilityState, + > kv::Gettable for Immutable { type Key = K; type Value = V; @@ -467,8 +522,9 @@ impl< V: VariableValue, H: CHasher, T: Translator, - S: State>, - > crate::qmdb::store::LogStore for Immutable + M: MerkleizationState>, + D: DurabilityState, + > crate::qmdb::store::LogStore for Immutable { type Value = V; @@ -490,25 +546,22 @@ impl< } } -impl - crate::qmdb::store::CleanStore for Immutable> +impl< + E: RStorage + Clock + Metrics, + K: Array, + V: VariableValue, + H: CHasher, + T: Translator, + D: DurabilityState, + > crate::qmdb::store::MerkleizedStore for Immutable, D> { type Digest = H::Digest; type Operation = Operation; - type Dirty = Immutable; fn root(&self) -> Self::Digest { self.root() } - async fn proof( - &self, - start_loc: Location, - max_ops: NonZeroU64, - ) -> Result<(Proof, Vec), Error> { - self.proof(start_loc, max_ops).await - } - async fn historical_proof( &self, historical_size: Location, @@ -518,21 +571,19 @@ impl Self::Dirty { - self.into_dirty() - } } -impl - crate::qmdb::store::DirtyStore for Immutable +impl< + E: RStorage + Clock + Metrics, + K: Array, + V: VariableValue, + H: CHasher, + T: Translator, + D: DurabilityState, + > crate::qmdb::store::PrunableStore for Immutable, D> { - type Digest = H::Digest; - type Operation = Operation; - type Clean = Immutable>; - - async fn merkleize(self) -> Result { - Ok(self.merkleize()) + async fn prune(&mut self, prune_loc: Location) -> Result<(), Error> { + self.prune(prune_loc).await } } @@ -585,7 +636,7 @@ pub(super) mod test { pub fn test_immutable_db_empty() { let executor = deterministic::Runner::default(); executor.start(|context| async move { - let mut db = open_db(context.clone()).await; + let db = open_db(context.clone()).await; assert_eq!(db.op_count(), 1); assert_eq!(db.oldest_retained_loc(), Location::new_unchecked(0)); assert!(db.get_metadata().await.unwrap().is_none()); @@ -594,15 +645,17 @@ pub(super) mod test { let k1 = Sha256::fill(1u8); let v1 = vec![4, 5, 6, 7]; let root = db.root(); + let mut db = db.into_mutable(); db.set(k1, v1).await.unwrap(); - db.sync().await.unwrap(); - drop(db); - let mut db = open_db(context.clone()).await; + drop(db); // Simulate failed commit + let db = open_db(context.clone()).await; assert_eq!(db.root(), root); assert_eq!(db.op_count(), 1); // Test calling commit on an empty db which should make it (durably) non-empty. - db.commit(None).await.unwrap(); + let db = db.into_mutable(); + let (durable_db, _) = db.commit(None).await.unwrap(); + let db = durable_db.into_merkleized(); assert_eq!(db.op_count(), 2); // commit op added let root = db.root(); drop(db); @@ -619,7 +672,7 @@ pub(super) mod test { let executor = deterministic::Runner::default(); executor.start(|context| async move { // Build a db with 2 keys. - let mut db = open_db(context.clone()).await; + let db = open_db(context.clone()).await; let k1 = Sha256::fill(1u8); let k2 = Sha256::fill(2u8); @@ -630,18 +683,21 @@ pub(super) mod test { assert!(db.get(&k2).await.unwrap().is_none()); // Set the first key. + let mut db = db.into_mutable(); db.set(k1, v1.clone()).await.unwrap(); assert_eq!(db.get(&k1).await.unwrap().unwrap(), v1); assert!(db.get(&k2).await.unwrap().is_none()); assert_eq!(db.op_count(), 2); // Commit the first key. let metadata = Some(vec![99, 100]); - db.commit(metadata.clone()).await.unwrap(); + let (durable_db, _) = db.commit(metadata.clone()).await.unwrap(); + let db = durable_db.into_merkleized(); assert_eq!(db.get(&k1).await.unwrap().unwrap(), v1); assert!(db.get(&k2).await.unwrap().is_none()); assert_eq!(db.op_count(), 3); assert_eq!(db.get_metadata().await.unwrap(), metadata.clone()); // Set the second key. + let mut db = db.into_mutable(); db.set(k2, v2.clone()).await.unwrap(); assert_eq!(db.get(&k1).await.unwrap().unwrap(), v1); assert_eq!(db.get(&k2).await.unwrap().unwrap(), v2); @@ -651,23 +707,23 @@ pub(super) mod test { assert_eq!(db.get_metadata().await.unwrap(), metadata); // Commit the second key. - db.commit(None).await.unwrap(); + let (durable_db, _) = db.commit(None).await.unwrap(); + let db = durable_db.into_merkleized(); assert_eq!(db.op_count(), 5); assert_eq!(db.get_metadata().await.unwrap(), None); // Capture state. let root = db.root(); - // Add an uncommitted op then close the db. + // Add an uncommitted op then simulate failure. let k3 = Sha256::fill(3u8); let v3 = vec![9, 10, 11]; + let mut db = db.into_mutable(); db.set(k3, v3).await.unwrap(); assert_eq!(db.op_count(), 6); - assert_ne!(db.root(), root); - // Drop & reopen, make sure state is restored to last commit point. - db.sync().await.unwrap(); - drop(db); + // Reopen, make sure state is restored to last commit point. + drop(db); // Simulate failed commit let db = open_db(context.clone()).await; assert!(db.get(&k3).await.unwrap().is_none()); assert_eq!(db.op_count(), 5); @@ -686,7 +742,8 @@ pub(super) mod test { const ELEMENTS: u64 = 2_000; executor.start(|context| async move { let mut hasher = Standard::::new(); - let mut db = open_db(context.clone()).await; + let db = open_db(context.clone()).await; + let mut db = db.into_mutable(); for i in 0u64..ELEMENTS { let k = Sha256::hash(&i.to_be_bytes()); @@ -696,7 +753,8 @@ pub(super) mod test { assert_eq!(db.op_count(), ELEMENTS + 1); - db.commit(None).await.unwrap(); + let (durable_db, _) = db.commit(None).await.unwrap(); + let db = durable_db.into_merkleized(); assert_eq!(db.op_count(), ELEMENTS + 2); // Drop & reopen the db, making sure it has exactly the same state. @@ -736,7 +794,8 @@ pub(super) mod test { executor.start(|context| async move { // Insert 1000 keys then sync. const ELEMENTS: u64 = 1000; - let mut db = open_db(context.clone()).await; + let db = open_db(context.clone()).await; + let mut db = db.into_mutable(); for i in 0u64..ELEMENTS { let k = Sha256::hash(&i.to_be_bytes()); @@ -745,29 +804,35 @@ pub(super) mod test { } assert_eq!(db.op_count(), ELEMENTS + 1); + let (durable_db, _) = db.commit(None).await.unwrap(); + let mut db = durable_db.into_merkleized(); db.sync().await.unwrap(); let halfway_root = db.root(); // Insert another 1000 keys then simulate a failed close and test recovery. + let mut db = db.into_mutable(); for i in 0u64..ELEMENTS { let k = Sha256::hash(&i.to_be_bytes()); let v = vec![i as u8; 100]; db.set(k, v).await.unwrap(); } - // We partially write only 101 of the cached MMR nodes to simulate a failure. - db.simulate_failed_commit_mmr(101).await.unwrap(); + // Commit without merkleizing the MMR, then drop to simulate failure. + // The commit persists the data to the journal, but the MMR is not synced. + let (durable_db, _) = db.commit(None).await.unwrap(); + drop(durable_db); // Drop before merkleizing - // Recovery should replay the log to regenerate the mmr. + // Recovery should replay the log to regenerate the MMR. + // op_count = 1002 (first batch + commit) + 1000 (second batch) + 1 (second commit) = 2003 let db = open_db(context.clone()).await; - assert_eq!(db.op_count(), 2002); + assert_eq!(db.op_count(), 2003); let root = db.root(); assert_ne!(root, halfway_root); // Drop & reopen could preserve the final commit. drop(db); let db = open_db(context.clone()).await; - assert_eq!(db.op_count(), 2002); + assert_eq!(db.op_count(), 2003); assert_eq!(db.root(), root); db.destroy().await.unwrap(); @@ -778,18 +843,20 @@ pub(super) mod test { pub fn test_immutable_db_recovery_from_failed_log_sync() { let executor = deterministic::Runner::default(); executor.start(|context| async move { - let mut db = open_db(context.clone()).await; + let mut db = open_db(context.clone()).await.into_mutable(); // Insert a single key and then commit to create a first commit point. let k1 = Sha256::fill(1u8); let v1 = vec![1, 2, 3]; db.set(k1, v1).await.unwrap(); - db.commit(None).await.unwrap(); + let (durable_db, _) = db.commit(None).await.unwrap(); + let db = durable_db.into_merkleized(); let first_commit_root = db.root(); // Insert 1000 keys then sync. const ELEMENTS: u64 = 1000; + let mut db = db.into_mutable(); for i in 0u64..ELEMENTS { let k = Sha256::hash(&i.to_be_bytes()); let v = vec![i as u8; 100]; @@ -797,7 +864,6 @@ pub(super) mod test { } assert_eq!(db.op_count(), ELEMENTS + 3); - db.sync().await.unwrap(); // Insert another 1000 keys then simulate a failed close and test recovery. for i in 0u64..ELEMENTS { @@ -806,8 +872,8 @@ pub(super) mod test { db.set(k, v).await.unwrap(); } - // Simulate failure to write the full locations map. - db.simulate_failed_commit_log().await.unwrap(); + // Simulate failure. + drop(db); // Recovery should back up to previous commit point. let db = open_db(context.clone()).await; @@ -825,7 +891,8 @@ pub(super) mod test { // Build a db with `ELEMENTS` key/value pairs then prune some of them. const ELEMENTS: u64 = 2_000; executor.start(|context| async move { - let mut db = open_db(context.clone()).await; + let db = open_db(context.clone()).await; + let mut db = db.into_mutable(); for i in 1u64..ELEMENTS+1 { let k = Sha256::hash(&i.to_be_bytes()); @@ -835,7 +902,8 @@ pub(super) mod test { assert_eq!(db.op_count(), ELEMENTS + 1); - db.commit(None).await.unwrap(); + let (durable_db, _) = db.commit(None).await.unwrap(); + let mut db = durable_db.into_merkleized(); assert_eq!(db.op_count(), ELEMENTS + 2); // Prune the db to the first half of the operations. @@ -933,22 +1001,24 @@ pub(super) mod test { let v2 = vec![2u8; 16]; let v3 = vec![3u8; 16]; + let mut db = db.into_mutable(); db.set(k1, v1.clone()).await.unwrap(); db.set(k2, v2.clone()).await.unwrap(); - db.commit(None).await.unwrap(); + let (durable_db, _) = db.commit(None).await.unwrap(); + let db = durable_db.into_merkleized(); + let mut db = db.into_mutable(); db.set(k3, v3.clone()).await.unwrap(); - // op_count is 4 (k1, k2, commit, k3), last_commit is at location 2 + // op_count is 5 (initial_commit, k1, k2, commit, k3), last_commit is at location 3 assert_eq!(*db.last_commit_loc, 3); - // Test valid prune (at last commit) - assert!(db.prune(db.last_commit_loc).await.is_ok()); - - // Add more and commit again - db.commit(None).await.unwrap(); - let new_last_commit = db.last_commit_loc; + // Test valid prune (at last commit) - need Merkleized state for prune + let (durable_db, _) = db.commit(None).await.unwrap(); + let mut db = durable_db.into_merkleized(); + assert!(db.prune(Location::new_unchecked(3)).await.is_ok()); // Test pruning beyond last commit + let new_last_commit = db.last_commit_loc; let beyond = new_last_commit + 1; let result = db.prune(beyond).await; assert!( diff --git a/storage/src/qmdb/immutable/sync/mod.rs b/storage/src/qmdb/immutable/sync/mod.rs index 1bddb19c94..3698541083 100644 --- a/storage/src/qmdb/immutable/sync/mod.rs +++ b/storage/src/qmdb/immutable/sync/mod.rs @@ -178,7 +178,7 @@ mod tests { sync::Arc, }; - /// Type alias for sync tests with simple codec config + /// Type alias for sync tests with simple codec config (Merkleized, Durable) type ImmutableSyncTest = immutable::Immutable< deterministic::Context, sha256::Digest, @@ -187,6 +187,17 @@ mod tests { crate::translator::TwoCap, >; + /// Type alias for mutable state (Unmerkleized, NonDurable) + type ImmutableSyncTestMutable = immutable::Immutable< + deterministic::Context, + sha256::Digest, + sha256::Digest, + Sha256, + crate::translator::TwoCap, + immutable::Unmerkleized, + immutable::NonDurable, + >; + /// Create a simple config for sync tests fn create_sync_config(suffix: &str) -> immutable::Config { const PAGE_SIZE: NonZeroUsize = NZUsize!(77); @@ -231,7 +242,7 @@ mod tests { /// Applies the given operations to the database. async fn apply_ops( - db: &mut ImmutableSyncTest, + db: &mut ImmutableSyncTestMutable, ops: Vec>, ) { for op in ops { @@ -239,8 +250,9 @@ mod tests { Operation::Set(key, value) => { db.set(key, value).await.unwrap(); } - Operation::Commit(metadata) => { - db.commit(metadata).await.unwrap(); + Operation::Commit(_metadata) => { + // Commit causes a state change, so it is not supported here. + panic!("Commit operation not supported in apply_ops"); } } } @@ -258,11 +270,12 @@ mod tests { fn test_sync(#[case] target_db_ops: usize, #[case] fetch_batch_size: NonZeroU64) { let executor = deterministic::Runner::default(); executor.start(|mut context| async move { - let mut target_db = create_test_db(context.clone()).await; + let mut target_db = create_test_db(context.clone()).await.into_mutable(); let target_db_ops = create_test_ops(target_db_ops); apply_ops(&mut target_db, target_db_ops.clone()).await; let metadata = Some(Sha256::fill(1)); - target_db.commit(metadata).await.unwrap(); + let (durable_db, _) = target_db.commit(metadata).await.unwrap(); + let target_db = durable_db.into_merkleized(); let target_op_count = target_db.op_count(); let target_oldest_retained_loc = target_db.oldest_retained_loc(); let target_root = target_db.root(); @@ -291,7 +304,7 @@ mod tests { max_outstanding_requests: 1, update_rx: None, }; - let mut got_db: ImmutableSyncTest = sync::sync(config).await.unwrap(); + let got_db: ImmutableSyncTest = sync::sync(config).await.unwrap(); // Verify database state assert_eq!(got_db.op_count(), target_op_count); @@ -317,30 +330,27 @@ mod tests { new_kvs.insert(key, value); } - // Apply new operations to both databases + // Apply new operations to both databases. + let mut got_db = got_db.into_mutable(); apply_ops(&mut got_db, new_ops.clone()).await; - { - let mut target_db = target_db.write().await; - apply_ops(&mut target_db, new_ops).await; - } + let mut target_db = Arc::try_unwrap(target_db).map_or_else( + |_| panic!("target_db should have no other references"), + |rw_lock| rw_lock.into_inner().into_mutable(), + ); + apply_ops(&mut target_db, new_ops.clone()).await; - // Verify both databases have the same state after additional operations + // Verify both databases have the new values for (key, expected_value) in &new_kvs { let synced_value = got_db.get(key).await.unwrap(); - let target_value = { - let target_db = target_db.read().await; - target_db.get(key).await.unwrap() - }; assert_eq!(synced_value, Some(*expected_value)); + let target_value = target_db.get(key).await.unwrap(); assert_eq!(target_value, Some(*expected_value)); } - got_db.destroy().await.unwrap(); - let target_db = Arc::try_unwrap(target_db).map_or_else( - |_| panic!("Failed to unwrap Arc - still has references"), - |rw_lock| rw_lock.into_inner(), - ); - target_db.destroy().await.unwrap(); + let (got_durable, _) = got_db.commit(None).await.unwrap(); + got_durable.into_merkleized().destroy().await.unwrap(); + let (target_durable, _) = target_db.commit(None).await.unwrap(); + target_durable.into_merkleized().destroy().await.unwrap(); }); } @@ -350,8 +360,10 @@ mod tests { let executor = deterministic::Runner::default(); executor.start(|mut context| async move { // Create an empty target database - let mut target_db = create_test_db(context.clone()).await; - target_db.commit(Some(Sha256::fill(1))).await.unwrap(); // Commit to establish a valid root + let target_db = create_test_db(context.clone()).await; + let target_db = target_db.into_mutable(); + let (durable_db, _) = target_db.commit(Some(Sha256::fill(1))).await.unwrap(); // Commit to establish a valid root + let target_db = durable_db.into_merkleized(); let target_op_count = target_db.op_count(); let target_oldest_retained_loc = target_db.oldest_retained_loc(); @@ -395,10 +407,12 @@ mod tests { let executor = deterministic::Runner::default(); executor.start(|context| async move { // Create and populate a simple target database - let mut target_db = create_test_db(context.clone()).await; + let target_db = create_test_db(context.clone()).await; + let mut target_db = target_db.into_mutable(); let target_ops = create_test_ops(10); apply_ops(&mut target_db, target_ops.clone()).await; - target_db.commit(Some(Sha256::fill(0))).await.unwrap(); + let (durable_db, _) = target_db.commit(Some(Sha256::fill(0))).await.unwrap(); + let target_db = durable_db.into_merkleized(); // Capture target state let target_root = target_db.root(); @@ -470,10 +484,12 @@ mod tests { let executor = deterministic::Runner::default(); executor.start(|mut context| async move { // Create and populate initial target database - let mut target_db = create_test_db(context.clone()).await; + let target_db = create_test_db(context.clone()).await; + let mut target_db = target_db.into_mutable(); let initial_ops = create_test_ops(50); apply_ops(&mut target_db, initial_ops.clone()).await; - target_db.commit(None).await.unwrap(); + let (durable_db, _) = target_db.commit(None).await.unwrap(); + let target_db = durable_db.into_merkleized(); // Capture the state after first commit let initial_lower_bound = target_db.oldest_retained_loc(); @@ -481,9 +497,11 @@ mod tests { let initial_root = target_db.root(); // Add more operations to create the extended target + let mut target_db = target_db.into_mutable(); let additional_ops = create_test_ops(25); apply_ops(&mut target_db, additional_ops.clone()).await; - target_db.commit(None).await.unwrap(); + let (durable_db, _) = target_db.commit(None).await.unwrap(); + let target_db = durable_db.into_merkleized(); let final_upper_bound = target_db.op_count(); let final_root = target_db.root(); @@ -603,19 +621,23 @@ mod tests { fn test_sync_subset_of_target_database() { let executor = deterministic::Runner::default(); executor.start(|mut context| async move { - let mut target_db = create_test_db(context.clone()).await; + let target_db = create_test_db(context.clone()).await; + let mut target_db = target_db.into_mutable(); let target_ops = create_test_ops(30); // Apply all but the last operation apply_ops(&mut target_db, target_ops[..29].to_vec()).await; - target_db.commit(None).await.unwrap(); + let (durable_db, _) = target_db.commit(None).await.unwrap(); + let target_db = durable_db.into_merkleized(); let target_root = target_db.root(); let lower_bound = target_db.oldest_retained_loc(); let op_count = target_db.op_count(); // Add final op after capturing the range + let mut target_db = target_db.into_mutable(); apply_ops(&mut target_db, target_ops[29..].to_vec()).await; - target_db.commit(None).await.unwrap(); + let (durable_db, _) = target_db.commit(None).await.unwrap(); + let target_db = durable_db.into_merkleized(); let target_db = Arc::new(commonware_runtime::RwLock::new(target_db)); let config = Config { @@ -654,25 +676,31 @@ mod tests { let original_ops = create_test_ops(50); // Create two databases - let mut target_db = create_test_db(context.clone()).await; + let target_db = create_test_db(context.clone()).await; + let mut target_db = target_db.into_mutable(); let sync_db_config = create_sync_config(&format!("partial_{}", context.next_u64())); - let mut sync_db: ImmutableSyncTest = + let sync_db: ImmutableSyncTest = immutable::Immutable::init(context.clone(), sync_db_config.clone()) .await .unwrap(); + let mut sync_db = sync_db.into_mutable(); // Apply the same operations to both databases apply_ops(&mut target_db, original_ops.clone()).await; apply_ops(&mut sync_db, original_ops.clone()).await; - target_db.commit(None).await.unwrap(); - sync_db.commit(None).await.unwrap(); + let (durable_db, _) = target_db.commit(None).await.unwrap(); + let target_db = durable_db.into_merkleized(); + let (durable_db, _) = sync_db.commit(None).await.unwrap(); + let sync_db = durable_db.into_merkleized(); drop(sync_db); // Add one more operation and commit the target database + let mut target_db = target_db.into_mutable(); let last_op = create_test_ops(1); apply_ops(&mut target_db, last_op.clone()).await; - target_db.commit(None).await.unwrap(); + let (durable_db, _) = target_db.commit(None).await.unwrap(); + let target_db = durable_db.into_merkleized(); let root = target_db.root(); let lower_bound = target_db.oldest_retained_loc(); let upper_bound = target_db.op_count(); // Up to the last operation @@ -714,18 +742,22 @@ mod tests { let target_ops = create_test_ops(40); // Create two databases - let mut target_db = create_test_db(context.clone()).await; + let target_db = create_test_db(context.clone()).await; + let mut target_db = target_db.into_mutable(); let sync_config = create_sync_config(&format!("exact_{}", context.next_u64())); - let mut sync_db: ImmutableSyncTest = + let sync_db: ImmutableSyncTest = immutable::Immutable::init(context.clone(), sync_config.clone()) .await .unwrap(); + let mut sync_db = sync_db.into_mutable(); // Apply the same operations to both databases apply_ops(&mut target_db, target_ops.clone()).await; apply_ops(&mut sync_db, target_ops.clone()).await; - target_db.commit(None).await.unwrap(); - sync_db.commit(None).await.unwrap(); + let (durable_db, _) = target_db.commit(None).await.unwrap(); + let target_db = durable_db.into_merkleized(); + let (durable_db, _) = sync_db.commit(None).await.unwrap(); + let sync_db = durable_db.into_merkleized(); drop(sync_db); @@ -768,10 +800,12 @@ mod tests { let executor = deterministic::Runner::default(); executor.start(|mut context| async move { // Create and populate target database - let mut target_db = create_test_db(context.clone()).await; + let target_db = create_test_db(context.clone()).await; + let mut target_db = target_db.into_mutable(); let target_ops = create_test_ops(100); apply_ops(&mut target_db, target_ops).await; - target_db.commit(None).await.unwrap(); + let (durable_db, _) = target_db.commit(None).await.unwrap(); + let mut target_db = durable_db.into_merkleized(); target_db.prune(Location::new_unchecked(10)).await.unwrap(); @@ -828,10 +862,12 @@ mod tests { let executor = deterministic::Runner::default(); executor.start(|mut context| async move { // Create and populate target database - let mut target_db = create_test_db(context.clone()).await; + let target_db = create_test_db(context.clone()).await; + let mut target_db = target_db.into_mutable(); let target_ops = create_test_ops(50); apply_ops(&mut target_db, target_ops).await; - target_db.commit(None).await.unwrap(); + let (durable_db, _) = target_db.commit(None).await.unwrap(); + let target_db = durable_db.into_merkleized(); // Capture initial target state let initial_lower_bound = target_db.oldest_retained_loc(); @@ -886,10 +922,12 @@ mod tests { let executor = deterministic::Runner::default(); executor.start(|mut context| async move { // Create and populate target database - let mut target_db = create_test_db(context.clone()).await; + let target_db = create_test_db(context.clone()).await; + let mut target_db = target_db.into_mutable(); let target_ops = create_test_ops(100); apply_ops(&mut target_db, target_ops.clone()).await; - target_db.commit(None).await.unwrap(); + let (durable_db, _) = target_db.commit(None).await.unwrap(); + let target_db = durable_db.into_merkleized(); // Capture initial target state let initial_lower_bound = target_db.oldest_retained_loc(); @@ -897,12 +935,16 @@ mod tests { let initial_root = target_db.root(); // Apply more operations to the target database + let mut target_db = target_db.into_mutable(); let more_ops = create_test_ops(5); apply_ops(&mut target_db, more_ops.clone()).await; - target_db.commit(None).await.unwrap(); + let (durable_db, _) = target_db.commit(None).await.unwrap(); + let mut target_db = durable_db.into_merkleized(); target_db.prune(Location::new_unchecked(10)).await.unwrap(); - target_db.commit(None).await.unwrap(); + let target_db = target_db.into_mutable(); + let (durable_db, _) = target_db.commit(None).await.unwrap(); + let target_db = durable_db.into_merkleized(); // Capture final target state let final_lower_bound = target_db.oldest_retained_loc(); @@ -962,10 +1004,12 @@ mod tests { let executor = deterministic::Runner::default(); executor.start(|mut context| async move { // Create and populate target database - let mut target_db = create_test_db(context.clone()).await; + let target_db = create_test_db(context.clone()).await; + let mut target_db = target_db.into_mutable(); let target_ops = create_test_ops(25); apply_ops(&mut target_db, target_ops).await; - target_db.commit(None).await.unwrap(); + let (durable_db, _) = target_db.commit(None).await.unwrap(); + let target_db = durable_db.into_merkleized(); // Capture initial target state let initial_lower_bound = target_db.oldest_retained_loc(); @@ -1018,10 +1062,12 @@ mod tests { let executor = deterministic::Runner::default(); executor.start(|mut context| async move { // Create and populate target database - let mut target_db = create_test_db(context.clone()).await; + let target_db = create_test_db(context.clone()).await; + let mut target_db = target_db.into_mutable(); let target_ops = create_test_ops(10); apply_ops(&mut target_db, target_ops).await; - target_db.commit(None).await.unwrap(); + let (durable_db, _) = target_db.commit(None).await.unwrap(); + let target_db = durable_db.into_merkleized(); // Capture target state let lower_bound = target_db.oldest_retained_loc(); diff --git a/storage/src/qmdb/keyless/mod.rs b/storage/src/qmdb/keyless/mod.rs index 1ec06d8c15..5821b74fbb 100644 --- a/storage/src/qmdb/keyless/mod.rs +++ b/storage/src/qmdb/keyless/mod.rs @@ -1,27 +1,22 @@ //! The [Keyless] qmdb allows for append-only storage of arbitrary variable-length data that can //! later be retrieved by its location. -//! -//! The implementation consists of an `mmr` over the operations applied to the database and an -//! operations `log` storing these operations. -//! -//! The state of the operations log up until the last commit point is the "source of truth". In the -//! event of unclean shutdown, the mmr will be brought back into alignment with the log on startup. use crate::{ journal::{ authenticated, contiguous::variable::{Config as JournalConfig, Journal as ContiguousJournal}, }, - mmr::{ - journaled::Config as MmrConfig, - mem::{Clean, Dirty, State}, - Location, Proof, + mmr::{journaled::Config as MmrConfig, Location, Proof}, + qmdb::{ + any::VariableValue, + operation::Committable, + store::{LogStore, MerkleizedStore, PrunableStore}, + DurabilityState, Durable, Error, MerkleizationState, Merkleized, NonDurable, Unmerkleized, }, - qmdb::{any::VariableValue, operation::Committable, Error}, }; use commonware_cryptography::{DigestOf, Hasher}; use commonware_runtime::{buffer::PoolRef, Clock, Metrics, Storage, ThreadPool}; -use core::ops::Range; +use core::{marker::PhantomData, ops::Range}; use std::num::{NonZeroU64, NonZeroUsize}; use tracing::{debug, warn}; @@ -68,21 +63,32 @@ pub struct Config { /// A keyless QMDB for variable length data. type Journal = authenticated::Journal>, H, S>; +/// A keyless authenticated database for variable-length data. pub struct Keyless< E: Storage + Clock + Metrics, V: VariableValue, H: Hasher, - S: State> = Dirty, + M: MerkleizationState> = Merkleized, + D: DurabilityState = Durable, > { /// Authenticated journal of operations. - journal: Journal, + journal: Journal, /// The location of the last commit, if any. last_commit_loc: Location, + + /// Marker for durability state. + _durability: PhantomData, } -impl>> - Keyless +// Impl block for functionality available in all states. +impl< + E: Storage + Clock + Metrics, + V: VariableValue, + H: Hasher, + M: MerkleizationState>, + D: DurabilityState, + > Keyless { /// Get the value at location `loc` in the database. /// @@ -117,12 +123,9 @@ impl Result { - self.journal - .append(Operation::Append(value)) - .await - .map_err(Into::into) + /// Return the oldest location that is no longer required to be retained. + pub fn inactivity_floor_loc(&self) -> Location { + self.journal.pruning_boundary() } /// Get the metadata associated with the last commit. @@ -136,7 +139,10 @@ impl Keyless> { +// Implementation for the Clean state. +impl + Keyless, Durable> +{ /// Returns a [Keyless] qmdb initialized from `cfg`. Any uncommitted operations will be discarded /// and the state of the db will be as of the last committed operation. pub async fn init(context: E, cfg: Config) -> Result { @@ -173,6 +179,7 @@ impl Keyless Keyless) -> Result, Error> { - let start_loc = self.last_commit_loc + 1; - self.last_commit_loc = self.journal.append(Operation::Commit(metadata)).await?; - self.journal.commit().await?; - debug!(size = ?self.op_count(), "committed db"); - - Ok(start_loc..self.op_count()) - } - /// Prune historical operations prior to `loc`. This does not affect the db's root. /// /// # Errors @@ -260,99 +251,94 @@ impl Keyless Result<(), Error> { - if sync_log { - self.journal.journal.sync().await.map_err(Error::Journal)?; - } - if sync_mmr { - self.journal.mmr.sync().await.map_err(Error::Mmr)?; + /// Convert this database into the Mutable state for accepting new operations. + pub fn into_mutable(self) -> Keyless { + Keyless { + journal: self.journal.into_dirty(), + last_commit_loc: self.last_commit_loc, + _durability: PhantomData, } - - Ok(()) } +} - #[cfg(test)] - /// Simulate pruning failure by consuming the db and abandoning pruning operation mid-flight. - pub(super) async fn simulate_prune_failure(mut self, loc: Location) -> Result<(), Error> { - if loc > self.last_commit_loc { - return Err(Error::PruneBeyondMinRequired(loc, self.last_commit_loc)); - } - // Perform the same steps as pruning except "crash" right after the log is pruned. - self.journal.mmr.sync().await.map_err(Error::Mmr)?; - assert!( - self.journal - .journal - .prune(*loc) - .await - .map_err(Error::Journal)?, - "nothing was pruned, so could not simulate failure" - ); - - // "fail" before mmr is pruned. - Ok(()) +// Implementation for the Mutable state. +impl + Keyless +{ + /// Append a value to the db, returning its location which can be used to retrieve it. + pub async fn append(&mut self, value: V) -> Result { + self.journal + .append(Operation::Append(value)) + .await + .map_err(Into::into) } - /// Convert this database into its dirty counterpart for batched updates. - pub fn into_dirty(self) -> Keyless { - Keyless { - journal: self.journal.into_dirty(), + /// Commits any pending operations and transitions the database to the Durable state. + /// + /// The caller can associate an arbitrary `metadata` value with the commit. Returns the + /// `(start_loc, end_loc]` location range of committed operations. The end of the returned + /// range includes the commit operation itself, and hence will always be equal to `op_count`. + pub async fn commit( + mut self, + metadata: Option, + ) -> Result<(Keyless, Range), Error> { + let start_loc = self.last_commit_loc + 1; + self.last_commit_loc = self.journal.append(Operation::Commit(metadata)).await?; + self.journal.commit().await?; + debug!(size = ?self.op_count(), "committed db"); + + let op_count = self.op_count(); + let durable = Keyless { + journal: self.journal, last_commit_loc: self.last_commit_loc, - } + _durability: PhantomData, + }; + + Ok((durable, start_loc..op_count)) } -} -impl Keyless { - /// Merkleize the database and compute the root digest. - pub fn merkleize(self) -> Keyless> { + pub fn into_merkleized(self) -> Keyless, Durable> { Keyless { journal: self.journal.merkleize(), last_commit_loc: self.last_commit_loc, + _durability: PhantomData, } } } -impl>> - crate::qmdb::store::LogStore for Keyless +// Implementation for the (Unmerkleized, Durable) state. +impl + Keyless { - type Value = V; - - fn op_count(&self) -> Location { - self.op_count() - } - - // All unpruned operations are active in a keyless store. - fn inactivity_floor_loc(&self) -> Location { - self.journal.pruning_boundary() - } - - fn is_empty(&self) -> bool { - self.op_count() == 0 + /// Convert this database into the Mutable state for accepting more operations without + /// re-merkleizing. + pub fn into_mutable(self) -> Keyless { + Keyless { + journal: self.journal, + last_commit_loc: self.last_commit_loc, + _durability: PhantomData, + } } - async fn get_metadata(&self) -> Result, Error> { - self.get_metadata().await + /// Compute the merkle root and transition to the Merkleized, Durable state. + pub fn into_merkleized(self) -> Keyless, Durable> { + Keyless { + journal: self.journal.merkleize(), + last_commit_loc: self.last_commit_loc, + _durability: PhantomData, + } } } -impl crate::qmdb::store::CleanStore - for Keyless> +// Implementation of MerkleizedStore for the Merkleized state (any durability). +impl MerkleizedStore + for Keyless, D> { type Digest = H::Digest; type Operation = Operation; - type Dirty = Keyless; fn root(&self) -> Self::Digest { - self.root() - } - - async fn proof( - &self, - start_loc: Location, - max_ops: NonZeroU64, - ) -> Result<(Proof, Vec), Error> { - self.proof(start_loc, max_ops).await + self.journal.root() } async fn historical_proof( @@ -361,24 +347,53 @@ impl crate::qmdb::sto start_loc: Location, max_ops: NonZeroU64, ) -> Result<(Proof, Vec), Error> { - self.historical_proof(historical_size, start_loc, max_ops) - .await + Ok(self + .journal + .historical_proof(historical_size, start_loc, max_ops) + .await?) } +} + +// Implementation of LogStore for all states. +impl< + E: Storage + Clock + Metrics, + V: VariableValue, + H: Hasher, + M: MerkleizationState>, + D: DurabilityState, + > LogStore for Keyless +{ + type Value = V; - fn into_dirty(self) -> Self::Dirty { - self.into_dirty() + fn is_empty(&self) -> bool { + // A keyless database is never "empty" in the traditional sense since it always + // has at least one commit operation. We consider it empty if there are no appends. + self.op_count() <= 1 + } + + fn op_count(&self) -> Location { + self.op_count() + } + + fn inactivity_floor_loc(&self) -> Location { + self.inactivity_floor_loc() + } + + async fn get_metadata(&self) -> Result, Error> { + self.get_metadata().await } } -impl crate::qmdb::store::DirtyStore - for Keyless +// Implementation of PrunableStore for the Merkleized state (any durability). +impl PrunableStore + for Keyless, D> { - type Digest = H::Digest; - type Operation = Operation; - type Clean = Keyless>; - - async fn merkleize(self) -> Result { - Ok(self.merkleize()) + async fn prune(&mut self, loc: Location) -> Result<(), Error> { + if loc > self.last_commit_loc { + return Err(Error::PruneBeyondMinRequired(loc, self.last_commit_loc)); + } + self.journal.prune(loc).await?; + Ok(()) } } @@ -412,19 +427,24 @@ mod test { } } - /// A type alias for the concrete [Keyless] type used in these unit tests. - type Db = Keyless, Sha256, Clean<::Digest>>; + /// Type alias for the Merkleized, Durable state. + type CleanDb = Keyless, Sha256, Merkleized, Durable>; + + /// Type alias for the Mutable (Unmerkleized, NonDurable) state. + type MutableDb = Keyless, Sha256, Unmerkleized, NonDurable>; /// Return a [Keyless] database initialized with a fixed config. - async fn open_db(context: deterministic::Context) -> Db { - Db::init(context, db_config("partition")).await.unwrap() + async fn open_db(context: deterministic::Context) -> CleanDb { + CleanDb::init(context, db_config("partition")) + .await + .unwrap() } #[test_traced("INFO")] pub fn test_keyless_db_empty() { let executor = deterministic::Runner::default(); executor.start(|context| async move { - let mut db = open_db(context.clone()).await; + let db = open_db(context.clone()).await; assert_eq!(db.op_count(), 1); // initial commit should exist assert_eq!(db.oldest_retained_loc(), Location::new_unchecked(0)); @@ -434,17 +454,19 @@ mod test { // Make sure closing/reopening gets us back to the same state, even after adding an uncommitted op. let v1 = vec![1u8; 8]; let root = db.root(); + let mut db = db.into_mutable(); db.append(v1).await.unwrap(); - db.sync().await.unwrap(); - drop(db); - let mut db = open_db(context.clone()).await; + drop(db); // Simulate failed commit + let db = open_db(context.clone()).await; assert_eq!(db.root(), root); assert_eq!(db.op_count(), 1); assert_eq!(db.get_metadata().await.unwrap(), None); // Test calling commit on an empty db which should make it (durably) non-empty. let metadata = vec![3u8; 10]; - db.commit(Some(metadata.clone())).await.unwrap(); + let db = db.into_mutable(); + let (durable, _) = db.commit(Some(metadata.clone())).await.unwrap(); + let db = durable.into_merkleized(); assert_eq!(db.op_count(), 2); // 2 commit ops assert_eq!(db.get_metadata().await.unwrap(), Some(metadata.clone())); assert_eq!( @@ -469,7 +491,8 @@ mod test { let executor = deterministic::Runner::default(); executor.start(|context| async move { // Build a db with 2 values and make sure we can get them back. - let mut db = open_db(context.clone()).await; + let db = open_db(context.clone()).await; + let mut db = db.into_mutable(); let v1 = vec![1u8; 8]; let v2 = vec![2u8; 20]; @@ -481,25 +504,27 @@ mod test { assert_eq!(db.get(loc2).await.unwrap().unwrap(), v2); // Make sure closing/reopening gets us back to the same state. - db.commit(None).await.unwrap(); + let (durable, _) = db.commit(None).await.unwrap(); + let mut db = durable.into_merkleized(); assert_eq!(db.op_count(), 4); // 2 appends, 1 commit + 1 initial commit assert_eq!(db.get_metadata().await.unwrap(), None); assert_eq!(db.get(Location::new_unchecked(3)).await.unwrap(), None); // the commit op let root = db.root(); + db.sync().await.unwrap(); drop(db); - let mut db = open_db(context.clone()).await; + let db = open_db(context.clone()).await; assert_eq!(db.op_count(), 4); assert_eq!(db.root(), root); assert_eq!(db.get(loc1).await.unwrap().unwrap(), v1); assert_eq!(db.get(loc2).await.unwrap().unwrap(), v2); + let mut db = db.into_mutable(); db.append(v2).await.unwrap(); db.append(v1).await.unwrap(); // Make sure uncommitted items get rolled back. - db.sync().await.unwrap(); - drop(db); + drop(db); // Simulate failed commit let db = open_db(context.clone()).await; assert_eq!(db.op_count(), 4); assert_eq!(db.root(), root); @@ -515,7 +540,7 @@ mod test { } // Helper function to append random elements to a database. - async fn append_elements(db: &mut Db, rng: &mut T, num_elements: usize) { + async fn append_elements(db: &mut MutableDb, rng: &mut T, num_elements: usize) { for _ in 0..num_elements { let value = vec![(rng.next_u32() % 255) as u8, (rng.next_u32() % 255) as u8]; db.append(value).await.unwrap(); @@ -527,50 +552,40 @@ mod test { let executor = deterministic::Runner::default(); const ELEMENTS: usize = 1000; executor.start(|mut context| async move { - let mut db = open_db(context.clone()).await; + let db = open_db(context.clone()).await; let root = db.root(); + let mut db = db.into_mutable(); append_elements(&mut db, &mut context, ELEMENTS).await; // Simulate a failure before committing. - db.simulate_failure(false, false).await.unwrap(); + drop(db); // Should rollback to the previous root. - let mut db = open_db(context.clone()).await; + let db = open_db(context.clone()).await; assert_eq!(root, db.root()); // Re-apply the updates and commit them this time. + let mut db = db.into_mutable(); append_elements(&mut db, &mut context, ELEMENTS).await; - db.commit(None).await.unwrap(); + let (durable, _) = db.commit(None).await.unwrap(); + let db = durable.into_merkleized(); let root = db.root(); // Append more values. + let mut db = db.into_mutable(); append_elements(&mut db, &mut context, ELEMENTS).await; // Simulate a failure. - db.simulate_failure(false, false).await.unwrap(); - // Should rollback to the previous root. - let mut db = open_db(context.clone()).await; - assert_eq!(root, db.root()); - - // Re-apply the updates. - append_elements(&mut db, &mut context, ELEMENTS).await; - // Simulate a failure after syncing log but not MMR. - db.simulate_failure(true, false).await.unwrap(); - // Should rollback to the previous root. - let mut db = open_db(context.clone()).await; - assert_eq!(root, db.root()); - - // Re-apply the updates. - append_elements(&mut db, &mut context, ELEMENTS).await; - // Simulate a failure after syncing MMR but not log. - db.simulate_failure(false, true).await.unwrap(); + drop(db); // Should rollback to the previous root. - let mut db = open_db(context.clone()).await; + let db = open_db(context.clone()).await; assert_eq!(root, db.root()); // Re-apply the updates and commit them this time. + let mut db = db.into_mutable(); append_elements(&mut db, &mut context, ELEMENTS).await; - db.commit(None).await.unwrap(); + let (durable, _) = db.commit(None).await.unwrap(); + let db = durable.into_merkleized(); let root = db.root(); // Make sure we can reopen and get back to the same state. @@ -589,12 +604,14 @@ mod test { fn test_keyless_db_non_empty_db_recovery() { let executor = deterministic::Runner::default(); executor.start(|mut context| async move { - let mut db = open_db(context.clone()).await; + let db = open_db(context.clone()).await; // Append many values then commit. const ELEMENTS: usize = 200; + let mut db = db.into_mutable(); append_elements(&mut db, &mut context, ELEMENTS).await; - db.commit(None).await.unwrap(); + let (durable, _) = db.commit(None).await.unwrap(); + let db = durable.into_merkleized(); let root = db.root(); let op_count = db.op_count(); @@ -605,31 +622,17 @@ mod test { assert_eq!(db.last_commit_loc(), op_count - 1); drop(db); - // Insert many operations without commit, then simulate various types of failures. + // Insert many operations without commit, then simulate failure. async fn recover_from_failure( mut context: deterministic::Context, root: ::Digest, op_count: Location, ) { - let mut db = open_db(context.clone()).await; + let mut db = open_db(context.clone()).await.into_mutable(); // Append operations and simulate failure. append_elements(&mut db, &mut context, ELEMENTS).await; - db.simulate_failure(false, false).await.unwrap(); - let mut db = open_db(context.clone()).await; - assert_eq!(db.op_count(), op_count); - assert_eq!(db.root(), root); - - // Append operations and simulate failure after syncing log but not MMR. - append_elements(&mut db, &mut context, ELEMENTS).await; - db.simulate_failure(true, false).await.unwrap(); - let mut db = open_db(context.clone()).await; - assert_eq!(db.op_count(), op_count); - assert_eq!(db.root(), root); - - // Append operations and simulate failure after syncing MMR but not log. - append_elements(&mut db, &mut context, ELEMENTS).await; - db.simulate_failure(false, true).await.unwrap(); + drop(db); let db = open_db(context.clone()).await; assert_eq!(db.op_count(), op_count); assert_eq!(db.root(), root); @@ -637,15 +640,6 @@ mod test { recover_from_failure(context.clone(), root, op_count).await; - // Simulate a failure during pruning and ensure we recover. - let db = open_db(context.clone()).await; - let last_commit_loc = db.last_commit_loc(); - db.simulate_prune_failure(last_commit_loc).await.unwrap(); - let db = open_db(context.clone()).await; - assert_eq!(db.op_count(), op_count); - assert_eq!(db.root(), root); - drop(db); - // Repeat recover_from_failure tests after successfully pruning to the last commit. let mut db = open_db(context.clone()).await; db.prune(db.last_commit_loc()).await.unwrap(); @@ -657,9 +651,9 @@ mod test { recover_from_failure(context.clone(), root, op_count).await; // Apply the ops one last time but fully commit them this time, then clean up. - let mut db = open_db(context.clone()).await; + let mut db = open_db(context.clone()).await.into_mutable(); append_elements(&mut db, &mut context, ELEMENTS).await; - db.commit(None).await.unwrap(); + let (_durable, _) = db.commit(None).await.unwrap(); let db = open_db(context.clone()).await; assert!(db.op_count() > op_count); assert_ne!(db.root(), root); @@ -680,72 +674,48 @@ mod test { let root = db.root(); // Reopen DB without clean shutdown and make sure the state is the same. - let mut db = open_db(context.clone()).await; + let db = open_db(context.clone()).await; assert_eq!(db.op_count(), 1); // initial commit should exist assert_eq!(db.root(), root); - async fn apply_ops(db: &mut Db) { + async fn apply_ops(db: &mut MutableDb) { for i in 0..ELEMENTS { let v = vec![(i % 255) as u8; ((i % 17) + 13) as usize]; db.append(v).await.unwrap(); } } - // Simulate various failure types after inserting operations without a commit. - apply_ops(&mut db).await; - db.simulate_failure(false, false).await.unwrap(); - let mut db = open_db(context.clone()).await; - assert_eq!(db.op_count(), 1); // initial commit should exist - assert_eq!(db.root(), root); - - apply_ops(&mut db).await; - db.simulate_failure(true, false).await.unwrap(); - let mut db = open_db(context.clone()).await; - assert_eq!(db.op_count(), 1); // initial commit should exist - assert_eq!(db.root(), root); - - apply_ops(&mut db).await; - db.simulate_failure(false, false).await.unwrap(); - let mut db = open_db(context.clone()).await; - assert_eq!(db.op_count(), 1); // initial commit should exist - assert_eq!(db.root(), root); - + // Simulate failure after inserting operations without a commit. + let mut db = db.into_mutable(); apply_ops(&mut db).await; - db.simulate_failure(false, true).await.unwrap(); - let mut db = open_db(context.clone()).await; - assert_eq!(db.op_count(), 1); // initial commit should exist - assert_eq!(db.root(), root); - - apply_ops(&mut db).await; - db.simulate_failure(true, false).await.unwrap(); - let mut db = open_db(context.clone()).await; - assert_eq!(db.op_count(), 1); // initial commit should exist - assert_eq!(db.root(), root); - - apply_ops(&mut db).await; - db.simulate_failure(true, true).await.unwrap(); - let mut db = open_db(context.clone()).await; + drop(db); + let db = open_db(context.clone()).await; assert_eq!(db.op_count(), 1); // initial commit should exist assert_eq!(db.root(), root); + // Repeat: simulate failure after inserting operations without a commit. + let mut db = db.into_mutable(); apply_ops(&mut db).await; - db.simulate_failure(false, true).await.unwrap(); - let mut db = open_db(context.clone()).await; + drop(db); + let db = open_db(context.clone()).await; assert_eq!(db.op_count(), 1); // initial commit should exist assert_eq!(db.root(), root); // One last check that re-open without proper shutdown still recovers the correct state. + let mut db = db.into_mutable(); apply_ops(&mut db).await; apply_ops(&mut db).await; apply_ops(&mut db).await; - let mut db = open_db(context.clone()).await; + drop(db); + let db = open_db(context.clone()).await; assert_eq!(db.op_count(), 1); // initial commit should exist assert_eq!(db.root(), root); assert_eq!(db.last_commit_loc(), Location::new_unchecked(0)); // Apply the ops one last time but fully commit them this time, then clean up. + let mut db = db.into_mutable(); apply_ops(&mut db).await; - db.commit(None).await.unwrap(); + let (_db, _) = db.commit(None).await.unwrap(); let db = open_db(context.clone()).await; assert!(db.op_count() > 1); assert_ne!(db.root(), root); @@ -759,7 +729,8 @@ mod test { let executor = deterministic::Runner::default(); executor.start(|context| async move { let mut hasher = Standard::::new(); - let mut db = open_db(context.clone()).await; + let db = open_db(context.clone()).await; + let mut db = db.into_mutable(); // Build a db with some values const ELEMENTS: u64 = 100; @@ -769,7 +740,8 @@ mod test { values.push(v.clone()); db.append(v).await.unwrap(); } - db.commit(None).await.unwrap(); + let (durable, _) = db.commit(None).await.unwrap(); + let db = durable.into_merkleized(); // Test that historical proof fails with op_count > number of operations assert!(matches!( @@ -857,7 +829,8 @@ mod test { let executor = deterministic::Runner::default(); executor.start(|context| async move { let mut hasher = Standard::::new(); - let mut db = open_db(context.clone()).await; + let db = open_db(context.clone()).await; + let mut db = db.into_mutable(); // Build a db with some values const ELEMENTS: u64 = 100; @@ -867,15 +840,17 @@ mod test { values.push(v.clone()); db.append(v).await.unwrap(); } - db.commit(None).await.unwrap(); + let (durable, _) = db.commit(None).await.unwrap(); // Add more elements and commit again + let mut db = durable.into_mutable(); for i in ELEMENTS..ELEMENTS * 2 { let v = vec![(i % 255) as u8; ((i % 17) + 5) as usize]; values.push(v.clone()); db.append(v).await.unwrap(); } - db.commit(None).await.unwrap(); + let (durable, _) = db.commit(None).await.unwrap(); + let mut db = durable.into_merkleized(); let root = db.root(); println!("last commit loc: {}", db.last_commit_loc()); @@ -984,26 +959,29 @@ mod test { let executor = deterministic::Runner::default(); executor.start(|context| async move { // Create initial database with committed data - let mut db = open_db(context.clone()).await; + let db = open_db(context.clone()).await; + let mut db = db.into_mutable(); // Add some initial operations and commit for i in 0..10 { let v = vec![i as u8; 10]; db.append(v).await.unwrap(); } - db.commit(None).await.unwrap(); + let (durable, _) = db.commit(None).await.unwrap(); + let db = durable.into_merkleized(); let committed_root = db.root(); let committed_size = db.op_count(); // Add exactly one more append (uncommitted) let uncommitted_value = vec![99u8; 20]; + let mut db = db.into_mutable(); db.append(uncommitted_value.clone()).await.unwrap(); - // Sync only the log (not MMR) - db.simulate_failure(true, false).await.unwrap(); + // Simulate failure without commit + drop(db); // Reopen database - let mut db = open_db(context.clone()).await; + let db = open_db(context.clone()).await; // Verify correct recovery assert_eq!( @@ -1020,6 +998,7 @@ mod test { // Verify the uncommitted append was properly discarded // We should be able to append new data without issues + let mut db = db.into_mutable(); let new_value = vec![77u8; 15]; let loc = db.append(new_value.clone()).await.unwrap(); assert_eq!( @@ -1031,18 +1010,20 @@ mod test { assert_eq!(db.get(loc).await.unwrap(), Some(new_value)); // Test with multiple trailing appends to ensure robustness - db.commit(None).await.unwrap(); + let (durable, _) = db.commit(None).await.unwrap(); + let db = durable.into_merkleized(); let new_committed_root = db.root(); let new_committed_size = db.op_count(); // Add multiple uncommitted appends + let mut db = db.into_mutable(); for i in 0..5 { let v = vec![(200 + i) as u8; 10]; db.append(v).await.unwrap(); } - // Simulate the same partial failure scenario - db.simulate_failure(true, false).await.unwrap(); + // Simulate failure without commit + drop(db); // Reopen and verify correct recovery let db = open_db(context.clone()).await; @@ -1070,7 +1051,7 @@ mod test { pub fn test_keyless_db_get_out_of_bounds() { let executor = deterministic::Runner::default(); executor.start(|context| async move { - let mut db = open_db(context.clone()).await; + let db = open_db(context.clone()).await; // Test getting from empty database let result = db.get(Location::new_unchecked(0)).await.unwrap(); @@ -1079,24 +1060,26 @@ mod test { // Add some values let v1 = vec![1u8; 8]; let v2 = vec![2u8; 8]; + let mut db = db.into_mutable(); db.append(v1.clone()).await.unwrap(); db.append(v2.clone()).await.unwrap(); - db.commit(None).await.unwrap(); + let (durable, _) = db.commit(None).await.unwrap(); // Test getting valid locations - should succeed - assert_eq!(db.get(Location::new_unchecked(1)).await.unwrap().unwrap(), v1); - assert_eq!(db.get(Location::new_unchecked(2)).await.unwrap().unwrap(), v2); + assert_eq!(durable.get(Location::new_unchecked(1)).await.unwrap().unwrap(), v1); + assert_eq!(durable.get(Location::new_unchecked(2)).await.unwrap().unwrap(), v2); // Test getting out of bounds location - let result = db.get(Location::new_unchecked(3)).await.unwrap(); + let result = durable.get(Location::new_unchecked(3)).await.unwrap(); assert!(result.is_none()); // Test getting out of bounds location - let result = db.get(Location::new_unchecked(4)).await; + let result = durable.get(Location::new_unchecked(4)).await; assert!( matches!(result, Err(Error::LocationOutOfBounds(loc, size)) if loc == Location::new_unchecked(4) && size == Location::new_unchecked(4)) ); + let db = durable.into_merkleized(); db.destroy().await.unwrap(); }); } @@ -1118,23 +1101,24 @@ mod test { let v1 = vec![1u8; 8]; let v2 = vec![2u8; 8]; let v3 = vec![3u8; 8]; + let mut db = db.into_mutable(); db.append(v1.clone()).await.unwrap(); db.append(v2.clone()).await.unwrap(); - db.commit(None).await.unwrap(); + let (db, _) = db.commit(None).await.unwrap(); + let mut db = db.into_mutable(); db.append(v3.clone()).await.unwrap(); - // op_count is 4 (v1, v2, commit, v3) + 1, last_commit_loc is 3 + // op_count is 5 (initial_commit, v1, v2, commit, v3), last_commit_loc is 3 let last_commit = db.last_commit_loc(); assert_eq!(last_commit, Location::new_unchecked(3)); - // Test valid prune (at last commit) - assert!(db.prune(last_commit).await.is_ok()); - - // Add more and commit again - db.commit(None).await.unwrap(); - let new_last_commit = db.last_commit_loc(); + // Test valid prune (at last commit) - need Clean state for prune + let (durable, _) = db.commit(None).await.unwrap(); + let mut db = durable.into_merkleized(); + assert!(db.prune(Location::new_unchecked(3)).await.is_ok()); // Test pruning beyond last commit + let new_last_commit = db.last_commit_loc(); let beyond = Location::new_unchecked(*new_last_commit + 1); let result = db.prune(beyond).await; assert!( diff --git a/storage/src/qmdb/mod.rs b/storage/src/qmdb/mod.rs index 233d48372d..619cb97cf1 100644 --- a/storage/src/qmdb/mod.rs +++ b/storage/src/qmdb/mod.rs @@ -2,29 +2,62 @@ //! //! # Terminology //! -//! A _key_ in an authenticated database either has a _value_ or it doesn't. Two types of -//! _operations_ can be applied to the db to modify the state of a specific key. A key that has a -//! value can change to one without a value through the _delete_ operation. The _update_ operation -//! gives a key a specific value whether it previously had no value or had a different value. +//! A database's state is derived from an append-only log of state-changing _operations_. +//! +//! In a _keyed_ database, a _key_ either has a _value_ or it doesn't, and different types of +//! operations modify the state of a specific key. A key that has a value can change to one without +//! a value through the _delete_ operation. The _update_ operation gives a key a specific value. We +//! sometimes call an update for a key that doesn't already have a value a _create_ operation, but +//! its representation in the log is the same. //! //! Keys with values are called _active_. An operation is called _active_ if (1) its key is active, //! (2) it is an update operation, and (3) it is the most recent operation for that key. //! +//! # Database States +//! +//! An _authenticated_ database can be in one of four states based on two orthogonal dimensions: +//! - Merkleization: [Merkleized] (has computed root) or [Unmerkleized] (root not yet computed) +//! - Durability : [Durable] (committed to disk) or [NonDurable] (uncommitted changes) +//! +//! We call the combined (Merkleized,Durable) state the _Clean_ state. +//! +//! We call the combined (Unmerkleized,NonDurable) state the _Mutable_ state since it's the only +//! state in which the database state (as reflected by its `root`) can be changed. +//! +//! State transitions result from `into_mutable()`, `into_merkleized()`, and `commit()`: +//! - `init()` → `Clean` +//! - `Clean.into_mutable()` → `Mutable` +//! - `(Unmerkleized,Durable).into_mutable()` → `Mutable` +//! - `(Merkleized,NonDurable).into_mutable()` → `Mutable` +//! - `(Unmerkleized,Durable).into_merkleized()` → `Clean` +//! - `Mutable.into_merkleized()` → `(Merkleized,NonDurable)` +//! - `Mutable.commit()` → `(Unmerkleized,Durable)` +//! +//! An authenticated database implements [store::LogStore] in every state, and keyed databases +//! additionally implement [crate::kv::Gettable]. Additional functionality in other states includes: +//! +//! - Clean: [store::MerkleizedStore], [store::PrunableStore], [super::Persistable] +//! - (Merkleized,NonDurable): [store::MerkleizedStore], [store::PrunableStore] +//! +//! Keyed databases additionally implement: +//! - Mutable: [crate::kv::Deletable], [crate::kv::Batchable] +//! //! # Acknowledgments //! //! The following resources were used as references when implementing this crate: //! //! * [QMDB: Quick Merkle Database](https://arxiv.org/abs/2501.05262) -//! * [Merkle Mountain Ranges](https://github.com/opentimestamps/opentimestamps-server/blob/master/doc/merkle-mountain-range.md) +//! * [Merkle Mountain +//! Ranges](https://github.com/opentimestamps/opentimestamps-server/blob/master/doc/merkle-mountain-range.md) use crate::{ index::{Cursor, Unordered as Index}, journal::contiguous::{Contiguous, MutableContiguous}, - mmr::Location, - qmdb::operation::Operation, + mmr::{mem::State as MerkleizationState, Location}, + qmdb::{operation::Operation, store::State as DurabilityState}, DirtyAuthenticatedBitMap, }; -use commonware_cryptography::Digest; +use commonware_cryptography::{Digest, DigestOf}; use commonware_utils::NZUsize; use core::num::NonZeroUsize; use futures::{pin_mut, StreamExt as _}; @@ -89,6 +122,15 @@ impl From for Error { } } +/// Type alias for merkleized state of a QMDB. +pub type Merkleized = crate::mmr::mem::Clean>; +/// Type alias for unmerkleized state of a QMDB. +pub type Unmerkleized = crate::mmr::mem::Dirty; +/// Type alias for durable state of a QMDB. +pub type Durable = store::Durable; +/// Type alias for non-durable state of a QMDB. +pub type NonDurable = store::NonDurable; + /// The size of the read buffer to use for replaying the operations log when rebuilding the /// snapshot. const SNAPSHOT_READ_BUFFER_SIZE: NonZeroUsize = NZUsize!(1 << 16); diff --git a/storage/src/qmdb/store/batch.rs b/storage/src/qmdb/store/batch.rs index 005f400362..63518fc336 100644 --- a/storage/src/qmdb/store/batch.rs +++ b/storage/src/qmdb/store/batch.rs @@ -3,12 +3,15 @@ #[cfg(test)] pub mod tests { use crate::{ - kv::{self, Batchable, Deletable as _, Gettable as _, Updatable as _}, - qmdb::Error, - Persistable, + kv::{Batchable, Deletable as _, Gettable, Updatable as _}, + qmdb::{ + any::states::{MutableAny, UnmerkleizedDurableAny as _}, + Error, + }, + Persistable as _, }; use commonware_codec::Codec; - use commonware_cryptography::{blake3, sha256}; + use commonware_cryptography::sha256; use commonware_runtime::{ deterministic::{self, Context}, Runner as _, @@ -18,29 +21,47 @@ pub mod tests { use rand::{rngs::StdRng, Rng, SeedableRng}; use std::collections::HashSet; - pub trait TestKey: Array { + pub trait TestKey: Array + Copy { fn from_seed(seed: u8) -> Self; } - pub trait TestValue: Codec + Clone + PartialEq + Debug { + pub trait TestValue: Codec + Eq + PartialEq + Debug { fn from_seed(seed: u8) -> Self; } + /// Helper trait for async closures that create a database. + pub trait NewDb: FnMut() -> Self::Fut { + type Fut: Future; + } + impl NewDb for F + where + F: FnMut() -> Fut, + Fut: Future, + { + type Fut = Fut; + } + + /// Destroy an MutableAny database by committing and then destroying. + async fn destroy_db(db: D) -> Result<(), Error> { + let db = db.commit(None).await?.0; + db.into_merkleized().await?.destroy().await + } + /// Run the batch test suite against a database factory within a deterministic executor twice, /// and test the auditor output for equality. pub fn test_batch(mut new_db: F) where F: FnMut(Context) -> Fut + Clone, Fut: Future, - D: Batchable + Persistable, + D: MutableAny, D::Key: TestKey, - D::Value: TestValue, + ::Value: TestValue, { let executor = deterministic::Runner::default(); let mut new_db_clone = new_db.clone(); let state1 = executor.start(|context| async move { let ctx = context.clone(); - run_batch_tests::(&mut || new_db_clone(ctx.clone())) + run_batch_tests(&mut || new_db_clone(ctx.clone())) .await .unwrap(); ctx.auditor().state() @@ -49,9 +70,7 @@ pub mod tests { let executor = deterministic::Runner::default(); let state2 = executor.start(|context| async move { let ctx = context.clone(); - run_batch_tests::(&mut || new_db(ctx.clone())) - .await - .unwrap(); + run_batch_tests(&mut || new_db(ctx.clone())).await.unwrap(); ctx.auditor().state() }); @@ -59,13 +78,12 @@ pub mod tests { } /// Run the shared batch test suite against a database factory. - pub async fn run_batch_tests(new_db: &mut F) -> Result<(), Error> + pub async fn run_batch_tests(new_db: &mut F) -> Result<(), Error> where - F: FnMut() -> Fut, - Fut: Future, - D: Batchable + Persistable, + F: NewDb, + D: MutableAny, D::Key: TestKey, - D::Value: TestValue, + ::Value: TestValue, { test_overlay_reads(new_db).await?; test_create(new_db).await?; @@ -77,132 +95,181 @@ pub mod tests { Ok(()) } - async fn test_overlay_reads(new_db: &mut F) -> Result<(), Error> + async fn test_overlay_reads(new_db: &mut F) -> Result<(), Error> where - F: FnMut() -> Fut, - Fut: Future, - D: Batchable + Persistable, + F: NewDb, + D: MutableAny, D::Key: TestKey, - D::Value: TestValue, + ::Value: TestValue, { let mut db = new_db().await; - let key = D::Key::from_seed(1); - db.update(key.clone(), D::Value::from_seed(1)).await?; + let key = TestKey::from_seed(1); + db.update(key, TestValue::from_seed(1)).await?; let mut batch = db.start_batch(); - assert_eq!(batch.get(&key).await?, Some(D::Value::from_seed(1))); + assert_eq!(batch.get(&key).await?, Some(TestValue::from_seed(1))); - batch.update(key.clone(), D::Value::from_seed(9)).await?; - assert_eq!(batch.get(&key).await?, Some(D::Value::from_seed(9))); + batch.update(key, TestValue::from_seed(9)).await?; + assert_eq!(batch.get(&key).await?, Some(TestValue::from_seed(9))); - db.destroy().await?; - Ok(()) + destroy_db(db).await } - async fn test_create(new_db: &mut F) -> Result<(), Error> + async fn test_create(new_db: &mut F) -> Result<(), Error> where - F: FnMut() -> Fut, - Fut: Future, - D: Batchable + Persistable, + F: NewDb, + D: MutableAny, D::Key: TestKey, - D::Value: TestValue, + ::Value: TestValue, { let mut db = new_db().await; let mut batch = db.start_batch(); - let key = D::Key::from_seed(2); - assert!(batch.create(key.clone(), D::Value::from_seed(1)).await?); - assert!(!batch.create(key.clone(), D::Value::from_seed(2)).await?); + let key = TestKey::from_seed(2); + assert!(batch.create(key, TestValue::from_seed(1)).await?); + assert!(!batch.create(key, TestValue::from_seed(2)).await?); + + batch.delete_unchecked(key).await?; + assert!(batch.create(key, TestValue::from_seed(3)).await?); + assert_eq!(batch.get(&key).await?, Some(TestValue::from_seed(3))); - batch.delete_unchecked(key.clone()).await?; - assert!(batch.create(key.clone(), D::Value::from_seed(3)).await?); - assert_eq!(batch.get(&key).await?, Some(D::Value::from_seed(3))); + let existing = TestKey::from_seed(3); + db.update(existing, TestValue::from_seed(4)).await?; - let existing = D::Key::from_seed(3); - db.update(existing.clone(), D::Value::from_seed(4)).await?; let mut batch = db.start_batch(); - assert!( - !batch - .create(existing.clone(), D::Value::from_seed(5)) - .await? - ); + assert!(!batch.create(existing, TestValue::from_seed(5)).await?); - db.destroy().await?; - Ok(()) + destroy_db(db).await } - async fn test_delete(new_db: &mut F) -> Result<(), Error> + async fn test_delete(new_db: &mut F) -> Result<(), Error> where - F: FnMut() -> Fut, - Fut: Future, - D: Batchable + Persistable, + F: NewDb, + D: MutableAny, D::Key: TestKey, - D::Value: TestValue, + ::Value: TestValue, { let mut db = new_db().await; - let base_key = D::Key::from_seed(4); - db.update(base_key.clone(), D::Value::from_seed(10)).await?; + let base_key = TestKey::from_seed(4); + db.update(base_key, TestValue::from_seed(10)).await?; let mut batch = db.start_batch(); - assert!(batch.delete(base_key.clone()).await?); + assert!(batch.delete(base_key).await?); assert_eq!(batch.get(&base_key).await?, None); - assert!(!batch.delete(base_key.clone()).await?); + assert!(!batch.delete(base_key).await?); let mut batch = db.start_batch(); - let overlay_key = D::Key::from_seed(5); - batch - .update(overlay_key.clone(), D::Value::from_seed(11)) - .await?; - assert!(batch.delete(overlay_key.clone()).await?); + let overlay_key = TestKey::from_seed(5); + batch.update(overlay_key, TestValue::from_seed(11)).await?; + assert!(batch.delete(overlay_key).await?); assert_eq!(batch.get(&overlay_key).await?, None); assert!(!batch.delete(overlay_key).await?); - db.destroy().await?; - Ok(()) + destroy_db(db).await } - async fn test_delete_unchecked(new_db: &mut F) -> Result<(), Error> + async fn test_delete_unchecked(new_db: &mut F) -> Result<(), Error> where - F: FnMut() -> Fut, - Fut: Future, - D: Batchable + Persistable, + F: NewDb, + D: MutableAny, D::Key: TestKey, - D::Value: TestValue, + ::Value: TestValue, { let mut db = new_db().await; - let key = D::Key::from_seed(6); + let key = TestKey::from_seed(6); let mut batch = db.start_batch(); - batch.update(key.clone(), D::Value::from_seed(12)).await?; - batch.delete_unchecked(key.clone()).await?; + batch.update(key, TestValue::from_seed(12)).await?; + batch.delete_unchecked(key).await?; assert_eq!(batch.get(&key).await?, None); - db.update(key.clone(), D::Value::from_seed(13)).await?; + db.update(key, TestValue::from_seed(13)).await?; let mut batch = db.start_batch(); - batch.delete_unchecked(key.clone()).await?; + batch.delete_unchecked(key).await?; assert_eq!(batch.get(&key).await?, None); - db.destroy().await?; - Ok(()) + destroy_db(db).await + } + + async fn test_write_batch_from_to_empty(new_db: &mut F) -> Result<(), Error> + where + F: NewDb, + D: MutableAny, + D::Key: TestKey, + ::Value: TestValue, + { + let mut db = new_db().await; + let mut batch = db.start_batch(); + for i in 0..100 { + batch + .update(TestKey::from_seed(i), TestValue::from_seed(i)) + .await?; + } + db.write_batch(batch.into_iter()).await?; + for i in 0..100 { + assert_eq!( + db.get(&TestKey::from_seed(i)).await?, + Some(TestValue::from_seed(i)) + ); + } + destroy_db(db).await + } + + async fn test_write_batch(new_db: &mut F) -> Result<(), Error> + where + F: NewDb, + D: MutableAny, + D::Key: TestKey, + ::Value: TestValue, + { + let mut db = new_db().await; + for i in 0..100 { + db.update(TestKey::from_seed(i), TestValue::from_seed(i)) + .await?; + } + + let mut batch = db.start_batch(); + for i in 0..100 { + batch.delete(TestKey::from_seed(i)).await?; + } + for i in 100..200 { + batch + .update(TestKey::from_seed(i), TestValue::from_seed(i)) + .await?; + } + + db.write_batch(batch.into_iter()).await?; + + for i in 0..100 { + assert!(db.get(&TestKey::from_seed(i)).await?.is_none()); + } + for i in 100..200 { + assert_eq!( + db.get(&TestKey::from_seed(i)).await?, + Some(TestValue::from_seed(i)) + ); + } + + destroy_db(db).await } /// Create an empty db, write a small # of keys, then delete half, then recreate those that were /// deleted. Also includes a delete_unchecked of an inactive key. - async fn test_update_delete_update(new_db: &mut F) -> Result<(), Error> + async fn test_update_delete_update(new_db: &mut F) -> Result<(), Error> where - F: FnMut() -> Fut, - Fut: Future, - D: Batchable + Persistable, + F: NewDb, + D: MutableAny, D::Key: TestKey, - D::Value: TestValue, + ::Value: TestValue, { let mut db = new_db().await; // Create 100 keys and commit them. for i in 0..100 { assert!( - db.create(D::Key::from_seed(i), D::Value::from_seed(i)) + db.create(TestKey::from_seed(i), TestValue::from_seed(i)) .await? ); } - db.commit().await?; + let (durable, _) = db.commit(None).await?; + let mut db = durable.into_mutable(); // Delete half of the keys at random. let mut rng = StdRng::seed_from_u64(1337); @@ -211,191 +278,64 @@ pub mod tests { for i in 0..100 { if rng.gen_bool(0.5) { deleted.insert(i); - assert!(batch.delete(D::Key::from_seed(i)).await?); + assert!(batch.delete(TestKey::from_seed(i)).await?); } } // Try to delete an inactive key. - batch.delete_unchecked(D::Key::from_seed(255)).await?; + batch.delete_unchecked(TestKey::from_seed(255)).await?; // Commit the batch then confirm output is as expected. db.write_batch(batch.into_iter()).await?; - db.commit().await?; + let (durable, _) = db.commit(None).await?; for i in 0..100 { if deleted.contains(&i) { - assert_eq!(kv::Gettable::get(&db, &D::Key::from_seed(i)).await?, None); + assert!(durable.get(&TestKey::from_seed(i)).await?.is_none()); } else { assert_eq!( - kv::Gettable::get(&db, &D::Key::from_seed(i)).await?, - Some(D::Value::from_seed(i)) + durable.get(&TestKey::from_seed(i)).await?, + Some(TestValue::from_seed(i)) ); } } + let mut db = durable.into_mutable(); // Recreate the deleted keys. let mut batch = db.start_batch(); - for i in 0..100 { - if deleted.contains(&i) { + for i in deleted.iter() { + assert!( batch - .create(D::Key::from_seed(i), D::Value::from_seed(i)) - .await?; - } + .create(TestKey::from_seed(*i), TestValue::from_seed(*i)) + .await? + ); } - - // Commit the batch then confirm output is as expected. db.write_batch(batch.into_iter()).await?; - db.commit().await?; + let (durable, _) = db.commit(None).await?; for i in 0..100 { assert_eq!( - kv::Gettable::get(&db, &D::Key::from_seed(i)).await?, - Some(D::Value::from_seed(i)) + durable.get(&TestKey::from_seed(i)).await?, + Some(TestValue::from_seed(i)) ); } - db.destroy().await?; - - Ok(()) - } - - /// Create an empty db, write a batch containing small # of keys, then write another batch deleting those - /// keys. - async fn test_write_batch_from_to_empty(new_db: &mut F) -> Result<(), Error> - where - F: FnMut() -> Fut, - Fut: Future, - D: Batchable + Persistable, - D::Key: TestKey, - D::Value: TestValue, - { - // 2 key test - let mut db = new_db().await; - let created1 = D::Key::from_seed(1); - let created2 = D::Key::from_seed(2); - let mut batch = db.start_batch(); - batch - .create(created1.clone(), D::Value::from_seed(1)) - .await?; - batch - .create(created2.clone(), D::Value::from_seed(2)) - .await?; - batch - .update(created1.clone(), D::Value::from_seed(3)) - .await?; - db.write_batch(batch.into_iter()).await?; - - assert_eq!( - kv::Gettable::get(&db, &created1).await?, - Some(D::Value::from_seed(3)) - ); - assert_eq!( - kv::Gettable::get(&db, &created2).await?, - Some(D::Value::from_seed(2)) - ); - - let mut delete_batch = db.start_batch(); - delete_batch.delete(created1.clone()).await?; - delete_batch.delete(created2.clone()).await?; - db.write_batch(delete_batch.into_iter()).await?; - assert_eq!(kv::Gettable::get(&db, &created1).await?, None); - assert_eq!(kv::Gettable::get(&db, &created2).await?, None); - - db.destroy().await?; - - // 1 key test - let mut db = new_db().await; - let created1 = D::Key::from_seed(1); - let mut batch = db.start_batch(); - batch - .create(created1.clone(), D::Value::from_seed(1)) - .await?; - db.write_batch(batch.into_iter()).await?; - assert_eq!( - kv::Gettable::get(&db, &created1).await?, - Some(D::Value::from_seed(1)) - ); - let mut delete_batch = db.start_batch(); - delete_batch.delete(created1.clone()).await?; - db.write_batch(delete_batch.into_iter()).await?; - assert_eq!(kv::Gettable::get(&db, &created1).await?, None); - - db.destroy().await?; - - Ok(()) - } - - async fn test_write_batch(new_db: &mut F) -> Result<(), Error> - where - F: FnMut() -> Fut, - Fut: Future, - D: Batchable + Persistable, - D::Key: TestKey, - D::Value: TestValue, - { - let mut db = new_db().await; - let existing = D::Key::from_seed(7); - db.update(existing.clone(), D::Value::from_seed(0)).await?; - - let created = D::Key::from_seed(8); - let mut batch = db.start_batch(); - batch - .update(existing.clone(), D::Value::from_seed(8)) - .await?; - batch - .create(created.clone(), D::Value::from_seed(9)) - .await?; - db.write_batch(batch.into_iter()).await?; - - assert_eq!( - kv::Gettable::get(&db, &existing).await?, - Some(D::Value::from_seed(8)) - ); - assert_eq!( - kv::Gettable::get(&db, &created).await?, - Some(D::Value::from_seed(9)) - ); - - let mut delete_batch = db.start_batch(); - delete_batch.delete(existing.clone()).await?; - db.write_batch(delete_batch.into_iter()).await?; - assert_eq!(kv::Gettable::get(&db, &existing).await?, None); - - db.destroy().await?; - Ok(()) - } - - fn seed_bytes(seed: u8) -> [u8; 32] { - let mut bytes = [0u8; 32]; - bytes[0] = seed; - bytes - } - - impl TestKey for blake3::Digest { - fn from_seed(seed: u8) -> Self { - Self::from(seed_bytes(seed)) - } + destroy_db(durable.into_mutable()).await } impl TestKey for sha256::Digest { fn from_seed(seed: u8) -> Self { - Self::from(seed_bytes(seed)) + commonware_cryptography::Sha256::fill(seed) } } - impl TestValue for Vec { + impl TestValue for D { fn from_seed(seed: u8) -> Self { - vec![seed] + D::from_seed(seed) } } - impl TestValue for blake3::Digest { - fn from_seed(seed: u8) -> Self { - Self::from(seed_bytes(seed)) - } - } - - impl TestValue for sha256::Digest { + impl TestValue for Vec { fn from_seed(seed: u8) -> Self { - Self::from(seed_bytes(seed)) + vec![seed; 32] } } } diff --git a/storage/src/qmdb/store/db.rs b/storage/src/qmdb/store/db.rs new file mode 100644 index 0000000000..9e34b99655 --- /dev/null +++ b/storage/src/qmdb/store/db.rs @@ -0,0 +1,1168 @@ +//! A mutable key-value database that supports variable-sized values, but without authentication. +//! +//! # Lifecycle +//! +//! Unlike authenticated stores which have 4 potential states, an unauthenticated store only has +//! two: +//! +//! - **Clean**: The store has no uncommitted operations and its key/value state is immutable. Use +//! `into_dirty` to transform it into a dirty state. +//! +//! - **Dirty**: The store has uncommitted operations and its key/value state is mutable. Use +//! `commit` to transform it into a clean state. +//! +//! # Example +//! +//! ```rust +//! use commonware_storage::{ +//! qmdb::store::db::{Config, Db}, +//! translator::TwoCap, +//! }; +//! use commonware_utils::{NZUsize, NZU64}; +//! use commonware_cryptography::{blake3::Digest, Digest as _}; +//! use commonware_math::algebra::Random; +//! use commonware_runtime::{buffer::PoolRef, deterministic::Runner, Metrics, Runner as _}; +//! +//! const PAGE_SIZE: usize = 8192; +//! const PAGE_CACHE_SIZE: usize = 100; +//! +//! let executor = Runner::default(); +//! executor.start(|mut ctx| async move { +//! let config = Config { +//! log_partition: "test_partition".to_string(), +//! log_write_buffer: NZUsize!(64 * 1024), +//! log_compression: None, +//! log_codec_config: (), +//! log_items_per_section: NZU64!(4), +//! translator: TwoCap, +//! buffer_pool: PoolRef::new(NZUsize!(PAGE_SIZE), NZUsize!(PAGE_CACHE_SIZE)), +//! }; +//! let db = +//! Db::<_, Digest, Digest, TwoCap>::init(ctx.with_label("store"), config) +//! .await +//! .unwrap(); +//! +//! // Insert a key-value pair +//! let k = Digest::random(&mut ctx); +//! let v = Digest::random(&mut ctx); +//! let mut db = db.into_dirty(); +//! db.update(k, v).await.unwrap(); +//! +//! // Fetch the value +//! let fetched_value = db.get(&k).await.unwrap(); +//! assert_eq!(fetched_value.unwrap(), v); +//! +//! // Commit the operation to make it persistent +//! let metadata = Some(Digest::random(&mut ctx)); +//! let (db, _) = db.commit(metadata).await.unwrap(); +//! +//! // Delete the key's value +//! let mut db = db.into_dirty(); +//! db.delete(k).await.unwrap(); +//! +//! // Fetch the value +//! let fetched_value = db.get(&k).await.unwrap(); +//! assert!(fetched_value.is_none()); +//! +//! // Commit the operation to make it persistent +//! let (db, _) = db.commit(None).await.unwrap(); +//! +//! // Destroy the store +//! db.destroy().await.unwrap(); +//! }); +//! ``` + +use crate::{ + index::{unordered::Index, Unordered as _}, + journal::contiguous::{ + variable::{Config as JournalConfig, Journal}, + MutableContiguous as _, + }, + kv::{Batchable, Deletable, Updatable}, + mmr::Location, + qmdb::{ + any::{ + unordered::{variable::Operation, Update}, + VariableValue, + }, + build_snapshot_from_log, create_key, delete_key, + operation::{Committable as _, Operation as _}, + store::{Durable, LogStore, NonDurable, PrunableStore, State}, + update_key, Error, FloorHelper, + }, + translator::Translator, + Persistable, +}; +use commonware_codec::Read; +use commonware_runtime::{buffer::PoolRef, Clock, Metrics, Storage}; +use commonware_utils::Array; +use core::ops::Range; +use std::num::{NonZeroU64, NonZeroUsize}; +use tracing::{debug, warn}; + +/// Configuration for initializing a [Db]. +#[derive(Clone)] +pub struct Config { + /// The name of the [Storage] partition used to persist the log of operations. + pub log_partition: String, + + /// The size of the write buffer to use for each blob in the [Journal]. + pub log_write_buffer: NonZeroUsize, + + /// Optional compression level (using `zstd`) to apply to log data before storing. + pub log_compression: Option, + + /// The codec configuration to use for encoding and decoding log items. + pub log_codec_config: C, + + /// The number of operations to store in each section of the [Journal]. + pub log_items_per_section: NonZeroU64, + + /// The [Translator] used by the [Index]. + pub translator: T, + + /// The [PoolRef] to use for caching data. + pub buffer_pool: PoolRef, +} + +/// An unauthenticated key-value database based off of an append-only [Journal] of operations. +pub struct Db +where + E: Storage + Clock + Metrics, + K: Array, + V: VariableValue, + T: Translator, + S: State, +{ + /// A log of all [Operation]s that have been applied to the store. + /// + /// # Invariants + /// + /// - There is always at least one commit operation in the log. + /// - The log is never pruned beyond the inactivity floor. + log: Journal>, + + /// A snapshot of all currently active operations in the form of a map from each key to the + /// location containing its most recent update. + /// + /// # Invariant + /// + /// Only references operations of type [Operation::Update]. + snapshot: Index, + + /// The number of active keys in the store. + active_keys: usize, + + /// A location before which all operations are "inactive" (that is, operations before this point + /// are over keys that have been updated by some operation at or after this point). + pub inactivity_floor_loc: Location, + + /// The location of the last commit operation. + pub last_commit_loc: Location, + + /// The state of the store. + pub state: S, +} + +impl Db +where + E: Storage + Clock + Metrics, + K: Array, + V: VariableValue, + T: Translator, + S: State, +{ + /// Get the value of `key` in the db, or None if it has no value. + pub async fn get(&self, key: &K) -> Result, Error> { + for &loc in self.snapshot.get(key) { + let Operation::Update(Update(k, v)) = self.get_op(loc).await? else { + unreachable!("location ({loc}) does not reference update operation"); + }; + + if &k == key { + return Ok(Some(v)); + } + } + + Ok(None) + } + + /// Whether the db currently has no active keys. + pub const fn is_empty(&self) -> bool { + self.active_keys == 0 + } + + /// Gets a [Operation] from the log at the given location. Returns [Error::OperationPruned] + /// if the location precedes the oldest retained location. The location is otherwise assumed + /// valid. + async fn get_op(&self, loc: Location) -> Result, Error> { + assert!(loc < self.op_count()); + + // Get the operation from the log at the specified position. + // The journal will return ItemPruned if the location is pruned. + self.log.read(*loc).await.map_err(|e| match e { + crate::journal::Error::ItemPruned(_) => Error::OperationPruned(loc), + e => Error::Journal(e), + }) + } + + /// The number of operations that have been applied to this db, including those that have been + /// pruned and those that are not yet committed. + pub const fn op_count(&self) -> Location { + Location::new_unchecked(self.log.size()) + } + + /// Return the inactivity floor location. This is the location before which all operations are + /// known to be inactive. Operations before this point can be safely pruned. + pub const fn inactivity_floor_loc(&self) -> Location { + self.inactivity_floor_loc + } + + /// Get the metadata associated with the last commit. + pub async fn get_metadata(&self) -> Result, Error> { + let Operation::CommitFloor(metadata, _) = self.log.read(*self.last_commit_loc).await? + else { + unreachable!("last commit should be a commit floor operation"); + }; + + Ok(metadata) + } + + /// Prune historical operations prior to `prune_loc`. This does not affect the db's root + /// or current snapshot. + pub async fn prune(&mut self, prune_loc: Location) -> Result<(), Error> { + if prune_loc > self.inactivity_floor_loc { + return Err(Error::PruneBeyondMinRequired( + prune_loc, + self.inactivity_floor_loc, + )); + } + + // Prune the log. The log will prune at section boundaries, so the actual oldest retained + // location may be less than requested. + if !self.log.prune(*prune_loc).await? { + return Ok(()); + } + + debug!( + log_size = ?self.op_count(), + oldest_retained_loc = ?self.log.oldest_retained_pos(), + ?prune_loc, + "pruned inactive ops" + ); + + Ok(()) + } +} + +impl Db +where + E: Storage + Clock + Metrics, + K: Array, + V: VariableValue, + T: Translator, +{ + /// Initializes a new [Db] with the given configuration. + pub async fn init( + context: E, + cfg: Config as Read>::Cfg>, + ) -> Result { + let mut log = Journal::>::init( + context.with_label("log"), + JournalConfig { + partition: cfg.log_partition, + items_per_section: cfg.log_items_per_section, + compression: cfg.log_compression, + codec_config: cfg.log_codec_config, + buffer_pool: cfg.buffer_pool, + write_buffer: cfg.log_write_buffer, + }, + ) + .await?; + + // Rewind log to remove uncommitted operations. + if log.rewind_to(|op| op.is_commit()).await? == 0 { + warn!("Log is empty, initializing new db"); + log.append(Operation::CommitFloor(None, Location::new_unchecked(0))) + .await?; + } + + // Sync the log to avoid having to repeat any recovery that may have been performed on next + // startup. + log.sync().await?; + + let last_commit_loc = + Location::new_unchecked(log.size().checked_sub(1).expect("commit should exist")); + let op = log.read(*last_commit_loc).await?; + let inactivity_floor_loc = op.has_floor().expect("last op should be a commit"); + + // Build the snapshot. + let mut snapshot = Index::new(context.with_label("snapshot"), cfg.translator); + let active_keys = + build_snapshot_from_log(inactivity_floor_loc, &log, &mut snapshot, |_, _| {}).await?; + + Ok(Self { + log, + snapshot, + active_keys, + inactivity_floor_loc, + last_commit_loc, + state: Durable, + }) + } + + /// Convert this clean store into its dirty counterpart for making updates. + pub fn into_dirty(self) -> Db { + Db { + log: self.log, + snapshot: self.snapshot, + active_keys: self.active_keys, + inactivity_floor_loc: self.inactivity_floor_loc, + last_commit_loc: self.last_commit_loc, + state: NonDurable::default(), + } + } + + /// Sync all database state to disk. While this isn't necessary to ensure durability of + /// committed operations, periodic invocation may reduce memory usage and the time required to + /// recover the database on restart. + pub async fn sync(&mut self) -> Result<(), Error> { + self.log.sync().await.map_err(Into::into) + } + + /// Destroy the db, removing all data from disk. + pub async fn destroy(self) -> Result<(), Error> { + self.log.destroy().await.map_err(Into::into) + } +} + +impl Db +where + E: Storage + Clock + Metrics, + K: Array, + V: VariableValue, + T: Translator, +{ + const fn as_floor_helper( + &mut self, + ) -> FloorHelper<'_, Index, Journal>> { + FloorHelper { + snapshot: &mut self.snapshot, + log: &mut self.log, + } + } + + /// Updates `key` to have value `value`. The operation is reflected in the snapshot, but will be + /// subject to rollback until the next successful `commit`. + pub async fn update(&mut self, key: K, value: V) -> Result<(), Error> { + let new_loc = self.op_count(); + if update_key(&mut self.snapshot, &self.log, &key, new_loc) + .await? + .is_some() + { + self.state.steps += 1; + } else { + self.active_keys += 1; + } + + self.log + .append(Operation::Update(Update(key, value))) + .await?; + + Ok(()) + } + + /// Creates a new key-value pair in the db. The operation is reflected in the snapshot, but will + /// be subject to rollback until the next successful `commit`. Returns true if the key was + /// created, false if it already existed. + pub async fn create(&mut self, key: K, value: V) -> Result { + let new_loc = self.op_count(); + if !create_key(&mut self.snapshot, &self.log, &key, new_loc).await? { + return Ok(false); + } + + self.active_keys += 1; + self.log + .append(Operation::Update(Update(key, value))) + .await?; + + Ok(true) + } + + /// Delete `key` and its value from the db. Deleting a key that already has no value is a no-op. + /// The operation is reflected in the snapshot, but will be subject to rollback until the next + /// successful `commit`. Returns true if the key was deleted, false if it was already inactive. + pub async fn delete(&mut self, key: K) -> Result { + let r = delete_key(&mut self.snapshot, &self.log, &key).await?; + if r.is_none() { + return Ok(false); + } + + self.log.append(Operation::Delete(key)).await?; + self.state.steps += 1; + self.active_keys -= 1; + + Ok(true) + } + + /// Commit any pending operations to the database, ensuring their durability upon return from + /// this function. Also raises the inactivity floor according to the schedule. Returns the + /// `(start_loc, end_loc]` location range of committed operations. The end of the returned range + /// includes the commit operation itself, and hence will always be equal to `op_count`. + /// + /// Failures after commit (but before `sync` or `close`) may still require reprocessing to + /// recover the database on restart. + /// + /// Consumes this dirty store and returns a clean one. + pub async fn commit( + mut self, + metadata: Option, + ) -> Result<(Db, Range), Error> { + let start_loc = self.last_commit_loc + 1; + + // Raise the inactivity floor by taking `self.state.steps` steps, plus 1 to account for the + // previous commit becoming inactive. + if self.is_empty() { + self.inactivity_floor_loc = self.op_count(); + debug!(tip = ?self.inactivity_floor_loc, "db is empty, raising floor to tip"); + } else { + let steps_to_take = self.state.steps + 1; + for _ in 0..steps_to_take { + let loc = self.inactivity_floor_loc; + self.inactivity_floor_loc = self.as_floor_helper().raise_floor(loc).await?; + } + } + + // Apply the commit operation with the new inactivity floor. + self.last_commit_loc = Location::new_unchecked( + self.log + .append(Operation::CommitFloor(metadata, self.inactivity_floor_loc)) + .await?, + ); + + let range = start_loc..self.op_count(); + + // Commit the log to ensure durability. + self.log.commit().await?; + + Ok(( + Db { + log: self.log, + snapshot: self.snapshot, + active_keys: self.active_keys, + inactivity_floor_loc: self.inactivity_floor_loc, + last_commit_loc: self.last_commit_loc, + state: Durable, + }, + range, + )) + } +} + +impl Persistable for Db +where + E: Storage + Clock + Metrics, + K: Array, + V: VariableValue, + T: Translator, +{ + type Error = Error; + + async fn commit(&mut self) -> Result<(), Error> { + // No-op, DB already in recoverable state. + Ok(()) + } + + async fn sync(&mut self) -> Result<(), Error> { + self.sync().await + } + + async fn destroy(self) -> Result<(), Error> { + self.destroy().await + } +} + +impl LogStore for Db +where + E: Storage + Clock + Metrics, + K: Array, + V: VariableValue, + T: Translator, + S: State, +{ + type Value = V; + + fn op_count(&self) -> Location { + self.op_count() + } + + fn inactivity_floor_loc(&self) -> Location { + self.inactivity_floor_loc() + } + + async fn get_metadata(&self) -> Result, Error> { + self.get_metadata().await + } + + fn is_empty(&self) -> bool { + self.is_empty() + } +} + +impl PrunableStore for Db +where + E: Storage + Clock + Metrics, + K: Array, + V: VariableValue, + T: Translator, + S: State, +{ + async fn prune(&mut self, prune_loc: Location) -> Result<(), Error> { + self.prune(prune_loc).await + } +} + +impl crate::kv::Gettable for Db +where + E: Storage + Clock + Metrics, + K: Array, + V: VariableValue, + T: Translator, + S: State, +{ + type Key = K; + type Value = V; + type Error = Error; + + async fn get(&self, key: &Self::Key) -> Result, Self::Error> { + self.get(key).await + } +} + +impl Updatable for Db +where + E: Storage + Clock + Metrics, + K: Array, + V: VariableValue, + T: Translator, +{ + async fn update(&mut self, key: Self::Key, value: Self::Value) -> Result<(), Self::Error> { + self.update(key, value).await + } +} + +impl Deletable for Db +where + E: Storage + Clock + Metrics, + K: Array, + V: VariableValue, + T: Translator, +{ + async fn delete(&mut self, key: Self::Key) -> Result { + self.delete(key).await + } +} + +impl Batchable for Db +where + E: Storage + Clock + Metrics, + K: Array, + V: VariableValue, + T: Translator, +{ + async fn write_batch( + &mut self, + iter: impl Iterator)>, + ) -> Result<(), Self::Error> { + for (key, value) in iter { + if let Some(value) = value { + self.update(key, value).await?; + } else { + self.delete(key).await?; + } + } + Ok(()) + } +} + +#[cfg(test)] +mod test { + use super::*; + use crate::{kv::Gettable as _, translator::TwoCap}; + use commonware_cryptography::{ + blake3::{Blake3, Digest}, + Hasher as _, + }; + use commonware_macros::test_traced; + use commonware_math::algebra::Random; + use commonware_runtime::{deterministic, Runner}; + use commonware_utils::{NZUsize, NZU64}; + + const PAGE_SIZE: usize = 77; + const PAGE_CACHE_SIZE: usize = 9; + + /// The type of the store used in tests. + type TestStore = Db, TwoCap, Durable>; + + async fn create_test_store(context: deterministic::Context) -> TestStore { + let cfg = Config { + log_partition: "journal".to_string(), + log_write_buffer: NZUsize!(64 * 1024), + log_compression: None, + log_codec_config: ((0..=10000).into(), ()), + log_items_per_section: NZU64!(7), + translator: TwoCap, + buffer_pool: PoolRef::new(NZUsize!(PAGE_SIZE), NZUsize!(PAGE_CACHE_SIZE)), + }; + TestStore::init(context, cfg).await.unwrap() + } + + #[test_traced("DEBUG")] + pub fn test_store_construct_empty() { + let executor = deterministic::Runner::default(); + executor.start(|mut context| async move { + let mut db = create_test_store(context.clone()).await; + assert_eq!(db.op_count(), 1); + assert_eq!(db.log.oldest_retained_pos(), Some(0)); + assert_eq!(db.inactivity_floor_loc(), 0); + assert!(matches!(db.prune(db.inactivity_floor_loc()).await, Ok(()))); + assert!(matches!( + db.prune(Location::new_unchecked(1)).await, + Err(Error::PruneBeyondMinRequired(_, _)) + )); + assert!(db.get_metadata().await.unwrap().is_none()); + + // Make sure closing/reopening gets us back to the same state, even after adding an uncommitted op. + let d1 = Digest::random(&mut context); + let v1 = vec![1, 2, 3]; + let mut dirty = db.into_dirty(); + dirty.update(d1, v1).await.unwrap(); + drop(dirty); + + let db = create_test_store(context.clone()).await.into_dirty(); + assert_eq!(db.op_count(), 1); + + // Test calling commit on an empty db which should make it (durably) non-empty. + let metadata = vec![1, 2, 3]; + let (mut db, range) = db.commit(Some(metadata.clone())).await.unwrap(); + assert_eq!(range.start, 1); + assert_eq!(range.end, 2); + assert_eq!(db.op_count(), 2); + assert!(matches!(db.prune(db.inactivity_floor_loc()).await, Ok(()))); + assert_eq!(db.get_metadata().await.unwrap(), Some(metadata.clone())); + + let mut db = create_test_store(context.clone()).await.into_dirty(); + assert_eq!(db.get_metadata().await.unwrap(), Some(metadata)); + + // Confirm the inactivity floor doesn't fall endlessly behind with multiple commits on a + // non-empty db. + db.update(Digest::random(&mut context), vec![1, 2, 3]) + .await + .unwrap(); + let (mut db, _) = db.commit(None).await.unwrap(); + for _ in 1..100 { + (db, _) = db.into_dirty().commit(None).await.unwrap(); + // Distance should equal 3 after the second commit, with inactivity_floor + // referencing the previous commit operation. + assert!(db.op_count() - db.inactivity_floor_loc <= 3); + assert!(db.get_metadata().await.unwrap().is_none()); + } + + db.destroy().await.unwrap(); + }); + } + + #[test_traced("DEBUG")] + fn test_store_construct_basic() { + let executor = deterministic::Runner::default(); + + executor.start(|mut ctx| async move { + let mut db = create_test_store(ctx.with_label("store")) + .await + .into_dirty(); + + // Ensure the store is empty + assert_eq!(db.op_count(), 1); + assert_eq!(db.inactivity_floor_loc, 0); + + let key = Digest::random(&mut ctx); + let value = vec![2, 3, 4, 5]; + + // Attempt to get a key that does not exist + let result = db.get(&key).await; + assert!(result.unwrap().is_none()); + + // Insert a key-value pair + db.update(key, value.clone()).await.unwrap(); + + assert_eq!(db.op_count(), 2); + assert_eq!(db.inactivity_floor_loc, 0); + + // Fetch the value + let fetched_value = db.get(&key).await.unwrap(); + assert_eq!(fetched_value.unwrap(), value); + + // Simulate commit failure. + drop(db); + + // Re-open the store + let mut db = create_test_store(ctx.with_label("store")) + .await + .into_dirty(); + + // Ensure the re-opened store removed the uncommitted operations + assert_eq!(db.op_count(), 1); + assert_eq!(db.inactivity_floor_loc, 0); + assert!(db.get_metadata().await.unwrap().is_none()); + + // Insert a key-value pair + db.update(key, value.clone()).await.unwrap(); + + assert_eq!(db.op_count(), 2); + assert_eq!(db.inactivity_floor_loc, 0); + + // Persist the changes + let metadata = vec![99, 100]; + let (db, range) = db.commit(Some(metadata.clone())).await.unwrap(); + assert_eq!(range.start, 1); + assert_eq!(range.end, 4); + assert_eq!(db.get_metadata().await.unwrap(), Some(metadata.clone())); + + // Even though the store was pruned, the inactivity floor was raised by 2, and + // the old operations remain in the same blob as an active operation, so they're + // retained. + assert_eq!(db.op_count(), 4); + assert_eq!(db.inactivity_floor_loc, 2); + + // Re-open the store + let mut db = create_test_store(ctx.with_label("store")) + .await + .into_dirty(); + + // Ensure the re-opened store retained the committed operations + assert_eq!(db.op_count(), 4); + assert_eq!(db.inactivity_floor_loc, 2); + + // Fetch the value, ensuring it is still present + let fetched_value = db.get(&key).await.unwrap(); + assert_eq!(fetched_value.unwrap(), value); + + // Insert two new k/v pairs to force pruning of the first section. + let (k1, v1) = (Digest::random(&mut ctx), vec![2, 3, 4, 5, 6]); + let (k2, v2) = (Digest::random(&mut ctx), vec![6, 7, 8]); + db.update(k1, v1.clone()).await.unwrap(); + db.update(k2, v2.clone()).await.unwrap(); + + assert_eq!(db.op_count(), 6); + assert_eq!(db.inactivity_floor_loc, 2); + + // Make sure we can still get metadata. + assert_eq!(db.get_metadata().await.unwrap(), Some(metadata)); + + let (db, range) = db.commit(None).await.unwrap(); + assert_eq!(range.start, 4); + assert_eq!(range.end, db.op_count()); + assert_eq!(db.get_metadata().await.unwrap(), None); + let mut db = db.into_dirty(); + + assert_eq!(db.op_count(), 8); + assert_eq!(db.inactivity_floor_loc, 3); + + // Ensure all keys can be accessed, despite the first section being pruned. + assert_eq!(db.get(&key).await.unwrap().unwrap(), value); + assert_eq!(db.get(&k1).await.unwrap().unwrap(), v1); + assert_eq!(db.get(&k2).await.unwrap().unwrap(), v2); + + // Ensure upsert works for existing key. + db.upsert(k1, |v| v.push(7)).await.unwrap(); + let (db, _) = db.commit(None).await.unwrap(); + assert_eq!(db.get(&k1).await.unwrap().unwrap(), vec![2, 3, 4, 5, 6, 7]); + + // Ensure upsert works for new key. + let mut db = db.into_dirty(); + let k3 = Digest::random(&mut ctx); + db.upsert(k3, |v| v.push(8)).await.unwrap(); + let (db, _) = db.commit(None).await.unwrap(); + assert_eq!(db.get(&k3).await.unwrap().unwrap(), vec![8]); + + // Destroy the store + db.destroy().await.unwrap(); + }); + } + + #[test_traced("DEBUG")] + fn test_store_log_replay() { + let executor = deterministic::Runner::default(); + + executor.start(|mut ctx| async move { + let mut db = create_test_store(ctx.with_label("store")) + .await + .into_dirty(); + + // Update the same key many times. + const UPDATES: u64 = 100; + let k = Digest::random(&mut ctx); + for _ in 0..UPDATES { + let v = vec![1, 2, 3, 4, 5]; + db.update(k, v.clone()).await.unwrap(); + } + + let iter = db.snapshot.get(&k); + assert_eq!(iter.count(), 1); + + let (mut db, _) = db.commit(None).await.unwrap(); + db.sync().await.unwrap(); + drop(db); + + // Re-open the store, prune it, then ensure it replays the log correctly. + let mut db = create_test_store(ctx.with_label("store")).await; + db.prune(db.inactivity_floor_loc()).await.unwrap(); + + let iter = db.snapshot.get(&k); + assert_eq!(iter.count(), 1); + + // 100 operations were applied, each triggering one step, plus the commit op. + assert_eq!(db.op_count(), UPDATES * 2 + 2); + // Only the highest `Update` operation is active, plus the commit operation above it. + let expected_floor = UPDATES * 2; + assert_eq!(db.inactivity_floor_loc, expected_floor); + + // All blobs prior to the inactivity floor are pruned, so the oldest retained location + // is the first in the last retained blob. + assert_eq!( + db.log.oldest_retained_pos(), + Some(expected_floor - expected_floor % 7) + ); + + db.destroy().await.unwrap(); + }); + } + + #[test_traced("DEBUG")] + fn test_store_build_snapshot_keys_with_shared_prefix() { + let executor = deterministic::Runner::default(); + + executor.start(|mut ctx| async move { + let mut db = create_test_store(ctx.with_label("store")) + .await + .into_dirty(); + + let (k1, v1) = (Digest::random(&mut ctx), vec![1, 2, 3, 4, 5]); + let (mut k2, v2) = (Digest::random(&mut ctx), vec![6, 7, 8, 9, 10]); + + // Ensure k2 shares 2 bytes with k1 (test DB uses `TwoCap` translator.) + k2.0[0..2].copy_from_slice(&k1.0[0..2]); + + db.update(k1, v1.clone()).await.unwrap(); + db.update(k2, v2.clone()).await.unwrap(); + + assert_eq!(db.get(&k1).await.unwrap().unwrap(), v1); + assert_eq!(db.get(&k2).await.unwrap().unwrap(), v2); + + let (mut db, _) = db.commit(None).await.unwrap(); + db.sync().await.unwrap(); + drop(db); + + // Re-open the store to ensure it builds the snapshot for the conflicting + // keys correctly. + let db = create_test_store(ctx.with_label("store")).await; + + assert_eq!(db.get(&k1).await.unwrap().unwrap(), v1); + assert_eq!(db.get(&k2).await.unwrap().unwrap(), v2); + + db.destroy().await.unwrap(); + }); + } + + #[test_traced("DEBUG")] + fn test_store_delete() { + let executor = deterministic::Runner::default(); + + executor.start(|mut ctx| async move { + let mut db = create_test_store(ctx.with_label("store")) + .await + .into_dirty(); + + // Insert a key-value pair + let k = Digest::random(&mut ctx); + let v = vec![1, 2, 3, 4, 5]; + db.update(k, v.clone()).await.unwrap(); + let (db, _) = db.commit(None).await.unwrap(); + + // Fetch the value + let fetched_value = db.get(&k).await.unwrap(); + assert_eq!(fetched_value.unwrap(), v); + + // Delete the key + let mut db = db.into_dirty(); + assert!(db.delete(k).await.unwrap()); + + // Ensure the key is no longer present + let fetched_value = db.get(&k).await.unwrap(); + assert!(fetched_value.is_none()); + assert!(!db.delete(k).await.unwrap()); + + // Commit the changes + let _ = db.commit(None).await.unwrap(); + + // Re-open the store and ensure the key is still deleted + let mut db = create_test_store(ctx.with_label("store")) + .await + .into_dirty(); + let fetched_value = db.get(&k).await.unwrap(); + assert!(fetched_value.is_none()); + + // Re-insert the key + db.update(k, v.clone()).await.unwrap(); + let fetched_value = db.get(&k).await.unwrap(); + assert_eq!(fetched_value.unwrap(), v); + + // Commit the changes + let _ = db.commit(None).await.unwrap(); + + // Re-open the store and ensure the snapshot restores the key, after processing + // the delete and the subsequent set. + let mut db = create_test_store(ctx.with_label("store")) + .await + .into_dirty(); + let fetched_value = db.get(&k).await.unwrap(); + assert_eq!(fetched_value.unwrap(), v); + + // Delete a non-existent key (no-op) + let k_n = Digest::random(&mut ctx); + db.delete(k_n).await.unwrap(); + + let (db, range) = db.commit(None).await.unwrap(); + assert_eq!(range.start, 9); + assert_eq!(range.end, 11); + + assert!(db.get(&k_n).await.unwrap().is_none()); + // Make sure k is still there + assert!(db.get(&k).await.unwrap().is_some()); + + db.destroy().await.unwrap(); + }); + } + + /// Tests the pruning example in the module documentation. + #[test_traced("DEBUG")] + fn test_store_pruning() { + let executor = deterministic::Runner::default(); + + executor.start(|mut ctx| async move { + let mut db = create_test_store(ctx.with_label("store")) + .await + .into_dirty(); + + let k_a = Digest::random(&mut ctx); + let k_b = Digest::random(&mut ctx); + + let v_a = vec![1]; + let v_b = vec![]; + let v_c = vec![4, 5, 6]; + + db.update(k_a, v_a.clone()).await.unwrap(); + db.update(k_b, v_b.clone()).await.unwrap(); + + let (db, _) = db.commit(None).await.unwrap(); + assert_eq!(db.op_count(), 5); + assert_eq!(db.inactivity_floor_loc, 2); + assert_eq!(db.get(&k_a).await.unwrap().unwrap(), v_a); + + let mut db = db.into_dirty(); + db.update(k_b, v_a.clone()).await.unwrap(); + db.update(k_a, v_c.clone()).await.unwrap(); + + let (db, _) = db.commit(None).await.unwrap(); + assert_eq!(db.op_count(), 11); + assert_eq!(db.inactivity_floor_loc, 8); + assert_eq!(db.get(&k_a).await.unwrap().unwrap(), v_c); + assert_eq!(db.get(&k_b).await.unwrap().unwrap(), v_a); + + db.destroy().await.unwrap(); + }); + } + + #[test_traced("WARN")] + pub fn test_store_db_recovery() { + let executor = deterministic::Runner::default(); + // Build a db with 1000 keys, some of which we update and some of which we delete. + const ELEMENTS: u64 = 1000; + executor.start(|context| async move { + let mut db = create_test_store(context.with_label("store")) + .await + .into_dirty(); + + for i in 0u64..ELEMENTS { + let k = Blake3::hash(&i.to_be_bytes()); + let v = vec![(i % 255) as u8; ((i % 13) + 7) as usize]; + db.update(k, v.clone()).await.unwrap(); + } + + // Simulate a failed commit and test that we rollback to the previous root. + drop(db); + let db = create_test_store(context.with_label("store")).await; + assert_eq!(db.op_count(), 1); + + // re-apply the updates and commit them this time. + let mut db = db.into_dirty(); + for i in 0u64..ELEMENTS { + let k = Blake3::hash(&i.to_be_bytes()); + let v = vec![(i % 255) as u8; ((i % 13) + 7) as usize]; + db.update(k, v.clone()).await.unwrap(); + } + let (db, _) = db.commit(None).await.unwrap(); + let op_count = db.op_count(); + + // Update every 3rd key + let mut db = db.into_dirty(); + for i in 0u64..ELEMENTS { + if i % 3 != 0 { + continue; + } + let k = Blake3::hash(&i.to_be_bytes()); + let v = vec![((i + 1) % 255) as u8; ((i % 13) + 8) as usize]; + db.update(k, v.clone()).await.unwrap(); + } + + // Simulate a failed commit and test that we rollback to the previous root. + drop(db); + let mut db = create_test_store(context.with_label("store")) + .await + .into_dirty(); + assert_eq!(db.op_count(), op_count); + + // Re-apply updates for every 3rd key and commit them this time. + for i in 0u64..ELEMENTS { + if i % 3 != 0 { + continue; + } + let k = Blake3::hash(&i.to_be_bytes()); + let v = vec![((i + 1) % 255) as u8; ((i % 13) + 8) as usize]; + db.update(k, v.clone()).await.unwrap(); + } + let (db, _) = db.commit(None).await.unwrap(); + let op_count = db.op_count(); + assert_eq!(op_count, 1673); + assert_eq!(db.snapshot.items(), 1000); + + // Delete every 7th key + let mut db = db.into_dirty(); + for i in 0u64..ELEMENTS { + if i % 7 != 1 { + continue; + } + let k = Blake3::hash(&i.to_be_bytes()); + db.delete(k).await.unwrap(); + } + + // Simulate a failed commit and test that we rollback to the previous root. + drop(db); + let db = create_test_store(context.with_label("store")).await; + assert_eq!(db.op_count(), op_count); + + // Sync and reopen the store to ensure the final commit is preserved. + let mut db = db; + db.sync().await.unwrap(); + drop(db); + let mut db = create_test_store(context.with_label("store")) + .await + .into_dirty(); + assert_eq!(db.op_count(), op_count); + + // Re-delete every 7th key and commit this time. + for i in 0u64..ELEMENTS { + if i % 7 != 1 { + continue; + } + let k = Blake3::hash(&i.to_be_bytes()); + db.delete(k).await.unwrap(); + } + let (mut db, _) = db.commit(None).await.unwrap(); + + assert_eq!(db.op_count(), 1961); + assert_eq!(db.inactivity_floor_loc, 756); + + db.prune(db.inactivity_floor_loc()).await.unwrap(); + assert_eq!(db.log.oldest_retained_pos(), Some(756 /*- 756 % 7 == 0*/)); + assert_eq!(db.snapshot.items(), 857); + + db.destroy().await.unwrap(); + }); + } + + #[test_traced("DEBUG")] + fn test_store_batchable() { + let executor = deterministic::Runner::default(); + + executor.start(|mut ctx| async move { + let mut db = create_test_store(ctx.with_label("store")) + .await + .into_dirty(); + + // Ensure the store is empty + assert_eq!(db.op_count(), 1); + assert_eq!(db.inactivity_floor_loc, 0); + + let key = Digest::random(&mut ctx); + let value = vec![2, 3, 4, 5]; + + let mut batch = db.start_batch(); + + // Attempt to get a key that does not exist + let result = batch.get(&key).await; + assert!(result.unwrap().is_none()); + + // Insert a key-value pair + batch.update(key, value.clone()).await.unwrap(); + + assert_eq!(db.op_count(), 1); // The batch is not applied yet + assert_eq!(db.inactivity_floor_loc, 0); + + // Fetch the value + let fetched_value = batch.get(&key).await.unwrap(); + assert_eq!(fetched_value.unwrap(), value); + db.write_batch(batch.into_iter()).await.unwrap(); + drop(db); + + // Re-open the store + let mut db = create_test_store(ctx.with_label("store")) + .await + .into_dirty(); + + // Ensure the batch was not applied since we didn't commit. + assert_eq!(db.op_count(), 1); + assert_eq!(db.inactivity_floor_loc, 0); + assert!(db.get_metadata().await.unwrap().is_none()); + + // Insert a key-value pair + let mut batch = db.start_batch(); + batch.update(key, value.clone()).await.unwrap(); + + // Persist the changes + db.write_batch(batch.into_iter()).await.unwrap(); + assert_eq!(db.op_count(), 2); + assert_eq!(db.inactivity_floor_loc, 0); + let metadata = vec![99, 100]; + let (db, range) = db.commit(Some(metadata.clone())).await.unwrap(); + assert_eq!(range.start, 1); + assert_eq!(range.end, 4); + assert_eq!(db.get_metadata().await.unwrap(), Some(metadata.clone())); + drop(db); + + // Re-open the store + let db = create_test_store(ctx.with_label("store")).await; + + // Ensure the re-opened store retained the committed operations + assert_eq!(db.op_count(), 4); + assert_eq!(db.inactivity_floor_loc, 2); + + // Fetch the value, ensuring it is still present + let fetched_value = db.get(&key).await.unwrap(); + assert_eq!(fetched_value.unwrap(), value); + + // Destroy the store + db.destroy().await.unwrap(); + }); + } +} diff --git a/storage/src/qmdb/store/mod.rs b/storage/src/qmdb/store/mod.rs index 9130190ef5..05e441b3a2 100644 --- a/storage/src/qmdb/store/mod.rs +++ b/storage/src/qmdb/store/mod.rs @@ -1,144 +1,52 @@ -//! A mutable key-value database that supports variable-sized values, but without authentication. -//! -//! # Terminology -//! -//! A _key_ in an unauthenticated database either has a _value_ or it doesn't. The _update_ -//! operation gives a key a specific value whether it previously had no value or had a different -//! value. -//! -//! Keys with values are called _active_. An operation is called _active_ if (1) its key is active, -//! (2) it is an update operation, and (3) it is the most recent operation for that key. -//! -//! # Lifecycle -//! -//! 1. **Initialization**: Create with [Store::init] using a [Config] -//! 2. **Insertion**: Use [Store::update] to assign a value to a given key -//! 3. **Deletions**: Use [Store::delete] to remove a key's value -//! 4. **Persistence**: Call [Store::commit] or [Store::sync] to make changes durable -//! 5. **Queries**: Use [Store::get] to retrieve current values -//! 6. **Cleanup**: Call [Store::destroy] to remove all data +//! Traits for interacting with stores whose state is derived from an append-only log of +//! state-changing operations. //! //! # Pruning //! -//! The database maintains a location before which all operations are inactive, called the -//! _inactivity floor_. These items can be cleaned from storage by calling [Store::prune]. -//! -//! # Example -//! -//! ```rust -//! use commonware_storage::{ -//! qmdb::store::{Config, Store}, -//! translator::TwoCap, -//! }; -//! use commonware_utils::{NZUsize, NZU64}; -//! use commonware_cryptography::{blake3::Digest, Digest as _}; -//! use commonware_math::algebra::Random; -//! use commonware_runtime::{buffer::PoolRef, deterministic::Runner, Metrics, Runner as _}; -//! -//! const PAGE_SIZE: usize = 77; -//! const PAGE_CACHE_SIZE: usize = 9; -//! -//! let executor = Runner::default(); -//! executor.start(|mut ctx| async move { -//! let config = Config { -//! log_partition: "test_partition".to_string(), -//! log_write_buffer: NZUsize!(64 * 1024), -//! log_compression: None, -//! log_codec_config: (), -//! log_items_per_section: NZU64!(4), -//! translator: TwoCap, -//! buffer_pool: PoolRef::new(NZUsize!(PAGE_SIZE), NZUsize!(PAGE_CACHE_SIZE)), -//! }; -//! let mut store = -//! Store::<_, Digest, Digest, TwoCap>::init(ctx.with_label("store"), config) -//! .await -//! .unwrap(); -//! -//! // Insert a key-value pair -//! let k = Digest::random(&mut ctx); -//! let v = Digest::random(&mut ctx); -//! store.update(k, v).await.unwrap(); -//! -//! // Fetch the value -//! let fetched_value = store.get(&k).await.unwrap(); -//! assert_eq!(fetched_value.unwrap(), v); -//! -//! // Commit the operation to make it persistent -//! let metadata = Some(Digest::random(&mut ctx)); -//! store.commit(metadata).await.unwrap(); -//! -//! // Delete the key's value -//! store.delete(k).await.unwrap(); -//! -//! // Fetch the value -//! let fetched_value = store.get(&k).await.unwrap(); -//! assert!(fetched_value.is_none()); -//! -//! // Commit the operation to make it persistent -//! store.commit(None).await.unwrap(); -//! -//! // Destroy the store -//! store.destroy().await.unwrap(); -//! }); -//! ``` +//! A log based store maintains a location before which all operations are inactive, called the +//! _inactivity floor_. These operations can be cleaned from storage by calling [PrunableStore::prune]. use crate::{ - index::{unordered::Index, Unordered as _}, - journal::contiguous::{ - variable::{Config as JournalConfig, Journal}, - MutableContiguous as _, - }, - kv, mmr::{Location, Proof}, - qmdb::{ - any::{ - unordered::{variable::Operation, Update}, - VariableValue, - }, - build_snapshot_from_log, create_key, delete_key, - operation::{Committable as _, Operation as _}, - update_key, Error, FloorHelper, - }, - translator::Translator, + qmdb::Error, }; -use commonware_codec::{Codec, Read}; +use commonware_codec::Codec; use commonware_cryptography::Digest; -use commonware_runtime::{buffer::PoolRef, Clock, Metrics, Storage}; -use commonware_utils::Array; -use core::{future::Future, ops::Range}; -use std::num::{NonZeroU64, NonZeroUsize}; -use tracing::{debug, warn}; +use core::future::Future; +use std::num::NonZeroU64; mod batch; +pub mod db; #[cfg(test)] pub use batch::tests as batch_tests; -/// Configuration for initializing a [Store] database. -#[derive(Clone)] -pub struct Config { - /// The name of the [`Storage`] partition used to persist the log of operations. - pub log_partition: String, - - /// The size of the write buffer to use for each blob in the [Journal]. - pub log_write_buffer: NonZeroUsize, - - /// Optional compression level (using `zstd`) to apply to log data before storing. - pub log_compression: Option, +/// Sealed trait for store state types. +mod private { + pub trait Sealed {} +} - /// The codec configuration to use for encoding and decoding log items. - pub log_codec_config: C, +/// Trait for valid store state types. +pub trait State: private::Sealed + Sized {} - /// The number of operations to store in each section of the [Journal]. - pub log_items_per_section: NonZeroU64, +/// Marker type for a store in a "durable" state (no uncommitted operations). +#[derive(Clone, Copy, Debug)] +pub struct Durable; - /// The [`Translator`] used by the compressed index. - pub translator: T, +impl private::Sealed for Durable {} +impl State for Durable {} - /// The buffer pool to use for caching data. - pub buffer_pool: PoolRef, +/// Marker type for a store in a "non-durable" state (may contain uncommitted operations). +#[derive(Clone, Debug, Default)] +pub struct NonDurable { + /// The number of _steps_ to raise the inactivity floor. Each step involves moving exactly one + /// active operation to tip. + pub(crate) steps: u64, } -/// A trait for any key-value store based on an append-only log of operations. +impl private::Sealed for NonDurable {} +impl State for NonDurable {} + +/// A trait for a store based on an append-only log of operations. pub trait LogStore { type Value: Codec + Clone; @@ -157,50 +65,20 @@ pub trait LogStore { fn get_metadata(&self) -> impl Future, Error>>; } -/// A trait for log stores that support pruning. -pub trait LogStorePrunable: LogStore { - /// Prune historical operations prior to `prune_loc`. - fn prune(&mut self, prune_loc: Location) -> impl Future>; -} - -/// A trait for authenticated stores in a "dirty" state with unmerkleized operations. -pub trait DirtyStore: LogStore { - /// The digest type used for authentication. - type Digest: Digest; - - /// The operation type stored in the log. - type Operation; - - /// The clean state type that this dirty store transitions to. - type Clean: CleanStore< - Digest = Self::Digest, - Operation = Self::Operation, - Dirty = Self, - Value = Self::Value, - >; - - /// Merkleize the store and compute the root digest. - /// - /// Consumes this dirty store and returns a clean store with the computed root. - fn merkleize(self) -> impl Future>; +/// A trait for stores that can be pruned. +pub trait PrunableStore: LogStore { + /// Prune historical operations prior to `loc`. + fn prune(&mut self, loc: Location) -> impl Future>; } -/// A trait for authenticated stores in a "clean" state where the MMR root is computed. -pub trait CleanStore: LogStore { +/// A trait for stores that support authentication through merkleization and inclusion proofs. +pub trait MerkleizedStore: LogStore { /// The digest type used for authentication. type Digest: Digest; /// The operation type stored in the log. type Operation; - /// The dirty state type that this clean store transitions to. - type Dirty: DirtyStore< - Digest = Self::Digest, - Operation = Self::Operation, - Clean = Self, - Value = Self::Value, - >; - /// Returns the root digest of the authenticated store. fn root(&self) -> Self::Digest; @@ -220,7 +98,9 @@ pub trait CleanStore: LogStore { &self, start_loc: Location, max_ops: NonZeroU64, - ) -> impl Future, Vec), Error>>; + ) -> impl Future, Vec), Error>> { + self.historical_proof(self.op_count(), start_loc, max_ops) + } /// Generate and return: /// 1. a proof of all operations applied to the store in the range starting at (and including) @@ -242,887 +122,4 @@ pub trait CleanStore: LogStore { start_loc: Location, max_ops: NonZeroU64, ) -> impl Future, Vec), Error>>; - - /// Convert this clean store into its dirty counterpart for batched updates. - fn into_dirty(self) -> Self::Dirty; -} - -/// An unauthenticated key-value database based off of an append-only [Journal] of operations. -pub struct Store -where - E: Storage + Clock + Metrics, - K: Array, - V: VariableValue, - T: Translator, -{ - /// A log of all [Operation]s that have been applied to the store. - /// - /// # Invariants - /// - /// - There is always at least one commit operation in the log. - /// - The log is never pruned beyond the inactivity floor. - log: Journal>, - - /// A snapshot of all currently active operations in the form of a map from each key to the - /// location containing its most recent update. - /// - /// # Invariant - /// - /// Only references operations of type [Operation::Update]. - snapshot: Index, - - /// The number of active keys in the store. - active_keys: usize, - - /// A location before which all operations are "inactive" (that is, operations before this point - /// are over keys that have been updated by some operation at or after this point). - inactivity_floor_loc: Location, - - /// The number of _steps_ to raise the inactivity floor. Each step involves moving exactly one - /// active operation to tip. - steps: u64, - - /// The location of the last commit operation. - last_commit_loc: Location, -} - -/// Type alias for the shared state wrapper used by this Any database variant. -type FloorHelperState<'a, E, K, V, T> = - FloorHelper<'a, Index, Journal>>; - -impl Store -where - E: Storage + Clock + Metrics, - K: Array, - V: VariableValue, - T: Translator, -{ - /// Initializes a new [`Store`] database with the given configuration. - /// - /// ## Rollback - /// - /// Any uncommitted operations will be rolled back if the [Store] was previously closed without - /// committing. - pub async fn init( - context: E, - cfg: Config as Read>::Cfg>, - ) -> Result { - let mut log = Journal::>::init( - context.with_label("log"), - JournalConfig { - partition: cfg.log_partition, - items_per_section: cfg.log_items_per_section, - compression: cfg.log_compression, - codec_config: cfg.log_codec_config, - buffer_pool: cfg.buffer_pool, - write_buffer: cfg.log_write_buffer, - }, - ) - .await?; - - // Rewind log to remove uncommitted operations. - if log.rewind_to(|op| op.is_commit()).await? == 0 { - warn!("Log is empty, initializing new db"); - log.append(Operation::CommitFloor(None, Location::new_unchecked(0))) - .await?; - } - - // Sync the log to avoid having to repeat any recovery that may have been performed on next - // startup. - log.sync().await?; - - let last_commit_loc = - Location::new_unchecked(log.size().checked_sub(1).expect("commit should exist")); - let op = log.read(*last_commit_loc).await?; - let inactivity_floor_loc = op.has_floor().expect("last op should be a commit"); - - // Build the snapshot. - let mut snapshot = Index::new(context.with_label("snapshot"), cfg.translator); - let active_keys = - build_snapshot_from_log(inactivity_floor_loc, &log, &mut snapshot, |_, _| {}).await?; - - Ok(Self { - log, - snapshot, - active_keys, - inactivity_floor_loc, - steps: 0, - last_commit_loc, - }) - } - - /// Get the value of `key` in the db, or None if it has no value. - pub async fn get(&self, key: &K) -> Result, Error> { - for &loc in self.snapshot.get(key) { - let Operation::Update(Update(k, v)) = self.get_op(loc).await? else { - unreachable!("location ({loc}) does not reference update operation"); - }; - - if &k == key { - return Ok(Some(v)); - } - } - - Ok(None) - } - - const fn as_floor_helper(&mut self) -> FloorHelperState<'_, E, K, V, T> { - FloorHelper { - snapshot: &mut self.snapshot, - log: &mut self.log, - } - } - - /// Whether the db currently has no active keys. - pub const fn is_empty(&self) -> bool { - self.active_keys == 0 - } - - /// Gets a [Operation] from the log at the given location. Returns [Error::OperationPruned] - /// if the location precedes the oldest retained location. The location is otherwise assumed - /// valid. - async fn get_op(&self, loc: Location) -> Result, Error> { - assert!(loc < self.op_count()); - - // Get the operation from the log at the specified position. - // The journal will return ItemPruned if the location is pruned. - self.log.read(*loc).await.map_err(|e| match e { - crate::journal::Error::ItemPruned(_) => Error::OperationPruned(loc), - e => Error::Journal(e), - }) - } - - /// The number of operations that have been applied to this db, including those that have been - /// pruned and those that are not yet committed. - pub const fn op_count(&self) -> Location { - Location::new_unchecked(self.log.size()) - } - - /// Return the inactivity floor location. This is the location before which all operations are - /// known to be inactive. Operations before this point can be safely pruned. - pub const fn inactivity_floor_loc(&self) -> Location { - self.inactivity_floor_loc - } - - /// Get the metadata associated with the last commit. - pub async fn get_metadata(&self) -> Result, Error> { - let Operation::CommitFloor(metadata, _) = self.log.read(*self.last_commit_loc).await? - else { - unreachable!("last commit should be a commit floor operation"); - }; - - Ok(metadata) - } - - /// Updates `key` to have value `value`. The operation is reflected in the snapshot, but will be - /// subject to rollback until the next successful `commit`. - pub async fn update(&mut self, key: K, value: V) -> Result<(), Error> { - let new_loc = self.op_count(); - if update_key(&mut self.snapshot, &self.log, &key, new_loc) - .await? - .is_some() - { - self.steps += 1; - } else { - self.active_keys += 1; - } - - self.log - .append(Operation::Update(Update(key, value))) - .await?; - - Ok(()) - } - - /// Creates a new key-value pair in the db. The operation is reflected in the snapshot, but will - /// be subject to rollback until the next successful `commit`. Returns true if the key was - /// created, false if it already existed. - pub async fn create(&mut self, key: K, value: V) -> Result { - let new_loc = self.op_count(); - if !create_key(&mut self.snapshot, &self.log, &key, new_loc).await? { - return Ok(false); - } - - self.active_keys += 1; - self.log - .append(Operation::Update(Update(key, value))) - .await?; - - Ok(true) - } - - /// Delete `key` and its value from the db. Deleting a key that already has no value is a no-op. - /// The operation is reflected in the snapshot, but will be subject to rollback until the next - /// successful `commit`. Returns true if the key was deleted, false if it was already inactive. - pub async fn delete(&mut self, key: K) -> Result { - let r = delete_key(&mut self.snapshot, &self.log, &key).await?; - if r.is_none() { - return Ok(false); - } - - self.log.append(Operation::Delete(key)).await?; - self.steps += 1; - self.active_keys -= 1; - - Ok(true) - } - - /// Commit any pending operations to the database, ensuring their durability upon return from - /// this function. Also raises the inactivity floor according to the schedule. Returns the - /// `(start_loc, end_loc]` location range of committed operations. The end of the returned range - /// includes the commit operation itself, and hence will always be equal to `op_count`. - /// - /// Failures after commit (but before `sync` or `close`) may still require reprocessing to - /// recover the database on restart. - pub async fn commit(&mut self, metadata: Option) -> Result, Error> { - let start_loc = self.last_commit_loc + 1; - - // Raise the inactivity floor by taking `self.steps` steps, plus 1 to account for the - // previous commit becoming inactive. - if self.is_empty() { - self.inactivity_floor_loc = self.op_count(); - debug!(tip = ?self.inactivity_floor_loc, "db is empty, raising floor to tip"); - } else { - let steps_to_take = self.steps + 1; - for _ in 0..steps_to_take { - let loc = self.inactivity_floor_loc; - self.inactivity_floor_loc = self.as_floor_helper().raise_floor(loc).await?; - } - } - self.steps = 0; - - // Apply the commit operation with the new inactivity floor. - self.last_commit_loc = Location::new_unchecked( - self.log - .append(Operation::CommitFloor(metadata, self.inactivity_floor_loc)) - .await?, - ); - - // Commit the log to ensure this commit is durable. - self.log.commit().await?; - - Ok(start_loc..self.op_count()) - } - - /// Sync all database state to disk. While this isn't necessary to ensure durability of - /// committed operations, periodic invocation may reduce memory usage and the time required to - /// recover the database on restart. - pub async fn sync(&mut self) -> Result<(), Error> { - self.log.sync().await.map_err(Into::into) - } - - /// Prune historical operations prior to `prune_loc`. This does not affect the db's root - /// or current snapshot. - pub async fn prune(&mut self, prune_loc: Location) -> Result<(), Error> { - if prune_loc > self.inactivity_floor_loc { - return Err(Error::PruneBeyondMinRequired( - prune_loc, - self.inactivity_floor_loc, - )); - } - - // Prune the log. The log will prune at section boundaries, so the actual oldest retained - // location may be less than requested. - if !self.log.prune(*prune_loc).await? { - return Ok(()); - } - - debug!( - log_size = ?self.op_count(), - oldest_retained_loc = ?self.log.oldest_retained_pos(), - ?prune_loc, - "pruned inactive ops" - ); - - Ok(()) - } - - /// Destroy the db, removing all data from disk. - pub async fn destroy(self) -> Result<(), Error> { - self.log.destroy().await.map_err(Into::into) - } -} - -impl LogStorePrunable for Store -where - E: Storage + Clock + Metrics, - K: Array, - V: VariableValue, - T: Translator, -{ - async fn prune(&mut self, prune_loc: Location) -> Result<(), Error> { - self.prune(prune_loc).await - } -} - -impl crate::Persistable for Store -where - E: Storage + Clock + Metrics, - K: Array, - V: VariableValue, - T: Translator, -{ - type Error = Error; - - async fn commit(&mut self) -> Result<(), Error> { - self.commit(None).await.map(|_| ()) - } - - async fn sync(&mut self) -> Result<(), Error> { - self.sync().await - } - - async fn destroy(self) -> Result<(), Error> { - self.destroy().await - } -} - -impl LogStore for Store -where - E: Storage + Clock + Metrics, - K: Array, - V: VariableValue, - T: Translator, -{ - type Value = V; - - fn op_count(&self) -> Location { - self.op_count() - } - - fn inactivity_floor_loc(&self) -> Location { - self.inactivity_floor_loc() - } - - async fn get_metadata(&self) -> Result, Error> { - self.get_metadata().await - } - - fn is_empty(&self) -> bool { - self.is_empty() - } -} - -impl kv::Gettable for Store -where - E: Storage + Clock + Metrics, - K: Array, - V: VariableValue, - T: Translator, -{ - type Key = K; - type Value = V; - type Error = Error; - - async fn get(&self, key: &Self::Key) -> Result, Self::Error> { - self.get(key).await - } -} - -impl kv::Updatable for Store -where - E: Storage + Clock + Metrics, - K: Array, - V: VariableValue, - T: Translator, -{ - async fn update(&mut self, key: Self::Key, value: Self::Value) -> Result<(), Self::Error> { - self.update(key, value).await - } -} - -impl kv::Deletable for Store -where - E: Storage + Clock + Metrics, - K: Array, - V: VariableValue, - T: Translator, -{ - async fn delete(&mut self, key: Self::Key) -> Result { - self.delete(key).await - } -} - -impl kv::Batchable for Store -where - E: Storage + Clock + Metrics, - K: Array, - V: VariableValue, - T: Translator, -{ -} - -#[cfg(test)] -mod test { - use super::*; - use crate::{kv::Updatable as _, qmdb::store::batch_tests, translator::TwoCap}; - use commonware_cryptography::{ - blake3::{Blake3, Digest}, - Hasher as _, - }; - use commonware_macros::test_traced; - use commonware_math::algebra::Random; - use commonware_runtime::{deterministic, Runner}; - use commonware_utils::{NZUsize, NZU64}; - - const PAGE_SIZE: usize = 77; - const PAGE_CACHE_SIZE: usize = 9; - - /// The type of the store used in tests. - type TestStore = Store, TwoCap>; - - async fn create_test_store(context: deterministic::Context) -> TestStore { - let cfg = Config { - log_partition: "journal".to_string(), - log_write_buffer: NZUsize!(64 * 1024), - log_compression: None, - log_codec_config: ((0..=10000).into(), ()), - log_items_per_section: NZU64!(7), - translator: TwoCap, - buffer_pool: PoolRef::new(NZUsize!(PAGE_SIZE), NZUsize!(PAGE_CACHE_SIZE)), - }; - Store::init(context, cfg).await.unwrap() - } - - #[test_traced("DEBUG")] - pub fn test_store_construct_empty() { - let executor = deterministic::Runner::default(); - executor.start(|mut context| async move { - let mut db = create_test_store(context.clone()).await; - assert_eq!(db.op_count(), 1); - assert_eq!(db.log.oldest_retained_pos(), Some(0)); - assert_eq!(db.inactivity_floor_loc(), 0); - assert!(matches!(db.prune(db.inactivity_floor_loc()).await, Ok(()))); - assert!(matches!( - db.prune(Location::new_unchecked(1)).await, - Err(Error::PruneBeyondMinRequired(_, _)) - )); - assert!(db.get_metadata().await.unwrap().is_none()); - - // Make sure closing/reopening gets us back to the same state, even after adding an uncommitted op. - let d1 = Digest::random(&mut context); - let v1 = vec![1, 2, 3]; - db.update(d1, v1).await.unwrap(); - db.sync().await.unwrap(); - drop(db); - let mut db = create_test_store(context.clone()).await; - assert_eq!(db.op_count(), 1); - - // Test calling commit on an empty db which should make it (durably) non-empty. - let metadata = vec![1, 2, 3]; - let range = db.commit(Some(metadata.clone())).await.unwrap(); - assert_eq!(range.start, 1); - assert_eq!(range.end, 2); - assert_eq!(db.op_count(), 2); - assert!(matches!(db.prune(db.inactivity_floor_loc()).await, Ok(()))); - assert_eq!(db.get_metadata().await.unwrap(), Some(metadata.clone())); - - let mut db = create_test_store(context.clone()).await; - assert_eq!(db.get_metadata().await.unwrap(), Some(metadata)); - - // Confirm the inactivity floor doesn't fall endlessly behind with multiple commits on a - // non-empty db. - db.update(Digest::random(&mut context), vec![1, 2, 3]) - .await - .unwrap(); - for _ in 1..100 { - db.commit(None).await.unwrap(); - // Distance should equal 3 after the second commit, with inactivity_floor - // referencing the previous commit operation. - assert!(db.op_count() - db.inactivity_floor_loc <= 3); - assert!(db.get_metadata().await.unwrap().is_none()); - } - - db.destroy().await.unwrap(); - }); - } - - #[test_traced("DEBUG")] - fn test_store_construct_basic() { - let executor = deterministic::Runner::default(); - - executor.start(|mut ctx| async move { - let mut store = create_test_store(ctx.with_label("store")).await; - - // Ensure the store is empty - assert_eq!(store.op_count(), 1); - assert_eq!(store.inactivity_floor_loc, 0); - - let key = Digest::random(&mut ctx); - let value = vec![2, 3, 4, 5]; - - // Attempt to get a key that does not exist - let result = store.get(&key).await; - assert!(result.unwrap().is_none()); - - // Insert a key-value pair - store.update(key, value.clone()).await.unwrap(); - - assert_eq!(store.op_count(), 2); - assert_eq!(store.inactivity_floor_loc, 0); - - // Fetch the value - let fetched_value = store.get(&key).await.unwrap(); - assert_eq!(fetched_value.unwrap(), value); - - // Sync the store to persist the changes - store.sync().await.unwrap(); - - // Re-open the store - let mut store = create_test_store(ctx.with_label("store")).await; - - // Ensure the re-opened store removed the uncommitted operations - assert_eq!(store.op_count(), 1); - assert_eq!(store.inactivity_floor_loc, 0); - assert!(store.get_metadata().await.unwrap().is_none()); - - // Insert a key-value pair - store.update(key, value.clone()).await.unwrap(); - - assert_eq!(store.op_count(), 2); - assert_eq!(store.inactivity_floor_loc, 0); - - // Persist the changes - let metadata = vec![99, 100]; - let range = store.commit(Some(metadata.clone())).await.unwrap(); - assert_eq!(range.start, 1); - assert_eq!(range.end, 4); - assert_eq!(store.get_metadata().await.unwrap(), Some(metadata.clone())); - - // Even though the store was pruned, the inactivity floor was raised by 2, and - // the old operations remain in the same blob as an active operation, so they're - // retained. - assert_eq!(store.op_count(), 4); - assert_eq!(store.inactivity_floor_loc, 2); - - // Re-open the store - let mut store = create_test_store(ctx.with_label("store")).await; - - // Ensure the re-opened store retained the committed operations - assert_eq!(store.op_count(), 4); - assert_eq!(store.inactivity_floor_loc, 2); - - // Fetch the value, ensuring it is still present - let fetched_value = store.get(&key).await.unwrap(); - assert_eq!(fetched_value.unwrap(), value); - - // Insert two new k/v pairs to force pruning of the first section. - let (k1, v1) = (Digest::random(&mut ctx), vec![2, 3, 4, 5, 6]); - let (k2, v2) = (Digest::random(&mut ctx), vec![6, 7, 8]); - store.update(k1, v1.clone()).await.unwrap(); - store.update(k2, v2.clone()).await.unwrap(); - - assert_eq!(store.op_count(), 6); - assert_eq!(store.inactivity_floor_loc, 2); - - // Make sure we can still get metadata. - assert_eq!(store.get_metadata().await.unwrap(), Some(metadata)); - - let range = store.commit(None).await.unwrap(); - assert_eq!(range.start, 4); - assert_eq!(range.end, store.op_count()); - assert_eq!(store.get_metadata().await.unwrap(), None); - - assert_eq!(store.op_count(), 8); - assert_eq!(store.inactivity_floor_loc, 3); - - // Ensure all keys can be accessed, despite the first section being pruned. - assert_eq!(store.get(&key).await.unwrap().unwrap(), value); - assert_eq!(store.get(&k1).await.unwrap().unwrap(), v1); - assert_eq!(store.get(&k2).await.unwrap().unwrap(), v2); - - // Ensure upsert works for existing key. - store.upsert(k1, |v| v.push(7)).await.unwrap(); - assert_eq!( - store.get(&k1).await.unwrap().unwrap(), - vec![2, 3, 4, 5, 6, 7] - ); - - // Ensure upsert works for new key. - let k3 = Digest::random(&mut ctx); - store.upsert(k3, |v| v.push(8)).await.unwrap(); - assert_eq!(store.get(&k3).await.unwrap().unwrap(), vec![8]); - - // Destroy the store - store.destroy().await.unwrap(); - }); - } - - #[test_traced("DEBUG")] - fn test_store_log_replay() { - let executor = deterministic::Runner::default(); - - executor.start(|mut ctx| async move { - let mut store = create_test_store(ctx.with_label("store")).await; - - // Update the same key many times. - const UPDATES: u64 = 100; - let k = Digest::random(&mut ctx); - for _ in 0..UPDATES { - let v = vec![1, 2, 3, 4, 5]; - store.update(k, v.clone()).await.unwrap(); - } - - let iter = store.snapshot.get(&k); - assert_eq!(iter.count(), 1); - - store.commit(None).await.unwrap(); - drop(store); - - // Re-open the store, prune it, then ensure it replays the log correctly. - let mut store = create_test_store(ctx.with_label("store")).await; - store.prune(store.inactivity_floor_loc()).await.unwrap(); - - let iter = store.snapshot.get(&k); - assert_eq!(iter.count(), 1); - - // 100 operations were applied, each triggering one step, plus the commit op. - assert_eq!(store.op_count(), UPDATES * 2 + 2); - // Only the highest `Update` operation is active, plus the commit operation above it. - let expected_floor = UPDATES * 2; - assert_eq!(store.inactivity_floor_loc, expected_floor); - - // All blobs prior to the inactivity floor are pruned, so the oldest retained location - // is the first in the last retained blob. - assert_eq!( - store.log.oldest_retained_pos(), - Some(expected_floor - expected_floor % 7) - ); - - store.destroy().await.unwrap(); - }); - } - - #[test_traced("DEBUG")] - fn test_store_build_snapshot_keys_with_shared_prefix() { - let executor = deterministic::Runner::default(); - - executor.start(|mut ctx| async move { - let mut store = create_test_store(ctx.with_label("store")).await; - - let (k1, v1) = (Digest::random(&mut ctx), vec![1, 2, 3, 4, 5]); - let (mut k2, v2) = (Digest::random(&mut ctx), vec![6, 7, 8, 9, 10]); - - // Ensure k2 shares 2 bytes with k1 (test DB uses `TwoCap` translator.) - k2.0[0..2].copy_from_slice(&k1.0[0..2]); - - store.update(k1, v1.clone()).await.unwrap(); - store.update(k2, v2.clone()).await.unwrap(); - - assert_eq!(store.get(&k1).await.unwrap().unwrap(), v1); - assert_eq!(store.get(&k2).await.unwrap().unwrap(), v2); - - store.commit(None).await.unwrap(); - drop(store); - - // Re-open the store to ensure it builds the snapshot for the conflicting - // keys correctly. - let store = create_test_store(ctx.with_label("store")).await; - - assert_eq!(store.get(&k1).await.unwrap().unwrap(), v1); - assert_eq!(store.get(&k2).await.unwrap().unwrap(), v2); - - store.destroy().await.unwrap(); - }); - } - - #[test_traced("DEBUG")] - fn test_store_delete() { - let executor = deterministic::Runner::default(); - - executor.start(|mut ctx| async move { - let mut store = create_test_store(ctx.with_label("store")).await; - - // Insert a key-value pair - let k = Digest::random(&mut ctx); - let v = vec![1, 2, 3, 4, 5]; - store.update(k, v.clone()).await.unwrap(); - - // Fetch the value - let fetched_value = store.get(&k).await.unwrap(); - assert_eq!(fetched_value.unwrap(), v); - - // Delete the key - assert!(store.delete(k).await.unwrap()); - - // Ensure the key is no longer present - let fetched_value = store.get(&k).await.unwrap(); - assert!(fetched_value.is_none()); - assert!(!store.delete(k).await.unwrap()); - - // Commit the changes - store.commit(None).await.unwrap(); - - // Re-open the store and ensure the key is still deleted - let mut store = create_test_store(ctx.with_label("store")).await; - let fetched_value = store.get(&k).await.unwrap(); - assert!(fetched_value.is_none()); - - // Re-insert the key - store.update(k, v.clone()).await.unwrap(); - let fetched_value = store.get(&k).await.unwrap(); - assert_eq!(fetched_value.unwrap(), v); - - // Commit the changes - store.commit(None).await.unwrap(); - - // Re-open the store and ensure the snapshot restores the key, after processing - // the delete and the subsequent set. - let mut store = create_test_store(ctx.with_label("store")).await; - let fetched_value = store.get(&k).await.unwrap(); - assert_eq!(fetched_value.unwrap(), v); - - // Delete a non-existent key (no-op) - let k_n = Digest::random(&mut ctx); - store.delete(k_n).await.unwrap(); - - let iter = store.snapshot.get(&k); - assert_eq!(iter.count(), 1); - - let iter = store.snapshot.get(&k_n); - assert_eq!(iter.count(), 0); - - store.destroy().await.unwrap(); - }); - } - - /// Tests the pruning example in the module documentation. - #[test_traced("DEBUG")] - fn test_store_pruning() { - let executor = deterministic::Runner::default(); - - executor.start(|mut ctx| async move { - let mut store = create_test_store(ctx.with_label("store")).await; - - let k_a = Digest::random(&mut ctx); - let k_b = Digest::random(&mut ctx); - - let v_a = vec![1]; - let v_b = vec![]; - let v_c = vec![4, 5, 6]; - - store.update(k_a, v_a.clone()).await.unwrap(); - store.update(k_b, v_b.clone()).await.unwrap(); - - store.commit(None).await.unwrap(); - assert_eq!(store.op_count(), 5); - assert_eq!(store.inactivity_floor_loc, 2); - assert_eq!(store.get(&k_a).await.unwrap().unwrap(), v_a); - - store.update(k_b, v_a.clone()).await.unwrap(); - store.update(k_a, v_c.clone()).await.unwrap(); - - store.commit(None).await.unwrap(); - assert_eq!(store.op_count(), 11); - assert_eq!(store.inactivity_floor_loc, 8); - assert_eq!(store.get(&k_a).await.unwrap().unwrap(), v_c); - assert_eq!(store.get(&k_b).await.unwrap().unwrap(), v_a); - - store.destroy().await.unwrap(); - }); - } - - #[test_traced("WARN")] - pub fn test_store_db_recovery() { - let executor = deterministic::Runner::default(); - // Build a db with 1000 keys, some of which we update and some of which we delete. - const ELEMENTS: u64 = 1000; - executor.start(|context| async move { - let mut db = create_test_store(context.with_label("store")).await; - - for i in 0u64..ELEMENTS { - let k = Blake3::hash(&i.to_be_bytes()); - let v = vec![(i % 255) as u8; ((i % 13) + 7) as usize]; - db.update(k, v.clone()).await.unwrap(); - } - - // Simulate a failed commit and test that we rollback to the previous root. - drop(db); - let mut db = create_test_store(context.with_label("store")).await; - assert_eq!(db.op_count(), 1); - - // re-apply the updates and commit them this time. - for i in 0u64..ELEMENTS { - let k = Blake3::hash(&i.to_be_bytes()); - let v = vec![(i % 255) as u8; ((i % 13) + 7) as usize]; - db.update(k, v.clone()).await.unwrap(); - } - db.commit(None).await.unwrap(); - let op_count = db.op_count(); - - // Update every 3rd key - for i in 0u64..ELEMENTS { - if i % 3 != 0 { - continue; - } - let k = Blake3::hash(&i.to_be_bytes()); - let v = vec![((i + 1) % 255) as u8; ((i % 13) + 8) as usize]; - db.update(k, v.clone()).await.unwrap(); - } - - // Simulate a failed commit and test that we rollback to the previous root. - drop(db); - let mut db = create_test_store(context.with_label("store")).await; - assert_eq!(db.op_count(), op_count); - - // Re-apply updates for every 3rd key and commit them this time. - for i in 0u64..ELEMENTS { - if i % 3 != 0 { - continue; - } - let k = Blake3::hash(&i.to_be_bytes()); - let v = vec![((i + 1) % 255) as u8; ((i % 13) + 8) as usize]; - db.update(k, v.clone()).await.unwrap(); - } - db.commit(None).await.unwrap(); - let op_count = db.op_count(); - assert_eq!(op_count, 1673); - assert_eq!(db.snapshot.items(), 1000); - - // Delete every 7th key - for i in 0u64..ELEMENTS { - if i % 7 != 1 { - continue; - } - let k = Blake3::hash(&i.to_be_bytes()); - db.delete(k).await.unwrap(); - } - - // Simulate a failed commit and test that we rollback to the previous root. - drop(db); - let db = create_test_store(context.with_label("store")).await; - assert_eq!(db.op_count(), op_count); - - // Reopen the store to ensure the final commit is preserved. - drop(db); - let mut db = create_test_store(context.with_label("store")).await; - assert_eq!(db.op_count(), op_count); - - // Re-delete every 7th key and commit this time. - for i in 0u64..ELEMENTS { - if i % 7 != 1 { - continue; - } - let k = Blake3::hash(&i.to_be_bytes()); - db.delete(k).await.unwrap(); - } - db.commit(None).await.unwrap(); - - assert_eq!(db.op_count(), 1961); - assert_eq!(db.inactivity_floor_loc, 756); - - db.prune(db.inactivity_floor_loc()).await.unwrap(); - assert_eq!(db.log.oldest_retained_pos(), Some(756 /*- 756 % 7 == 0*/)); - assert_eq!(db.snapshot.items(), 857); - - db.destroy().await.unwrap(); - }); - } - - #[test_traced("DEBUG")] - fn test_batch() { - batch_tests::test_batch( - |ctx| async move { create_test_store(ctx.with_label("batch")).await }, - ); - } } diff --git a/storage/src/qmdb/sync/journal.rs b/storage/src/qmdb/sync/journal.rs index ac8a06fbc4..60e0f3a5b9 100644 --- a/storage/src/qmdb/sync/journal.rs +++ b/storage/src/qmdb/sync/journal.rs @@ -35,7 +35,27 @@ where } async fn append(&mut self, op: Self::Op) -> Result<(), Self::Error> { - Self::append(self, op).await?; - Ok(()) + self.append(op).await.map(|_| ()) + } +} + +impl Journal for crate::journal::contiguous::fixed::Journal +where + E: commonware_runtime::Storage + commonware_runtime::Metrics, + A: commonware_codec::CodecFixed, +{ + type Op = A; + type Error = crate::journal::Error; + + async fn sync(&mut self) -> Result<(), Self::Error> { + Self::sync(self).await + } + + async fn size(&self) -> u64 { + Self::size(self) + } + + async fn append(&mut self, op: Self::Op) -> Result<(), Self::Error> { + self.append(op).await.map(|_| ()) } } diff --git a/storage/src/qmdb/sync/resolver.rs b/storage/src/qmdb/sync/resolver.rs index a24efde335..88eb1755ca 100644 --- a/storage/src/qmdb/sync/resolver.rs +++ b/storage/src/qmdb/sync/resolver.rs @@ -10,7 +10,7 @@ use crate::{ FixedValue, VariableValue, }, immutable::{Immutable, Operation as ImmutableOp}, - store::CleanStore as _, + Durable, Merkleized, }, translator::Translator, }; @@ -63,7 +63,7 @@ pub trait Resolver: Send + Sync + Clone + 'static { ) -> impl Future, Self::Error>> + Send + 'a; } -impl Resolver for Arc> +impl Resolver for Arc, Durable>> where E: Storage + Clock + Metrics, K: Array, @@ -95,7 +95,7 @@ where /// Implement Resolver directly for `Arc>` to eliminate the need for wrapper types /// while allowing direct database access. -impl Resolver for Arc>> +impl Resolver for Arc, Durable>>> where E: Storage + Clock + Metrics, K: Array, @@ -126,7 +126,7 @@ where } } -impl Resolver for Arc> +impl Resolver for Arc, Durable>> where E: Storage + Clock + Metrics, K: Array, @@ -158,7 +158,7 @@ where /// Implement Resolver directly for `Arc>` to eliminate the need for wrapper /// types while allowing direct database access. -impl Resolver for Arc>> +impl Resolver for Arc, Durable>>> where E: Storage + Clock + Metrics, K: Array, @@ -189,7 +189,76 @@ where } } -impl Resolver for Arc> +/// Implement Resolver for `Arc>>` to allow taking ownership during sync. +impl Resolver for Arc, Durable>>>> +where + E: Storage + Clock + Metrics, + K: Array, + V: FixedValue + Send + Sync + 'static, + H: Hasher, + T: Translator + Send + Sync + 'static, + T::Key: Send + Sync, +{ + type Digest = H::Digest; + type Op = FixedOperation; + type Error = qmdb::Error; + + async fn get_operations( + &self, + op_count: Location, + start_loc: Location, + max_ops: NonZeroU64, + ) -> Result, qmdb::Error> { + let guard = self.read().await; + let db: &FixedDb, Durable> = + guard.as_ref().ok_or(qmdb::Error::KeyNotFound)?; + db.historical_proof(op_count, start_loc, max_ops) + .await + .map(|(proof, operations)| FetchResult { + proof, + operations, + // Result of proof verification isn't used by this implementation. + success_tx: oneshot::channel().0, + }) + } +} + +/// Implement Resolver for `Arc>>` to allow taking ownership during sync. +impl Resolver + for Arc, Durable>>>> +where + E: Storage + Clock + Metrics, + K: Array, + V: VariableValue + Send + Sync + 'static, + H: Hasher, + T: Translator + Send + Sync + 'static, + T::Key: Send + Sync, +{ + type Digest = H::Digest; + type Op = VariableOperation; + type Error = qmdb::Error; + + async fn get_operations( + &self, + op_count: Location, + start_loc: Location, + max_ops: NonZeroU64, + ) -> Result, qmdb::Error> { + let guard = self.read().await; + let db: &VariableDb, Durable> = + guard.as_ref().ok_or(qmdb::Error::KeyNotFound)?; + db.historical_proof(op_count, start_loc, max_ops) + .await + .map(|(proof, operations)| FetchResult { + proof, + operations, + // Result of proof verification isn't used by this implementation. + success_tx: oneshot::channel().0, + }) + } +} + +impl Resolver for Arc, Durable>> where E: Storage + Clock + Metrics, K: Array, @@ -221,7 +290,7 @@ where /// Implement Resolver directly for `Arc>` to eliminate the need for wrapper /// types while allowing direct database access. -impl Resolver for Arc>> +impl Resolver for Arc, Durable>>> where E: Storage + Clock + Metrics, K: Array,