diff --git a/Cargo.toml b/Cargo.toml index 7616adb84..93df3316f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ members = [ "grovedb-bulk-append-tree", "grovedb-dense-fixed-sized-merkle-tree", "grovedb-query", + "grovedb-private-document-store", ] [workspace.dependencies] diff --git a/grovedb-bulk-append-tree/Cargo.toml b/grovedb-bulk-append-tree/Cargo.toml index 196061260..4583b0f91 100644 --- a/grovedb-bulk-append-tree/Cargo.toml +++ b/grovedb-bulk-append-tree/Cargo.toml @@ -18,12 +18,16 @@ storage = [ "grovedb-dense-fixed-sized-merkle-tree/storage", ] mem_store = ["grovedb-merkle-mountain-range/mem_store"] +# Exposes `test_utils::MemStorageContext` to other crates' test builds so the +# in-memory StorageContext harness lives in exactly one place. +test-utils = ["storage"] [dependencies] grovedb-merkle-mountain-range = { version = "5.0.1", path = "../grovedb-merkle-mountain-range", default-features = false } grovedb-dense-fixed-sized-merkle-tree = { version = "5.0.1", path = "../grovedb-dense-fixed-sized-merkle-tree", default-features = false } grovedb-query = { version = "5.0.1", path = "../grovedb-query" } grovedb-costs = { version = "5.0.1", path = "../costs" } +grovedb-version = { version = "5.0.1", path = "../grovedb-version" } grovedb-storage = { version = "5.0.1", path = "../storage", optional = true } blake3 = { workspace = true } bincode = { workspace = true, features = ["derive"] } diff --git a/grovedb-bulk-append-tree/src/cost/mod.rs b/grovedb-bulk-append-tree/src/cost/mod.rs new file mode 100644 index 000000000..7b3a412db --- /dev/null +++ b/grovedb-bulk-append-tree/src/cost/mod.rs @@ -0,0 +1,43 @@ +//! Versioned cost accounting for the bulk-append tree. +//! +//! Only the reported hash count is versioned; chunk bytes, roots and stored +//! state are identical under every version. This gate reaches a live fee — +//! `CommitmentTree` adds the compaction hash count straight into its own +//! `hash_node_calls` — so the corrected figure arrives as a new version rather +//! than replacing the old one. + +mod v0; +mod v1; + +use grovedb_version::{error::GroveVersionError, version::GroveVersion}; + +use crate::BulkAppendError; + +/// Hashes to report for a compacting append. +/// +/// `leaf_count` is the MMR leaf count BEFORE the push (what +/// `hash_count_for_push` expects); `mmr_size_after_push` is the size the MMR +/// reached, which determines how many peaks the compaction's `get_root` had +/// to fold. +pub(crate) fn compaction_hash_count( + leaf_count: u64, + mmr_size_after_push: u64, + grove_version: &GroveVersion, +) -> Result { + match grove_version + .bulk_append_tree_versions + .cost + .compaction_hash_count + { + 0 => Ok(v0::compaction_hash_count(leaf_count)), + 1 => Ok(v1::compaction_hash_count(leaf_count, mmr_size_after_push)), + version => Err(BulkAppendError::VersionError( + GroveVersionError::UnknownVersionMismatch { + method: "BulkAppendTree compaction hash count".to_string(), + known_versions: vec![0, 1], + received: version, + } + .to_string(), + )), + } +} diff --git a/grovedb-bulk-append-tree/src/cost/v0.rs b/grovedb-bulk-append-tree/src/cost/v0.rs new file mode 100644 index 000000000..b66a16456 --- /dev/null +++ b/grovedb-bulk-append-tree/src/cost/v0.rs @@ -0,0 +1,14 @@ +//! Shipped compaction hash count. +//! +//! The chunk-blob leaf hash plus one per peak the MMR push collapses. It omits +//! the peak bagging the compaction's own `get_root` performs, so a compaction +//! landing on a multi-peak MMR under-reports by `peaks - 1`. +//! +//! Locked: GROVE_V1..V3 are released and CommitmentTree has been billing this +//! figure on mainnet. + +use grovedb_merkle_mountain_range::hash_count_for_push; + +pub(super) fn compaction_hash_count(leaf_count: u64) -> u32 { + hash_count_for_push(leaf_count) +} diff --git a/grovedb-bulk-append-tree/src/cost/v1.rs b/grovedb-bulk-append-tree/src/cost/v1.rs new file mode 100644 index 000000000..d7862b44d --- /dev/null +++ b/grovedb-bulk-append-tree/src/cost/v1.rs @@ -0,0 +1,13 @@ +//! Corrected compaction hash count. +//! +//! Adds the peak-bagging merges [`v0`](super::v0) omitted. Used from GROVE_V4. + +use grovedb_merkle_mountain_range::{hash_count_for_push, hash_count_for_root_bagging}; + +pub(super) fn compaction_hash_count(leaf_count: u64, mmr_size_after_push: u64) -> u32 { + // Derived from the MMR shape rather than read back out of the accumulated + // `OperationCost`, so this stays correct regardless of whether + // `MMR::get_root`'s own charge is enabled for the caller's version — the + // two gates are independent. + hash_count_for_push(leaf_count).saturating_add(hash_count_for_root_bagging(mmr_size_after_push)) +} diff --git a/grovedb-bulk-append-tree/src/error.rs b/grovedb-bulk-append-tree/src/error.rs index 28689bd58..db05c573a 100644 --- a/grovedb-bulk-append-tree/src/error.rs +++ b/grovedb-bulk-append-tree/src/error.rs @@ -15,4 +15,8 @@ pub enum BulkAppendError { InvalidProof(String), #[error("invalid input: {0}")] InvalidInput(String), + /// The grove version selected an unknown version for a versioned + /// bulk-append method. + #[error("version error: {0}")] + VersionError(String), } diff --git a/grovedb-bulk-append-tree/src/lib.rs b/grovedb-bulk-append-tree/src/lib.rs index f46436b85..0100c00a2 100644 --- a/grovedb-bulk-append-tree/src/lib.rs +++ b/grovedb-bulk-append-tree/src/lib.rs @@ -9,12 +9,13 @@ //! CDN-cacheable. pub mod chunk; +mod cost; mod error; pub mod proof; mod tree; -#[cfg(all(test, feature = "storage"))] -pub(crate) mod test_utils; +#[cfg(all(feature = "storage", any(test, feature = "test-utils")))] +pub mod test_utils; // Re-export main types pub use chunk::{deserialize_chunk_blob, serialize_chunk_blob}; diff --git a/grovedb-bulk-append-tree/src/proof/tests.rs b/grovedb-bulk-append-tree/src/proof/tests.rs index 83b6f30d3..d51452025 100644 --- a/grovedb-bulk-append-tree/src/proof/tests.rs +++ b/grovedb-bulk-append-tree/src/proof/tests.rs @@ -2,6 +2,7 @@ mod proof_tests { use grovedb_merkle_mountain_range::MmrTreeProof; use grovedb_query::{Query, QueryItem}; + use grovedb_version::version::GroveVersion; use crate::{proof::*, test_utils::MemStorageContext, BulkAppendTree}; @@ -17,7 +18,9 @@ mod proof_tests { let mut last_state_root = [0u8; 32]; for value in values { - let result = tree.append(value).expect("append value"); + let result = tree + .append(value, GroveVersion::latest()) + .expect("append value"); last_state_root = result.state_root; } diff --git a/grovedb-bulk-append-tree/src/test_utils.rs b/grovedb-bulk-append-tree/src/test_utils.rs index fbf79a715..d68915dbe 100644 --- a/grovedb-bulk-append-tree/src/test_utils.rs +++ b/grovedb-bulk-append-tree/src/test_utils.rs @@ -1,6 +1,9 @@ //! Test utilities: in-memory StorageContext for BulkAppendTree tests. -use std::{cell::RefCell, collections::HashMap}; +use std::{ + cell::{Cell, RefCell}, + collections::HashMap, +}; use grovedb_costs::{ storage_cost::key_value_cost::KeyValueStorageCost, ChildrenSizesWithIsSumTree, CostContext, @@ -13,15 +16,38 @@ use grovedb_storage::{Batch, RawIterator, StorageContext}; /// Immediate reads and writes backed by a `HashMap`. Only `get` and `put` /// (data storage) have real implementations; all other `StorageContext` /// methods panic if called. +/// +/// `fail_get` / `fail_put` inject storage faults. Tree code is full of arms +/// that can only be reached when the backing store errors mid-operation — +/// exactly the paths that decide whether a fault surfaces or is silently +/// swallowed — and without injection those arms are untestable. #[derive(Default)] -pub(crate) struct MemStorageContext { +pub struct MemStorageContext { pub data: RefCell, Vec>>, + pub fail_get: Cell, + pub fail_put: Cell, } impl MemStorageContext { pub fn new() -> Self { Self::default() } + + /// Make every subsequent `get` return a storage error. + pub fn fail_reads(&self) { + self.fail_get.set(true); + } + + /// Make every subsequent `put` return a storage error. + pub fn fail_writes(&self) { + self.fail_put.set(true); + } + + /// Resume normal operation. + pub fn heal(&self) { + self.fail_get.set(false); + self.fail_put.set(false); + } } impl<'db> StorageContext<'db> for MemStorageContext { @@ -29,6 +55,12 @@ impl<'db> StorageContext<'db> for MemStorageContext { type RawIterator = MemRawIterator; fn get>(&self, key: K) -> CostResult>, grovedb_storage::Error> { + if self.fail_get.get() { + return Err(grovedb_storage::Error::StorageError( + "injected read failure".to_string(), + )) + .wrap_with_cost(OperationCost::default()); + } Ok(self.data.borrow().get(key.as_ref()).cloned()).wrap_with_cost(OperationCost::default()) } @@ -39,6 +71,12 @@ impl<'db> StorageContext<'db> for MemStorageContext { _children_sizes: ChildrenSizesWithIsSumTree, _cost_info: Option, ) -> CostResult<(), grovedb_storage::Error> { + if self.fail_put.get() { + return Err(grovedb_storage::Error::StorageError( + "injected write failure".to_string(), + )) + .wrap_with_cost(OperationCost::default()); + } self.data .borrow_mut() .insert(key.as_ref().to_vec(), value.to_vec()); @@ -141,7 +179,7 @@ impl<'db> StorageContext<'db> for MemStorageContext { // ── Batch and RawIterator stubs ─────────────────────────────────────── /// No-op batch (never used — MemStorageContext does immediate writes). -pub(crate) struct MemBatch; +pub struct MemBatch; impl Batch for MemBatch { fn put>( @@ -186,7 +224,7 @@ impl Batch for MemBatch { } /// Stub iterator (never used by the bulk append tree). -pub(crate) struct MemRawIterator; +pub struct MemRawIterator; impl RawIterator for MemRawIterator { fn seek_to_first(&mut self) -> CostContext<()> { diff --git a/grovedb-bulk-append-tree/src/tree/append.rs b/grovedb-bulk-append-tree/src/tree/append.rs index a3ca53b5c..89819e2a1 100644 --- a/grovedb-bulk-append-tree/src/tree/append.rs +++ b/grovedb-bulk-append-tree/src/tree/append.rs @@ -1,15 +1,15 @@ //! Append and compaction logic for BulkAppendTree. -use grovedb_merkle_mountain_range::{ - hash_count_for_push, mmr_size_to_leaf_count, MmrKeySize, MmrNode, MmrStore, MMR, -}; +use grovedb_costs::{CostResult, CostsExt, OperationCost}; +use grovedb_merkle_mountain_range::{mmr_size_to_leaf_count, MmrKeySize, MmrNode, MmrStore, MMR}; use grovedb_storage::StorageContext; +use grovedb_version::version::GroveVersion; use super::{ capacity_for_height, hash::compute_state_root, AppendNoStateRootResult, AppendResult, BulkAppendTree, }; -use crate::{chunk::serialize_chunk_blob, BulkAppendError}; +use crate::{chunk::serialize_chunk_blob, cost::compaction_hash_count, BulkAppendError}; impl<'db, S: StorageContext<'db>> BulkAppendTree { /// Create a new empty tree. @@ -61,8 +61,12 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { /// state root computation. For batched inserts prefer /// [`append_many`](Self::append_many) or [`append_no_state_root`](Self::append_no_state_root) /// — they skip the per-leaf state-root blake3 call. - pub fn append(&mut self, value: &[u8]) -> Result { - let r = self.append_no_state_root(value)?; + pub fn append( + &mut self, + value: &[u8], + grove_version: &GroveVersion, + ) -> Result { + let r = self.append_no_state_root(value, grove_version)?; let state_root = self.compute_current_state_root()?; Ok(AppendResult { state_root, @@ -83,9 +87,14 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { /// Storage mutation is identical to [`append`](Self::append). /// /// [`CommitmentTree::append_many_raw`]: ../../grovedb_commitment_tree/struct.CommitmentTree.html#method.append_many_raw + /// + /// Stored bytes, chunks and roots are identical under every grove + /// version; only the reported `hash_count` differs, and only for an append + /// that compacts. pub fn append_no_state_root( &mut self, value: &[u8], + grove_version: &GroveVersion, ) -> Result { let mut hash_count: u32 = 0; let global_position = self.total_count; @@ -111,7 +120,7 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { // Dense tree is full — compact existing entries + new value. // Must run before incrementing total_count so that // self.mmr_size() reflects the pre-compaction state. - let (compact_hashes, mmr_root) = self.compact_with_value(value)?; + let (compact_hashes, mmr_root) = self.compact_with_value(value, grove_version)?; hash_count += compact_hashes; // MMR mutated by the compaction — refresh the cached root. self.last_mmr_root = Some(mmr_root); @@ -128,6 +137,91 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { }) } + /// Append a value deferring **both** the dense-tree root and the + /// state root. + /// + /// Storage effect is identical to [`append`](Self::append), but the + /// per-insert `compute_root_hash` walk over the dense buffer is skipped. + /// [`append_no_state_root`](Self::append_no_state_root) still pays that + /// walk on every call (via `try_insert`), which makes a run of N appends + /// O(N^2) in hash calls — 65,535 entries at `height = 16` costs ~4.3 + /// billion. This variant is O(N) plus one final root computation. + /// + /// The caller MUST recover the state root once at the end via + /// [`compute_current_state_root`](Self::compute_current_state_root); + /// until then the dense root is stale in-memory only (it is always + /// recomputed from stored values, never cached). + /// + /// Compaction still happens inline when the buffer fills, because the + /// chunk blob is built from the stored values, not from the root. + pub fn append_deferred_roots( + &mut self, + value: &[u8], + grove_version: &GroveVersion, + ) -> CostResult { + let mut cost = OperationCost::default(); + let global_position = self.total_count; + + let try_result = match self + .dense_tree + .try_insert_no_root(value) + .unwrap_add_cost(&mut cost) + { + Ok(r) => r, + Err(e) => { + return Err(BulkAppendError::StorageError(format!( + "dense tree insert failed: {}", + e + ))) + .wrap_with_cost(cost); + } + }; + + let compacted = match try_result { + // Inserted into the buffer; no root walk, so no hashes yet. + Some(_position) => false, + None => { + // Buffer full — compact existing entries plus this value. + // Must run before incrementing total_count so self.mmr_size() + // reflects the pre-compaction state. + // The model counter this returns is deliberately unused — see + // the `hash_count` derivation below. + let (_model_hash_count, mmr_root) = match self + .compact_with_value_with_cost(value, grove_version) + .unwrap_add_cost(&mut cost) + { + Ok(r) => r, + Err(e) => return Err(e).wrap_with_cost(cost), + }; + self.last_mmr_root = Some(mmr_root); + true + } + }; + + self.total_count += 1; + + // Derive the reported counter from what was actually billed rather + // than from `hash_count_for_push`. That helper covers the eager leaf + // hash and the merges `push` performs, but NOT the peak-bagging + // merges `get_root` performs during a compaction, so the model + // counter falls below the true figure as soon as the MMR has more + // than one peak. Everything accumulated in `cost` here is this + // append's own hashing, so the two cannot disagree. + // + // Scoped to this deferred path on purpose: `compact_with_value` and + // `append_no_state_root` keep returning the model counter, because + // the live CommitmentTree adds that value straight into its own + // `hash_node_calls` and changing it would move a released cost. + let hash_count = cost.hash_node_calls; + + Ok(AppendNoStateRootResult { + global_position, + hash_count, + compacted, + }) + .wrap_with_cost(cost) + } + /// Compute the current state root without modifying the tree. /// /// Uses the cached MMR root when available, so this is O(1) on the @@ -145,29 +239,105 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { Ok(compute_state_root(&mmr_root, &dense_root)) } + /// Cost-propagating variant of + /// [`compute_current_state_root`](Self::compute_current_state_root). + /// + /// Identical result; the difference is that the dense-tree root walk's + /// storage reads and hash calls reach the caller instead of being + /// discarded, and the final state-root blake3 is charged on top of them. + /// Callers that bill work — anything returning a `CostResult` — should + /// prefer this. + pub fn compute_current_state_root_with_cost( + &self, + grove_version: &GroveVersion, + ) -> CostResult<[u8; 32], BulkAppendError> { + let mut cost = OperationCost::default(); + let mmr_root = match self.last_mmr_root { + Some(r) => r, + // Lazy path: a reopened tree has no cached root, so this read is + // real I/O and must be billed. + None => match self + .get_mmr_root_with_cost(grove_version) + .unwrap_add_cost(&mut cost) + { + Ok(r) => r, + Err(e) => return Err(e).wrap_with_cost(cost), + }, + }; + let dense_root = match self.dense_tree.root_hash().unwrap_add_cost(&mut cost) { + Ok(r) => r, + Err(e) => { + return Err(BulkAppendError::StorageError(format!( + "dense tree root_hash failed: {}", + e + ))) + .wrap_with_cost(cost); + } + }; + // `root_hash` already charged the walk itself: `hash_node` bills a + // value hash and a node hash for every filled position it visits, and + // those reached us through `unwrap_add_cost` above. Only the final + // blake3 combining the MMR and dense roots is still unbilled. + cost.hash_node_calls = cost.hash_node_calls.saturating_add(1); + Ok(compute_state_root(&mmr_root, &dense_root)).wrap_with_cost(cost) + } + /// Compact all dense tree entries plus a new value into a chunk blob /// and append to the chunk MMR. Resets the dense tree. /// Returns `(hash_count, mmr_root)`. - fn compact_with_value(&mut self, new_value: &[u8]) -> Result<(u32, [u8; 32]), BulkAppendError> { + /// + /// Cost-discarding wrapper over + /// [`compact_with_value_with_cost`](Self::compact_with_value_with_cost). + /// Kept so the released `append_no_state_root` path bills exactly what it + /// always has — its costs are dropped here, not at the call site. + fn compact_with_value( + &mut self, + new_value: &[u8], + grove_version: &GroveVersion, + ) -> Result<(u32, [u8; 32]), BulkAppendError> { + // The accumulated `OperationCost` is discarded here — this path + // reports its work through the returned `hash_count` instead, which + // IS version-dependent and which CommitmentTree bills. + self.compact_with_value_with_cost(new_value, grove_version) + .unwrap() + } + + /// Compact the buffer plus `new_value` into a chunk, propagating cost. + /// + /// Compaction is the expensive branch of an append: it reads every + /// buffered entry back out of storage, hashes the serialized blob, and + /// pushes it through the MMR. All of that was previously discarded, so a + /// compacting append looked no more expensive than a buffered one. + fn compact_with_value_with_cost( + &mut self, + new_value: &[u8], + grove_version: &GroveVersion, + ) -> CostResult<(u32, [u8; 32]), BulkAppendError> { + let mut cost = OperationCost::default(); let mut hash_count: u32 = 0; let count = self.dense_tree.count(); // Read all existing entries from dense tree let mut entries: Vec> = Vec::with_capacity(count as usize + 1); for i in 0..count { - let value = self - .dense_tree - .get(i) - .unwrap() - .map_err(|e| { - BulkAppendError::StorageError(format!("dense tree get at {} failed: {}", i, e)) - })? - .ok_or_else(|| { - BulkAppendError::CorruptedData(format!( + let read = self.dense_tree.get(i).unwrap_add_cost(&mut cost); + let value = match read { + Ok(Some(v)) => v, + Ok(None) => { + return Err(BulkAppendError::CorruptedData(format!( "dense tree missing value at position {} (count={})", i, count - )) - })?; + ))) + .wrap_with_cost(cost); + } + Err(e) => { + return Err(BulkAppendError::StorageError(format!( + "dense tree get at {} failed: {}", + i, e + ))) + .wrap_with_cost(cost); + } + }; entries.push(value); } @@ -175,13 +345,20 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { entries.push(new_value.to_vec()); // Serialize chunk blob as a standard MMR leaf — hash = blake3(0x00 || blob) - let blob = serialize_chunk_blob(&entries)?; + let blob = match serialize_chunk_blob(&entries) { + Ok(b) => b, + Err(e) => return Err(e).wrap_with_cost(cost), + }; + // `MmrNode::leaf` hashes the blob eagerly. + cost.hash_node_calls = cost.hash_node_calls.saturating_add(1); let leaf = MmrNode::leaf(blob); // Append chunk root to MMR let mmr_size = self.mmr_size(); let leaf_count = mmr_size_to_leaf_count(mmr_size); - hash_count += hash_count_for_push(leaf_count); + // Assigned once inside the MMR block below, after the push, so the + // bagging term is computed from the shape `get_root` actually folded. + let mmr_size_after_push; // Create MmrStore on the fly from the dense tree's storage. // Use the overlay from previous compactions so cross-compaction @@ -192,14 +369,15 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { let mut mmr = MMR::new_with_overlay(mmr_size, &mmr_store, std::mem::take(&mut self.mmr_overlay)); - let push_result = mmr.push(leaf).unwrap(); + let push_result = mmr.push(leaf, grove_version).unwrap_add_cost(&mut cost); if let Err(e) = push_result { // Restore overlay before returning error self.mmr_overlay = mmr.batch.take_overlay(); - return Err(BulkAppendError::MmrError(format!("MMR push failed: {}", e))); + return Err(BulkAppendError::MmrError(format!("MMR push failed: {}", e))) + .wrap_with_cost(cost); } - let root_result = mmr.get_root().unwrap(); + let root_result = mmr.get_root(grove_version).unwrap_add_cost(&mut cost); let root = match root_result { Ok(node) => node.hash(), Err(e) => { @@ -207,11 +385,13 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { return Err(BulkAppendError::MmrError(format!( "MMR get_root failed: {}", e - ))); + ))) + .wrap_with_cost(cost); } }; // Take overlay back instead of committing + mmr_size_after_push = mmr.mmr_size; self.mmr_overlay = mmr.batch.take_overlay(); root @@ -220,22 +400,50 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { // Reset dense tree (old values stay in store, overwritten on next cycle) self.dense_tree.reset(); - Ok((hash_count, mmr_root)) + // The reported count is version-gated: v0 is the shipped figure (leaf + // hash + push collapses), v1 adds the peak bagging the `get_root` + // above performed. `mmr.mmr_size` is read after the push, which is + // the shape that root had to fold. + hash_count = match compaction_hash_count(leaf_count, mmr_size_after_push, grove_version) { + Ok(h) => hash_count.saturating_add(h), + Err(e) => return Err(e).wrap_with_cost(cost), + }; + + Ok((hash_count, mmr_root)).wrap_with_cost(cost) } /// Get the MMR root hash, or `[0; 32]` if no chunks exist. pub(crate) fn get_mmr_root(&self) -> Result<[u8; 32], BulkAppendError> { + // Cost discarded here, so the version is unobservable; pinned to the + // shipped accounting to match the released callers. + self.get_mmr_root_with_cost(GroveVersion::first()).unwrap() + } + + /// Cost-propagating variant of [`get_mmr_root`](Self::get_mmr_root). + /// + /// Matters on the lazy path: `from_state` leaves `last_mmr_root` as + /// `None`, so a REOPENED non-empty tree resolves its root through here — + /// exactly the case proof binding and the integrity walk hit. Discarding + /// the read cost there undercharges their storage I/O. + pub(crate) fn get_mmr_root_with_cost( + &self, + grove_version: &GroveVersion, + ) -> CostResult<[u8; 32], BulkAppendError> { + let mut cost = OperationCost::default(); let mmr_size = self.mmr_size(); if mmr_size == 0 { - return Ok([0u8; 32]); + return Ok([0u8; 32]).wrap_with_cost(cost); } let mmr_store = MmrStore::with_key_size(&self.dense_tree.storage, MmrKeySize::U32); let mmr = MMR::new_with_overlay(mmr_size, &mmr_store, self.mmr_overlay.clone()); - let root_node = mmr - .get_root() - .unwrap() - .map_err(|e| BulkAppendError::MmrError(format!("MMR get_root failed: {}", e)))?; - Ok(root_node.hash()) + match mmr.get_root(grove_version).unwrap_add_cost(&mut cost) { + Ok(root_node) => Ok(root_node.hash()).wrap_with_cost(cost), + Err(e) => Err(BulkAppendError::MmrError(format!( + "MMR get_root failed: {}", + e + ))) + .wrap_with_cost(cost), + } } /// Flush the MMR overlay to storage. @@ -269,3 +477,98 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { Ok(()) } } + +#[cfg(test)] +mod compaction_hash_count_gate_tests { + use grovedb_version::version::{v1::GROVE_V1, v3::GROVE_V3, v4::GROVE_V4}; + + use super::*; + use crate::test_utils::MemStorageContext; + + /// The reported hash count for a compacting append is version-gated: v0 + /// (V1..V3) omits the peak bagging the compaction's own `get_root` + /// performs, v1 (V4) includes it. Everything else about the append — + /// stored bytes, chunk contents, roots — must be identical. + #[test] + fn compaction_hash_count_gains_the_bagging_term_at_v4() { + // chunk_power 2 -> epoch 4. Enough appends to compact repeatedly so + // the MMR passes through single- and multi-peak shapes. + let build = |version: &_| { + let mut t = BulkAppendTree::new(2, MemStorageContext::new()).expect("new"); + let mut counts = Vec::new(); + let mut roots = Vec::new(); + for i in 0..20u8 { + let r = t.append_no_state_root(&[i; 8], version).expect("append"); + if r.compacted { + counts.push(r.hash_count); + } + } + roots.push(t.compute_current_state_root().expect("root")); + (counts, roots) + }; + + let (v0_counts, v0_roots) = build(&GROVE_V3); + let (v1_counts, v1_roots) = build(&GROVE_V4); + + assert_eq!( + v0_roots, v1_roots, + "the tree itself must not depend on the cost version" + ); + assert_eq!( + v0_counts.len(), + v1_counts.len(), + "same number of compactions" + ); + + // v1 is never cheaper, and is strictly dearer on at least one + // compaction — the ones that landed on a multi-peak MMR. + let mut saw_increase = false; + for (i, (a, b)) in v0_counts.iter().zip(v1_counts.iter()).enumerate() { + assert!( + b >= a, + "v1 must never report fewer hashes (compaction {}): v0={} v1={}", + i, + a, + b + ); + if b > a { + saw_increase = true; + } + } + assert!( + saw_increase, + "expected at least one multi-peak compaction to gain the bagging \ + term: v0={:?} v1={:?}", + v0_counts, v1_counts + ); + + // V1 and V3 are both v0, so they must agree exactly. + let (v1_ver_counts, _) = build(&GROVE_V1); + assert_eq!( + v0_counts, v1_ver_counts, + "GROVE_V1 and GROVE_V3 both select the shipped figure" + ); + } + + /// An unknown charge version must be rejected, not silently treated as one + /// of the implemented ones. + #[test] + fn compaction_hash_count_rejects_unknown_version() { + let mut bad = GROVE_V4.clone(); + bad.bulk_append_tree_versions.cost.compaction_hash_count = 99; + + let mut t = BulkAppendTree::new(2, MemStorageContext::new()).expect("new"); + // Fill the buffer so the next append compacts and reaches the gate. + for i in 0..3u8 { + t.append_no_state_root(&[i; 8], &bad) + .expect("buffered appends do not reach the gate"); + } + assert!( + matches!( + t.append_no_state_root(&[9u8; 8], &bad), + Err(BulkAppendError::VersionError(_)) + ), + "a compacting append must reject an unknown charge version" + ); + } +} diff --git a/grovedb-bulk-append-tree/src/tree/fetch.rs b/grovedb-bulk-append-tree/src/tree/fetch.rs index 3ad8bed63..7a3baaea5 100644 --- a/grovedb-bulk-append-tree/src/tree/fetch.rs +++ b/grovedb-bulk-append-tree/src/tree/fetch.rs @@ -1,5 +1,6 @@ //! Read operations for BulkAppendTree. +use grovedb_costs::{CostResult, CostsExt, OperationCost}; use grovedb_dense_fixed_sized_merkle_tree::DenseTreeProof; use grovedb_merkle_mountain_range::{leaf_to_pos, MmrKeySize, MmrStore, MMR}; use grovedb_query::Query; @@ -7,6 +8,7 @@ use grovedb_storage::StorageContext; use super::BulkAppendTree; use crate::{chunk::deserialize_chunk_blob, BulkAppendError}; +use grovedb_version::version::GroveVersion; /// Result of querying the dense tree buffer. #[derive(Debug, Clone)] @@ -38,12 +40,34 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { /// from completed chunks. The position is relative to the current buffer /// cycle (0-based). pub fn get_buffer_value(&self, position: u16) -> Result>, BulkAppendError> { + self.get_buffer_value_with_cost(position).unwrap() + } + + /// Cost-propagating variant of + /// [`get_buffer_value`](Self::get_buffer_value). + /// + /// Identical behavior; the difference is that the dense-tree read's + /// `OperationCost` (seek count and loaded bytes) reaches the caller + /// instead of being discarded. Callers that bill reads — anything + /// returning a `CostResult` to GroveDB — must use this variant, or the + /// read is served for free. + pub fn get_buffer_value_with_cost( + &self, + position: u16, + ) -> CostResult>, BulkAppendError> { + let mut cost = OperationCost::default(); if position >= self.buffer_count() { - return Ok(None); + return Ok(None).wrap_with_cost(cost); } - self.dense_tree.get(position).unwrap().map_err(|e| { - BulkAppendError::StorageError(format!("dense tree get at {} failed: {}", position, e)) - }) + let result = self.dense_tree.get(position).unwrap_add_cost(&mut cost); + result + .map_err(|e| { + BulkAppendError::StorageError(format!( + "dense tree get at {} failed: {}", + position, e + )) + }) + .wrap_with_cost(cost) } /// Query the buffer using a dense tree query. @@ -73,28 +97,45 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { /// Uses the MMR overlay to find nodes that were pushed during this session /// but not yet committed to storage. pub fn get_chunk_value(&self, chunk_index: u64) -> Result>, BulkAppendError> { + self.get_chunk_value_with_cost(chunk_index).unwrap() + } + + /// Cost-propagating variant of + /// [`get_chunk_value`](Self::get_chunk_value). See + /// [`get_buffer_value_with_cost`](Self::get_buffer_value_with_cost) for + /// why the cost-bearing form exists. + pub fn get_chunk_value_with_cost( + &self, + chunk_index: u64, + ) -> CostResult>, BulkAppendError> { + let mut cost = OperationCost::default(); if chunk_index >= self.chunk_count() { - return Ok(None); + return Ok(None).wrap_with_cost(cost); } let mmr_pos = leaf_to_pos(chunk_index); let mmr_store = MmrStore::with_key_size(&self.dense_tree.storage, MmrKeySize::U32); let mmr = MMR::new_with_overlay(self.mmr_size(), &mmr_store, self.mmr_overlay.clone()); - let node = mmr + let node = match mmr .batch .element_at_position(mmr_pos) - .unwrap() - .map_err(|e| { - BulkAppendError::MmrError(format!( + .unwrap_add_cost(&mut cost) + { + Ok(n) => n, + Err(e) => { + return Err(BulkAppendError::MmrError(format!( "failed to read MMR node for chunk {}: {}", chunk_index, e - )) - })?; + ))) + .wrap_with_cost(cost); + } + }; match node { - Some(n) => Ok(n.into_value()), + Some(n) => Ok(n.into_value()).wrap_with_cost(cost), None => Err(BulkAppendError::CorruptedData(format!( "missing MMR leaf for chunk {}", chunk_index - ))), + ))) + .wrap_with_cost(cost), } } @@ -138,14 +179,17 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { let mmr = MMR::new_with_overlay(mmr_size, &mmr_store, self.mmr_overlay.clone()); let positions: Vec = chunk_indices.iter().map(|&idx| leaf_to_pos(idx)).collect(); - let proof = mmr.gen_proof(positions).unwrap().map_err(|e| { - BulkAppendError::MmrError(format!("chunk MMR gen_proof failed: {}", e)) - })?; + let proof = mmr + .gen_proof(positions, GroveVersion::latest()) + .unwrap() + .map_err(|e| { + BulkAppendError::MmrError(format!("chunk MMR gen_proof failed: {}", e)) + })?; let proof_items: Vec<[u8; 32]> = proof.proof_items().iter().map(|node| node.hash()).collect(); - let root = mmr.get_root().unwrap().map_err(|e| { + let root = mmr.get_root(GroveVersion::latest()).unwrap().map_err(|e| { BulkAppendError::MmrError(format!("chunk MMR get_root failed: {}", e)) })?; diff --git a/grovedb-bulk-append-tree/src/tree/tests.rs b/grovedb-bulk-append-tree/src/tree/tests.rs index 3c3f2e490..f2bb87c72 100644 --- a/grovedb-bulk-append-tree/src/tree/tests.rs +++ b/grovedb-bulk-append-tree/src/tree/tests.rs @@ -2,6 +2,7 @@ use super::BulkAppendTree; use crate::{chunk::deserialize_chunk_blob, test_utils::MemStorageContext}; +use grovedb_version::version::GroveVersion; #[test] fn new_tree() { @@ -49,7 +50,7 @@ fn cached_mmr_root_matches_recomputation_across_compactions() { // height=2 → epoch_size=4, so 20 appends span 5 compaction cycles. let mut tree = BulkAppendTree::new(2u8, MemStorageContext::new()).expect("create tree"); for i in 0..20u8 { - tree.append(&[i]).expect("append"); + tree.append(&[i], GroveVersion::latest()).expect("append"); let fresh = tree.get_mmr_root().expect("recompute mmr root"); assert_eq!( tree.last_mmr_root, @@ -64,7 +65,9 @@ fn cached_mmr_root_matches_recomputation_across_compactions() { fn single_append() { let mut tree = BulkAppendTree::new(2u8, MemStorageContext::new()).expect("create tree"); - let result = tree.append(b"hello").expect("append hello"); + let result = tree + .append(b"hello", GroveVersion::latest()) + .expect("append hello"); assert_eq!(result.global_position, 0); assert!(!result.compacted); assert_eq!(tree.total_count, 1); @@ -82,7 +85,9 @@ fn multiple_appends_no_compaction() { // Height=2, capacity=3. Append 2 values (no compaction). for i in 0..2 { - let result = tree.append(&[i]).expect("append entry"); + let result = tree + .append(&[i], GroveVersion::latest()) + .expect("append entry"); assert_eq!(result.global_position, i as u64); assert!(!result.compacted); } @@ -98,10 +103,14 @@ fn compaction_trigger() { // Height=2, capacity=3, epoch_size=4. First 3 appends fill the buffer, // 4th triggers compaction (try_insert returns None when buffer is full). for i in 0..3u8 { - let r = tree.append(&[i]).expect("append pre-compaction entry"); + let r = tree + .append(&[i], GroveVersion::latest()) + .expect("append pre-compaction entry"); assert!(!r.compacted); } - let result = tree.append(&[3]).expect("append compacting entry"); + let result = tree + .append(&[3], GroveVersion::latest()) + .expect("append compacting entry"); assert!(result.compacted); assert_eq!(result.global_position, 3); assert_eq!(tree.total_count, 4); @@ -119,7 +128,8 @@ fn multi_chunk() { // append 2 → buffer (count=1), append 3 → compaction (chunk has [2,3]) // 4 appends = 2 chunks + 0 buffer for i in 0..4u8 { - tree.append(&[i]).expect("append entry"); + tree.append(&[i], GroveVersion::latest()) + .expect("append entry"); } assert_eq!(tree.total_count, 4); assert_eq!(tree.chunk_count(), 2); @@ -135,10 +145,10 @@ fn get_chunk_value_from_mmr() { // append b → try_insert fails (full), compact [a, b] → chunk 0 // append c → buffer (count=1) // append d → try_insert fails, compact [c, d] → chunk 1 - tree.append(b"a").expect("append a"); - tree.append(b"b").expect("append b"); - tree.append(b"c").expect("append c"); - tree.append(b"d").expect("append d"); + tree.append(b"a", GroveVersion::latest()).expect("append a"); + tree.append(b"b", GroveVersion::latest()).expect("append b"); + tree.append(b"c", GroveVersion::latest()).expect("append c"); + tree.append(b"d", GroveVersion::latest()).expect("append d"); assert_eq!(tree.chunk_count(), 2); assert_eq!(tree.buffer_count(), 0); @@ -167,8 +177,8 @@ fn get_buffer_value_from_dense_tree() { // capacity=3 let mut tree = BulkAppendTree::new(2u8, MemStorageContext::new()).expect("create tree"); - tree.append(b"a").expect("append a"); - tree.append(b"b").expect("append b"); + tree.append(b"a", GroveVersion::latest()).expect("append a"); + tree.append(b"b", GroveVersion::latest()).expect("append b"); // Both from the buffer (dense tree) assert_eq!( @@ -189,8 +199,8 @@ fn get_chunk_blob() { let mut tree = BulkAppendTree::new(1u8, MemStorageContext::new()).expect("create tree"); // Need 2 appends to trigger compaction (epoch_size=2) - tree.append(b"x").expect("append x"); - tree.append(b"y").expect("append y"); // compacts [x, y] + tree.append(b"x", GroveVersion::latest()).expect("append x"); + tree.append(b"y", GroveVersion::latest()).expect("append y"); // compacts [x, y] let blob = tree.get_chunk_value(0).expect("get chunk 0"); assert!(blob.is_some()); @@ -207,8 +217,8 @@ fn query_buffer_entries() { // capacity=3 let mut tree = BulkAppendTree::new(2u8, MemStorageContext::new()).expect("create tree"); - tree.append(b"a").expect("append a"); - tree.append(b"b").expect("append b"); + tree.append(b"a", GroveVersion::latest()).expect("append a"); + tree.append(b"b", GroveVersion::latest()).expect("append b"); // Query all buffer entries with RangeFull let query = grovedb_query::Query::new_range_full(); @@ -225,10 +235,10 @@ fn query_chunks_from_mmr() { let mut tree = BulkAppendTree::new(1u8, MemStorageContext::new()).expect("create tree"); // 4 appends → 2 chunks: chunk 0 = [a,b], chunk 1 = [c,d] - tree.append(b"a").expect("append a"); - tree.append(b"b").expect("append b"); - tree.append(b"c").expect("append c"); - tree.append(b"d").expect("append d"); + tree.append(b"a", GroveVersion::latest()).expect("append a"); + tree.append(b"b", GroveVersion::latest()).expect("append b"); + tree.append(b"c", GroveVersion::latest()).expect("append c"); + tree.append(b"d", GroveVersion::latest()).expect("append d"); // Query both chunks let result = tree.query_chunks(&[0, 1]).expect("query chunks"); @@ -268,8 +278,12 @@ fn state_root_determinism() { let mut tree2 = BulkAppendTree::new(2u8, MemStorageContext::new()).expect("create tree2"); for i in 0..5u8 { - tree1.append(&[i]).expect("append to tree1"); - tree2.append(&[i]).expect("append to tree2"); + tree1 + .append(&[i], GroveVersion::latest()) + .expect("append to tree1"); + tree2 + .append(&[i], GroveVersion::latest()) + .expect("append to tree2"); } let root1 = tree1.compute_current_state_root().expect("state root 1"); @@ -292,15 +306,17 @@ fn hash_count_accuracy() { let mut tree = BulkAppendTree::new(2u8, MemStorageContext::new()).expect("create tree"); // Non-compacting append includes dense tree hashing + state root - let r = tree.append(b"a").expect("append a"); + let r = tree.append(b"a", GroveVersion::latest()).expect("append a"); assert!(r.hash_count > 0); - tree.append(b"b").expect("append b"); - tree.append(b"c").expect("append c"); + tree.append(b"b", GroveVersion::latest()).expect("append b"); + tree.append(b"c", GroveVersion::latest()).expect("append c"); // 4th append triggers compaction: should have more hash calls (dense + mmr + // state root) - let r = tree.append(b"d").expect("append d (compaction)"); + let r = tree + .append(b"d", GroveVersion::latest()) + .expect("append d (compaction)"); assert!(r.compacted); assert!(r.hash_count > 1); } @@ -309,8 +325,10 @@ fn hash_count_accuracy() { fn from_state_roundtrip() { let mut tree = BulkAppendTree::new(2u8, MemStorageContext::new()).expect("create tree"); - tree.append(b"hello").expect("append hello"); - tree.append(b"world").expect("append world"); + tree.append(b"hello", GroveVersion::latest()) + .expect("append hello"); + tree.append(b"world", GroveVersion::latest()) + .expect("append world"); let total_count = tree.total_count; let mmr_size = tree.mmr_size(); @@ -334,7 +352,7 @@ fn compaction_and_continue() { // Fill one epoch and continue for i in 0..5u8 { - tree.append(&[i]).expect("append"); + tree.append(&[i], GroveVersion::latest()).expect("append"); } assert_eq!(tree.total_count, 5); assert_eq!(tree.chunk_count(), 1); // 5/4 = 1 full chunk @@ -362,7 +380,7 @@ fn multiple_compaction_cycles() { // 8 values = 2 full chunks (8/4 = 2) for i in 0..8u8 { - tree.append(&[i]).expect("append"); + tree.append(&[i], GroveVersion::latest()).expect("append"); } assert_eq!(tree.total_count, 8); assert_eq!(tree.chunk_count(), 2); @@ -382,8 +400,8 @@ fn multiple_compaction_cycles() { #[test] fn query_chunks_empty_indices_returns_empty_proof() { let mut tree = BulkAppendTree::new(1u8, MemStorageContext::new()).expect("create tree"); - tree.append(b"a").expect("append a"); - tree.append(b"b").expect("append b"); // one completed chunk exists + tree.append(b"a", GroveVersion::latest()).expect("append a"); + tree.append(b"b", GroveVersion::latest()).expect("append b"); // one completed chunk exists let result = tree.query_chunks(&[]).expect("query with empty indices"); assert!(result.chunks.is_empty()); diff --git a/grovedb-commitment-tree/Cargo.toml b/grovedb-commitment-tree/Cargo.toml index 84689b7c0..caa1e7ed8 100644 --- a/grovedb-commitment-tree/Cargo.toml +++ b/grovedb-commitment-tree/Cargo.toml @@ -22,6 +22,7 @@ incrementalmerkletree = "0.8" shardtree = { version = "0.6", optional = true } rusqlite = { version = "0.38", features = ["bundled"], optional = true } grovedb-costs = { version = "5.0.1", path = "../costs" } +grovedb-version = { version = "5.0.1", path = "../grovedb-version" } grovedb-storage = { version = "5.0.1", path = "../storage", optional = true } grovedb-bulk-append-tree = { version = "5.0.1", path = "../grovedb-bulk-append-tree", default-features = false } blake3 = { workspace = true } diff --git a/grovedb-commitment-tree/benches/seeding.rs b/grovedb-commitment-tree/benches/seeding.rs index 7e22a42b0..3e6356961 100644 --- a/grovedb-commitment-tree/benches/seeding.rs +++ b/grovedb-commitment-tree/benches/seeding.rs @@ -26,6 +26,7 @@ fn main() { }; use grovedb_path::SubtreePath; use grovedb_storage::{rocksdb_storage::RocksDbStorage, Storage, StorageBatch}; + use grovedb_version::version::GroveVersion; use rand::{rngs::StdRng, Rng, SeedableRng}; let n: u64 = std::env::var("SEED_N") @@ -87,7 +88,7 @@ fn main() { // at the end, not per leaf. let t_seed = Instant::now(); let result = ct - .append_many_raw(notes) + .append_many_raw(notes, GroveVersion::latest()) .value .expect("batched commitment-tree seeding"); // Flush the MMR overlay into the storage batch now that we're done diff --git a/grovedb-commitment-tree/src/commitment_tree/mod.rs b/grovedb-commitment-tree/src/commitment_tree/mod.rs index 8649211d0..9f674e583 100644 --- a/grovedb-commitment-tree/src/commitment_tree/mod.rs +++ b/grovedb-commitment-tree/src/commitment_tree/mod.rs @@ -14,6 +14,7 @@ use std::marker::PhantomData; use grovedb_bulk_append_tree::BulkAppendTree; use grovedb_costs::{CostResult, CostsExt, OperationCost}; use grovedb_storage::StorageContext; +use grovedb_version::version::GroveVersion; use orchard::{ memo::{DashMemo, MemoSize}, note::TransmittedNoteCiphertext, @@ -250,15 +251,19 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { /// decrypts `out_ciphertext`, and it cannot be recomputed from the note. /// /// Call [`save`](Self::save) afterwards to persist the updated frontier. + /// The note, chunk and roots are identical under every grove version; + /// what the version selects is the hash count a compacting append + /// reports, which this method bills. pub fn append( &mut self, cmx: [u8; 32], rho: [u8; 32], cv_net: [u8; 32], ciphertext: &TransmittedNoteCiphertext, + grove_version: &GroveVersion, ) -> CostResult { let payload = serialize_ciphertext(ciphertext); - self.append_raw(cmx, rho, cv_net, &payload) + self.append_raw(cmx, rho, cv_net, &payload, grove_version) } /// Append a note commitment and raw payload bytes to the commitment tree. @@ -287,6 +292,7 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { rho: [u8; 32], cv_net: [u8; 32], payload: &[u8], + grove_version: &GroveVersion, ) -> CostResult { let mut cost = OperationCost::default(); @@ -314,7 +320,7 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { item_value.extend_from_slice(&cv_net); item_value.extend_from_slice(payload); - let bulk_result = match self.bulk_tree.append(&item_value) { + let bulk_result = match self.bulk_tree.append(&item_value, grove_version) { Ok(r) => r, // codecov:ignore — requires BulkAppendTree::append to fail, which only happens on // storage faults (put/get errors) during dense tree insert or MMR compaction; @@ -418,6 +424,7 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { pub fn append_many_raw( &mut self, entries: I, + grove_version: &GroveVersion, ) -> CostResult where I: IntoIterator, @@ -463,7 +470,10 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { item_value.extend_from_slice(&cv_net); item_value.extend_from_slice(&payload); - let r = match self.bulk_tree.append_no_state_root(&item_value) { + let r = match self + .bulk_tree + .append_no_state_root(&item_value, grove_version) + { Ok(r) => r, // codecov:ignore — only reachable on a storage fault during the // dense-tree insert or MMR compaction (see `append_raw`'s diff --git a/grovedb-commitment-tree/src/commitment_tree/tests.rs b/grovedb-commitment-tree/src/commitment_tree/tests.rs index 78b9c2489..5e8c65ebe 100644 --- a/grovedb-commitment-tree/src/commitment_tree/tests.rs +++ b/grovedb-commitment-tree/src/commitment_tree/tests.rs @@ -1,5 +1,6 @@ #[cfg(test)] mod storage_tests { + use grovedb_version::version::GroveVersion; use std::{collections::BTreeMap, marker::PhantomData}; use grovedb_bulk_append_tree::BulkAppendTree; @@ -515,6 +516,7 @@ mod storage_tests { test_rho(i as u8), test_cv_net(i as u8), &test_ciphertext(i as u8), + GroveVersion::latest(), ) .value .expect("append should succeed"); @@ -567,6 +569,7 @@ mod storage_tests { test_rho(0), test_cv_net(0), &test_ciphertext(0), + GroveVersion::latest(), ) .value .expect("append should succeed"); @@ -669,6 +672,7 @@ mod storage_tests { test_rho(i as u8), test_cv_net(i as u8), &test_ciphertext(i as u8), + GroveVersion::latest(), ) .value .expect("append should succeed"); @@ -708,6 +712,7 @@ mod storage_tests { test_rho(0), test_cv_net(0), &test_ciphertext(0), + GroveVersion::latest(), ) .value .expect("first append"); @@ -724,6 +729,7 @@ mod storage_tests { test_rho(1), test_cv_net(1), &test_ciphertext(1), + GroveVersion::latest(), ) .value .expect("second append"); @@ -753,7 +759,13 @@ mod storage_tests { .expect("open should succeed"); // Too small - let result = ct.append_raw(test_leaf(0), test_rho(0), test_cv_net(0), &[0u8; 10]); + let result = ct.append_raw( + test_leaf(0), + test_rho(0), + test_cv_net(0), + &[0u8; 10], + GroveVersion::latest(), + ); let err = result.value.expect_err("should reject wrong size"); let msg = format!("{}", err); assert!( @@ -763,7 +775,13 @@ mod storage_tests { ); // Too large - let result = ct.append_raw(test_leaf(0), test_rho(0), test_cv_net(0), &[0u8; 300]); + let result = ct.append_raw( + test_leaf(0), + test_rho(0), + test_cv_net(0), + &[0u8; 300], + GroveVersion::latest(), + ); assert!( result.value.is_err(), "should reject payload that is too large" @@ -776,6 +794,7 @@ mod storage_tests { test_rho(0), test_cv_net(0), &vec![0u8; expected_size], + GroveVersion::latest(), ); assert!(result.value.is_ok(), "correct size should succeed"); } @@ -854,6 +873,7 @@ mod storage_tests { test_rho(0), test_cv_net(0), &test_ciphertext(0), + GroveVersion::latest(), ) .value .expect("append should succeed"); @@ -896,6 +916,7 @@ mod storage_tests { test_rho(0), test_cv_net(0), &test_ciphertext(0), + GroveVersion::latest(), ) .value .expect("append 0"); @@ -905,6 +926,7 @@ mod storage_tests { test_rho(1), test_cv_net(1), &test_ciphertext(1), + GroveVersion::latest(), ) .value .expect("append 1"); @@ -948,6 +970,7 @@ mod storage_tests { test_rho(0), test_cv_net(0), &test_ciphertext(0), + GroveVersion::latest(), ) .value .expect("append 0"); @@ -979,6 +1002,7 @@ mod storage_tests { test_rho(0), test_cv_net(0), &test_ciphertext(0), + GroveVersion::latest(), ) .value .expect("append 0"); @@ -987,6 +1011,7 @@ mod storage_tests { test_rho(1), test_cv_net(1), &test_ciphertext(1), + GroveVersion::latest(), ) .value .expect("append 1"); @@ -999,6 +1024,7 @@ mod storage_tests { test_rho(2), test_cv_net(2), &test_ciphertext(2), + GroveVersion::latest(), ) .value .expect("append 2"); @@ -1007,6 +1033,7 @@ mod storage_tests { test_rho(3), test_cv_net(3), &test_ciphertext(3), + GroveVersion::latest(), ) .value .expect("append 3"); @@ -1033,6 +1060,7 @@ mod storage_tests { test_rho(0), test_cv_net(0), &test_ciphertext(0), + GroveVersion::latest(), ) .value .expect("append 0"); @@ -1067,6 +1095,7 @@ mod storage_tests { test_rho(0), test_cv_net(0), &test_ciphertext(0), + GroveVersion::latest(), ) .value .expect("append should succeed"); @@ -1105,6 +1134,7 @@ mod storage_tests { test_rho(0), test_cv_net(0), &test_ciphertext(0), + GroveVersion::latest(), ) .value .expect("append should succeed"); @@ -1133,7 +1163,13 @@ mod storage_tests { // All 0xFF is not a valid Pallas field element let payload = vec![0u8; ciphertext_payload_size::()]; - let result = ct.append_raw([0xFF; 32], test_rho(0), test_cv_net(0), &payload); + let result = ct.append_raw( + [0xFF; 32], + test_rho(0), + test_cv_net(0), + &payload, + GroveVersion::latest(), + ); assert!( result.value.is_err(), "should reject invalid cmx field element" @@ -1170,9 +1206,15 @@ mod storage_tests { CommitmentTree::<_, DashMemo>::new(TEST_CHUNK_POWER, MockDataStorageContext::new()) .expect("new should succeed"); for entry in entries { - ct.append_raw(entry.cmx, entry.rho, entry.cv_net, &entry.payload) - .value - .expect("per-leaf append_raw should succeed"); + ct.append_raw( + entry.cmx, + entry.rho, + entry.cv_net, + &entry.payload, + GroveVersion::latest(), + ) + .value + .expect("per-leaf append_raw should succeed"); } ct } @@ -1184,7 +1226,7 @@ mod storage_tests { let mut ct = CommitmentTree::<_, DashMemo>::new(TEST_CHUNK_POWER, MockDataStorageContext::new()) .expect("new should succeed"); - ct.append_many_raw(entries) + ct.append_many_raw(entries, GroveVersion::latest()) .value .expect("append_many_raw should succeed"); ct @@ -1438,7 +1480,7 @@ mod storage_tests { }, ]; let err = ct - .append_many_raw(entries) + .append_many_raw(entries, GroveVersion::latest()) .value .expect_err("invalid cmx must propagate out of the batch"); assert!( @@ -1472,7 +1514,7 @@ mod storage_tests { }, ]; let err = ct - .append_many_raw(entries) + .append_many_raw(entries, GroveVersion::latest()) .value .expect_err("bad payload size must propagate out of the batch"); assert!( @@ -1502,7 +1544,7 @@ mod storage_tests { payload: seed_payload(0), }]; let err = ct - .append_many_raw(entries) + .append_many_raw(entries, GroveVersion::latest()) .value .expect_err("bulk storage failure should surface"); assert!( @@ -1547,7 +1589,9 @@ mod storage_tests { cv_net: test_cv_net(i), payload: seed_payload(i), }); - ct.append_many_raw(notes).value.expect("warmup"); + ct.append_many_raw(notes, GroveVersion::latest()) + .value + .expect("warmup"); ct.commit_mmr().expect("commit_mmr should flush cleanly"); } @@ -1571,7 +1615,7 @@ mod storage_tests { // A single append keeps the item in the dense buffer (epoch_size = 2 for // TEST_CHUNK_POWER = 1), so we can read the raw item bytes straight back. - ct.append(cmx, rho, cv_net, &ciphertext) + ct.append(cmx, rho, cv_net, &ciphertext, GroveVersion::latest()) .value .expect("append should succeed"); @@ -1643,9 +1687,15 @@ mod storage_tests { .expect("new should succeed"); for (i, cmx) in fixed_cmx.iter().enumerate() { let idx = i as u8; - ct.append_raw(*cmx, test_rho(idx), test_cv_net(idx), &seed_payload(idx)) - .value - .expect("append_raw should succeed"); + ct.append_raw( + *cmx, + test_rho(idx), + test_cv_net(idx), + &seed_payload(idx), + GroveVersion::latest(), + ) + .value + .expect("append_raw should succeed"); } // Build a pure frontier from the SAME cmx sequence (no rho/cv_net/payload). diff --git a/grovedb-dense-fixed-sized-merkle-tree/src/tree.rs b/grovedb-dense-fixed-sized-merkle-tree/src/tree.rs index 905e74088..53e8a8a30 100644 --- a/grovedb-dense-fixed-sized-merkle-tree/src/tree.rs +++ b/grovedb-dense-fixed-sized-merkle-tree/src/tree.rs @@ -187,6 +187,35 @@ impl<'db, S: StorageContext<'db>> DenseFixedSizedMerkleTree { } } + /// Insert a value **without** recomputing the root hash. + /// + /// Same storage effect as [`try_insert`](Self::try_insert) — the value is + /// written at the next free position and `count` is incremented — but the + /// O(count) `compute_root_hash` walk is skipped. Returns the position, or + /// `None` when the tree is already full. + /// + /// Use this when inserting a run of values and only the FINAL root + /// matters: calling [`try_insert`](Self::try_insert) in a loop is + /// O(n^2) in hash calls, because every insert re-walks every filled + /// position. Recover the root once at the end with + /// [`root_hash`](Self::root_hash). + pub fn try_insert_no_root( + &mut self, + value: &[u8], + ) -> CostResult, DenseMerkleError> { + let mut cost = OperationCost::default(); + + if self.count >= self.capacity() { + return Ok(None).wrap_with_cost(cost); + } + + let position = self.count; + cost_return_on_error!(cost, self.put_value(position, value)); + self.count += 1; + + Ok(Some(position)).wrap_with_cost(cost) + } + /// Get a value by position. /// /// Returns `None` if position >= count. Returns an error if position < diff --git a/grovedb-element/src/element/constructor.rs b/grovedb-element/src/element/constructor.rs index 8949ca1aa..b426397f7 100644 --- a/grovedb-element/src/element/constructor.rs +++ b/grovedb-element/src/element/constructor.rs @@ -500,6 +500,70 @@ impl Element { Element::DenseAppendOnlyFixedSizeTree(count, height, flags) } + /// Set element to an empty private document store. + /// + /// Returns `InvalidInput` unless `entry_size` is in `1..=65535` and + /// `chunk_power` is in `1..=16` (the underlying `BulkAppendTree` dense-buffer height + /// range). Unlike `empty_commitment_tree` / `empty_bulk_append_tree`, + /// the constraints are enforced eagerly here: the configuration is + /// committed into the state root, so an unusable config must never be + /// constructible. + pub fn empty_private_document_store( + entry_size: u32, + chunk_power: u8, + ) -> Result { + Self::empty_private_document_store_with_flags(entry_size, chunk_power, None) + } + + /// Set element to an empty private document store with flags. + /// + /// Same validation as [`Element::empty_private_document_store`]. + pub fn empty_private_document_store_with_flags( + entry_size: u32, + chunk_power: u8, + flags: Option, + ) -> Result { + if entry_size == 0 || entry_size > u16::MAX as u32 { + // Upper bound keeps `2^16 * entry_size` (the worst-case + // compaction blob) inside the u32 `added_bytes` field, so the + // worst-case storage estimate stays a real bound. An entry + // larger than 64 KiB is outside this type's design envelope + // anyway. + return Err(ElementError::InvalidInput( + "private document store entry_size must be in 1..=65535", + )); + } + if !(1..=16).contains(&chunk_power) { + return Err(ElementError::InvalidInput( + "private document store chunk_power must be between 1 and 16", + )); + } + Ok(Element::PrivateDocumentStore( + 0, + entry_size, + chunk_power, + flags, + )) + } + + /// Set element to a private document store with all fields. + /// + /// Restoration constructor: unchecked, mirroring `new_commitment_tree` / + /// `new_bulk_append_tree` — it rebuilds an element from already-validated + /// state (stored bytes, batch metadata). Invalid configurations are + /// rejected at every real ingress: the `empty_*` constructors, the direct + /// and batch insert paths, and both (de)serialization codecs + /// (`Element::serialize` / `Element::deserialize` / serde) via + /// [`Element::validate_private_document_store_config`]. + pub fn new_private_document_store( + total_count: u64, + entry_size: u32, + chunk_power: u8, + flags: Option, + ) -> Self { + Element::PrivateDocumentStore(total_count, entry_size, chunk_power, flags) + } + /// Set element to an empty provable sum-indexed tree without flags. pub fn empty_provable_sum_indexed_tree() -> Self { Element::ProvableSumIndexedTree(None, None, 0, None) diff --git a/grovedb-element/src/element/helpers.rs b/grovedb-element/src/element/helpers.rs index 623726c1d..a4774c7da 100644 --- a/grovedb-element/src/element/helpers.rs +++ b/grovedb-element/src/element/helpers.rs @@ -358,6 +358,7 @@ impl Element { | Element::ProvableSumIndexedTree(..) | Element::ProvableCountIndexedTree(..) | Element::ProvableCountProvableSumIndexedTree(..) + | Element::PrivateDocumentStore(..) ) } @@ -414,6 +415,12 @@ impl Element { matches!(self.underlying(), Element::DenseAppendOnlyFixedSizeTree(..)) } + /// Check if the element is a private document store. Looks through + /// `NonCounted`. + pub fn is_private_document_store(&self) -> bool { + matches!(self.underlying(), Element::PrivateDocumentStore(..)) + } + /// Check if the element is a tree type that stores data in the data /// namespace as non-Merk entries. These tree types have an always-empty /// Merk (root_key = None) and never contain child subtrees. The data @@ -430,6 +437,7 @@ impl Element { | Element::MmrTree(..) | Element::BulkAppendTree(..) | Element::DenseAppendOnlyFixedSizeTree(..) + | Element::PrivateDocumentStore(..) ) } @@ -443,6 +451,7 @@ impl Element { Element::MmrTree(mmr_size, _) => Some(*mmr_size), Element::BulkAppendTree(count, ..) => Some(*count), Element::DenseAppendOnlyFixedSizeTree(count, ..) => Some(*count as u64), + Element::PrivateDocumentStore(count, ..) => Some(*count), _ => None, } } @@ -470,6 +479,7 @@ impl Element { | Element::MmrTree(..) | Element::BulkAppendTree(..) | Element::DenseAppendOnlyFixedSizeTree(..) + | Element::PrivateDocumentStore(..) | Element::ProvableSumIndexedTree(Some(_), ..) | Element::ProvableSumIndexedTree(_, Some(_), ..) | Element::ProvableCountIndexedTree(Some(_), ..) @@ -663,6 +673,7 @@ impl Element { | Element::MmrTree(.., flags) | Element::BulkAppendTree(.., flags) | Element::DenseAppendOnlyFixedSizeTree(.., flags) + | Element::PrivateDocumentStore(.., flags) | Element::ProvableSumIndexedTree(.., flags) | Element::ProvableCountIndexedTree(.., flags) | Element::ReferenceWithSumItem(.., flags) => flags, @@ -695,6 +706,7 @@ impl Element { | Element::MmrTree(.., flags) | Element::BulkAppendTree(.., flags) | Element::DenseAppendOnlyFixedSizeTree(.., flags) + | Element::PrivateDocumentStore(.., flags) | Element::ProvableSumIndexedTree(.., flags) | Element::ProvableCountIndexedTree(.., flags) | Element::ReferenceWithSumItem(.., flags) => flags, @@ -727,6 +739,7 @@ impl Element { | Element::MmrTree(.., flags) | Element::BulkAppendTree(.., flags) | Element::DenseAppendOnlyFixedSizeTree(.., flags) + | Element::PrivateDocumentStore(.., flags) | Element::ProvableSumIndexedTree(.., flags) | Element::ProvableCountIndexedTree(.., flags) | Element::ReferenceWithSumItem(.., flags) => flags, @@ -758,6 +771,7 @@ impl Element { | Element::MmrTree(.., flags) | Element::BulkAppendTree(.., flags) | Element::DenseAppendOnlyFixedSizeTree(.., flags) + | Element::PrivateDocumentStore(.., flags) | Element::ProvableSumIndexedTree(.., flags) | Element::ProvableCountIndexedTree(.., flags) | Element::ReferenceWithSumItem(.., flags) => *flags = new_flags, @@ -1320,6 +1334,12 @@ mod flag_accessor_tests { check_accessors_round_trip(e); } + #[test] + fn private_document_store_flag_accessors() { + let e = Element::PrivateDocumentStore(0, 64, 4, flags()); + check_accessors_round_trip(e); + } + #[test] fn dense_append_only_tree_flag_accessors() { let e = Element::DenseAppendOnlyFixedSizeTree(0, 0, flags()); diff --git a/grovedb-element/src/element/mod.rs b/grovedb-element/src/element/mod.rs index 3fc5e9022..a8b9c1013 100644 --- a/grovedb-element/src/element/mod.rs +++ b/grovedb-element/src/element/mod.rs @@ -323,6 +323,31 @@ pub enum Element { Vec<(u8, Option>)>, Option, ), + /// Private document store: an append-only store of fixed-size opaque + /// entries, a thin wrapper over a `BulkAppendTree` (the same + /// relationship `CommitmentTree` has to it, minus the Sinsemilla + /// frontier). Entries are write-once — there is no per-entry delete or + /// update; immutability is enforced by the type. + /// + /// Fields: `(total_count, entry_size, chunk_power, flags)` + /// - `total_count`: Number of entries appended so far. + /// - `entry_size`: Committed byte length of every entry, in `1..=65535`; + /// appends of any other length are rejected. + /// - `chunk_power`: Log2 of the chunk size (actual size = `1 << + /// chunk_power`). + /// - `flags`: Optional per-element metadata. + /// + /// The state root + /// (`blake3("pds_state" || config_hash || bulk_state_root)`, where + /// `config_hash` commits to `{entry_size, chunk_power}`) flows through + /// the Merk child hash mechanism (`insert_subtree`'s + /// `subtree_root_hash` parameter), so the declared configuration is + /// consensus-visible and a proof can never be reinterpreted under a + /// different config. + /// + /// Variant order in this enum determines bincode's variant-index + /// encoding on disk. This variant gets index 24. + PrivateDocumentStore(u64, u32, u8, Option), } pub fn hex_to_ascii(hex_value: &[u8]) -> String { @@ -590,6 +615,18 @@ impl fmt::Display for Element { .map_or(String::new(), |f| format!(", flags: {:?}", f)) ) } + Element::PrivateDocumentStore(total_count, entry_size, chunk_power, flags) => { + write!( + f, + "PrivateDocumentStore(count: {}, entry_size: {}, chunk_power: {}{})", + total_count, + entry_size, + chunk_power, + flags + .as_ref() + .map_or(String::new(), |f| format!(", flags: {:?}", f)) + ) + } Element::NotSummed(inner) => { write!(f, "NotSummed({})", inner) } @@ -667,6 +704,7 @@ impl Element { Element::ProvableCountProvableSumIndexedTree(..) => { ElementType::ProvableCountProvableSumIndexedTree } + Element::PrivateDocumentStore(..) => ElementType::PrivateDocumentStore, Element::NonCounted(inner) => match inner.element_type() { ElementType::Item => ElementType::NonCountedItem, ElementType::Reference => ElementType::NonCountedReference, @@ -699,6 +737,7 @@ impl Element { ElementType::ProvableCountProvableSumIndexedTree => { ElementType::NonCountedProvableCountProvableSumIndexedTree } + ElementType::PrivateDocumentStore => ElementType::NonCountedPrivateDocumentStore, // Inner is always a base type — nested wrappers are // forbidden at construction and (de)serialization. already_non_counted => already_non_counted, @@ -740,6 +779,38 @@ impl Element { self.element_type().as_str() } + /// Validate the committed configuration of a `PrivateDocumentStore` + /// element, looking through `NonCounted`: `entry_size` must be in + /// `1..=65535` + /// (the upper bound keeps `2^16 * entry_size` — the worst-case + /// compaction blob — representable in the u32 storage-cost field, so the + /// worst-case estimate stays a real bound), and `chunk_power` must be in + /// `1..=16` (the underlying `BulkAppendTree` dense-buffer height range). + /// Returns `Ok(())` for every other variant. + /// + /// The configuration is committed into the store's state root, so an + /// unusable configuration must not be representable: the checked + /// constructors, the insert paths, and both (de)serialization codecs + /// (bincode and serde) all enforce this. `new_private_document_store` + /// itself stays unchecked — it is the restoration constructor used to + /// rebuild elements from already-validated on-disk state, mirroring + /// `new_commitment_tree` / `new_bulk_append_tree`. + pub fn validate_private_document_store_config(&self) -> Result<(), crate::error::ElementError> { + if let Element::PrivateDocumentStore(_, entry_size, chunk_power, _) = self.underlying() { + if *entry_size == 0 || *entry_size > u16::MAX as u32 { + return Err(crate::error::ElementError::InvalidInput( + "private document store entry_size must be in 1..=65535", + )); + } + if !(1..=16).contains(chunk_power) { + return Err(crate::error::ElementError::InvalidInput( + "private document store chunk_power must be between 1 and 16", + )); + } + } + Ok(()) + } + /// Verify the wrapper invariants for `self`: /// - `NonCounted`, `NotSummed`, and `NotCountedOrSummed` may not nest /// in any combination. @@ -892,6 +963,7 @@ mod serde_impl { Vec<(u8, Option>)>, Option, ), + PrivateDocumentStore(u64, u32, u8, Option), } impl From for Element { @@ -941,6 +1013,9 @@ mod serde_impl { ElementShadow::ProvableCountProvableSumIndexedTree(pk, c, s, axes, f) => { Element::ProvableCountProvableSumIndexedTree(pk, c, s, axes, f) } + ElementShadow::PrivateDocumentStore(c, e, p, f) => { + Element::PrivateDocumentStore(c, e, p, f) + } } } } @@ -956,6 +1031,11 @@ mod serde_impl { // built by recursive `From` calls, so the check // at each level catches a violation at any depth. Self::check_recursive_wrapper_invariants(&element).map_err(D::Error::custom)?; + // A PrivateDocumentStore's committed config must be valid at + // every ingress — including this external-tooling codec. + element + .validate_private_document_store_config() + .map_err(D::Error::custom)?; Ok(element) } } @@ -990,6 +1070,7 @@ mod serde_impl { let cases = vec![ Element::Item(b"abc".to_vec(), None), Element::SumTree(Some(b"r".to_vec()), 42, None), + Element::PrivateDocumentStore(9, 64, 4, Some(vec![1])), Element::new_non_counted(Element::Item(b"x".to_vec(), None)).unwrap(), Element::new_not_summed(Element::SumTree(None, 100, None)).unwrap(), Element::new_not_counted_or_summed(Element::CountSumTree(None, 3, 100, None)) @@ -1059,6 +1140,42 @@ mod serde_impl { assert_eq!(back, pcpsit); } + /// A `PrivateDocumentStore` carrying an unusable committed config + /// must be rejected by the serde codec as well as bincode: the + /// config is bound into the store's state root, so an invalid one + /// must not be representable through any ingress. + #[test] + fn serde_rejects_invalid_private_document_store_config() { + for payload in [ + // entry_size = 0 + r#"{"PrivateDocumentStore":[0,0,4,null]}"#, + // entry_size above the 65535 cap + r#"{"PrivateDocumentStore":[0,65536,4,null]}"#, + // chunk_power = 0 and 17 (outside 1..=16) + r#"{"PrivateDocumentStore":[0,64,0,null]}"#, + r#"{"PrivateDocumentStore":[0,64,17,null]}"#, + // the same violations behind a NonCounted wrapper + r#"{"NonCounted":{"PrivateDocumentStore":[0,64,0,null]}}"#, + r#"{"NonCounted":{"PrivateDocumentStore":[0,0,4,null]}}"#, + ] { + let result: Result = serde_json::from_str(payload); + assert!( + result.is_err(), + "serde must reject {}; got {:?}", + payload, + result + ); + } + + // A valid config still round-trips. + let valid = Element::PrivateDocumentStore(3, 64, 4, None); + let json = serde_json::to_string(&valid).expect("serialize"); + assert_eq!( + serde_json::from_str::(&json).expect("deserialize"), + valid + ); + } + /// `NotCountedOrSummed(NotCountedOrSummed(_))` and cross-nestings /// involving the new wrapper must be rejected. #[test] diff --git a/grovedb-element/src/element/serialize.rs b/grovedb-element/src/element/serialize.rs index 3ea841bcb..16a7ad338 100644 --- a/grovedb-element/src/element/serialize.rs +++ b/grovedb-element/src/element/serialize.rs @@ -79,6 +79,16 @@ impl Element { } } } + // A PrivateDocumentStore's committed config is bound into its state + // root; an unusable config must never reach disk. The checked + // constructors and insert paths already enforce this — the codec + // check closes the caller-built-element gap. + if let Err(e) = self.validate_private_document_store_config() { + return Err(ElementError::CorruptedData(format!( + "invalid private document store config: {}", + e + ))); + } let config = config::standard().with_big_endian().with_no_limit(); bincode::encode_to_vec(self, config) .map_err(|e| ElementError::CorruptedData(format!("unable to serialize element {}", e))) @@ -188,6 +198,18 @@ impl Element { } } } + // Reject a PrivateDocumentStore with an unusable committed config + // (entry_size 0 or chunk_power outside 1..=16). No such bytes can + // legitimately exist — serialization and every insert path enforce + // the same bound — so this cannot reject previously-valid data; + // it makes the invalid configuration unrepresentable, mirroring + // the wrapper-invariant checks above. + if let Err(e) = elem.validate_private_document_store_config() { + return Err(ElementError::CorruptedData(format!( + "deserialized private document store with invalid config: {}", + e + ))); + } Ok(elem) } } diff --git a/grovedb-element/src/element/visualize.rs b/grovedb-element/src/element/visualize.rs index b2e26d83a..652706072 100644 --- a/grovedb-element/src/element/visualize.rs +++ b/grovedb-element/src/element/visualize.rs @@ -176,6 +176,21 @@ impl Visualize for Element { drawer = f.visualize(drawer)?; } } + Element::PrivateDocumentStore(total_count, entry_size, chunk_power, flags) => { + drawer.write( + format!( + "private_document_store: count: {total_count} entry_size: {entry_size} \ + chunk_power: {chunk_power}", + ) + .as_bytes(), + )?; + + if let Some(f) = flags + && !f.is_empty() + { + drawer = f.visualize(drawer)?; + } + } Element::NonCounted(inner) => { drawer.write(b"non_counted(")?; drawer = inner.visualize(drawer)?; diff --git a/grovedb-element/src/element_type.rs b/grovedb-element/src/element_type.rs index e7d0b9c0e..5d4400a6c 100644 --- a/grovedb-element/src/element_type.rs +++ b/grovedb-element/src/element_type.rs @@ -293,6 +293,10 @@ pub enum ElementType { /// `ProvableCountProvableSumTree` + 1..=3 secondaries keyed by axis /// sort-prefix) - discriminant 23. The new dual/triple-axis variant. ProvableCountProvableSumIndexedTree = 23, + /// Private document store (append-only fixed-size opaque entries over a + /// `BulkAppendTree`, with the `{entry_size, chunk_power}` config bound + /// into the state root) - discriminant 24. + PrivateDocumentStore = 24, /// Non-counted wrapper around `Item` - discriminant 128 NonCountedItem = 128, /// Non-counted wrapper around `Reference` - discriminant 129 @@ -339,6 +343,9 @@ pub enum ElementType { /// Non-counted wrapper around `ProvableCountProvableSumIndexedTree` - /// discriminant 151 (`0x80 | 23`) NonCountedProvableCountProvableSumIndexedTree = 151, + /// Non-counted wrapper around `PrivateDocumentStore` - discriminant 152 + /// (`0x80 | 24`) + NonCountedPrivateDocumentStore = 152, /// Not-summed wrapper around `SumTree` - discriminant 180 (`0xB4`) NotSummedSumTree = 180, /// Not-summed wrapper around `BigSumTree` - discriminant 181 (`0xB5`) @@ -407,10 +414,11 @@ impl ElementType { // those are `0..=14`, `18` (ReferenceWithSumItem), `19` // (ProvableSumTree), `20` (ProvableCountProvableSumTree), // `21` (ProvableSumIndexedTree), `22` (ProvableCountIndexedTree), - // and `23` (ProvableCountProvableSumIndexedTree). + // `23` (ProvableCountProvableSumIndexedTree), and `24` + // (PrivateDocumentStore). // Bytes 15, 16, and 17 are the wrapper bytes themselves - // (nested wrappers forbidden in either direction); 24..=127 - // are unallocated; 128..=151 are the synthetic NonCountedXxx + // (nested wrappers forbidden in either direction); 25..=127 + // are unallocated; 128..=152 are the synthetic NonCountedXxx // twins which never appear on disk. // Without this check, the bitwise OR below would collapse // `0x80 | inner_byte` into `inner_byte` and a payload like @@ -419,10 +427,10 @@ impl ElementType { // Use an explicit allowlist so the next base-variant addition // is a one-line edit here and the check stays robust to new // variants landing without updating the guard. - if !matches!(inner_byte, 0..=14 | 18 | 19 | 20 | 21 | 22 | 23) { + if !matches!(inner_byte, 0..=14 | 18 | 19 | 20 | 21 | 22 | 23 | 24) { return Err(ElementError::CorruptedData(format!( "NonCounted inner discriminant must be a base type \ - (0..=14, 18, 19, 20, 21, 22, or 23), got {}", + (0..=14, 18, 19, 20, 21, 22, 23, or 24), got {}", inner_byte ))); } @@ -738,6 +746,7 @@ impl ElementType { | ElementType::ProvableSumIndexedTree | ElementType::ProvableCountIndexedTree | ElementType::ProvableCountProvableSumIndexedTree + | ElementType::PrivateDocumentStore ) } @@ -814,6 +823,7 @@ impl ElementType { ElementType::ProvableCountProvableSumIndexedTree => { "provable count provable sum indexed tree" } + ElementType::PrivateDocumentStore => "private_document_store", ElementType::NonCountedItem => "non_counted item", ElementType::NonCountedReference => "non_counted reference", ElementType::NonCountedTree => "non_counted tree", @@ -843,6 +853,7 @@ impl ElementType { ElementType::NonCountedProvableCountProvableSumIndexedTree => { "non_counted provable count provable sum indexed tree" } + ElementType::NonCountedPrivateDocumentStore => "non_counted private_document_store", ElementType::NotSummedSumTree => "not_summed sum tree", ElementType::NotSummedBigSumTree => "not_summed big sum tree", ElementType::NotSummedCountSumTree => "not_summed count sum tree", @@ -902,6 +913,7 @@ impl TryFrom for ElementType { 21 => Ok(ElementType::ProvableSumIndexedTree), 22 => Ok(ElementType::ProvableCountIndexedTree), 23 => Ok(ElementType::ProvableCountProvableSumIndexedTree), + 24 => Ok(ElementType::PrivateDocumentStore), 128 => Ok(ElementType::NonCountedItem), 129 => Ok(ElementType::NonCountedReference), 130 => Ok(ElementType::NonCountedTree), @@ -923,6 +935,7 @@ impl TryFrom for ElementType { 149 => Ok(ElementType::NonCountedProvableSumIndexedTree), 150 => Ok(ElementType::NonCountedProvableCountIndexedTree), 151 => Ok(ElementType::NonCountedProvableCountProvableSumIndexedTree), + 152 => Ok(ElementType::NonCountedPrivateDocumentStore), // NotSummed twins occupy the 0xB0..=0xBF family range; slots // are assigned explicitly per variant. 177 => Ok(ElementType::NotSummedProvableSumTree), @@ -1032,8 +1045,12 @@ mod tests { ElementType::try_from(23).unwrap(), ElementType::ProvableCountProvableSumIndexedTree ); - // 24..=127 are unallocated and invalid. - assert!(ElementType::try_from(24).is_err()); + assert_eq!( + ElementType::try_from(24).unwrap(), + ElementType::PrivateDocumentStore + ); + // 25..=127 are unallocated and invalid. + assert!(ElementType::try_from(25).is_err()); assert!(ElementType::try_from(100).is_err()); // NonCounted twins (0x80 | base): 128..142, plus 146 (= 0x80|18 = @@ -1075,6 +1092,10 @@ mod tests { ElementType::try_from(151).unwrap(), ElementType::NonCountedProvableCountProvableSumIndexedTree ); + assert_eq!( + ElementType::try_from(152).unwrap(), + ElementType::NonCountedPrivateDocumentStore + ); // Bytes between the base and NonCounted-twin ranges are invalid. assert!(ElementType::try_from(127).is_err()); // 143 (= 0x80|15), 144 (= 0x80|16), 145 (= 0x80|17): wrapper bytes @@ -1082,10 +1103,10 @@ mod tests { assert!(ElementType::try_from(143).is_err()); assert!(ElementType::try_from(144).is_err()); assert!(ElementType::try_from(145).is_err()); - // 152..=176 (between NonCounted-twin and NotSummed-twin ranges) are + // 153..=176 (between NonCounted-twin and NotSummed-twin ranges) are // invalid (with the exception of 177 = NotSummedProvableSumTree and // 178 = NotSummedProvableCountProvableSumTree). - assert!(ElementType::try_from(152).is_err()); + assert!(ElementType::try_from(153).is_err()); assert!(ElementType::try_from(176).is_err()); // NotSummed twins live in 0xB0..=0xBF with explicit per-variant @@ -1726,16 +1747,16 @@ mod tests { assert!(ElementType::from_serialized_value(&[15, 200]).is_err()); // Wrapper whose inner byte is itself a synthetic twin discriminant // (high bit set) is rejected — only base discriminants 0..=14, 18, - // 19, 20, 21, 22, 23 are legal on-disk inner bytes. Without this + // 19, 20, 21, 22, 23, 24 are legal on-disk inner bytes. Without this // guard, `0x80 | 128 == 128` would silently parse as `NonCountedItem`. assert!(ElementType::from_serialized_value(&[15, 128]).is_err()); assert!(ElementType::from_serialized_value(&[15, 142]).is_err()); // Wrapper with an unallocated mid-range inner byte (16, 17, - // 24..=127) is also rejected, even though it has no high bit + // 25..=127) is also rejected, even though it has no high bit // set. assert!(ElementType::from_serialized_value(&[15, 16]).is_err()); assert!(ElementType::from_serialized_value(&[15, 17]).is_err()); - assert!(ElementType::from_serialized_value(&[15, 24]).is_err()); + assert!(ElementType::from_serialized_value(&[15, 25]).is_err()); assert!(ElementType::from_serialized_value(&[15, 100]).is_err()); // Inner byte 18 (ReferenceWithSumItem) IS a legal base; resolves to @@ -1772,6 +1793,12 @@ mod tests { ElementType::from_serialized_value(&[15, 23]).unwrap(), ElementType::NonCountedProvableCountProvableSumIndexedTree ); + // Inner byte 24 (PrivateDocumentStore) resolves to its NonCounted + // twin at slot 152. + assert_eq!( + ElementType::from_serialized_value(&[15, 24]).unwrap(), + ElementType::NonCountedPrivateDocumentStore + ); } #[test] @@ -1796,6 +1823,8 @@ mod tests { assert!(ElementType::ProvableSumIndexedTree.is_tree()); assert!(ElementType::ProvableCountIndexedTree.is_tree()); assert!(ElementType::ProvableCountProvableSumIndexedTree.is_tree()); + assert!(ElementType::PrivateDocumentStore.is_tree()); + assert!(ElementType::NonCountedPrivateDocumentStore.is_tree()); // ReferenceWithSumItem is a reference, not a tree and not an item. assert!(!ElementType::ReferenceWithSumItem.is_tree()); assert!(ElementType::ReferenceWithSumItem.is_reference()); @@ -1982,16 +2011,22 @@ mod tests { ElementType::ProvableCountProvableSumIndexedTree, "ProvableCountProvableSumIndexedTree", ), + // discriminant 24 + ( + Element::PrivateDocumentStore(0, 64, 4, None), + ElementType::PrivateDocumentStore, + "PrivateDocumentStore", + ), ]; - // Verify we're testing all 21 base discriminants: 0..=14, 18, 19, - // 20, 21, 22, 23. (15 = NonCounted wrapper byte, 16 = NotSummed + // Verify we're testing all 22 base discriminants: 0..=14, 18, 19, + // 20, 21, 22, 23, 24. (15 = NonCounted wrapper byte, 16 = NotSummed // wrapper byte, 17 = NotCountedOrSummed wrapper byte — none has // a base ElementType variant.) assert_eq!( test_cases.len(), - 21, - "Expected 21 base Element variants in test, got {}", + 22, + "Expected 22 base Element variants in test, got {}", test_cases.len() ); diff --git a/grovedb-element/tests/element_display_and_serialization.rs b/grovedb-element/tests/element_display_and_serialization.rs index 40aed6c23..c65396315 100644 --- a/grovedb-element/tests/element_display_and_serialization.rs +++ b/grovedb-element/tests/element_display_and_serialization.rs @@ -445,3 +445,65 @@ fn element_display_without_flags_covers_none_branches() { ); } } + +#[test] +fn private_document_store_invalid_config_is_unrepresentable() { + let grove_version = GroveVersion::latest(); + // entry_size 0, chunk_power 0, chunk_power 17 must be rejected by both + // directions of the bincode codec (the config is committed into the + // store's state root, so an unusable config must never round-trip). + for bad in [ + Element::new_private_document_store(0, 0, 4, None), + Element::new_private_document_store(0, 64, 0, None), + Element::new_private_document_store(0, 64, 17, None), + ] { + assert!( + bad.serialize(grove_version).is_err(), + "serialize must reject {:?}", + bad + ); + } + // Craft the bytes directly (serialize refuses) and check deserialize + // rejects them: a valid element re-encoded with a zeroed entry_size. + let good = Element::new_private_document_store(0, 64, 4, None); + let mut bytes = good.serialize(grove_version).expect("serialize valid"); + // Layout: [24 (discriminant), total_count varint (0 = 1 byte), + // entry_size varint (64 = 1 byte), chunk_power, flags None] + assert_eq!(bytes[0], 24); + assert_eq!(bytes[2], 64); + bytes[2] = 0; // entry_size -> 0 + assert!( + Element::deserialize(&bytes, grove_version).is_err(), + "deserialize must reject a zero entry_size" + ); + // NonCounted-wrapped invalid config is rejected too (validation looks + // through the wrapper). + let wrapped = Element::NonCounted(Box::new(Element::new_private_document_store(0, 0, 4, None))); + assert!(wrapped.serialize(grove_version).is_err()); +} + +#[test] +fn private_document_store_bincode_rejects_invalid_chunk_power() { + let grove_version = GroveVersion::latest(); + // Craft bytes from a valid element, then corrupt chunk_power. serialize + // refuses to produce these, so deserialize is the ingress under test. + let good = Element::new_private_document_store(0, 64, 4, None); + let bytes = good.serialize(grove_version).expect("serialize valid"); + // Layout: [24 (discriminant), total_count, entry_size, chunk_power, flags] + assert_eq!(bytes[0], 24); + assert_eq!(bytes[3], 4, "chunk_power is the fourth byte for this shape"); + + for bad_power in [0u8, 17, 255] { + let mut corrupted = bytes.clone(); + corrupted[3] = bad_power; + assert!( + Element::deserialize(&corrupted, grove_version).is_err(), + "deserialize must reject chunk_power {}", + bad_power + ); + } + + // And the constructors reject the same values up front. + assert!(Element::empty_private_document_store(64, 0).is_err()); + assert!(Element::empty_private_document_store(64, 17).is_err()); +} diff --git a/grovedb-merkle-mountain-range/Cargo.toml b/grovedb-merkle-mountain-range/Cargo.toml index 5b445d7d1..36ac59a0c 100644 --- a/grovedb-merkle-mountain-range/Cargo.toml +++ b/grovedb-merkle-mountain-range/Cargo.toml @@ -22,6 +22,7 @@ blake3 = { workspace = true } bincode = { workspace = true, features = ["derive"] } grovedb-costs = { version = "5.0.1", path = "../costs" } grovedb-storage = { version = "5.0.1", path = "../storage", optional = true } +grovedb-version = { version = "5.0.1", path = "../grovedb-version" } [dev-dependencies] faster-hex = "0.10.0" diff --git a/grovedb-merkle-mountain-range/benches/mmr_benchmark.rs b/grovedb-merkle-mountain-range/benches/mmr_benchmark.rs index 75a75f947..14dd96713 100644 --- a/grovedb-merkle-mountain-range/benches/mmr_benchmark.rs +++ b/grovedb-merkle-mountain-range/benches/mmr_benchmark.rs @@ -3,6 +3,7 @@ extern crate criterion; use criterion::{BenchmarkId, Criterion}; use grovedb_merkle_mountain_range::{MMRStoreReadOps, MemStore, MmrNode, MMR}; +use grovedb_version::version::GroveVersion; use rand::seq::IndexedRandom; /// Create an MmrNode leaf from an integer (for benchmarking). @@ -14,7 +15,11 @@ fn prepare_mmr(count: u32) -> (u64, MemStore, Vec) { let store = MemStore::default(); let mut mmr = MMR::new(0, &store); let positions: Vec = (0u32..count) - .map(|i| mmr.push(leaf_from_u32(i)).unwrap().expect("push")) + .map(|i| { + mmr.push(leaf_from_u32(i), GroveVersion::latest()) + .unwrap() + .expect("push") + }) .collect(); let mmr_size = mmr.mmr_size; mmr.commit().unwrap().expect("write to store"); @@ -37,8 +42,11 @@ fn bench(c: &mut Criterion) { let mmr = MMR::new(mmr_size, &store); let mut rng = rand::rng(); b.iter(|| { - mmr.gen_proof(vec![*positions.choose(&mut rng).unwrap()]) - .unwrap() + mmr.gen_proof( + vec![*positions.choose(&mut rng).unwrap()], + GroveVersion::latest(), + ) + .unwrap() }); }); @@ -46,7 +54,10 @@ fn bench(c: &mut Criterion) { let (mmr_size, store, positions) = prepare_mmr(100_0000); let mmr = MMR::new(mmr_size, &store); let mut rng = rand::rng(); - let root = mmr.get_root().unwrap().expect("get root"); + let root = mmr + .get_root(GroveVersion::latest()) + .unwrap() + .expect("get root"); let proofs: Vec<_> = (0..10_000) .map(|_| { let pos = positions.choose(&mut rng).unwrap(); @@ -55,7 +66,10 @@ fn bench(c: &mut Criterion) { .unwrap() .expect("read") .expect("exists"); - let proof = mmr.gen_proof(vec![*pos]).unwrap().expect("gen proof"); + let proof = mmr + .gen_proof(vec![*pos], GroveVersion::latest()) + .unwrap() + .expect("gen proof"); (pos, elem, proof) }) .collect(); diff --git a/grovedb-merkle-mountain-range/src/cost/mod.rs b/grovedb-merkle-mountain-range/src/cost/mod.rs new file mode 100644 index 000000000..483f85626 --- /dev/null +++ b/grovedb-merkle-mountain-range/src/cost/mod.rs @@ -0,0 +1,102 @@ +//! Versioned hash charges for the MMR operations that merge internally. +//! +//! `push`, `get_root` and `gen_proof` all compute blake3 merges — collapsing +//! peaks on a push, folding peaks into a root or a proof. The shipped +//! accounting billed the storage reads those merges consume but not the +//! merges themselves. +//! +//! Correcting that changes `hash_node_calls`, and costs become fees, so the +//! correction cannot simply replace the old behaviour: a node replaying a +//! historical block has to charge what that block was admitted under. Each +//! charge is therefore version-dispatched, v0 being the shipped (uncharged) +//! accounting and v1 the corrected one. The values the operations return are +//! bit-identical either way. + +mod v0; +mod v1; + +use grovedb_version::{error::GroveVersionError, version::GroveVersion}; + +use crate::Error; + +/// Hashes to charge for the peak collapses a `push` performs. +pub(crate) fn push_merge_hashes(merges: u32, grove_version: &GroveVersion) -> Result { + match grove_version.mmr_versions.cost.push { + 0 => Ok(v0::merge_hashes(merges)), + 1 => Ok(v1::merge_hashes(merges)), + version => Err(Error::VersionError( + GroveVersionError::UnknownVersionMismatch { + method: "MMR::push hash charge".to_string(), + known_versions: vec![0, 1], + received: version, + } + .to_string(), + )), + } +} + +/// Hashes to charge for the peak bagging a `get_root` performs. +pub(crate) fn get_root_bagging_hashes( + peaks: usize, + grove_version: &GroveVersion, +) -> Result { + match grove_version.mmr_versions.cost.get_root { + 0 => Ok(v0::bagging_hashes(peaks)), + 1 => Ok(v1::bagging_hashes(peaks)), + version => Err(Error::VersionError( + GroveVersionError::UnknownVersionMismatch { + method: "MMR::get_root hash charge".to_string(), + known_versions: vec![0, 1], + received: version, + } + .to_string(), + )), + } +} + +/// Hashes to charge for the peak bagging a `gen_proof` performs. +pub(crate) fn gen_proof_bagging_hashes( + peaks: usize, + grove_version: &GroveVersion, +) -> Result { + match grove_version.mmr_versions.cost.gen_proof { + 0 => Ok(v0::bagging_hashes(peaks)), + 1 => Ok(v1::bagging_hashes(peaks)), + version => Err(Error::VersionError( + GroveVersionError::UnknownVersionMismatch { + method: "MMR::gen_proof hash charge".to_string(), + known_versions: vec![0, 1], + received: version, + } + .to_string(), + )), + } +} + +/// Hashes a CALLER must charge for a push, on top of what +/// [`MMR::push`](crate::MMR::push) charges itself. +/// +/// A caller that hashes the leaf before calling `push` — the ops layer does — +/// has to make up whatever `push` does not bill for the version in play: +/// +/// - v0: `push` charges no merges, so the caller owes the leaf hash AND the +/// collapses, i.e. `hash_count_for_push` +/// - v1: `push` charges the merges, so the caller owes only the leaf hash +/// +/// The invariant across both is `call_site + push == 1 + merges`, which is +/// why an MmrTree push costs the same under either version. Getting this +/// wrong in either direction double-charges or under-charges every merge. +pub fn push_call_site_hashes(leaf_count: u64, grove_version: &GroveVersion) -> Result { + match grove_version.mmr_versions.cost.push { + 0 => Ok(v0::call_site_hashes(leaf_count)), + 1 => Ok(v1::call_site_hashes(leaf_count)), + version => Err(Error::VersionError( + GroveVersionError::UnknownVersionMismatch { + method: "MMR push call-site hash charge".to_string(), + known_versions: vec![0, 1], + received: version, + } + .to_string(), + )), + } +} diff --git a/grovedb-merkle-mountain-range/src/cost/v0.rs b/grovedb-merkle-mountain-range/src/cost/v0.rs new file mode 100644 index 000000000..121f28ff8 --- /dev/null +++ b/grovedb-merkle-mountain-range/src/cost/v0.rs @@ -0,0 +1,22 @@ +//! Shipped MMR hash accounting: merges are not charged. +//! +//! Used by GROVE_V1..V3. These are released versions, so this is locked — +//! see [`super`] for why a cost correction cannot replace it in place. + +use crate::helper::hash_count_for_push; + +/// Charge for the peak collapses of a push: none. +pub(super) fn merge_hashes(_merges: u32) -> u32 { + 0 +} + +/// Charge for peak bagging: none. +pub(super) fn bagging_hashes(_peaks: usize) -> u32 { + 0 +} + +/// The caller owes the leaf hash and every collapse, because `push` bills +/// none of them under this version. +pub(super) fn call_site_hashes(leaf_count: u64) -> u32 { + hash_count_for_push(leaf_count) +} diff --git a/grovedb-merkle-mountain-range/src/cost/v1.rs b/grovedb-merkle-mountain-range/src/cost/v1.rs new file mode 100644 index 000000000..be7999180 --- /dev/null +++ b/grovedb-merkle-mountain-range/src/cost/v1.rs @@ -0,0 +1,19 @@ +//! Corrected MMR hash accounting: one charge per blake3 merge computed. +//! +//! Used from GROVE_V4. + +/// A push calls `MmrNode::merge` once per peak it collapses. +pub(super) fn merge_hashes(merges: u32) -> u32 { + merges +} + +/// Bagging folds the peaks right-to-left, so `n` peaks cost `n - 1` merges. +/// One peak (or none) folds nothing. +pub(super) fn bagging_hashes(peaks: usize) -> u32 { + peaks.saturating_sub(1) as u32 +} + +/// The caller owes only the eager leaf hash; `push` bills its own merges. +pub(super) fn call_site_hashes(_leaf_count: u64) -> u32 { + 1 +} diff --git a/grovedb-merkle-mountain-range/src/error.rs b/grovedb-merkle-mountain-range/src/error.rs index ec972ce87..9b828a932 100644 --- a/grovedb-merkle-mountain-range/src/error.rs +++ b/grovedb-merkle-mountain-range/src/error.rs @@ -26,6 +26,10 @@ pub enum Error { InvalidInput(String), /// Invalid proof during verification. InvalidProof(String), + /// The grove version selected an unknown version for a versioned MMR + /// method. Reaching this means a version set names a variant this build + /// does not implement. + VersionError(String), } impl core::fmt::Display for Error { @@ -41,6 +45,7 @@ impl core::fmt::Display for Error { InvalidData(msg) => write!(f, "Invalid MMR data: {}", msg), InvalidInput(msg) => write!(f, "Invalid input: {}", msg), InvalidProof(msg) => write!(f, "Invalid proof: {}", msg), + VersionError(e) => write!(f, "Version error: {}", e), } } } diff --git a/grovedb-merkle-mountain-range/src/helper.rs b/grovedb-merkle-mountain-range/src/helper.rs index 8d948855c..61db0c163 100644 --- a/grovedb-merkle-mountain-range/src/helper.rs +++ b/grovedb-merkle-mountain-range/src/helper.rs @@ -107,6 +107,24 @@ pub fn get_peak_map(mmr_size: u64) -> u64 { /// 2 /// / \ /// 0 1 3 +/// Blake3 merges [`MMR::get_root`](crate::MMR::get_root) performs for an MMR +/// of this size. +/// +/// Bagging folds the peaks right-to-left with one merge per additional peak, +/// so `n` peaks cost `n - 1`. Sizes 0 and 1 take the empty and single-element +/// paths, which fold nothing. +/// +/// Exposed so callers that report a hash count for an operation containing a +/// root computation can account for the bagging WITHOUT depending on whether +/// `get_root`'s own charge is enabled for their grove version — the two are +/// separately gated. +pub fn hash_count_for_root_bagging(mmr_size: u64) -> u32 { + if mmr_size <= 1 { + return 0; + } + get_peaks(mmr_size).len().saturating_sub(1) as u32 +} + pub fn get_peaks(mmr_size: u64) -> Vec { if mmr_size == 0 { return vec![]; diff --git a/grovedb-merkle-mountain-range/src/lib.rs b/grovedb-merkle-mountain-range/src/lib.rs index 1bb8b58e9..1ea9b28ba 100644 --- a/grovedb-merkle-mountain-range/src/lib.rs +++ b/grovedb-merkle-mountain-range/src/lib.rs @@ -20,6 +20,8 @@ #![deny(missing_docs)] +mod cost; +pub use cost::push_call_site_hashes; mod error; /// MMR helper functions for position arithmetic, storage keys, and cost /// calculations. @@ -39,9 +41,10 @@ mod tests; pub use error::{Error, Result}; pub use grovedb_costs::{CostResult, CostsExt, OperationCost}; pub use helper::{ - hash_count_for_push, leaf_index_to_mmr_size, leaf_index_to_mmr_size as leaf_to_mmr_size, - leaf_index_to_pos, leaf_index_to_pos as leaf_to_pos, mmr_node_key, mmr_node_key_sized, - mmr_size_to_leaf_count, MmrKey, MmrKeySize, MAX_U32_MMR_POSITION, + hash_count_for_push, hash_count_for_root_bagging, leaf_index_to_mmr_size, + leaf_index_to_mmr_size as leaf_to_mmr_size, leaf_index_to_pos, + leaf_index_to_pos as leaf_to_pos, mmr_node_key, mmr_node_key_sized, mmr_size_to_leaf_count, + MmrKey, MmrKeySize, MAX_U32_MMR_POSITION, }; #[cfg(any(test, feature = "mem_store"))] pub use mem_store::MemStore; diff --git a/grovedb-merkle-mountain-range/src/mmr.rs b/grovedb-merkle-mountain-range/src/mmr.rs index 880c8f9aa..8b47144c5 100644 --- a/grovedb-merkle-mountain-range/src/mmr.rs +++ b/grovedb-merkle-mountain-range/src/mmr.rs @@ -3,8 +3,10 @@ use std::{borrow::Cow, collections::VecDeque}; use grovedb_costs::{CostResult, CostsExt, OperationCost}; +use grovedb_version::version::GroveVersion; use crate::{ + cost::{gen_proof_bagging_hashes, get_root_bagging_hashes, push_merge_hashes}, helper::{get_peak_map, get_peaks, parent_offset, pos_height_in_tree, sibling_offset}, mmr_store::{MMRBatch, MMRStoreReadOps, MMRStoreWriteOps}, proof::{take_while_vec, MerkleProof}, @@ -87,8 +89,12 @@ impl MMR { /// /// This may also create internal (merged) nodes. The new nodes are /// buffered until [`MMR::commit`] is called. - pub fn push(&mut self, elem: MmrNode) -> CostResult { + /// + /// The MMR it builds is identical under every grove version; only the + /// hash charge differs — see [`crate::cost`]. + pub fn push(&mut self, elem: MmrNode, grove_version: &GroveVersion) -> CostResult { let mut cost = OperationCost::default(); + let mut merges: u32 = 0; let mut elems = vec![elem]; let elem_pos = self.mmr_size; let peak_map = get_peak_map(self.mmr_size); @@ -107,19 +113,30 @@ impl MMR { }; let right_elem = elems.last().expect("checked"); let parent_elem = MmrNode::merge(&left_elem, right_elem); + merges = merges.saturating_add(1); elems.push(parent_elem); } // store hashes self.batch.append(elem_pos, elems); // update mmr_size self.mmr_size = pos + 1; + // Each `MmrNode::merge` above is a blake3. Whether it is billed is + // version-gated: the shipped accounting charged the sibling reads the + // merges consume but not the merges themselves. + match push_merge_hashes(merges, grove_version) { + Ok(h) => cost.hash_node_calls = cost.hash_node_calls.saturating_add(h), + Err(e) => return Err(e).wrap_with_cost(cost), + } Ok(elem_pos).wrap_with_cost(cost) } /// Compute the root hash by bagging all peaks right-to-left. /// /// Returns [`Error::GetRootOnEmpty`] for an empty MMR. - pub fn get_root(&self) -> CostResult { + /// + /// The root is identical under every grove version; only the hash charge + /// differs. + pub fn get_root(&self, grove_version: &GroveVersion) -> CostResult { let mut cost = OperationCost::default(); if self.mmr_size == 0 { return Err(Error::GetRootOnEmpty).wrap_with_cost(cost); @@ -145,6 +162,14 @@ impl MMR { Ok(p) => p, Err(e) => return Err(e).wrap_with_cost(cost), }; + // `bag_peaks` folds the peaks right-to-left with one `MmrNode::merge` + // — a blake3 — per fold, so it performs `peaks - 1` hashes. Whether + // those are billed is version-gated; the shipped accounting billed + // the peak reads only. + match get_root_bagging_hashes(peaks.len(), grove_version) { + Ok(h) => cost.hash_node_calls = cost.hash_node_calls.saturating_add(h), + Err(e) => return Err(e).wrap_with_cost(cost), + } match bag_peaks(peaks) { Ok(Some(root)) => Ok(root).wrap_with_cost(cost), Ok(None) => Err(Error::InconsistentStore).wrap_with_cost(cost), @@ -224,7 +249,14 @@ impl MMR { /// Positions are sorted and deduplicated internally. Returns /// [`Error::GenProofForInvalidLeaves`] if any position is out of range /// or the list is empty. - pub fn gen_proof(&self, mut pos_list: Vec) -> CostResult { + /// + /// The proof is identical under every grove version; only the hash charge + /// differs. + pub fn gen_proof( + &self, + mut pos_list: Vec, + grove_version: &GroveVersion, + ) -> CostResult { let mut cost = OperationCost::default(); if pos_list.is_empty() { return Err(Error::GenProofForInvalidLeaves).wrap_with_cost(cost); @@ -260,6 +292,20 @@ impl MMR { return Err(Error::GenProofForInvalidLeaves).wrap_with_cost(cost); } + // Same shared `bag_peaks` the root computation uses, and the same + // `bagging_track - 1` blake3 merges. Version-gated identically, so + // proof generation neither gets the folds for free nor starts + // charging them on a released version. + // + // Dispatched unconditionally, not inside the `bagging_track > 1` + // branch below: with nothing to bag the charge is zero under every + // version, but an unknown version must still be rejected rather than + // slipping through whenever a proof happens not to fold peaks. + match gen_proof_bagging_hashes(bagging_track, grove_version) { + Ok(h) => cost.hash_node_calls = cost.hash_node_calls.saturating_add(h), + Err(e) => return Err(e).wrap_with_cost(cost), + } + if bagging_track > 1 { let rhs_peaks = proof.split_off(proof.len() - bagging_track); match bag_peaks(rhs_peaks) { diff --git a/grovedb-merkle-mountain-range/src/proof.rs b/grovedb-merkle-mountain-range/src/proof.rs index 659d9ce5c..97a04f55e 100644 --- a/grovedb-merkle-mountain-range/src/proof.rs +++ b/grovedb-merkle-mountain-range/src/proof.rs @@ -12,6 +12,7 @@ use std::{ use bincode::{Decode, Encode}; use grovedb_costs::{CostResult, CostsExt, OperationCost}; +use grovedb_version::version::GroveVersion; use crate::{ helper::{ @@ -440,7 +441,7 @@ impl MmrTreeProof { // Generate the MerkleProof (drops zero costs from lazy store) let mmr = crate::MMR::new(mmr_size, &store); - let proof_result = mmr.gen_proof(positions).unwrap(); + let proof_result = mmr.gen_proof(positions, GroveVersion::latest()).unwrap(); // Check deferred storage errors first — if the store failed, the // error (e.g. InconsistentStore) is a symptom, not the root cause. @@ -715,7 +716,7 @@ mod tests { let store = MemStore::default(); let mut mmr = MMR::new(0, &store); for v in values { - mmr.push(MmrNode::leaf(v.to_vec())) + mmr.push(MmrNode::leaf(v.to_vec()), GroveVersion::latest()) .unwrap() .expect("push should succeed"); } @@ -727,7 +728,7 @@ mod tests { /// Get root hash from a MemStore + mmr_size. fn root_hash(store: &MemStore, mmr_size: u64) -> [u8; 32] { let mmr = MMR::new(mmr_size, store); - mmr.get_root() + mmr.get_root(GroveVersion::latest()) .unwrap() .expect("get_root should succeed") .hash() diff --git a/grovedb-merkle-mountain-range/src/tests/test_coverage.rs b/grovedb-merkle-mountain-range/src/tests/test_coverage.rs index 2d008694c..0604388fd 100644 --- a/grovedb-merkle-mountain-range/src/tests/test_coverage.rs +++ b/grovedb-merkle-mountain-range/src/tests/test_coverage.rs @@ -8,6 +8,7 @@ use crate::{ MMR, }; use grovedb_costs::{CostResult, CostsExt, OperationCost}; +use grovedb_version::version::GroveVersion; /// Create an MmrNode leaf from an integer. fn leaf(i: u32) -> MmrNode { @@ -59,8 +60,12 @@ fn batch_into_iterator() { let store = MemStore::default(); let mut mmr = MMR::new(0, &store); - mmr.push(leaf(10)).unwrap().expect("push"); - mmr.push(leaf(11)).unwrap().expect("push"); + mmr.push(leaf(10), GroveVersion::latest()) + .unwrap() + .expect("push"); + mmr.push(leaf(11), GroveVersion::latest()) + .unwrap() + .expect("push"); let entries: Vec<(u64, Vec)> = mmr.batch.into_iter().collect(); assert_eq!(entries.len(), 2); @@ -88,7 +93,9 @@ fn batch_commit_surfaces_store_error() { let store = FailingWriteStore; let mut mmr = MMR::new(0, &store); - mmr.push(leaf(0)).unwrap().expect("push to batch"); + mmr.push(leaf(0), GroveVersion::latest()) + .unwrap() + .expect("push to batch"); let result = mmr.commit().unwrap(); assert!(result.is_err(), "commit should surface store write error"); @@ -107,7 +114,9 @@ fn mmr_is_empty() { assert!(mmr.is_empty()); let mut mmr2 = MMR::new(0, &store); - mmr2.push(leaf(0)).unwrap().expect("push"); + mmr2.push(leaf(0), GroveVersion::latest()) + .unwrap() + .expect("push"); assert!(!mmr2.is_empty()); } @@ -145,7 +154,7 @@ impl MMRStoreWriteOps for &ErrorStore { fn get_root_single_element_missing_returns_inconsistent() { let store = EmptyStore; let mmr = MMR::new(1, &store); - let result = mmr.get_root().unwrap(); + let result = mmr.get_root(GroveVersion::latest()).unwrap(); assert_eq!(result, Err(Error::InconsistentStore)); } @@ -153,7 +162,7 @@ fn get_root_single_element_missing_returns_inconsistent() { fn get_root_single_element_store_error_propagates() { let store = ErrorStore; let mmr = MMR::new(1, &store); - let result = mmr.get_root().unwrap(); + let result = mmr.get_root(GroveVersion::latest()).unwrap(); assert!(result.is_err()); let msg = format!("{}", result.unwrap_err()); assert!(msg.contains("read error"), "error: {}", msg); @@ -163,7 +172,7 @@ fn get_root_single_element_store_error_propagates() { fn get_root_multi_peak_store_error_propagates() { let store = ErrorStore; let mmr = MMR::new(4, &store); - let result = mmr.get_root().unwrap(); + let result = mmr.get_root(GroveVersion::latest()).unwrap(); assert!(result.is_err()); } @@ -234,7 +243,7 @@ fn push_propagates_store_read_error_during_merge() { let mut mmr = MMR::new(1, &store); // Push triggers merge with element at position 0 → store read fails - let result = mmr.push(leaf(1)).unwrap(); + let result = mmr.push(leaf(1), GroveVersion::latest()).unwrap(); assert!(result.is_err()); let msg = format!("{}", result.unwrap_err()); assert!( @@ -254,7 +263,7 @@ fn push_returns_inconsistent_store_when_merge_element_missing() { // Push triggers merge with element at position 0, but EmptyStore // returns Ok(None) → InconsistentStore - let result = mmr.push(leaf(1)).unwrap(); + let result = mmr.push(leaf(1), GroveVersion::latest()).unwrap(); assert_eq!(result, Err(Error::InconsistentStore)); } @@ -265,7 +274,9 @@ fn push_returns_inconsistent_store_when_merge_element_missing() { fn batch_element_at_position_break_falls_through_to_store() { let store = MemStore::default(); let mut mmr = MMR::new(0, &store); - mmr.push(leaf(0)).unwrap().expect("push"); + mmr.push(leaf(0), GroveVersion::latest()) + .unwrap() + .expect("push"); // batch has entry (0, [leaf(0)]). Position 5 is past this range, // triggering the break and falling through to the store. let result = mmr @@ -295,3 +306,190 @@ fn verify_and_get_root_surfaces_calculate_root_error() { msg ); } + +// ============================================================================= +// mmr.rs: versioned hash charges for the internal blake3 merges +// ============================================================================= + +/// `get_root` bags peaks with one blake3 merge per extra peak. v0 (shipped) +/// charges none of them; v1 charges `peaks - 1`. The root itself is identical +/// under both, so only the cost may differ. +#[test] +fn get_root_peak_bagging_charge_is_versioned() { + use grovedb_version::version::{v1::GROVE_V1, v4::GROVE_V4}; + + // (leaves, expected v1 merges): 1 and 2 leaves leave a single peak (or + // take the single-element path), 3 leaves give two peaks, 7 give three. + for (leaves, expected) in [(1u32, 0u32), (2, 0), (3, 1), (7, 2)] { + let store = MemStore::default(); + let mut mmr = MMR::new(0, &store); + for i in 0..leaves { + mmr.push(leaf(i), GroveVersion::latest()) + .unwrap() + .expect("push"); + } + + let v0 = mmr.get_root(&GROVE_V1); + let v0_root = v0.value.expect("root"); + assert_eq!( + v0.cost.hash_node_calls, 0, + "v0 charges no bagging merges ({} leaves)", + leaves + ); + + let v1 = mmr.get_root(&GROVE_V4); + let v1_root = v1.value.expect("root"); + assert_eq!( + v1.cost.hash_node_calls, expected, + "v1 charges one hash per fold ({} leaves), got {:?}", + leaves, v1.cost + ); + + assert_eq!( + v0_root, v1_root, + "the root must not depend on the cost version ({} leaves)", + leaves + ); + } +} + +/// `push` merges once per peak it collapses. v0 billed the sibling reads +/// those merges consume but not the merges; v1 charges them. +#[test] +fn push_peak_collapse_charge_is_versioned() { + use grovedb_version::version::{v1::GROVE_V1, v4::GROVE_V4}; + + // Collapses for the first four leaves: 0, 1, 0, 2. + let expected = [0u32, 1, 0, 2]; + + let store_v0 = MemStore::default(); + let mut mmr_v0 = MMR::new(0, &store_v0); + let store_v1 = MemStore::default(); + let mut mmr_v1 = MMR::new(0, &store_v1); + + for (i, exp) in expected.iter().enumerate() { + let c0 = mmr_v0.push(leaf(i as u32), &GROVE_V1); + c0.value.expect("push"); + assert_eq!( + c0.cost.hash_node_calls, 0, + "v0 charges no merges (leaf {})", + i + ); + + let c1 = mmr_v1.push(leaf(i as u32), &GROVE_V4); + c1.value.expect("push"); + assert_eq!( + c1.cost.hash_node_calls, *exp, + "v1 charges one hash per collapse (leaf {}), got {:?}", + i, c1.cost + ); + } + + // Same MMR either way. + assert_eq!( + mmr_v0 + .get_root(GroveVersion::latest()) + .unwrap() + .expect("root"), + mmr_v1 + .get_root(GroveVersion::latest()) + .unwrap() + .expect("root"), + "the MMR must not depend on the cost version" + ); +} + +/// `gen_proof` folds right-hand peaks through the same `bag_peaks` helper, +/// so it carries the same versioned charge. +#[test] +fn gen_proof_peak_bagging_charge_is_versioned() { + use grovedb_version::version::{v1::GROVE_V1, v4::GROVE_V4}; + + // 7 leaves gives three peaks (4 + 2 + 1); a proof for the first leaf + // leaves the two right-hand peaks to bag: one merge. + let store = MemStore::default(); + let mut mmr = MMR::new(0, &store); + let mut positions = Vec::new(); + for i in 0..7 { + positions.push( + mmr.push(leaf(i), GroveVersion::latest()) + .unwrap() + .expect("push"), + ); + } + + let v0 = mmr.gen_proof(vec![positions[0]], &GROVE_V1); + let v0_proof = v0.value.expect("proof"); + assert_eq!(v0.cost.hash_node_calls, 0, "v0 charges no bagging merges"); + + let v1 = mmr.gen_proof(vec![positions[0]], &GROVE_V4); + let v1_proof = v1.value.expect("proof"); + assert_eq!( + v1.cost.hash_node_calls, 1, + "v1 charges the one fold, got {:?}", + v1.cost + ); + + assert_eq!( + v0_proof.proof_items(), + v1_proof.proof_items(), + "the proof must not depend on the cost version" + ); + + // A single perfect peak has nothing to bag under either version. + let store = MemStore::default(); + let mut mmr = MMR::new(0, &store); + let mut positions = Vec::new(); + for i in 0..4 { + positions.push( + mmr.push(leaf(i), GroveVersion::latest()) + .unwrap() + .expect("push"), + ); + } + let ctx = mmr.gen_proof(vec![positions[0]], &GROVE_V4); + ctx.value.expect("proof"); + assert_eq!(ctx.cost.hash_node_calls, 0, "one peak means no bagging"); +} + +/// An unknown version must be rejected rather than silently falling back to +/// one of the implemented charges. +#[test] +fn mmr_cost_dispatch_rejects_unknown_version() { + use grovedb_version::version::{v4::GROVE_V4, GroveVersion}; + + let store = MemStore::default(); + let mut mmr = MMR::new(0, &store); + for i in 0..3 { + mmr.push(leaf(i), GroveVersion::latest()) + .unwrap() + .expect("push"); + } + + let mut bad: GroveVersion = GROVE_V4.clone(); + bad.mmr_versions.cost.get_root = 99; + assert!( + matches!(mmr.get_root(&bad).unwrap(), Err(Error::VersionError(_))), + "an unknown get_root charge version must error" + ); + + let mut bad: GroveVersion = GROVE_V4.clone(); + bad.mmr_versions.cost.push = 99; + assert!( + matches!( + mmr.push(leaf(9), &bad).unwrap(), + Err(Error::VersionError(_)) + ), + "an unknown push charge version must error" + ); + + let mut bad: GroveVersion = GROVE_V4.clone(); + bad.mmr_versions.cost.gen_proof = 99; + assert!( + matches!( + mmr.gen_proof(vec![0], &bad).unwrap(), + Err(Error::VersionError(_)) + ), + "an unknown gen_proof charge version must error" + ); +} diff --git a/grovedb-merkle-mountain-range/src/tests/test_helper.rs b/grovedb-merkle-mountain-range/src/tests/test_helper.rs index 0162410bf..7336b8018 100644 --- a/grovedb-merkle-mountain-range/src/tests/test_helper.rs +++ b/grovedb-merkle-mountain-range/src/tests/test_helper.rs @@ -1,3 +1,4 @@ +use grovedb_version::version::GroveVersion; use lazy_static::lazy_static; use proptest::prelude::*; @@ -19,7 +20,7 @@ lazy_static! { let store = MemStore::default(); let mut mmr = MMR::new(0, &store); (0u32..100_000) - .map(|i| mmr.push(leaf_from_u32(i)).unwrap().expect("push")) + .map(|i| mmr.push(leaf_from_u32(i), GroveVersion::latest()).unwrap().expect("push")) .collect() }; /// mmr size when 0..100_000 elem @@ -28,7 +29,7 @@ lazy_static! { let mut mmr = MMR::new(0, &store); (0u32..100_000) .map(|i| { - mmr.push(leaf_from_u32(i)).unwrap().expect("push"); + mmr.push(leaf_from_u32(i), GroveVersion::latest()).unwrap().expect("push"); mmr.mmr_size }) .collect() diff --git a/grovedb-merkle-mountain-range/src/tests/test_incremental.rs b/grovedb-merkle-mountain-range/src/tests/test_incremental.rs index 3db97b18d..52f3ebcd1 100644 --- a/grovedb-merkle-mountain-range/src/tests/test_incremental.rs +++ b/grovedb-merkle-mountain-range/src/tests/test_incremental.rs @@ -1,6 +1,7 @@ use proptest::proptest; use crate::{mem_store::MemStore, MmrNode, MMR}; +use grovedb_version::version::GroveVersion; /// Create an MmrNode leaf from an integer (for test convenience). fn leaf_from_u32(i: u32) -> MmrNode { @@ -22,7 +23,10 @@ fn test_incremental_with_params(start: u32, steps: usize, turns: usize) { let _positions: Vec = (0u32..start) .map(|_| { - let pos = mmr.push(leaf_from_u32(curr)).unwrap().expect("push"); + let pos = mmr + .push(leaf_from_u32(curr), GroveVersion::latest()) + .unwrap() + .expect("push"); curr += 1; pos }) @@ -30,12 +34,18 @@ fn test_incremental_with_params(start: u32, steps: usize, turns: usize) { mmr.commit().unwrap().expect("commit changes"); for turn in 0..turns { - let prev_root = mmr.get_root().unwrap().expect("get root"); + let prev_root = mmr + .get_root(GroveVersion::latest()) + .unwrap() + .expect("get root"); let (positions, leaves) = (0..steps).fold( (Vec::new(), Vec::new()), |(mut positions, mut leaves), _| { let leaf = leaf_from_u32(curr); - let pos = mmr.push(leaf.clone()).unwrap().expect("push"); + let pos = mmr + .push(leaf.clone(), GroveVersion::latest()) + .unwrap() + .expect("push"); curr += 1; positions.push(pos); leaves.push(leaf); @@ -43,8 +53,14 @@ fn test_incremental_with_params(start: u32, steps: usize, turns: usize) { }, ); mmr.commit().unwrap().expect("commit changes"); - let proof = mmr.gen_proof(positions).unwrap().expect("gen proof"); - let root = mmr.get_root().unwrap().expect("get root"); + let proof = mmr + .gen_proof(positions, GroveVersion::latest()) + .unwrap() + .expect("gen proof"); + let root = mmr + .get_root(GroveVersion::latest()) + .unwrap() + .expect("get root"); let result = proof .verify_incremental(root, prev_root, leaves) .expect("verify_incremental"); diff --git a/grovedb-merkle-mountain-range/src/tests/test_mmr.rs b/grovedb-merkle-mountain-range/src/tests/test_mmr.rs index a269e8a5b..3a3df26bc 100644 --- a/grovedb-merkle-mountain-range/src/tests/test_mmr.rs +++ b/grovedb-merkle-mountain-range/src/tests/test_mmr.rs @@ -1,4 +1,5 @@ use faster_hex::hex_string; +use grovedb_version::version::GroveVersion; use proptest::prelude::*; use rand::{seq::SliceRandom, RngExt}; @@ -16,15 +17,23 @@ fn test_mmr(count: u32, proof_elem: Vec) { let store = MemStore::default(); let mut mmr = MMR::new(0, &store); let positions: Vec = (0u32..count) - .map(|i| mmr.push(leaf_from_u32(i)).unwrap().expect("push")) + .map(|i| { + mmr.push(leaf_from_u32(i), GroveVersion::latest()) + .unwrap() + .expect("push") + }) .collect(); - let root = mmr.get_root().unwrap().expect("get root"); + let root = mmr + .get_root(GroveVersion::latest()) + .unwrap() + .expect("get root"); let proof = mmr .gen_proof( proof_elem .iter() .map(|elem| positions[*elem as usize]) .collect(), + GroveVersion::latest(), ) .unwrap() .expect("gen proof"); @@ -45,17 +54,27 @@ fn test_gen_new_root_from_proof(count: u32) { let store = MemStore::default(); let mut mmr = MMR::new(0, &store); let positions: Vec = (0u32..count) - .map(|i| mmr.push(leaf_from_u32(i)).unwrap().expect("push")) + .map(|i| { + mmr.push(leaf_from_u32(i), GroveVersion::latest()) + .unwrap() + .expect("push") + }) .collect(); let elem = count - 1; let pos = positions[elem as usize]; - let proof = mmr.gen_proof(vec![pos]).unwrap().expect("gen proof"); + let proof = mmr + .gen_proof(vec![pos], GroveVersion::latest()) + .unwrap() + .expect("gen proof"); let new_elem = count; let new_pos = mmr - .push(leaf_from_u32(new_elem)) + .push(leaf_from_u32(new_elem), GroveVersion::latest()) .unwrap() .expect("push new"); - let root = mmr.get_root().unwrap().expect("get root"); + let root = mmr + .get_root(GroveVersion::latest()) + .unwrap() + .expect("get root"); mmr.commit().unwrap().expect("commit changes"); let calculated_root = proof .calculate_root_with_new_leaf( @@ -73,9 +92,14 @@ fn test_mmr_root() { let store = MemStore::default(); let mut mmr = MMR::new(0, &store); (0u32..11).for_each(|i| { - mmr.push(leaf_from_u32(i)).unwrap().expect("push"); + mmr.push(leaf_from_u32(i), GroveVersion::latest()) + .unwrap() + .expect("push"); }); - let root = mmr.get_root().unwrap().expect("get root"); + let root = mmr + .get_root(GroveVersion::latest()) + .unwrap() + .expect("get root"); let hex_root = hex_string(&root.hash()); // This is the deterministic root for 11 leaves with MmrNode/blake3 assert_eq!(hex_root.len(), 64, "root hash should be 32 bytes hex"); @@ -85,7 +109,10 @@ fn test_mmr_root() { fn test_empty_mmr_root() { let store = MemStore::default(); let mmr = MMR::new(0, &store); - assert_eq!(Err(Error::GetRootOnEmpty), mmr.get_root().unwrap()); + assert_eq!( + Err(Error::GetRootOnEmpty), + mmr.get_root(GroveVersion::latest()).unwrap() + ); } #[test] @@ -165,10 +192,16 @@ fn test_invalid_proof_verification( let mut mmr = MMR::new(0, &store); let mut positions: Vec = Vec::new(); for i in 0u32..leaf_count { - let pos = mmr.push(leaf_from_u32(i)).unwrap().expect("push"); + let pos = mmr + .push(leaf_from_u32(i), GroveVersion::latest()) + .unwrap() + .expect("push"); positions.push(pos); } - let root = mmr.get_root().unwrap().expect("get root"); + let root = mmr + .get_root(GroveVersion::latest()) + .unwrap() + .expect("get root"); let entries_to_verify: Vec<(u64, MmrNode)> = positions_to_verify .iter() @@ -216,7 +249,10 @@ fn test_invalid_proof_verification( assert!(handrolled_proof_result.is_err() || !handrolled_proof_result.expect("verify")); } - match mmr.gen_proof(positions_to_verify.clone()).unwrap() { + match mmr + .gen_proof(positions_to_verify.clone(), GroveVersion::latest()) + .unwrap() + { Ok(proof) => { assert!(proof .verify(root.clone(), entries_to_verify) @@ -265,7 +301,10 @@ fn test_batch_cache_hit_returns_nonzero_cost() { // Push a leaf — it goes into MMRBatch.memory_batch let leaf = MmrNode::leaf(b"test value".to_vec()); let expected_size = leaf.serialized_size(); - let pos = mmr.push(leaf).unwrap().expect("push should succeed"); + let pos = mmr + .push(leaf, GroveVersion::latest()) + .unwrap() + .expect("push should succeed"); // Before commit, read from the batch (cache hit) let cost_result = mmr.batch.element_at_position(pos); @@ -290,7 +329,7 @@ fn test_push_cost_includes_read_costs() { let mut mmr = MMR::new(0, &store); // First push — no merging needed, no sibling reads - mmr.push(MmrNode::leaf(b"leaf0".to_vec())) + mmr.push(MmrNode::leaf(b"leaf0".to_vec()), GroveVersion::latest()) .unwrap() .expect("push should succeed"); @@ -299,10 +338,10 @@ fn test_push_cost_includes_read_costs() { let mut mmr = MMR::new(0, &store); // Push two leaves — second push triggers a merge with the first - let push0_result = mmr.push(MmrNode::leaf(b"leaf0".to_vec())); + let push0_result = mmr.push(MmrNode::leaf(b"leaf0".to_vec()), GroveVersion::latest()); let push0_cost = push0_result.cost; - let push1_result = mmr.push(MmrNode::leaf(b"leaf1".to_vec())); + let push1_result = mmr.push(MmrNode::leaf(b"leaf1".to_vec()), GroveVersion::latest()); let push1_cost = push1_result.cost; // Second push should have higher cost (reads the first leaf for merging) @@ -323,12 +362,12 @@ fn test_get_root_cost_reflects_peak_reads() { // Push 3 leaves → mmr_size=4, 2 peaks (pos 2 and pos 3) for i in 0..3u8 { - mmr.push(MmrNode::leaf(vec![i])) + mmr.push(MmrNode::leaf(vec![i]), GroveVersion::latest()) .unwrap() .expect("push should succeed"); } - let root_result = mmr.get_root(); + let root_result = mmr.get_root(GroveVersion::latest()); let root_cost = root_result.cost; // With 2 peaks, get_root reads 2 nodes → at least 2 seeks @@ -351,14 +390,20 @@ fn test_mmr_tree_proof_standard_leaf_verify_succeeds() { // Push standard leaves — leaf_hash(value) matches the stored hash for i in 0u32..5 { - mmr.push(MmrNode::leaf(i.to_le_bytes().to_vec())) - .unwrap() - .expect("push should succeed"); + mmr.push( + MmrNode::leaf(i.to_le_bytes().to_vec()), + GroveVersion::latest(), + ) + .unwrap() + .expect("push should succeed"); } mmr.commit().unwrap().expect("commit should succeed"); let mmr_size = mmr.mmr_size; - let root = mmr.get_root().unwrap().expect("get root should succeed"); + let root = mmr + .get_root(GroveVersion::latest()) + .unwrap() + .expect("get root should succeed"); let get_node = |pos: u64| -> crate::Result> { (&store) @@ -388,9 +433,14 @@ fn test_single_element_mmr_root() { let mut mmr = MMR::new(0, &store); let leaf = MmrNode::leaf(b"only leaf".to_vec()); let expected_hash = leaf.hash(); - mmr.push(leaf).unwrap().expect("push should succeed"); + mmr.push(leaf, GroveVersion::latest()) + .unwrap() + .expect("push should succeed"); - let root = mmr.get_root().unwrap().expect("get_root should succeed"); + let root = mmr + .get_root(GroveVersion::latest()) + .unwrap() + .expect("get_root should succeed"); assert_eq!( root.hash(), expected_hash, @@ -403,13 +453,13 @@ fn test_single_element_mmr_root() { fn test_gen_proof_empty_positions() { let store = MemStore::default(); let mut mmr = MMR::new(0, &store); - mmr.push(MmrNode::leaf(b"leaf".to_vec())) + mmr.push(MmrNode::leaf(b"leaf".to_vec()), GroveVersion::latest()) .unwrap() .expect("push should succeed"); assert!( matches!( - mmr.gen_proof(vec![]).unwrap(), + mmr.gen_proof(vec![], GroveVersion::latest()).unwrap(), Err(Error::GenProofForInvalidLeaves) ), "should reject empty positions" @@ -422,12 +472,14 @@ fn test_gen_proof_rejects_internal_positions() { let store = MemStore::default(); let mut mmr = MMR::new(0, &store); for i in 0u32..4 { - mmr.push(leaf_from_u32(i)).unwrap().expect("push"); + mmr.push(leaf_from_u32(i), GroveVersion::latest()) + .unwrap() + .expect("push"); } // Position 2 is an internal node (height 1, merge of pos 0 and 1) assert!( matches!( - mmr.gen_proof(vec![2]).unwrap(), + mmr.gen_proof(vec![2], GroveVersion::latest()).unwrap(), Err(Error::NodeProofsNotSupported) ), "should reject internal node positions" @@ -440,12 +492,14 @@ fn test_gen_proof_out_of_range_leaf_positions() { let store = MemStore::default(); let mut mmr = MMR::new(0, &store); for i in 0u32..4 { - mmr.push(leaf_from_u32(i)).unwrap().expect("push"); + mmr.push(leaf_from_u32(i), GroveVersion::latest()) + .unwrap() + .expect("push"); } // mmr_size = 7. Position 7 is a leaf position (height 0) but beyond range. assert!( matches!( - mmr.gen_proof(vec![7]).unwrap(), + mmr.gen_proof(vec![7], GroveVersion::latest()).unwrap(), Err(Error::GenProofForInvalidLeaves) ), "should reject leaf positions beyond MMR range" @@ -460,14 +514,21 @@ fn test_gen_proof_bags_trailing_peaks() { let mut mmr = MMR::new(0, &store); // 11 leaves → mmr_size=19, 3 peaks at positions [14, 17, 18] let positions: Vec = (0u32..11) - .map(|i| mmr.push(leaf_from_u32(i)).unwrap().expect("push")) + .map(|i| { + mmr.push(leaf_from_u32(i), GroveVersion::latest()) + .unwrap() + .expect("push") + }) .collect(); - let root = mmr.get_root().unwrap().expect("get_root"); + let root = mmr + .get_root(GroveVersion::latest()) + .unwrap() + .expect("get_root"); // Prove only leaf 0 (under peak 14). Peaks 17, 18 have no proved leaves // → bagging_track = 2 → triggers bag_peaks of the trailing peaks. let proof = mmr - .gen_proof(vec![positions[0]]) + .gen_proof(vec![positions[0]], GroveVersion::latest()) .unwrap() .expect("gen_proof should succeed"); let valid = proof @@ -496,9 +557,14 @@ fn test_verify_incremental_success() { // Build initial MMR with 4 leaves → single peak at position 6 for i in 0u32..4 { - mmr.push(leaf_from_u32(i)).unwrap().expect("push"); + mmr.push(leaf_from_u32(i), GroveVersion::latest()) + .unwrap() + .expect("push"); } - let prev_root = mmr.get_root().unwrap().expect("prev root"); + let prev_root = mmr + .get_root(GroveVersion::latest()) + .unwrap() + .expect("prev root"); let peak_node = mmr .batch .element_at_position(6) @@ -509,9 +575,14 @@ fn test_verify_incremental_success() { // Add 3 more incremental leaves let incremental_leaves: Vec = (4u32..7).map(leaf_from_u32).collect(); for leaf in &incremental_leaves { - mmr.push(leaf.clone()).unwrap().expect("push incremental"); + mmr.push(leaf.clone(), GroveVersion::latest()) + .unwrap() + .expect("push incremental"); } - let current_root = mmr.get_root().unwrap().expect("current root"); + let current_root = mmr + .get_root(GroveVersion::latest()) + .unwrap() + .expect("current root"); // Proof items = previous peak hashes (just [peak at 6]) let proof = MerkleProof::new(mmr.mmr_size, vec![peak_node]); @@ -529,7 +600,9 @@ fn test_verify_incremental_wrong_prev_root() { let mut mmr = MMR::new(0, &store); for i in 0u32..4 { - mmr.push(leaf_from_u32(i)).unwrap().expect("push"); + mmr.push(leaf_from_u32(i), GroveVersion::latest()) + .unwrap() + .expect("push"); } let peak_node = mmr .batch @@ -540,9 +613,14 @@ fn test_verify_incremental_wrong_prev_root() { let incremental_leaves: Vec = (4u32..7).map(leaf_from_u32).collect(); for leaf in &incremental_leaves { - mmr.push(leaf.clone()).unwrap().expect("push incremental"); + mmr.push(leaf.clone(), GroveVersion::latest()) + .unwrap() + .expect("push incremental"); } - let current_root = mmr.get_root().unwrap().expect("current root"); + let current_root = mmr + .get_root(GroveVersion::latest()) + .unwrap() + .expect("current root"); let proof = MerkleProof::new(mmr.mmr_size, vec![peak_node]); let wrong_prev = MmrNode::internal([0xFFu8; 32]); diff --git a/grovedb-merkle-mountain-range/src/tests/test_storage_adapter.rs b/grovedb-merkle-mountain-range/src/tests/test_storage_adapter.rs index af856bb17..fd0cda146 100644 --- a/grovedb-merkle-mountain-range/src/tests/test_storage_adapter.rs +++ b/grovedb-merkle-mountain-range/src/tests/test_storage_adapter.rs @@ -5,6 +5,7 @@ use grovedb_costs::{ CostResult, CostsExt, OperationCost, }; use grovedb_storage::StorageContext; +use grovedb_version::version::GroveVersion; use crate::{ helper::{mmr_node_key_sized, MmrKeySize}, @@ -485,18 +486,27 @@ fn mmr_store_full_mmr_roundtrip() { // Use MmrStore as the backend for a full MMR let mut mmr = MMR::new(0, &store); for i in 0u32..7 { - mmr.push(MmrNode::leaf(i.to_le_bytes().to_vec())) - .unwrap() - .expect("push should succeed"); - } - let root_before_commit = mmr.get_root().unwrap().expect("get_root should succeed"); + mmr.push( + MmrNode::leaf(i.to_le_bytes().to_vec()), + GroveVersion::latest(), + ) + .unwrap() + .expect("push should succeed"); + } + let root_before_commit = mmr + .get_root(GroveVersion::latest()) + .unwrap() + .expect("get_root should succeed"); // Commit to storage mmr.commit().unwrap().expect("commit should succeed"); // Re-open MMR from the same store and verify root let mmr2 = MMR::new(mmr.mmr_size, &store); - let root_after_reopen = mmr2.get_root().unwrap().expect("get_root should succeed"); + let root_after_reopen = mmr2 + .get_root(GroveVersion::latest()) + .unwrap() + .expect("get_root should succeed"); assert_eq!( root_before_commit.hash(), diff --git a/grovedb-private-document-store/Cargo.toml b/grovedb-private-document-store/Cargo.toml new file mode 100644 index 000000000..25df42d92 --- /dev/null +++ b/grovedb-private-document-store/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "grovedb-private-document-store" +version = "5.0.1" +authors = ["Samuel Westrich "] +edition = "2024" +license = "MIT" +description = "PrivateDocumentStore: append-only store of fixed-size opaque entries with committed config for GroveDB" +homepage = "https://www.grovedb.org" +repository = "https://github.com/dashpay/grovedb" +readme = "../README.md" +documentation = "https://docs.rs/grovedb" + +[features] +default = ["storage"] +storage = ["grovedb-storage", "grovedb-bulk-append-tree/storage"] + +[dependencies] +grovedb-bulk-append-tree = { version = "5.0.1", path = "../grovedb-bulk-append-tree", default-features = false } +grovedb-costs = { version = "5.0.1", path = "../costs" } +grovedb-version = { version = "5.0.1", path = "../grovedb-version" } +grovedb-storage = { version = "5.0.1", path = "../storage", optional = true } +blake3 = { workspace = true } +thiserror = { workspace = true } + +[dev-dependencies] +# Reuses the in-memory StorageContext harness rather than copying it; see the +# `test-utils` feature on grovedb-bulk-append-tree. +grovedb-bulk-append-tree = { version = "5.0.1", path = "../grovedb-bulk-append-tree", features = ["test-utils"] } diff --git a/grovedb-private-document-store/src/error.rs b/grovedb-private-document-store/src/error.rs new file mode 100644 index 000000000..d7e1565b2 --- /dev/null +++ b/grovedb-private-document-store/src/error.rs @@ -0,0 +1,27 @@ +//! Error type for the private document store. + +/// Errors returned by [`PrivateDocumentStore`](crate::PrivateDocumentStore) +/// operations. +#[derive(Debug, thiserror::Error)] +pub enum PrivateDocumentStoreError { + /// The store configuration is invalid (zero entry size, bad chunk power). + #[error("invalid private document store config: {0}")] + InvalidConfig(String), + + /// An entry's byte length does not match the committed `entry_size`. + #[error("invalid entry size: expected {expected} bytes, got {actual}")] + InvalidEntrySize { + /// The committed entry size of the store. + expected: u32, + /// The actual length of the offered entry. + actual: usize, + }, + + /// Underlying data is missing or inconsistent. + #[error("corrupted private document store data: {0}")] + CorruptedData(String), + + /// Wrapped storage / bulk tree failure. + #[error("private document store data error: {0}")] + InvalidData(String), +} diff --git a/grovedb-private-document-store/src/lib.rs b/grovedb-private-document-store/src/lib.rs new file mode 100644 index 000000000..0e484f5bb --- /dev/null +++ b/grovedb-private-document-store/src/lib.rs @@ -0,0 +1,160 @@ +#![deny(missing_docs)] +//! PrivateDocumentStore: an append-only store of fixed-size opaque entries +//! for GroveDB. +//! +//! This crate is a thin wrapper over [`BulkAppendTree`] — the same +//! relationship `grovedb-commitment-tree` has to it, but with **no +//! Sinsemilla frontier**: entries here are write-once and never proven +//! against later, so no anchor is needed. +//! +//! # Committed configuration +//! +//! The store's configuration `{entry_size, chunk_power}` is bound into the +//! state root: +//! +//! ```text +//! pds_state_root = blake3("pds_state" || config_hash || bulk_state_root) +//! config_hash = blake3("pds_config" || entry_size_be(4) || chunk_power(1)) +//! ``` +//! +//! so the declared entry size is consensus-visible and a proof can never be +//! reinterpreted under a different configuration. +//! +//! # Platform use +//! +//! Dash Platform stores each private document as a hiding commitment plus a +//! ciphertext of uniform, contract-declared size. GroveDB never interprets a +//! "document" — behaviorally this is a fully generic append-only log of +//! fixed-size opaque entries; the name simply describes its Platform use. + +mod error; +#[cfg(feature = "storage")] +mod store; + +pub use error::PrivateDocumentStoreError; +pub use grovedb_bulk_append_tree::{ + deserialize_chunk_blob, serialize_chunk_blob, BulkAppendError, BulkAppendTree, +}; +#[cfg(feature = "storage")] +pub use store::{ + PrivateDocumentStore, PrivateDocumentStoreAppendManyResult, PrivateDocumentStoreAppendResult, +}; + +/// Pre-computed state root of an empty [`BulkAppendTree`]: +/// `blake3("bulk_state" || [0; 32] || [0; 32])`. +/// +/// The empty bulk root is independent of `chunk_power` (an empty MMR and an +/// empty dense tree both contribute a zero hash regardless of height), so it +/// can be a true constant. The full empty *store* root additionally binds the +/// configuration and is therefore a function — +/// [`empty_private_document_store_state_root`]. +/// +/// The `test_empty_bulk_append_tree_state_root_constant` test pins this +/// constant to the runtime computation. +pub const EMPTY_BULK_APPEND_TREE_STATE_ROOT: [u8; 32] = [ + 0x41, 0xe0, 0x80, 0xa7, 0xfc, 0x26, 0x32, 0x3a, 0x1a, 0x44, 0x90, 0x5d, 0xa2, 0x0d, 0x6d, 0x59, + 0x85, 0x11, 0xf8, 0x39, 0xef, 0xd7, 0x03, 0x42, 0xe2, 0x1e, 0x7e, 0xdc, 0xd5, 0xc3, 0xff, 0x61, +]; + +/// Compute the committed-configuration hash for a private document store. +/// +/// `config_hash = blake3("pds_config" || entry_size.to_be_bytes() || [chunk_power])` +/// +/// The fixed-width big-endian encoding (4 bytes for `entry_size`, 1 byte for +/// `chunk_power`) makes the preimage unambiguous. +pub fn private_document_store_config_hash(entry_size: u32, chunk_power: u8) -> [u8; 32] { + let mut hasher = blake3::Hasher::new(); + hasher.update(b"pds_config"); + hasher.update(&entry_size.to_be_bytes()); + hasher.update(&[chunk_power]); + *hasher.finalize().as_bytes() +} + +/// Compute the combined PrivateDocumentStore state root that binds the +/// committed configuration to the [`BulkAppendTree`] data root. +/// +/// `pds_state_root = blake3("pds_state" || config_hash || bulk_state_root)` +/// +/// This is the value that flows as the Merk child hash, ensuring both the +/// configuration (entry size, chunk power) and the appended data are +/// authenticated by the GroveDB root hash. +pub fn compute_private_document_store_state_root( + config_hash: &[u8; 32], + bulk_state_root: &[u8; 32], +) -> [u8; 32] { + let mut hasher = blake3::Hasher::new(); + hasher.update(b"pds_state"); + hasher.update(config_hash); + hasher.update(bulk_state_root); + *hasher.finalize().as_bytes() +} + +/// The state root of an empty private document store with the given +/// configuration. +/// +/// Unlike `EMPTY_COMMITMENT_TREE_STATE_ROOT` (a constant — the commitment +/// tree root does not bind its configuration), the empty PDS root depends on +/// `{entry_size, chunk_power}`, so it is a function built from the +/// pre-computed [`EMPTY_BULK_APPEND_TREE_STATE_ROOT`] constant: two blake3 +/// calls instead of opening the tree. +pub fn empty_private_document_store_state_root(entry_size: u32, chunk_power: u8) -> [u8; 32] { + compute_private_document_store_state_root( + &private_document_store_config_hash(entry_size, chunk_power), + &EMPTY_BULK_APPEND_TREE_STATE_ROOT, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_empty_bulk_append_tree_state_root_constant() { + let null = [0u8; 32]; + let computed = grovedb_bulk_append_tree::compute_state_root(&null, &null); + assert_eq!( + computed, EMPTY_BULK_APPEND_TREE_STATE_ROOT, + "EMPTY_BULK_APPEND_TREE_STATE_ROOT constant does not match runtime computation" + ); + } + + /// Pin the full empty-store root for one canonical configuration so any + /// accidental change to a domain tag or preimage encoding is caught. + #[test] + fn test_empty_private_document_store_state_root_pinned_vector() { + let expected: [u8; 32] = [ + 0x56, 0x45, 0xf9, 0x7a, 0xe8, 0x5d, 0xba, 0xec, 0x29, 0x55, 0x47, 0x7a, 0x61, 0x65, + 0xdc, 0xbd, 0x17, 0xa6, 0x40, 0x71, 0xd5, 0x5a, 0xed, 0x9f, 0x0f, 0xf7, 0xe5, 0xff, + 0x41, 0xe0, 0x7f, 0xd9, + ]; + assert_eq!(empty_private_document_store_state_root(64, 4), expected); + } + + /// The state root must change when either configuration parameter + /// changes — that is the whole point of binding the config. + #[test] + fn test_state_root_binds_configuration() { + let base = empty_private_document_store_state_root(64, 4); + assert_ne!(base, empty_private_document_store_state_root(65, 4)); + assert_ne!(base, empty_private_document_store_state_root(64, 5)); + // Field boundaries are unambiguous: (entry_size, chunk_power) + // pairs that would collide under a length-prefix-free encoding + // must still differ thanks to the fixed-width layout. + assert_ne!( + private_document_store_config_hash(0x0102, 0x03), + private_document_store_config_hash(0x01, 0x02), + ); + } + + /// The PDS domain tags must not collide with the commitment tree's + /// `"ct_state"` or the bulk tree's `"bulk_state"` domains for identical + /// 64-byte payloads. + #[test] + fn test_domain_separation() { + let a = [7u8; 32]; + let b = [9u8; 32]; + let pds = compute_private_document_store_state_root(&a, &b); + let bulk = grovedb_bulk_append_tree::compute_state_root(&a, &b); + assert_ne!(pds, bulk); + } +} diff --git a/grovedb-private-document-store/src/store.rs b/grovedb-private-document-store/src/store.rs new file mode 100644 index 000000000..d28bcb0bb --- /dev/null +++ b/grovedb-private-document-store/src/store.rs @@ -0,0 +1,1435 @@ +//! Storage adapter bridging GroveDB's `StorageContext` to the private +//! document store. +//! +//! Provides [`PrivateDocumentStore`], a thin wrapper owning a +//! [`BulkAppendTree`] plus the committed configuration `{entry_size, +//! chunk_power}`. Unlike the commitment tree there is no Sinsemilla +//! frontier and no extra persisted state: everything lives in the bulk +//! tree; the wrapper adds entry-size validation and the config-binding +//! state root. + +use grovedb_bulk_append_tree::BulkAppendTree; +use grovedb_costs::{CostResult, CostsExt, OperationCost}; +use grovedb_storage::StorageContext; +use grovedb_version::version::GroveVersion; + +use crate::{ + compute_private_document_store_state_root, private_document_store_config_hash, + PrivateDocumentStoreError, +}; + +/// Result of appending to a [`PrivateDocumentStore`]. +#[derive(Debug, Clone)] +pub struct PrivateDocumentStoreAppendResult { + /// The new composite state root + /// (`blake3("pds_state" || config_hash || bulk_state_root)`). + /// This flows as the Merk child hash via `insert_subtree`. + pub state_root: [u8; 32], + /// The underlying BulkAppendTree state root. + pub bulk_state_root: [u8; 32], + /// The 0-based global position of the appended entry. + pub global_position: u64, + /// Number of blake3 hash calls performed during the bulk append. + pub hash_count: u32, + /// Whether compaction (epoch flush) occurred during this append. + pub compacted: bool, +} + +/// Result of [`PrivateDocumentStore::append_many`]. +/// +/// Distinct from [`PrivateDocumentStoreAppendResult`] because a batch may be +/// empty, and there is then no appended position to report. Reporting a +/// sentinel (position 0 on a fresh store, or the previous last entry on a +/// populated one) would be indistinguishable from a real append. +#[derive(Debug, Clone)] +pub struct PrivateDocumentStoreAppendManyResult { + /// The new composite state root. + pub state_root: [u8; 32], + /// The underlying BulkAppendTree state root. + pub bulk_state_root: [u8; 32], + /// Position of the last entry appended BY THIS CALL, or `None` when the + /// input was empty and nothing was written. + pub last_global_position: Option, + /// How many entries this call appended. + pub appended: u64, + /// Number of blake3 hash calls performed. + pub hash_count: u32, + /// Whether compaction occurred during this call. + pub compacted: bool, +} + +/// An append-only store of fixed-size opaque entries. +/// +/// Thin wrapper over [`BulkAppendTree`]: appends are validated against the +/// committed `entry_size` and the state root binds the configuration, so a +/// proof can never be reinterpreted under a different config. There is no +/// per-entry delete or update — immutability is enforced by the type. +pub struct PrivateDocumentStore { + entry_size: u32, + config_hash: [u8; 32], + pub(crate) bulk_tree: BulkAppendTree, +} + +impl std::fmt::Debug for PrivateDocumentStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PrivateDocumentStore") + .field("entry_size", &self.entry_size) + .field("chunk_power", &self.bulk_tree.height()) + .field("total_count", &self.bulk_tree.total_count) + .finish_non_exhaustive() + } +} + +impl<'db, S: StorageContext<'db>> PrivateDocumentStore { + /// Create a new empty private document store. + /// + /// `entry_size` is the committed byte length of every entry (must be + /// non-zero). `chunk_power` is the log2 of the epoch size for the + /// underlying [`BulkAppendTree`] (its dense-buffer height, 1–16). + pub fn new( + entry_size: u32, + chunk_power: u8, + storage: S, + ) -> CostResult { + Self::from_state(0, entry_size, chunk_power, storage) + } + + /// Restore a private document store from persisted state. + /// + /// Purely in-memory reconstruction: the bulk tree derives its chunk and + /// buffer counts from `total_count` and `chunk_power`; no storage reads + /// happen until an entry is accessed or appended. + pub fn from_state( + total_count: u64, + entry_size: u32, + chunk_power: u8, + storage: S, + ) -> CostResult { + let mut cost = OperationCost::default(); + if entry_size == 0 || entry_size > u16::MAX as u32 { + return Err(PrivateDocumentStoreError::InvalidConfig( + "entry_size must be in 1..=65535".to_string(), + )) + .wrap_with_cost(cost); + } + let bulk_tree = match BulkAppendTree::from_state(total_count, chunk_power, storage) { + Ok(t) => t, + Err(e) => { + return Err(PrivateDocumentStoreError::InvalidData(format!( + "bulk tree: {}", + e + ))) + .wrap_with_cost(cost); + } + }; + // Opening the store derives the committed-config hash, which is one + // blake3 call — charged here so every path that opens a store pays + // for it rather than getting it for free. + cost.hash_node_calls += 1; + Ok(Self { + entry_size, + config_hash: private_document_store_config_hash(entry_size, chunk_power), + bulk_tree, + }) + .wrap_with_cost(cost) + } + + /// Append an entry to the store. + /// + /// Validates that `entry.len()` equals the committed `entry_size` before + /// any mutation, then appends to the underlying [`BulkAppendTree`] and + /// returns the new composite state root. + pub fn append( + &mut self, + entry: &[u8], + grove_version: &GroveVersion, + ) -> CostResult { + let mut cost = OperationCost::default(); + + // Size validation is NOT repeated here: `append_many` validates every + // entry before writing any, and returns the same + // `InvalidEntrySize { expected, actual }` with the same (empty) cost. + // A second copy here would be one more place to drift. + // + // Run the single entry through the batch path rather than + // `BulkAppendTree::append`. + // + // `BulkAppendTree::append` walks the dense buffer TWICE per entry: + // once inside `append_no_state_root`, whose dense root nothing here + // reads, and again in `compute_current_state_root` to derive the root + // we actually return. It also returns a plain `Result`, so the second + // walk's storage reads and hash calls are discarded outright — a + // one-entry append billed 4 hash calls while performing 6. The + // deferred path walks once and bills what it walks, and sharing one + // implementation keeps the single and batch paths from drifting. + let many = match self + .append_many(core::iter::once(entry), grove_version) + .unwrap_add_cost(&mut cost) + { + Ok(r) => r, + Err(e) => return Err(e).wrap_with_cost(cost), + }; + + // Exactly one entry was supplied and `append_many` returned `Ok`, so + // it appended it and recorded the position. `expect` rather than an + // error arm: a `None` here would mean `append_many` reported success + // without appending, which is a broken postcondition in this file, not + // a runtime condition a caller can produce or handle. + let global_position = many + .last_global_position + .expect("append_many returned Ok for one entry without a position"); + + Ok(PrivateDocumentStoreAppendResult { + state_root: many.state_root, + bulk_state_root: many.bulk_state_root, + global_position, + hash_count: many.hash_count, + compacted: many.compacted, + }) + .wrap_with_cost(cost) + } + + /// Get an entry by its global 0-based position. + /// + /// Returns `None` when `global_position >= total_count`. Dispatches to + /// the current dense-tree buffer or a completed chunk blob as + /// appropriate. + pub fn get_value( + &self, + global_position: u64, + ) -> CostResult>, PrivateDocumentStoreError> { + let mut cost = OperationCost::default(); + + if global_position >= self.bulk_tree.total_count { + return Ok(None).wrap_with_cost(cost); + } + + let epoch_size = self.bulk_tree.epoch_size(); + let chunk_count = self.bulk_tree.chunk_count(); + let buffer_start = chunk_count * epoch_size; + + let value = if global_position >= buffer_start { + // Entry is in the current buffer. + let buffer_pos = (global_position - buffer_start) as u16; + match self + .bulk_tree + .get_buffer_value_with_cost(buffer_pos) + .unwrap_add_cost(&mut cost) + { + Ok(v) => v, + Err(e) => { + return Err(PrivateDocumentStoreError::InvalidData(format!("{}", e))) + .wrap_with_cost(cost); + } + } + } else { + // Entry is in a completed chunk. + let chunk_idx = global_position / epoch_size; + let pos_in_chunk = (global_position % epoch_size) as usize; + let blob = match self + .bulk_tree + .get_chunk_value_with_cost(chunk_idx) + .unwrap_add_cost(&mut cost) + { + Ok(Some(b)) => b, + Ok(None) => { + return Err(PrivateDocumentStoreError::CorruptedData(format!( + "missing chunk blob for index {}", + chunk_idx + ))) + .wrap_with_cost(cost); + } + Err(e) => { + return Err(PrivateDocumentStoreError::InvalidData(format!("{}", e))) + .wrap_with_cost(cost); + } + }; + let entries = match grovedb_bulk_append_tree::deserialize_chunk_blob(&blob) { + Ok(e) => e, + Err(e) => { + return Err(PrivateDocumentStoreError::CorruptedData(format!("{}", e))) + .wrap_with_cost(cost); + } + }; + // The bounds check above already proved this position exists, so + // a completed chunk that holds fewer than `epoch_size` entries is + // corruption — NOT absence. Enforce the same invariant + // `verify_entry_sizes` checks, otherwise a truncated blob would + // be reported to the caller as "no such document". + if entries.len() as u64 != epoch_size { + return Err(PrivateDocumentStoreError::CorruptedData(format!( + "chunk {} has {} entries, expected {}", + chunk_idx, + entries.len(), + epoch_size + ))) + .wrap_with_cost(cost); + } + Some(entries[pos_in_chunk].clone()) + }; + + // Defensive: a stored entry that violates the committed entry size + // indicates corruption (the append path rejects such entries). + if let Some(v) = &value + && v.len() != self.entry_size as usize + { + return Err(PrivateDocumentStoreError::CorruptedData(format!( + "entry at position {} has size {}, committed entry size is {}", + global_position, + v.len(), + self.entry_size + ))) + .wrap_with_cost(cost); + } + + Ok(value).wrap_with_cost(cost) + } + + /// Append many entries in one pass, computing roots once at the end. + /// + /// Byte-for-byte equivalent to calling [`append`](Self::append) once per + /// entry — same stored values, same chunk blobs, same final state root — + /// but O(N) in hash calls instead of O(N^2). + /// + /// [`append`](Self::append) recomputes the dense-buffer Merkle root on + /// every insert, so a run of N buffered appends re-walks every filled + /// position N times: at `chunk_power = 16` filling one epoch costs about + /// 4.3 billion blake3 calls for 65,535 entries. This method defers the + /// dense root, the bulk state root, and the composite `pds_state` root + /// until the whole run is written. + /// + /// # Failure semantics + /// + /// EVERY entry is size-validated before ANY is written, so a wrong-sized + /// entry anywhere in the input leaves the store completely untouched — + /// no partial write, nothing to roll back. + /// + /// A failure that can only be detected mid-run — a storage fault during + /// a dense-tree write or an MMR compaction — is NOT rolled back: entries + /// already written stay written. This method has no rollback path of its + /// own, so a caller that needs all-or-nothing under storage faults must + /// discard the surrounding transaction, which is what the GroveDB batch + /// path does. Callers holding no transaction should treat a mid-run + /// storage error as leaving the store in an indeterminate state. + /// + /// Returns post-run roots, the last position appended BY THIS CALL (or + /// `None` for empty input), how many entries landed, the summed + /// hash count, and whether any compaction occurred. For an empty input + /// nothing is written and the current state root is returned. + pub fn append_many<'e, I>( + &mut self, + entries: I, + grove_version: &GroveVersion, + ) -> CostResult + where + I: IntoIterator, + { + let mut cost = OperationCost::default(); + + // Validate EVERY entry before writing any of them. Validating inline + // would let a valid entry land and a later wrong-sized one fail, + // leaving the store mutated behind an error — which breaks atomicity + // for a direct caller that has no surrounding transaction to discard. + let entries: Vec<&[u8]> = entries.into_iter().collect(); + for entry in &entries { + if entry.len() != self.entry_size as usize { + return Err(PrivateDocumentStoreError::InvalidEntrySize { + expected: self.entry_size, + actual: entry.len(), + }) + .wrap_with_cost(cost); + } + } + + let mut hash_count: u32 = 0; + let mut any_compacted = false; + let starting_total = self.bulk_tree.total_count; + let mut last_global_position = None; + + for entry in &entries { + // The append's own cost — the buffer write, and on a compacting + // append the read-back of every buffered entry plus the MMR push + // and root — is merged into `cost` here. Copying only + // `r.hash_count` into the result field would leave that I/O and + // the MMR's bagging hashes free. + let r = match self + .bulk_tree + .append_deferred_roots(entry, grove_version) + .unwrap_add_cost(&mut cost) + { + Ok(r) => r, + Err(e) => { + return Err(PrivateDocumentStoreError::InvalidData(format!( + "bulk append: {}", + e + ))) + .wrap_with_cost(cost); + } + }; + hash_count = hash_count.saturating_add(r.hash_count); + any_compacted |= r.compacted; + last_global_position = Some(r.global_position); + } + + // Pay the deferred roots exactly once, through the cost-aware path + // so the dense-buffer walk's real storage reads and hashes are + // billed rather than re-derived from a hand-rolled model. + let root_ctx = self + .bulk_tree + .compute_current_state_root_with_cost(grove_version); + let root_cost = root_ctx.cost; + let bulk_state_root = match root_ctx.value { + Ok(r) => r, + Err(e) => { + cost += root_cost; + return Err(PrivateDocumentStoreError::InvalidData(format!( + "state root: {}", + e + ))) + .wrap_with_cost(cost); + } + }; + hash_count = hash_count.saturating_add(root_cost.hash_node_calls); + cost += root_cost; + + let state_root = + compute_private_document_store_state_root(&self.config_hash, &bulk_state_root); + // The composite root is computed on EVERY call, including an empty + // one, so it is charged unconditionally. + hash_count = hash_count.saturating_add(1); + cost.hash_node_calls = cost.hash_node_calls.saturating_add(1); + + Ok(PrivateDocumentStoreAppendManyResult { + state_root, + bulk_state_root, + last_global_position, + appended: self.bulk_tree.total_count - starting_total, + hash_count, + compacted: any_compacted, + }) + .wrap_with_cost(cost) + } + + /// Verify that all stored entries respect the committed `entry_size`. + /// + /// Walks the current buffer and every completed chunk blob; returns an + /// error naming the first violating position. Used by GroveDB's + /// `verify_grovedb` integrity walk. O(total_count) reads — intended for + /// integrity audits, not hot paths. + pub fn verify_entry_sizes(&self) -> Result<(), PrivateDocumentStoreError> { + let expected = self.entry_size as usize; + + // Completed chunks: each blob must deserialize to exactly + // `epoch_size` entries of `entry_size` bytes. + let epoch_size = self.bulk_tree.epoch_size(); + for chunk_idx in 0..self.bulk_tree.chunk_count() { + let blob = self + .bulk_tree + .get_chunk_value(chunk_idx) + .map_err(|e| PrivateDocumentStoreError::InvalidData(format!("{}", e)))? + .ok_or_else(|| { + PrivateDocumentStoreError::CorruptedData(format!( + "missing chunk blob for index {}", + chunk_idx + )) + })?; + let entries = grovedb_bulk_append_tree::deserialize_chunk_blob(&blob) + .map_err(|e| PrivateDocumentStoreError::CorruptedData(format!("{}", e)))?; + if entries.len() as u64 != epoch_size { + return Err(PrivateDocumentStoreError::CorruptedData(format!( + "chunk {} has {} entries, expected {}", + chunk_idx, + entries.len(), + epoch_size + ))); + } + for (i, entry) in entries.iter().enumerate() { + if entry.len() != expected { + return Err(PrivateDocumentStoreError::CorruptedData(format!( + "entry at position {} has size {}, committed entry size is {}", + chunk_idx * epoch_size + i as u64, + entry.len(), + expected + ))); + } + } + } + + // Current buffer. + let buffer_start = self.bulk_tree.chunk_count() * epoch_size; + for pos in 0..self.bulk_tree.buffer_count() { + let entry = self + .bulk_tree + .get_buffer_value(pos) + .map_err(|e| PrivateDocumentStoreError::InvalidData(format!("{}", e)))? + .ok_or_else(|| { + PrivateDocumentStoreError::CorruptedData(format!( + "missing buffer entry at position {}", + pos + )) + })?; + if entry.len() != expected { + return Err(PrivateDocumentStoreError::CorruptedData(format!( + "entry at position {} has size {}, committed entry size is {}", + buffer_start + pos as u64, + entry.len(), + expected + ))); + } + } + + Ok(()) + } + + /// Compute the composite state root + /// (`blake3("pds_state" || config_hash || bulk_state_root)`) without + /// modifying the store. + pub fn compute_current_state_root(&self) -> Result<[u8; 32], PrivateDocumentStoreError> { + let bulk_root = self + .bulk_tree + .compute_current_state_root() + .map_err(|e| PrivateDocumentStoreError::InvalidData(format!("state root: {}", e)))?; + Ok(compute_private_document_store_state_root( + &self.config_hash, + &bulk_root, + )) + } + + /// Cost-propagating variant of + /// [`compute_current_state_root`](Self::compute_current_state_root): + /// charges the underlying dense-root walk (reads and hashes) plus the + /// composite `pds_state` blake3. + pub fn compute_current_state_root_with_cost( + &self, + grove_version: &GroveVersion, + ) -> CostResult<[u8; 32], PrivateDocumentStoreError> { + let mut cost = OperationCost::default(); + let bulk_root = match self + .bulk_tree + .compute_current_state_root_with_cost(grove_version) + .unwrap_add_cost(&mut cost) + { + Ok(r) => r, + Err(e) => { + return Err(PrivateDocumentStoreError::InvalidData(format!( + "state root: {}", + e + ))) + .wrap_with_cost(cost); + } + }; + cost.hash_node_calls = cost.hash_node_calls.saturating_add(1); + Ok(compute_private_document_store_state_root( + &self.config_hash, + &bulk_root, + )) + .wrap_with_cost(cost) + } + + /// Flush the MMR overlay to storage. + /// + /// Delegates to [`BulkAppendTree::commit_mmr`]. Call this at the end of + /// a session to persist MMR nodes buffered during compaction cycles. + pub fn commit_mmr(&mut self) -> Result<(), PrivateDocumentStoreError> { + self.bulk_tree + .commit_mmr() + .map_err(|e| PrivateDocumentStoreError::InvalidData(format!("MMR commit: {}", e))) + } + + // ── Accessors ───────────────────────────────────────────────────── + + /// The committed entry size in bytes. + pub fn entry_size(&self) -> u32 { + self.entry_size + } + + /// The chunk power (dense-buffer height) of the underlying bulk tree. + pub fn chunk_power(&self) -> u8 { + self.bulk_tree.height() + } + + /// Total number of entries appended so far. + pub fn total_count(&self) -> u64 { + self.bulk_tree.total_count + } + + /// The number of entries per completed chunk (epoch). + pub fn epoch_size(&self) -> u64 { + self.bulk_tree.epoch_size() + } + + /// Number of completed chunks in the MMR. + pub fn chunk_count(&self) -> u64 { + self.bulk_tree.chunk_count() + } +} + +#[cfg(test)] +mod append_many_tests { + use super::*; + use grovedb_bulk_append_tree::test_utils::MemStorageContext; + + /// `append_many` must be byte-for-byte equivalent to `append` in a loop: + /// same state root, same positions, same stored bytes. It only differs in + /// how many hashes it takes to get there. + #[test] + fn append_many_matches_per_entry_append() { + // 10 entries at chunk_power 2 (epoch size 4) spans two compactions + // plus a partial buffer, so both storage tiers are exercised. + let entries: Vec> = (0..10u8).map(|i| vec![i; 8]).collect(); + + let mut one_by_one = PrivateDocumentStore::new(8, 2, MemStorageContext::new()) + .unwrap() + .expect("a"); + let mut last = None; + for e in &entries { + last = Some( + one_by_one + .append(e, GroveVersion::latest()) + .unwrap() + .expect("append"), + ); + } + let per_entry = last.expect("appended"); + + let mut batched = PrivateDocumentStore::new(8, 2, MemStorageContext::new()) + .unwrap() + .expect("b"); + let many = batched + .append_many(entries.iter().map(|e| e.as_slice()), GroveVersion::latest()) + .unwrap() + .expect("append_many"); + + assert_eq!(many.state_root, per_entry.state_root); + assert_eq!(many.bulk_state_root, per_entry.bulk_state_root); + assert_eq!(many.last_global_position, Some(per_entry.global_position)); + assert_eq!(many.appended, 10); + assert_eq!(batched.total_count(), one_by_one.total_count()); + assert_eq!( + batched.compute_current_state_root().expect("root"), + one_by_one.compute_current_state_root().expect("root") + ); + for i in 0..10u64 { + assert_eq!( + batched.get_value(i).unwrap().expect("get"), + one_by_one.get_value(i).unwrap().expect("get"), + "position {}", + i + ); + } + batched.verify_entry_sizes().expect("sizes ok"); + + // The whole point: the batched path does asymptotically less hashing. + assert!( + many.hash_count < per_entry.hash_count * 10, + "append_many should not pay the per-entry dense walk" + ); + } + + #[test] + fn append_many_validates_entry_size_and_handles_empty() { + let mut store = PrivateDocumentStore::new(8, 2, MemStorageContext::new()) + .unwrap() + .expect("new"); + + // Empty input writes nothing, reports NO appended position (rather + // than a sentinel indistinguishable from a real append), and still + // charges the two root hashes it actually performs. + let empty: Vec> = Vec::new(); + let ctx = store.append_many(empty.iter().map(|e| e.as_slice()), GroveVersion::latest()); + let empty_cost = ctx.cost.clone(); + let r = ctx.value.expect("empty append_many"); + assert_eq!(store.total_count(), 0); + assert_eq!(r.last_global_position, None); + assert_eq!(r.appended, 0); + assert_eq!( + r.state_root, + store.compute_current_state_root().expect("root") + ); + assert_eq!( + empty_cost.hash_node_calls, 2, + "an empty batch still computes the bulk and composite roots" + ); + + // A wrong-size entry is rejected. + let bad = [vec![0u8; 8], vec![0u8; 7]]; + assert!(matches!( + store + .append_many(bad.iter().map(|e| e.as_slice()), GroveVersion::latest()) + .unwrap(), + Err(PrivateDocumentStoreError::InvalidEntrySize { + expected: 8, + actual: 7 + }) + )); + } +} + +#[cfg(test)] +mod atomicity_tests { + use grovedb_bulk_append_tree::test_utils::MemStorageContext; + + use super::*; + + /// A reopened store resolves its MMR root through the lazy (uncached) + /// path, which is what proof binding and the integrity walk hit. That + /// read must be billed, not silently free. + #[test] + fn reopened_store_charges_the_uncached_mmr_root_read() { + let storage = MemStorageContext::new(); + let mut store = PrivateDocumentStore::new(8, 2, storage) + .unwrap() + .expect("new"); + // Past a compaction so the MMR actually holds a chunk. + for i in 0..6u8 { + store + .append(&[i; 8], GroveVersion::latest()) + .unwrap() + .expect("append"); + } + store.commit_mmr().expect("commit mmr"); + let storage = PrivateDocumentStore::into_storage_for_test(store); + + // `from_state` leaves the MMR root cache empty, so this computation + // takes the lazy path. + let reopened = PrivateDocumentStore::from_state(6, 8, 2, storage) + .unwrap() + .expect("reopen"); + let ctx = reopened.compute_current_state_root_with_cost(GroveVersion::latest()); + let cost = ctx.cost.clone(); + ctx.value.expect("state root"); + // NOTE: `MemStorageContext::get` reports `OperationCost::default()`, + // so storage seeks and loaded bytes are invisible to this harness — + // only the hash accounting is observable here. That the underlying + // storage reads are genuinely billed is covered against real + // RocksDB storage by + // `test_private_document_store_reopened_reads_are_billed` in the + // grovedb crate. + // Pinned exactly rather than `> 0`: a loose bound is what let a + // double-charge of the dense walk sit here unnoticed. With + // `chunk_power = 2` the buffer holds 3 and an epoch is 4, so 6 total + // entries leave one completed chunk (mmr_size 1, which takes the + // single-element path and bags no peaks) and 2 live buffer + // positions. `hash_node` bills a value hash and a node hash per + // filled position, so the dense walk is 4; the bulk state root and + // the composite `pds_state` root are one each. + assert_eq!( + cost.hash_node_calls, 6, + "expected 2*2 dense-walk hashes + 1 bulk state root + 1 composite \ + pds_state root, got {:?}", + cost + ); + } + + /// The hash accounting for an append, pinned entry by entry. + /// + /// Every figure here is derived from what the code actually hashes, so a + /// change to either the model or the charging breaks this test rather + /// than silently shifting fees. + #[test] + fn append_bills_exactly_the_hashes_it_performs() { + // `chunk_power = 4` gives a 15-slot buffer, so none of these appends + // compacts and the MMR stays empty (its root is the zero hash, taken + // without hashing). + let mut store = PrivateDocumentStore::new(8, 4, MemStorageContext::new()) + .unwrap() + .expect("new"); + + // First append: the dense walk visits 1 filled position (2 hashes), + // then the bulk state root (1) and the composite pds_state root (1). + let ctx = store.append(&[1u8; 8], GroveVersion::latest()); + ctx.value.expect("append"); + assert_eq!( + ctx.cost.hash_node_calls, 4, + "2 dense + 1 bulk root + 1 composite, got {:?}", + ctx.cost + ); + + // Second append: 2 filled positions now, so the walk costs 4. + let ctx = store.append(&[2u8; 8], GroveVersion::latest()); + ctx.value.expect("append"); + assert_eq!( + ctx.cost.hash_node_calls, 6, + "4 dense + 1 bulk root + 1 composite, got {:?}", + ctx.cost + ); + + // Third: 6 dense + 2 roots. + let ctx = store.append(&[3u8; 8], GroveVersion::latest()); + ctx.value.expect("append"); + assert_eq!( + ctx.cost.hash_node_calls, 8, + "6 dense + 1 bulk root + 1 composite, got {:?}", + ctx.cost + ); + } + + /// Compaction is the expensive branch of an append — it reads every + /// buffered entry back out of storage, hashes the chunk blob, and pushes + /// it through the MMR — and all of that used to be discarded, so a + /// compacting append billed no more I/O than a buffered one. + #[test] + fn compacting_append_bills_its_reads_and_hashes() { + // chunk_power 2: the buffer holds 3, so the 4th append compacts. + let mut store = PrivateDocumentStore::new(8, 2, MemStorageContext::new()) + .unwrap() + .expect("new"); + for i in 0..3u8 { + store + .append(&[i; 8], GroveVersion::latest()) + .unwrap() + .expect("append"); + } + + // The 4th append does not fit the buffer, so it compacts. + let compacting = store.append(&[3u8; 8], GroveVersion::latest()); + compacting.value.expect("compacting append"); + let compacting_cost = compacting.cost; + + assert!( + compacting_cost.seek_count > 0 && compacting_cost.storage_loaded_bytes > 0, + "compaction reads every buffered entry; those reads must be billed, got {:?}", + compacting_cost + ); + // 3 buffered entries read back, at the committed 8 bytes each. + assert!( + compacting_cost.storage_loaded_bytes >= 24, + "expected at least the 3 x 8 bytes compaction reads back, got {:?}", + compacting_cost + ); + // 1 chunk-blob leaf hash + 1 bulk state root + 1 composite root. The + // MMR push collapses no peaks at size 0 and the root takes the + // single-element path, so neither adds a hash here. + assert_eq!( + compacting_cost.hash_node_calls, 3, + "1 leaf + 1 bulk root + 1 composite, got {:?}", + compacting_cost + ); + + // A plain buffered append afterwards reads nothing back. + let plain = store.append(&[4u8; 8], GroveVersion::latest()); + plain.value.expect("buffered append"); + assert!( + plain.cost.storage_loaded_bytes < compacting_cost.storage_loaded_bytes, + "a buffered append must be cheaper in loaded bytes than a \ + compacting one (buffered {:?} vs compacting {:?})", + plain.cost, + compacting_cost + ); + } + + /// The reported `hash_count` and the billed `hash_node_calls` describe the + /// same work, so they must agree. They diverged once the MMR started + /// bagging peaks: the counter was derived from `hash_count_for_push`, + /// which covers the leaf hash and `push`'s merges but not `get_root`'s + /// peak folds, while the cost picked the folds up. + #[test] + fn append_many_hash_count_matches_the_billed_hashes() { + // chunk_power 2 gives an epoch of 4, so 12 entries compact three + // times. The third compaction leaves the MMR with two peaks, which is + // the case that used to be under-counted. + let mut store = PrivateDocumentStore::new(8, 2, MemStorageContext::new()) + .unwrap() + .expect("new"); + let entries: Vec> = (0..12u8).map(|i| vec![i; 8]).collect(); + let ctx = store.append_many(entries.iter().map(|e| e.as_slice()), GroveVersion::latest()); + let cost = ctx.cost.clone(); + let r = ctx.value.expect("append_many"); + + assert_eq!( + r.hash_count, cost.hash_node_calls, + "the reported hash count and the billed hashes describe the same \ + work: reported {}, billed {:?}", + r.hash_count, cost + ); + assert!(r.compacted, "12 entries at epoch 4 must compact"); + + // Same invariant on the single-append path, which forwards the counter. + let ctx = store.append(&[99u8; 8], GroveVersion::latest()); + let cost = ctx.cost.clone(); + let r = ctx.value.expect("append"); + assert_eq!( + r.hash_count, cost.hash_node_calls, + "reported {}, billed {:?}", + r.hash_count, cost + ); + } + + /// Opening a store derives the committed-config hash, which is real work + /// and must not be free. + #[test] + fn opening_a_store_bills_the_config_hash() { + let ctx = PrivateDocumentStore::new(8, 4, MemStorageContext::new()); + ctx.value.expect("new"); + assert_eq!( + ctx.cost.hash_node_calls, 1, + "the committed-config blake3, got {:?}", + ctx.cost + ); + } + + /// A wrong-sized entry anywhere in the batch must leave the store + /// completely untouched — not partially appended behind an error. A + /// direct caller has no surrounding transaction to discard. + #[test] + fn append_many_is_atomic_on_a_bad_entry() { + let mut store = PrivateDocumentStore::new(8, 2, MemStorageContext::new()) + .unwrap() + .expect("new"); + store + .append(&[1u8; 8], GroveVersion::latest()) + .unwrap() + .expect("seed"); + + let count_before = store.total_count(); + let root_before = store.compute_current_state_root().expect("root"); + let value_before = store.get_value(0).unwrap().expect("get"); + + // First entry is valid, second is the wrong size. + let batch = [vec![2u8; 8], vec![3u8; 7]]; + assert!(matches!( + store + .append_many(batch.iter().map(|e| e.as_slice()), GroveVersion::latest()) + .unwrap(), + Err(PrivateDocumentStoreError::InvalidEntrySize { + expected: 8, + actual: 7 + }) + )); + + assert_eq!(store.total_count(), count_before, "count must be unchanged"); + assert_eq!( + store.compute_current_state_root().expect("root"), + root_before, + "state root must be unchanged" + ); + assert_eq!( + store.get_value(0).unwrap().expect("get"), + value_before, + "stored values must be unchanged" + ); + assert_eq!( + store.get_value(1).unwrap().expect("get"), + None, + "the valid entry preceding the bad one must not have landed" + ); + } +} + +#[cfg(test)] +mod error_path_tests { + use super::*; + use grovedb_bulk_append_tree::test_utils::MemStorageContext; + + #[test] + fn test_debug_and_error_display() { + let store = PrivateDocumentStore::new(64, 4, MemStorageContext::new()) + .unwrap() + .expect("new"); + let dbg = format!("{:?}", store); + assert!(dbg.contains("PrivateDocumentStore") && dbg.contains("entry_size: 64")); + assert_eq!(store.entry_size(), 64); + assert_eq!(store.chunk_power(), 4); + assert_eq!(store.epoch_size(), 16); + + assert!(format!( + "{}", + PrivateDocumentStoreError::InvalidEntrySize { + expected: 8, + actual: 9 + } + ) + .contains("expected 8 bytes, got 9")); + assert!( + format!("{}", PrivateDocumentStoreError::InvalidConfig("x".into())).contains("config") + ); + assert!( + format!("{}", PrivateDocumentStoreError::CorruptedData("x".into())) + .contains("corrupted") + ); + assert!(format!("{}", PrivateDocumentStoreError::InvalidData("x".into())).contains("data")); + } + + #[test] + fn test_reads_error_on_wiped_storage() { + // Populate past a compaction so both a chunk and the buffer exist, + // then wipe the backing storage and reopen with the same claimed + // state: chunk reads and the integrity walk must surface errors + // rather than fabricate data. + let mut store = PrivateDocumentStore::new(8, 2, MemStorageContext::new()) + .unwrap() + .expect("new"); + for i in 0..6u8 { + store + .append(&[i; 8], GroveVersion::latest()) + .unwrap() + .expect("append"); + } + store.commit_mmr().expect("commit mmr"); + let storage = PrivateDocumentStore::into_storage_for_test(store); + storage.data.borrow_mut().clear(); + + let broken = PrivateDocumentStore::from_state(6, 8, 2, storage) + .unwrap() + .expect("reopen"); + // Position 0 lives in the (now missing) completed chunk. + assert!(broken.get_value(0).unwrap().is_err()); + // The integrity walk fails on the missing chunk too. + assert!(broken.verify_entry_sizes().is_err()); + // Buffer positions read as missing entries in the walk; direct + // get_value returns the underlying error or None consistently. + assert!( + broken.compute_current_state_root().is_err() || broken.get_value(5).unwrap().is_err() + ); + } + + /// Build a store, then reopen the same bytes under a DIFFERENT declared + /// config. This is the attack the config-binding state root exists to stop, + /// and the read paths must refuse rather than reinterpret the bytes. + #[test] + fn test_reopen_under_wrong_config_is_reported_as_corruption() { + // 10 entries at chunk_power 3 (epoch 8): one completed chunk of 8, + // two live in the buffer. + let mut store = PrivateDocumentStore::new(8, 3, MemStorageContext::new()) + .unwrap() + .expect("new"); + for i in 0..10u8 { + store + .append(&[i; 8], GroveVersion::latest()) + .unwrap() + .expect("append"); + } + store.commit_mmr().expect("commit mmr"); + let storage = PrivateDocumentStore::into_storage_for_test(store); + + // Wrong chunk_power: epoch is now 4, so the stored 8-entry chunk no + // longer matches the declared epoch. A truncated/oversized chunk must + // read as corruption, NOT as a missing document. + let wrong_power = PrivateDocumentStore::from_state(10, 8, 2, storage) + .unwrap() + .expect("reopen"); + assert!( + matches!( + wrong_power.get_value(0).unwrap(), + Err(PrivateDocumentStoreError::CorruptedData(ref m)) + if m.contains("expected 4") + ), + "got {:?}", + wrong_power.get_value(0).unwrap() + ); + assert!(matches!( + wrong_power.verify_entry_sizes(), + Err(PrivateDocumentStoreError::CorruptedData(_)) + )); + + // Wrong entry_size: the chunk deserializes and has the right count, + // but every entry is the wrong width. + let storage = PrivateDocumentStore::into_storage_for_test(wrong_power); + let wrong_size = PrivateDocumentStore::from_state(10, 16, 3, storage) + .unwrap() + .expect("reopen"); + assert!( + matches!( + wrong_size.verify_entry_sizes(), + Err(PrivateDocumentStoreError::CorruptedData(ref m)) + if m.contains("committed entry size is 16") + ), + "got {:?}", + wrong_size.verify_entry_sizes() + ); + // The same violation is caught on the direct read path. + assert!(matches!( + wrong_size.get_value(0).unwrap(), + Err(PrivateDocumentStoreError::CorruptedData(_)) + )); + } + + /// A store that claims more entries than its storage holds must report + /// the absence as corruption on every path — a claimed-but-absent chunk + /// and a claimed-but-absent buffer slot are different branches. + #[test] + fn test_claimed_entries_beyond_storage_are_corruption() { + let mut store = PrivateDocumentStore::new(8, 2, MemStorageContext::new()) + .unwrap() + .expect("new"); + // 6 entries at chunk_power 2 (epoch 4): one chunk, two buffered. + for i in 0..6u8 { + store + .append(&[i; 8], GroveVersion::latest()) + .unwrap() + .expect("append"); + } + store.commit_mmr().expect("commit mmr"); + let storage = PrivateDocumentStore::into_storage_for_test(store); + + // Claim 20 entries: chunks 1..=3 and their buffer slots do not exist. + let overclaimed = PrivateDocumentStore::from_state(20, 8, 2, storage) + .unwrap() + .expect("reopen"); + // Position 8 sits in chunk 2, which was never written. + assert!( + overclaimed.get_value(8).unwrap().is_err(), + "a claimed-but-absent chunk must not read as success" + ); + assert!(overclaimed.verify_entry_sizes().is_err()); + + // A store claiming buffer entries it never stored: 2 written, 3 + // claimed, and no completed chunk involved. + let mut store = PrivateDocumentStore::new(8, 3, MemStorageContext::new()) + .unwrap() + .expect("new"); + for i in 0..2u8 { + store + .append(&[i; 8], GroveVersion::latest()) + .unwrap() + .expect("append"); + } + let storage = PrivateDocumentStore::into_storage_for_test(store); + let overclaimed = PrivateDocumentStore::from_state(3, 8, 3, storage) + .unwrap() + .expect("reopen"); + // Surfaces as `InvalidData`, not `CorruptedData`: the dense tree + // detects the shortfall against its own count and errors before + // returning `None`, so the store's own "missing buffer entry" arm is + // defensive rather than reachable from here. What matters is that the + // walk refuses rather than reporting a short store as intact. + let err = overclaimed + .verify_entry_sizes() + .expect_err("claimed buffer entry does not exist"); + assert!( + format!("{}", err).contains("position 2"), + "error should name the missing position, got {:?}", + err + ); + } + + /// Deriving the state root over storage that cannot satisfy the claimed + /// state must surface the error, not a plausible-looking root. + #[test] + fn test_state_root_over_missing_storage_errors() { + let mut store = PrivateDocumentStore::new(8, 2, MemStorageContext::new()) + .unwrap() + .expect("new"); + for i in 0..6u8 { + store + .append(&[i; 8], GroveVersion::latest()) + .unwrap() + .expect("append"); + } + store.commit_mmr().expect("commit mmr"); + let storage = PrivateDocumentStore::into_storage_for_test(store); + storage.data.borrow_mut().clear(); + + let broken = PrivateDocumentStore::from_state(6, 8, 2, storage) + .unwrap() + .expect("reopen"); + let ctx = broken.compute_current_state_root_with_cost(GroveVersion::latest()); + assert!( + ctx.value.is_err(), + "a state root over wiped storage must be an error, got {:?}", + ctx.value + ); + // And appending onto that broken state fails rather than writing. + let mut broken = broken; + assert!(broken + .append(&[9u8; 8], GroveVersion::latest()) + .unwrap() + .is_err()); + } + + /// Every read path must surface a storage fault instead of reporting the + /// document as absent. "Not found" and "could not be read" are different + /// answers, and conflating them on an append-only store would let an I/O + /// fault look like a legitimately empty position. + #[test] + fn test_read_faults_surface_rather_than_reading_as_absent() { + let mut store = PrivateDocumentStore::new(8, 2, MemStorageContext::new()) + .unwrap() + .expect("new"); + // Past a compaction so both a completed chunk and the buffer exist. + for i in 0..6u8 { + store + .append(&[i; 8], GroveVersion::latest()) + .unwrap() + .expect("append"); + } + store.commit_mmr().expect("commit mmr"); + + // Reopen before injecting the fault. The dense tree keeps a + // write-through cache, so on the original handle a live buffer read is + // served from memory and never reaches storage — a fault there would + // prove nothing. A reopened store has a cold cache, which is also the + // state every real read after a restart is in. + let storage = PrivateDocumentStore::into_storage_for_test(store); + storage.fail_reads(); + let mut store = PrivateDocumentStore::from_state(6, 8, 2, storage) + .unwrap() + .expect("reopen"); + + // Position 5 is in the live buffer, position 0 in a completed chunk: + // these take different branches and both must error. + for pos in [5u64, 0] { + let r = store.get_value(pos).unwrap(); + assert!( + r.is_err(), + "position {} must report the read fault, got {:?}", + pos, + r + ); + } + + // The integrity walk and the state-root derivation likewise. + assert!(store.verify_entry_sizes().is_err()); + assert!(store + .compute_current_state_root_with_cost(GroveVersion::latest()) + .value + .is_err()); + + // An out-of-range position is answered before any storage is touched, + // so it still reports absence rather than the fault. + assert_eq!(store.get_value(99).unwrap().expect("in-range check"), None); + + store.bulk_tree.dense_tree.storage.heal(); + assert!(store.get_value(0).unwrap().expect("healed read").is_some()); + } + + /// A write fault during an append must fail the append, not report a + /// success whose state root does not match what was stored. + #[test] + fn test_write_faults_fail_the_append() { + let mut store = PrivateDocumentStore::new(8, 2, MemStorageContext::new()) + .unwrap() + .expect("new"); + store + .append(&[0u8; 8], GroveVersion::latest()) + .unwrap() + .expect("append"); + + store.bulk_tree.dense_tree.storage.fail_writes(); + let r = store.append(&[1u8; 8], GroveVersion::latest()).unwrap(); + assert!( + r.is_err(), + "a failed write must fail the append, got {:?}", + r + ); + + // The batch path too, and its size prevalidation still runs first: a + // wrong-sized entry is rejected on its own terms, not as a storage + // fault. + let bad = [vec![0u8; 7]]; + assert!(matches!( + store + .append_many(bad.iter().map(|e| e.as_slice()), GroveVersion::latest()) + .unwrap(), + Err(PrivateDocumentStoreError::InvalidEntrySize { .. }) + )); + let good = [vec![2u8; 8], vec![3u8; 8]]; + assert!(store + .append_many(good.iter().map(|e| e.as_slice()), GroveVersion::latest()) + .unwrap() + .is_err()); + } +} + +#[cfg(test)] +impl PrivateDocumentStore { + /// Test helper: tear down the store and recover its storage context so a + /// reopen can be simulated against the same in-memory backing. + pub(crate) fn into_storage_for_test(store: Self) -> S { + store.bulk_tree.dense_tree.storage + } +} + +#[cfg(test)] +mod tests { + use super::*; + use grovedb_bulk_append_tree::test_utils::MemStorageContext; + + use crate::{empty_private_document_store_state_root, EMPTY_BULK_APPEND_TREE_STATE_ROOT}; + + #[test] + fn test_empty_store_state_root_matches_helper() { + let store = PrivateDocumentStore::new(64, 4, MemStorageContext::new()) + .unwrap() + .expect("new store"); + assert_eq!( + store.compute_current_state_root().expect("state root"), + empty_private_document_store_state_root(64, 4), + ); + // And the inner bulk root of an empty store matches the constant. + assert_eq!( + store.bulk_tree.compute_current_state_root().expect("bulk"), + EMPTY_BULK_APPEND_TREE_STATE_ROOT, + ); + } + + #[test] + fn test_zero_entry_size_rejected() { + assert!(matches!( + PrivateDocumentStore::new(0, 4, MemStorageContext::new()).unwrap(), + Err(PrivateDocumentStoreError::InvalidConfig(_)) + )); + } + + #[test] + fn test_invalid_chunk_power_rejected() { + assert!(PrivateDocumentStore::new(64, 0, MemStorageContext::new()) + .unwrap() + .is_err()); + assert!(PrivateDocumentStore::new(64, 17, MemStorageContext::new()) + .unwrap() + .is_err()); + } + + #[test] + fn test_append_validates_entry_size() { + let mut store = PrivateDocumentStore::new(8, 2, MemStorageContext::new()) + .unwrap() + .expect("new store"); + assert!(matches!( + store.append(&[0u8; 7], GroveVersion::latest()).unwrap(), + Err(PrivateDocumentStoreError::InvalidEntrySize { + expected: 8, + actual: 7 + }) + )); + assert!(matches!( + store.append(&[0u8; 9], GroveVersion::latest()).unwrap(), + Err(PrivateDocumentStoreError::InvalidEntrySize { + expected: 8, + actual: 9 + }) + )); + // A rejected append must not mutate the store. + assert_eq!(store.total_count(), 0); + let ok = store + .append(&[1u8; 8], GroveVersion::latest()) + .unwrap() + .expect("valid append"); + assert_eq!(ok.global_position, 0); + assert_eq!(store.total_count(), 1); + } + + #[test] + fn test_append_get_roundtrip_across_compaction() { + // chunk_power 2 → capacity 3, epoch size 4: 10 appends span two + // completed chunks plus a partial buffer. + let mut store = PrivateDocumentStore::new(8, 2, MemStorageContext::new()) + .unwrap() + .expect("new store"); + let mut roots = Vec::new(); + for i in 0..10u8 { + let entry = [i; 8]; + let r = store + .append(&entry, GroveVersion::latest()) + .unwrap() + .expect("append"); + assert_eq!(r.global_position, i as u64); + roots.push(r.state_root); + } + // Every append must move the state root. + for w in roots.windows(2) { + assert_ne!(w[0], w[1]); + } + assert_eq!(store.total_count(), 10); + assert_eq!(store.chunk_count(), 2); + + for i in 0..10u8 { + let v = store.get_value(i as u64).unwrap().expect("get"); + assert_eq!(v, Some(vec![i; 8]), "position {}", i); + } + assert_eq!(store.get_value(10).unwrap().expect("get"), None); + + // The append-path state root matches a fresh computation. + assert_eq!( + store.compute_current_state_root().expect("state root"), + *roots.last().unwrap() + ); + + // And the whole store passes the entry-size integrity walk. + store.verify_entry_sizes().expect("sizes ok"); + } + + #[test] + fn test_state_root_differs_from_raw_bulk_root() { + // The composite root must bind the config: it can never equal the + // raw bulk root, and two stores with identical data but different + // configs must have different roots. + let mut a = PrivateDocumentStore::new(8, 2, MemStorageContext::new()) + .unwrap() + .expect("a"); + let mut b = PrivateDocumentStore::new(8, 3, MemStorageContext::new()) + .unwrap() + .expect("b"); + let ra = a + .append(&[7u8; 8], GroveVersion::latest()) + .unwrap() + .expect("append a"); + let rb = b + .append(&[7u8; 8], GroveVersion::latest()) + .unwrap() + .expect("append b"); + assert_ne!(ra.state_root, ra.bulk_state_root); + assert_ne!(ra.state_root, rb.state_root); + } + + #[test] + fn test_reopen_from_state() { + let storage = MemStorageContext::new(); + let mut store = PrivateDocumentStore::new(8, 2, storage) + .unwrap() + .expect("new store"); + for i in 0..6u8 { + store + .append(&[i; 8], GroveVersion::latest()) + .unwrap() + .expect("append"); + } + store.commit_mmr().expect("commit mmr"); + let root_before = store.compute_current_state_root().expect("root"); + let storage = PrivateDocumentStore::into_storage_for_test(store); + + let reopened = PrivateDocumentStore::from_state(6, 8, 2, storage) + .unwrap() + .expect("reopen"); + assert_eq!( + reopened.compute_current_state_root().expect("root"), + root_before + ); + for i in 0..6u8 { + assert_eq!( + reopened.get_value(i as u64).unwrap().expect("get"), + Some(vec![i; 8]) + ); + } + reopened.verify_entry_sizes().expect("sizes ok"); + } + + #[test] + fn test_verify_entry_sizes_detects_wrong_config() { + // Write entries of size 8, then reopen claiming entry_size 16 — the + // integrity walk must flag the first entry. + let storage = MemStorageContext::new(); + let mut store = PrivateDocumentStore::new(8, 2, storage) + .unwrap() + .expect("new store"); + for i in 0..6u8 { + store + .append(&[i; 8], GroveVersion::latest()) + .unwrap() + .expect("append"); + } + store.commit_mmr().expect("commit mmr"); + let storage = PrivateDocumentStore::into_storage_for_test(store); + + let reopened = PrivateDocumentStore::from_state(6, 16, 2, storage) + .unwrap() + .expect("reopen"); + assert!(matches!( + reopened.verify_entry_sizes(), + Err(PrivateDocumentStoreError::CorruptedData(_)) + )); + // get_value performs the same defensive check. + assert!(reopened.get_value(0).unwrap().is_err()); + } +} diff --git a/grovedb-version/src/tests.rs b/grovedb-version/src/tests.rs index e58a85c6d..2467d7911 100644 --- a/grovedb-version/src/tests.rs +++ b/grovedb-version/src/tests.rs @@ -3,6 +3,7 @@ use crate::version::grovedb_versions::*; use crate::version::merk_versions::*; use crate::version::v1::GROVE_V1; use crate::version::v2::GROVE_V2; +use crate::version::v3::GROVE_V3; use crate::version::v4::GROVE_V4; use crate::version::{GroveVersion, GROVE_VERSIONS}; use crate::{TryFromVersioned, TryIntoVersioned}; @@ -74,6 +75,26 @@ fn grove_versions_count() { assert_eq!(GROVE_VERSIONS.len(), 4); } +#[test] +fn private_document_store_slots_are_gated_to_v4() { + // The PrivateDocumentStore family fails closed on every released + // version: all slots must be 0 on V1..V3 and 1 on V4. Changing a V1..V3 + // value would retroactively enable (or a V4 value disable) the element + // type on a live protocol version — a consensus break. + for v in [&GROVE_V1, &GROVE_V2, &GROVE_V3] { + let pds = &v.grovedb_versions.operations.private_document_store; + assert_eq!(pds.element_creation, 0, "v{}", v.protocol_version); + assert_eq!(pds.insert, 0, "v{}", v.protocol_version); + assert_eq!(pds.get_value, 0, "v{}", v.protocol_version); + assert_eq!(pds.count, 0, "v{}", v.protocol_version); + } + let pds = &GROVE_V4.grovedb_versions.operations.private_document_store; + assert_eq!(pds.element_creation, 1); + assert_eq!(pds.insert, 1); + assert_eq!(pds.get_value, 1); + assert_eq!(pds.count, 1); +} + #[test] fn grove_versions_ordered_by_protocol_version() { for window in GROVE_VERSIONS.windows(2) { @@ -528,3 +549,80 @@ fn feature_version_bounds_default() { assert!(bounds.check_version(0)); assert!(!bounds.check_version(1)); } + +// ── MMR cost version table ─────────────────────────────────────────── + +/// The MMR hash charges are consensus-visible through fees, so which version +/// each protocol version selects is pinned here rather than left to whoever +/// edits a version constant next. V1..V3 are released and must stay on the +/// shipped accounting; the corrected charges arrive with V4. +#[test] +fn mmr_cost_versions_are_pinned_per_protocol_version() { + for (name, version) in [ + ("GROVE_V1", &GROVE_V1), + ("GROVE_V2", &GROVE_V2), + ("GROVE_V3", &GROVE_V3), + ] { + let cost = &version.mmr_versions.cost; + assert_eq!(cost.push, 0, "{name} must keep the shipped push charge"); + assert_eq!( + cost.get_root, 0, + "{name} must keep the shipped get_root charge" + ); + assert_eq!( + cost.gen_proof, 0, + "{name} must keep the shipped gen_proof charge" + ); + } + + let cost = &GROVE_V4.mmr_versions.cost; + assert_eq!(cost.push, 1, "GROVE_V4 charges push merges"); + assert_eq!(cost.get_root, 1, "GROVE_V4 charges get_root bagging"); + assert_eq!(cost.gen_proof, 1, "GROVE_V4 charges gen_proof bagging"); +} + +/// `GroveVersion::first()` is what the unversioned MMR entry points delegate +/// to, so it has to be the shipped accounting — otherwise every caller that +/// predates the gate would silently pick up the new charges. +#[test] +fn grove_version_first_selects_shipped_mmr_costs() { + let cost = &GroveVersion::first().mmr_versions.cost; + assert_eq!(cost.push, 0); + assert_eq!(cost.get_root, 0); + assert_eq!(cost.gen_proof, 0); +} + +// ── Bulk-append cost version table ─────────────────────────────────── + +/// The compaction hash count reaches a live fee through CommitmentTree, so +/// which version each protocol version selects is pinned here. +#[test] +fn bulk_append_cost_versions_are_pinned_per_protocol_version() { + for (name, version) in [ + ("GROVE_V1", &GROVE_V1), + ("GROVE_V2", &GROVE_V2), + ("GROVE_V3", &GROVE_V3), + ] { + assert_eq!( + version.bulk_append_tree_versions.cost.compaction_hash_count, 0, + "{name} must keep the shipped compaction hash count" + ); + } + assert_eq!( + GROVE_V4 + .bulk_append_tree_versions + .cost + .compaction_hash_count, + 1, + "GROVE_V4 adds the peak-bagging term" + ); + assert_eq!( + GroveVersion::first() + .bulk_append_tree_versions + .cost + .compaction_hash_count, + 0, + "the unversioned entry points delegate here and must stay on the \ + shipped figure" + ); +} diff --git a/grovedb-version/src/version/bulk_append_tree_versions.rs b/grovedb-version/src/version/bulk_append_tree_versions.rs new file mode 100644 index 000000000..a9b3bfbe3 --- /dev/null +++ b/grovedb-version/src/version/bulk_append_tree_versions.rs @@ -0,0 +1,30 @@ +//! Version gates for the bulk-append tree. +//! +//! As in [`super::mmr_versions`], only cost accounting is versioned: the +//! chunks, roots and stored bytes are identical under every version. These +//! gates matter more than the MMR ones, though, because the value they change +//! reaches a live fee — `CommitmentTree` adds the reported hash count straight +//! into its own `hash_node_calls`, and the shielded pool has been running on +//! it since mainnet activation. + +use versioned_feature_core::FeatureVersion; + +#[derive(Clone, Debug, Default)] +pub struct BulkAppendTreeVersions { + pub cost: BulkAppendTreeCostVersions, +} + +#[derive(Clone, Debug, Default)] +pub struct BulkAppendTreeCostVersions { + /// The `hash_count` a compacting append reports, which + /// `append_no_state_root` forwards and `CommitmentTree` bills. + /// + /// Version 0 reports `hash_count_for_push` — the chunk-blob leaf hash plus + /// one per peak the MMR push collapses. That omits the peak bagging the + /// compaction's own `get_root` performs, so a compaction landing on a + /// multi-peak MMR under-reports by `peaks - 1`. + /// + /// Version 1 adds that bagging term. Shipped chunk bytes and roots are + /// unaffected; what moves is the fee a compacting append is charged. + pub compaction_hash_count: FeatureVersion, +} diff --git a/grovedb-version/src/version/grovedb_versions.rs b/grovedb-version/src/version/grovedb_versions.rs index 5f3b2b8af..d1c418637 100644 --- a/grovedb-version/src/version/grovedb_versions.rs +++ b/grovedb-version/src/version/grovedb_versions.rs @@ -158,6 +158,36 @@ pub struct GroveDBOperationsVersions { pub indexed_axis: GroveDBOperationsIndexedAxisVersions, pub average_case: GroveDBOperationsAverageCaseVersions, pub worst_case: GroveDBOperationsWorstCaseVersions, + pub private_document_store: GroveDBOperationsPrivateDocumentStoreVersions, +} + +/// Version slots for the PrivateDocumentStore operation family. +/// +/// Unlike most families, these slots act as **capability gates**, not +/// implementation selectors: slot value `0` means the operation (and the +/// element type itself, via `element_creation`) is unavailable and fails +/// closed with a version-mismatch error; `1` means the v1 implementation is +/// active. `GROVE_V1`..`GROVE_V3` hold every slot at `0` — the +/// `PrivateDocumentStore` element (discriminant 24) cannot be created or +/// operated on under released protocol versions. `GROVE_V4` flips them to +/// `1`. +/// +/// Note this does NOT gate `Element::deserialize` — the element bincode +/// codec stays protocol-independent (append-only discriminants, see the +/// doc on `GroveDBElementMethodVersions::serialize`). The gate lives at +/// the write/read operation entry points and at element insertion. +#[derive(Clone, Debug, Default)] +pub struct GroveDBOperationsPrivateDocumentStoreVersions { + /// Creating a `PrivateDocumentStore` element (direct or batch insert of + /// the element itself). + pub element_creation: FeatureVersion, + /// Appending an entry (`private_document_store_insert`, the + /// `PrivateDocumentStoreInsert` batch op). + pub insert: FeatureVersion, + /// Reading an entry by position (`private_document_store_get_value`). + pub get_value: FeatureVersion, + /// Reading the entry count (`private_document_store_count`). + pub count: FeatureVersion, } #[derive(Clone, Debug, Default)] @@ -203,9 +233,11 @@ pub struct GroveDBOperationsProofVersions { pub verify_query_get_parent_tree_info_with_options: FeatureVersion, /// Whether a V1 proof binds the element bytes of a **terminally-reported /// non-Merk tree** — `CommitmentTree`, `MmrTree`, `BulkAppendTree`, - /// `DenseAppendOnlyFixedSizeTree` — to the `value_hash` its parent Merk - /// commits to. "Terminal" means the query targets the tree element itself - /// and the prover emits no lower layer. + /// `DenseAppendOnlyFixedSizeTree`, `PrivateDocumentStore` — to the + /// `value_hash` its parent Merk commits to. "Terminal" means the query + /// targets the tree element itself and the prover emits no lower layer. + /// (`PrivateDocumentStore` cannot exist before V4, so it is always + /// bound.) /// /// - `0` (V1..V3): the prover emits a bare `KVValueHash` node and the /// verifier does not require a child hash. That node hashes only diff --git a/grovedb-version/src/version/mmr_versions.rs b/grovedb-version/src/version/mmr_versions.rs new file mode 100644 index 000000000..168a8c3e2 --- /dev/null +++ b/grovedb-version/src/version/mmr_versions.rs @@ -0,0 +1,38 @@ +//! Version gates for the Merkle Mountain Range crate. +//! +//! Only cost accounting is versioned here. Every gate below leaves the +//! returned hashes, roots and proofs bit-identical — what changes is how many +//! `hash_node_calls` the operation reports. That still has to be gated, +//! because costs become fees: a node replaying a historical block must charge +//! what the block was admitted under, so a corrected charge cannot simply +//! replace the old one. + +use versioned_feature_core::FeatureVersion; + +#[derive(Clone, Debug, Default)] +pub struct MmrVersions { + pub cost: MmrCostVersions, +} + +/// Hash-charge versions for the three MMR operations that perform blake3 +/// merges internally. +/// +/// In every case version 0 is the shipped behaviour, which billed the storage +/// reads an operation performed but not the merges those reads fed, and +/// version 1 charges one hash per merge actually computed. +#[derive(Clone, Debug, Default)] +pub struct MmrCostVersions { + /// `MMR::push`. A push collapses one peak per set trailing bit of the + /// leaf count, calling `MmrNode::merge` — a blake3 — each time. Version 0 + /// billed the sibling reads those merges consume but not the merges. + /// Version 1 charges one hash per collapse. + pub push: FeatureVersion, + /// `MMR::get_root`. Bagging folds the peaks right-to-left with one + /// `MmrNode::merge` per additional peak. Version 0 billed the peak reads + /// only; version 1 charges `peaks - 1` merges. + pub get_root: FeatureVersion, + /// `MMR::gen_proof`. Proof generation folds the right-hand peaks through + /// the same `bag_peaks` helper `get_root` uses. Version 0 charged none of + /// those merges; version 1 charges `bagging_track - 1`. + pub gen_proof: FeatureVersion, +} diff --git a/grovedb-version/src/version/mod.rs b/grovedb-version/src/version/mod.rs index 7965d6489..148dce001 100644 --- a/grovedb-version/src/version/mod.rs +++ b/grovedb-version/src/version/mod.rs @@ -1,5 +1,7 @@ +pub mod bulk_append_tree_versions; pub mod grovedb_versions; pub mod merk_versions; +pub mod mmr_versions; pub mod v1; pub mod v2; pub mod v3; @@ -10,7 +12,8 @@ pub use versioned_feature_core::*; use crate::version::v3::GROVE_V3; use crate::version::v4::GROVE_V4; use crate::version::{ - grovedb_versions::GroveDBVersions, merk_versions::MerkVersions, v1::GROVE_V1, v2::GROVE_V2, + bulk_append_tree_versions::BulkAppendTreeVersions, grovedb_versions::GroveDBVersions, + merk_versions::MerkVersions, mmr_versions::MmrVersions, v1::GROVE_V1, v2::GROVE_V2, }; #[derive(Clone, Debug, Default)] @@ -18,6 +21,8 @@ pub struct GroveVersion { pub protocol_version: u32, pub grovedb_versions: GroveDBVersions, pub merk_versions: MerkVersions, + pub mmr_versions: MmrVersions, + pub bulk_append_tree_versions: BulkAppendTreeVersions, } impl GroveVersion { diff --git a/grovedb-version/src/version/v1.rs b/grovedb-version/src/version/v1.rs index a1b1f6cdb..43b129b2f 100644 --- a/grovedb-version/src/version/v1.rs +++ b/grovedb-version/src/version/v1.rs @@ -1,17 +1,20 @@ use crate::version::grovedb_versions::GroveDBAggregateSumPathQueryMethodVersions; use crate::version::{ + bulk_append_tree_versions::{BulkAppendTreeCostVersions, BulkAppendTreeVersions}, grovedb_versions::{ GroveDBApplyBatchVersions, GroveDBElementMethodVersions, GroveDBOperationsAverageCaseVersions, GroveDBOperationsDeleteUpTreeVersions, GroveDBOperationsDeleteVersions, GroveDBOperationsGetVersions, GroveDBOperationsIndexedAxisVersions, GroveDBOperationsInsertVersions, - GroveDBOperationsProofVersions, GroveDBOperationsQueryVersions, GroveDBOperationsVersions, + GroveDBOperationsPrivateDocumentStoreVersions, GroveDBOperationsProofVersions, + GroveDBOperationsQueryVersions, GroveDBOperationsVersions, GroveDBOperationsWorstCaseVersions, GroveDBPathQueryMethodVersions, GroveDBQueryLimits, GroveDBReplicationVersions, GroveDBVersions, }, merk_versions::{ MerkAverageCaseCostsVersions, MerkBatchVersions, MerkProofVersions, MerkVersions, }, + mmr_versions::{MmrCostVersions, MmrVersions}, GroveVersion, }; @@ -208,6 +211,14 @@ pub const GROVE_V1: GroveVersion = GroveVersion { add_worst_case_get_cost: 0, worst_case_commitment_tree_insert: 0, }, + // PrivateDocumentStore is unavailable before GROVE_V4: every + // slot is 0 and the operations fail closed. + private_document_store: GroveDBOperationsPrivateDocumentStoreVersions { + element_creation: 0, + insert: 0, + get_value: 0, + count: 0, + }, }, aggregate_sum_path_query_methods: GroveDBAggregateSumPathQueryMethodVersions { merge: 0 }, path_query_methods: GroveDBPathQueryMethodVersions { @@ -245,4 +256,23 @@ pub const GROVE_V1: GroveVersion = GroveVersion { prove_count_offset_on_range: 0, }, }, + // MMR hash charges: the shipped accounting, which billed the + // storage reads each operation performed but not the blake3 merges + // those reads fed. Locked here — these versions are released and a + // replayed block must be charged what it was admitted under. + mmr_versions: MmrVersions { + cost: MmrCostVersions { + push: 0, + get_root: 0, + gen_proof: 0, + }, + }, + // Compaction hash count: the shipped figure, which omits the peak + // bagging a compaction's own `get_root` performs. Locked — the + // shielded pool has been charged this since mainnet activation. + bulk_append_tree_versions: BulkAppendTreeVersions { + cost: BulkAppendTreeCostVersions { + compaction_hash_count: 0, + }, + }, }; diff --git a/grovedb-version/src/version/v2.rs b/grovedb-version/src/version/v2.rs index 5788c54e3..d817794ed 100644 --- a/grovedb-version/src/version/v2.rs +++ b/grovedb-version/src/version/v2.rs @@ -1,17 +1,20 @@ use crate::version::grovedb_versions::GroveDBAggregateSumPathQueryMethodVersions; use crate::version::{ + bulk_append_tree_versions::{BulkAppendTreeCostVersions, BulkAppendTreeVersions}, grovedb_versions::{ GroveDBApplyBatchVersions, GroveDBElementMethodVersions, GroveDBOperationsAverageCaseVersions, GroveDBOperationsDeleteUpTreeVersions, GroveDBOperationsDeleteVersions, GroveDBOperationsGetVersions, GroveDBOperationsIndexedAxisVersions, GroveDBOperationsInsertVersions, - GroveDBOperationsProofVersions, GroveDBOperationsQueryVersions, GroveDBOperationsVersions, + GroveDBOperationsPrivateDocumentStoreVersions, GroveDBOperationsProofVersions, + GroveDBOperationsQueryVersions, GroveDBOperationsVersions, GroveDBOperationsWorstCaseVersions, GroveDBPathQueryMethodVersions, GroveDBQueryLimits, GroveDBReplicationVersions, GroveDBVersions, }, merk_versions::{ MerkAverageCaseCostsVersions, MerkBatchVersions, MerkProofVersions, MerkVersions, }, + mmr_versions::{MmrCostVersions, MmrVersions}, GroveVersion, }; @@ -208,6 +211,14 @@ pub const GROVE_V2: GroveVersion = GroveVersion { add_worst_case_get_cost: 0, worst_case_commitment_tree_insert: 0, }, + // PrivateDocumentStore is unavailable before GROVE_V4: every + // slot is 0 and the operations fail closed. + private_document_store: GroveDBOperationsPrivateDocumentStoreVersions { + element_creation: 0, + insert: 0, + get_value: 0, + count: 0, + }, }, aggregate_sum_path_query_methods: GroveDBAggregateSumPathQueryMethodVersions { merge: 0 }, path_query_methods: GroveDBPathQueryMethodVersions { @@ -244,4 +255,23 @@ pub const GROVE_V2: GroveVersion = GroveVersion { prove_count_offset_on_range: 0, }, }, + // MMR hash charges: the shipped accounting, which billed the + // storage reads each operation performed but not the blake3 merges + // those reads fed. Locked here — these versions are released and a + // replayed block must be charged what it was admitted under. + mmr_versions: MmrVersions { + cost: MmrCostVersions { + push: 0, + get_root: 0, + gen_proof: 0, + }, + }, + // Compaction hash count: the shipped figure, which omits the peak + // bagging a compaction's own `get_root` performs. Locked — the + // shielded pool has been charged this since mainnet activation. + bulk_append_tree_versions: BulkAppendTreeVersions { + cost: BulkAppendTreeCostVersions { + compaction_hash_count: 0, + }, + }, }; diff --git a/grovedb-version/src/version/v3.rs b/grovedb-version/src/version/v3.rs index 88e486f7b..6a141dce3 100644 --- a/grovedb-version/src/version/v3.rs +++ b/grovedb-version/src/version/v3.rs @@ -1,17 +1,20 @@ use crate::version::grovedb_versions::GroveDBAggregateSumPathQueryMethodVersions; use crate::version::{ + bulk_append_tree_versions::{BulkAppendTreeCostVersions, BulkAppendTreeVersions}, grovedb_versions::{ GroveDBApplyBatchVersions, GroveDBElementMethodVersions, GroveDBOperationsAverageCaseVersions, GroveDBOperationsDeleteUpTreeVersions, GroveDBOperationsDeleteVersions, GroveDBOperationsGetVersions, GroveDBOperationsIndexedAxisVersions, GroveDBOperationsInsertVersions, - GroveDBOperationsProofVersions, GroveDBOperationsQueryVersions, GroveDBOperationsVersions, + GroveDBOperationsPrivateDocumentStoreVersions, GroveDBOperationsProofVersions, + GroveDBOperationsQueryVersions, GroveDBOperationsVersions, GroveDBOperationsWorstCaseVersions, GroveDBPathQueryMethodVersions, GroveDBQueryLimits, GroveDBReplicationVersions, GroveDBVersions, }, merk_versions::{ MerkAverageCaseCostsVersions, MerkBatchVersions, MerkProofVersions, MerkVersions, }, + mmr_versions::{MmrCostVersions, MmrVersions}, GroveVersion, }; @@ -212,6 +215,14 @@ pub const GROVE_V3: GroveVersion = GroveVersion { add_worst_case_get_cost: 0, worst_case_commitment_tree_insert: 0, }, + // PrivateDocumentStore is unavailable before GROVE_V4: every + // slot is 0 and the operations fail closed. + private_document_store: GroveDBOperationsPrivateDocumentStoreVersions { + element_creation: 0, + insert: 0, + get_value: 0, + count: 0, + }, }, aggregate_sum_path_query_methods: GroveDBAggregateSumPathQueryMethodVersions { merge: 0 }, path_query_methods: GroveDBPathQueryMethodVersions { @@ -259,4 +270,23 @@ pub const GROVE_V3: GroveVersion = GroveVersion { prove_count_offset_on_range: 0, }, }, + // MMR hash charges: the shipped accounting, which billed the + // storage reads each operation performed but not the blake3 merges + // those reads fed. Locked here — these versions are released and a + // replayed block must be charged what it was admitted under. + mmr_versions: MmrVersions { + cost: MmrCostVersions { + push: 0, + get_root: 0, + gen_proof: 0, + }, + }, + // Compaction hash count: the shipped figure, which omits the peak + // bagging a compaction's own `get_root` performs. Locked — the + // shielded pool has been charged this since mainnet activation. + bulk_append_tree_versions: BulkAppendTreeVersions { + cost: BulkAppendTreeCostVersions { + compaction_hash_count: 0, + }, + }, }; diff --git a/grovedb-version/src/version/v4.rs b/grovedb-version/src/version/v4.rs index 352737ad4..e08c49f27 100644 --- a/grovedb-version/src/version/v4.rs +++ b/grovedb-version/src/version/v4.rs @@ -101,18 +101,21 @@ use crate::version::grovedb_versions::GroveDBAggregateSumPathQueryMethodVersions; use crate::version::{ + bulk_append_tree_versions::{BulkAppendTreeCostVersions, BulkAppendTreeVersions}, grovedb_versions::{ GroveDBApplyBatchVersions, GroveDBElementMethodVersions, GroveDBOperationsAverageCaseVersions, GroveDBOperationsDeleteUpTreeVersions, GroveDBOperationsDeleteVersions, GroveDBOperationsGetVersions, GroveDBOperationsIndexedAxisVersions, GroveDBOperationsInsertVersions, - GroveDBOperationsProofVersions, GroveDBOperationsQueryVersions, GroveDBOperationsVersions, + GroveDBOperationsPrivateDocumentStoreVersions, GroveDBOperationsProofVersions, + GroveDBOperationsQueryVersions, GroveDBOperationsVersions, GroveDBOperationsWorstCaseVersions, GroveDBPathQueryMethodVersions, GroveDBQueryLimits, GroveDBReplicationVersions, GroveDBVersions, }, merk_versions::{ MerkAverageCaseCostsVersions, MerkBatchVersions, MerkProofVersions, MerkVersions, }, + mmr_versions::{MmrCostVersions, MmrVersions}, GroveVersion, }; @@ -313,6 +316,13 @@ pub const GROVE_V4: GroveVersion = GroveVersion { add_worst_case_get_cost: 0, worst_case_commitment_tree_insert: 1, }, + // PrivateDocumentStore activates in GROVE_V4. + private_document_store: GroveDBOperationsPrivateDocumentStoreVersions { + element_creation: 1, + insert: 1, + get_value: 1, + count: 1, + }, }, aggregate_sum_path_query_methods: GroveDBAggregateSumPathQueryMethodVersions { merge: 0 }, path_query_methods: GroveDBPathQueryMethodVersions { @@ -360,4 +370,24 @@ pub const GROVE_V4: GroveVersion = GroveVersion { prove_count_offset_on_range: 0, }, }, + // MMR hash charges: one hash per blake3 merge actually computed — + // `push` per collapsed peak, `get_root` and `gen_proof` per peak + // folded during bagging. V1..V3 bill the reads but not these + // merges. Roots and proofs are bit-identical across both versions; + // only `hash_node_calls` differs. + mmr_versions: MmrVersions { + cost: MmrCostVersions { + push: 1, + get_root: 1, + gen_proof: 1, + }, + }, + // Compaction hash count: adds the peak-bagging merges the shipped + // figure omitted, so a compacting append is charged the hashes it + // actually performs. Chunk bytes and roots are unchanged. + bulk_append_tree_versions: BulkAppendTreeVersions { + cost: BulkAppendTreeCostVersions { + compaction_hash_count: 1, + }, + }, }; diff --git a/grovedb/Cargo.toml b/grovedb/Cargo.toml index 6358fbba8..a3b527f45 100644 --- a/grovedb/Cargo.toml +++ b/grovedb/Cargo.toml @@ -26,6 +26,7 @@ grovedb-element = { version = "5.0.1", path = "../grovedb-element" } grovedb-commitment-tree = { version = "5.0.1", path = "../grovedb-commitment-tree", optional = true } grovedb-merkle-mountain-range = { version = "5.0.1", path = "../grovedb-merkle-mountain-range", optional = true, default-features = false } grovedb-bulk-append-tree = { version = "5.0.1", path = "../grovedb-bulk-append-tree", optional = true, default-features = false } +grovedb-private-document-store = { version = "5.0.1", path = "../grovedb-private-document-store", optional = true, default-features = false } grovedb-dense-fixed-sized-merkle-tree = { version = "5.0.1", path = "../grovedb-dense-fixed-sized-merkle-tree", optional = true, default-features = false } grovedb-query = { version = "5.0.1", path = "../grovedb-query" } @@ -93,6 +94,8 @@ minimal = [ "grovedb-merkle-mountain-range/storage", "grovedb-bulk-append-tree", "grovedb-bulk-append-tree/storage", + "grovedb-private-document-store", + "grovedb-private-document-store/storage", "grovedb-dense-fixed-sized-merkle-tree", "grovedb-dense-fixed-sized-merkle-tree/storage", "thiserror", diff --git a/grovedb/src/batch/batch_structure.rs b/grovedb/src/batch/batch_structure.rs index 765f7e64f..e3dcfebeb 100644 --- a/grovedb/src/batch/batch_structure.rs +++ b/grovedb/src/batch/batch_structure.rs @@ -242,6 +242,7 @@ where | GroveOp::MmrTreeAppend { .. } | GroveOp::BulkAppend { .. } | GroveOp::DenseTreeInsert { .. } + | GroveOp::PrivateDocumentStoreInsert { .. } | GroveOp::ReplaceNonMerkTreeRoot { .. } => { // User-facing tree ops are preprocessed before batch // execution into ReplaceNonMerkTreeRoot ops, which must diff --git a/grovedb/src/batch/estimated_costs/average_case_costs.rs b/grovedb/src/batch/estimated_costs/average_case_costs.rs index d5f1b7c88..1ef3fa5fa 100644 --- a/grovedb/src/batch/estimated_costs/average_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/average_case_costs.rs @@ -47,11 +47,13 @@ impl GroveOp { &self, key: &KeyInfo, layer_element_estimates: &EstimatedLayerInformation, - // The declared chunk power of the commitment tree a - // `CommitmentTreeInsert` op targets (from the tree's own layer in - // the estimation paths), or `None` to charge the constructor- - // enforced cap. Ignored by every other op type. - ct_chunk_power: Option, + // The declared chunk power of the append-only tree this op targets + // — a `CommitmentTreeInsert`'s commitment tree or a + // `PrivateDocumentStoreInsert`'s store — read from that tree's own + // layer in the estimation paths. Both types size their dense-recompute + // and compaction terms by `2^chunk_power`, which the op itself does + // not carry. Ignored by every other op type. + append_tree_chunk_power: Option, propagate: bool, grove_version: &GroveVersion, ) -> CostResult<(), Error> { @@ -219,7 +221,7 @@ impl GroveOp { payload, key, layer_element_estimates, - ct_chunk_power, + append_tree_chunk_power, propagate, grove_version, ) @@ -287,6 +289,83 @@ impl GroveOp { sinsemilla_hash_calls: 0, }) } + GroveOp::PrivateDocumentStoreInsert { entry } => { + // The dense-recompute and compaction terms scale with + // `2^chunk_power`, which the op does not carry, so the + // store's own layer MUST be declared with + // `TreeType::PrivateDocumentStore(chunk_power)` — the same + // declare-your-layers contract `CommitmentTreeInsert` + // follows. A silent fallback would either under-bound or + // grotesquely over-reserve; both are worse than a loud error + // at integration time. + let Some(chunk_power) = append_tree_chunk_power else { + return Err(Error::PathNotFoundInCacheForEstimatedCosts( + "PrivateDocumentStoreInsert estimation requires the store's own layer \ + declared with TreeType::PrivateDocumentStore(chunk_power) in the \ + estimated layer information" + .to_string(), + )) + .wrap_with_cost(OperationCost::default()); + }; + let item_cost = GroveDb::average_case_merk_replace_tree( + key, + layer_element_estimates, + TreeType::PrivateDocumentStore(chunk_power), + propagate, + grove_version, + ); + use grovedb_costs::storage_cost::{removal::StorageRemovedBytes, StorageCost}; + // `entry.len()` IS the store's committed entry size — the + // append path rejects any other length — so the byte terms + // need no separate declaration. + let entry_size = entry.len() as u32; + let epoch_entries: u32 = 1u32 << chunk_power.min(16) as u32; + // Amortized over one epoch: every entry is written once to + // the buffer, and once more into the chunk blob when the + // epoch compacts. + let amortized_compaction_bytes = entry_size; + // The dense-buffer root walk costs two hashes per filled + // position and runs on every append, so across an epoch it + // averages half the buffer. + let avg_dense_hashes = epoch_entries.saturating_sub(1); + // Bulk state root + composite pds_state root + the + // committed-config hash paid when the store is opened. + const ROOT_AND_CONFIG_HASHES: u32 = 3; + // MMR push work, amortized across the epoch it serves. + const AMORTIZED_MMR_HASHES: u32 = 1; + // The dense-root walk does not just hash: it READS every + // filled position. Averaged across an epoch that is about + // half a buffer per append, and compaction adds one more + // pass, so the I/O terms have to scale with the epoch too — + // charging one seek and zero loaded bytes understated this by + // O(epoch size). + let avg_dense_reads = epoch_entries / 2; + // A NonCounted-wrapped store serializes one byte wider, and + // the apply path now preserves that wrapper. Neither the op + // nor the declared layer records whether this store is + // wrapped, so charge the byte unconditionally: over-charging + // one byte is harmless, whereas omitting it understates every + // append to a non-counted store. + const NON_COUNTED_WRAPPER_BYTE: u32 = 1; + item_cost.add_cost(OperationCost { + // 1 buffer entry write + the root walk's reads. + seek_count: 1u32.saturating_add(avg_dense_reads), + storage_cost: StorageCost { + added_bytes: entry_size + .saturating_add(amortized_compaction_bytes) + .saturating_add(NON_COUNTED_WRAPPER_BYTE), + replaced_bytes: 0, + removed_bytes: StorageRemovedBytes::NoStorageRemoval, + }, + storage_loaded_bytes: (avg_dense_reads as u64) + .saturating_mul(entry_size as u64), + hash_node_calls: avg_dense_hashes + .saturating_add(ROOT_AND_CONFIG_HASHES) + .saturating_add(AMORTIZED_MMR_HASHES), + sinsemilla_hash_calls: 0, + }) + } + GroveOp::DenseTreeInsert { value } => { // Cost of updating parent element in the Merk let item_cost = GroveDb::average_case_merk_replace_tree( @@ -398,7 +477,7 @@ impl GroveOp { payload: &[u8], key: &KeyInfo, layer_element_estimates: &EstimatedLayerInformation, - ct_chunk_power: Option, + append_tree_chunk_power: Option, propagate: bool, grove_version: &GroveVersion, ) -> CostResult<(), Error> { @@ -419,7 +498,7 @@ impl GroveOp { payload, key, layer_element_estimates, - ct_chunk_power, + append_tree_chunk_power, propagate, grove_version, ), @@ -486,7 +565,7 @@ impl GroveOp { payload: &[u8], key: &KeyInfo, layer_element_estimates: &EstimatedLayerInformation, - ct_chunk_power: Option, + append_tree_chunk_power: Option, propagate: bool, grove_version: &GroveVersion, ) -> CostResult<(), Error> { @@ -497,7 +576,7 @@ impl GroveOp { // other estimated op follows. A silent fallback here would either // under-bound (too small) or grotesquely over-reserve (the physical // ceiling), both worse than a loud error at integration time. - let Some(chunk_power) = ct_chunk_power else { + let Some(chunk_power) = append_tree_chunk_power else { return Err(Error::PathNotFoundInCacheForEstimatedCosts( "CommitmentTreeInsert estimation requires the commitment tree's own layer \ declared with TreeType::CommitmentTree(chunk_power) in the estimated layer \ @@ -715,14 +794,65 @@ impl TreeCache for AverageCaseTreeCacheKnownPaths { // `TreeType::CommitmentTree(chunk_power)` — as Dash Platform // does — the estimate uses the tree's ACTUAL epoch scale // instead of the constructor-enforced cap. - let ct_chunk_power = if matches!(op, GroveOp::CommitmentTreeInsert { .. }) { + let append_tree_chunk_power = if matches!( + op, + GroveOp::CommitmentTreeInsert { .. } | GroveOp::PrivateDocumentStoreInsert { .. } + ) { crate::batch::batch_structure::keyless_op_tree_key(&key).and_then(|tree_key| { - let mut tree_path = path.clone(); - tree_path.push(KeyInfo::KnownKey(tree_key.to_vec())); - match self.paths.get(&tree_path).map(|layer| layer.tree_type) { - Some(TreeType::CommitmentTree(chunk_power)) => Some(chunk_power), - _ => None, - } + // Match the declared layer by KEY BYTES, not by + // `KeyInfo` equality. A caller may legitimately declare a + // layer with `KeyInfo::MaxKeySize { unique_id, .. }`, and + // `KeyInfo`'s `PartialEq` deliberately reports + // `KnownKey` and `MaxKeySize` as unequal — so an exact + // `paths.get` with a synthesized `KnownKey` misses such a + // declaration and the estimate then fails with + // `PathNotFoundInCacheForEstimatedCosts` even though the + // layer WAS declared. `KeyInfo::as_slice` yields the key + // for `KnownKey` and the `unique_id` for `MaxKeySize`, + // which is the identity callers declare with in both + // cases. + let parent_segments: Vec<&[u8]> = path.0.iter().map(|k| k.as_slice()).collect(); + self.paths + .iter() + .find(|(declared, _)| { + declared.0.len() == parent_segments.len() + 1 + && declared + .0 + .iter() + .map(|k| k.as_slice()) + .zip( + parent_segments + .iter() + .copied() + .chain(std::iter::once(tree_key)), + ) + .all(|(a, b)| a == b) + }) + .and_then(|(_, layer)| { + // The declared layer must be the RIGHT KIND of + // append tree for this op. Accepting either kind + // would let a store's epoch be estimated from a + // commitment tree's declaration (or vice versa), + // silently producing a confident but wrong + // figure; a mismatch should fall through to the + // loud "declare your layer" error instead. + let chunk_power = match (&op, layer.tree_type) { + ( + GroveOp::CommitmentTreeInsert { .. }, + TreeType::CommitmentTree(cp), + ) + | ( + GroveOp::PrivateDocumentStoreInsert { .. }, + TreeType::PrivateDocumentStore(cp), + ) => cp, + _ => return None, + }; + // A declared chunk power outside the range the + // constructors accept cannot describe a real + // tree, so treat it as undeclared rather than + // estimating from it. + (1..=16).contains(&chunk_power).then_some(chunk_power) + }) }) } else { None @@ -732,7 +862,7 @@ impl TreeCache for AverageCaseTreeCacheKnownPaths { op.average_case_cost( &key, layer_element_estimates, - ct_chunk_power, + append_tree_chunk_power, false, grove_version ) @@ -1910,6 +2040,86 @@ mod tests { ); } + #[test] + fn test_private_document_store_insert_average_case_cost_direct() { + let grove_version = GroveVersion::latest(); + let op = GroveOp::PrivateDocumentStoreInsert { + entry: vec![42u8; 64], + }; + let key = KeyInfo::KnownKey(b"pds_key".to_vec()); + let layer_info = EstimatedLayerInformation { + tree_type: TreeType::NormalTree, + estimated_layer_count: ApproximateElements(5), + estimated_layer_sizes: AllSubtrees(4, NoSumTrees, None), + }; + let cost = op + .average_case_cost(&key, &layer_info, Some(4), false, grove_version) + .cost_as_result() + .expect("expected cost for private document store insert"); + // PrivateDocumentStoreInsert mirrors BulkAppend: parent replace cost + // plus buffer write + running hash; added bytes are entry-size + // parametrized (the entry length is the committed entry_size). + assert!( + cost.seek_count > 0, + "expected seek_count > 0, got {}", + cost.seek_count + ); + assert!( + cost.hash_node_calls > 0, + "expected hash_node_calls > 0, got {}", + cost.hash_node_calls + ); + assert!( + cost.storage_cost.added_bytes >= 64, + "expected added_bytes >= entry size, got {}", + cost.storage_cost.added_bytes + ); + + // Entry-size parametrization: doubling the entry length grows the + // added bytes by exactly the difference. + let op_large = GroveOp::PrivateDocumentStoreInsert { + entry: vec![42u8; 128], + }; + let cost_large = op_large + .average_case_cost(&key, &layer_info, Some(4), false, grove_version) + .cost_as_result() + .expect("expected cost for larger entry"); + + // Undeclared layer must fail loudly rather than silently guessing a + // chunk power, matching the CommitmentTreeInsert contract. + assert!( + op.average_case_cost(&key, &layer_info, None, false, grove_version) + .cost_as_result() + .is_err(), + "estimation without a declared PrivateDocumentStore layer must error" + ); + + // The estimate tracks the declared chunk power: a larger epoch means + // a deeper dense-buffer walk. + let small = op + .average_case_cost(&key, &layer_info, Some(2), false, grove_version) + .cost_as_result() + .expect("cost at chunk_power 2"); + let big = op + .average_case_cost(&key, &layer_info, Some(10), false, grove_version) + .cost_as_result() + .expect("cost at chunk_power 10"); + assert!( + big.hash_node_calls > small.hash_node_calls, + "a larger declared epoch must cost more hashing ({} vs {})", + big.hash_node_calls, + small.hash_node_calls + ); + // Each entry is written TWICE across its lifetime: once into the + // dense buffer and once more into the chunk blob when the epoch + // compacts. The amortized per-append charge is therefore 2x the + // entry size, so doubling the entry grows added_bytes by 2 x 64. + assert_eq!( + cost_large.storage_cost.added_bytes - cost.storage_cost.added_bytes, + 128 + ); + } + #[test] fn test_dense_tree_insert_average_case_cost_direct() { let grove_version = GroveVersion::latest(); diff --git a/grovedb/src/batch/estimated_costs/worst_case_costs.rs b/grovedb/src/batch/estimated_costs/worst_case_costs.rs index 18cd586ba..eb8b8d247 100644 --- a/grovedb/src/batch/estimated_costs/worst_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/worst_case_costs.rs @@ -278,6 +278,109 @@ impl GroveOp { sinsemilla_hash_calls: 0, }) } + GroveOp::PrivateDocumentStoreInsert { entry } => { + // Cost of updating parent element in the Merk. + let item_cost = GroveDb::worst_case_merk_replace_tree( + key, + TreeType::PrivateDocumentStore(0), + in_parent_tree_type, + worst_case_layer_element_estimates, + propagate, + grove_version, + ); + // A genuine UPPER BOUND over every configuration the type + // permits, not a typical-case figure. `chunk_power` is + // validated to 1..=16, so the dense buffer holds at most + // `2^16 - 1 = 65535` entries and an epoch is at most + // `2^16 = 65536` entries. + // + // The costly op is the compacting append: the dense-root walk + // hashes every filled position twice, the epoch is serialized + // into one chunk blob, and the blob is pushed to the MMR. + // + // This deliberately OVER-estimates smaller configurations — + // a `chunk_power = 4` store pays the `chunk_power = 16` + // bound — because `GroveOp::PrivateDocumentStoreInsert` + // carries only the entry, not the store's committed config, + // and the config is not knowable here. Over-estimating is the + // safe direction for a fee admission bound (an under-estimate + // makes a legitimate block fail replay). Threading the + // committed `{entry_size, chunk_power}` into the estimate is + // the way to tighten this; it needs the op or the layer + // estimate to carry the config. + use grovedb_costs::storage_cost::{removal::StorageRemovedBytes, StorageCost}; + let entry_size = entry.len() as u32; + /// Largest epoch the type permits: `2^16` entries. + const MAX_EPOCH_ENTRIES: u32 = 1 << 16; + /// Largest dense buffer: `2^16 - 1` filled positions, each + /// costing a value hash and a node hash on the root walk. + const MAX_DENSE_HASHES: u32 = 2 * (MAX_EPOCH_ENTRIES - 1); + /// MMR push merges (bounded by the 64-bit position space). + const MAX_MMR_MERGES: u32 = 65; + /// Bulk state root + composite pds_state root + the + /// committed-config hash paid when the store is opened. + const ROOT_AND_CONFIG_HASHES: u32 = 3; + const MAX_HASH_CALLS: u32 = + MAX_DENSE_HASHES + MAX_MMR_MERGES + ROOT_AND_CONFIG_HASHES; + // Writes: buffer entry + chunk blob + MMR nodes + const MAX_WRITES: u32 = 1 + 1 + MAX_MMR_MERGES; + const MAX_MMR_READS: u32 = 64; // MMR sibling reads + /// Dense-buffer reads, which dominate the seek count and are + /// easy to miss: the buffer lives in storage, so BOTH the + /// dense-root walk and compaction read it position by + /// position. A reopened store at `chunk_power = 16` walks up + /// to `2^16 - 1` filled positions to derive the root, and a + /// compacting append reads the whole epoch again to build the + /// chunk blob. Counting only the MMR's 64 sibling reads left + /// `seek_count` three orders of magnitude below the true + /// worst case, which is not an upper bound at all. + const MAX_DENSE_READS: u32 = 2 * (MAX_EPOCH_ENTRIES - 1); + // A compacted epoch is not stored as a bare payload: the + // chunk blob carries a 9-byte header, sits inside a 37-byte + // MMR leaf envelope, and every internal MMR node the push + // creates costs a further 33 bytes. Counting only the raw + // payload made the "upper bound" fall short — for + // `entry_size = 1` the very first compaction already exceeded + // it. + const CHUNK_HEADER_BYTES: u32 = 9; + const MMR_LEAF_ENVELOPE_BYTES: u32 = 37; + const MMR_INTERNAL_NODE_BYTES: u32 = 33; + const MMR_SERIALIZATION_OVERHEAD: u32 = CHUNK_HEADER_BYTES + + MMR_LEAF_ENVELOPE_BYTES + + MMR_INTERNAL_NODE_BYTES * MAX_MMR_MERGES; + // The compaction blob holds a whole epoch of entries, so its + // size scales with the committed entry size — a flat byte + // constant is not a bound. `entry_size` is capped at + // `u16::MAX` at every creation site precisely so this product + // stays representable in the u32 `added_bytes` field. + let max_compaction_blob = MAX_EPOCH_ENTRIES + .saturating_mul(entry_size) + .saturating_add(MMR_SERIALIZATION_OVERHEAD); + item_cost.add_cost(OperationCost { + seek_count: MAX_WRITES + .saturating_add(MAX_MMR_READS) + .saturating_add(MAX_DENSE_READS), + storage_cost: StorageCost { + // +1 for the NonCounted wrapper byte a preserved + // wrapper adds to the replaced parent element; the op + // does not record whether this store is wrapped, so + // the bound charges it unconditionally. + added_bytes: entry_size + .saturating_add(max_compaction_blob) + .saturating_add(1), + replaced_bytes: 0, + removed_bytes: StorageRemovedBytes::NoStorageRemoval, + }, + // Each dense read returns one entry, so the bytes those + // reads load scale with the committed entry size. This + // product exceeds u32, hence the u64 arithmetic. + storage_loaded_bytes: (33 * MAX_MMR_READS) as u64 + + max_compaction_blob as u64 + + (MAX_DENSE_READS as u64).saturating_mul(entry_size as u64), + hash_node_calls: MAX_HASH_CALLS, + sinsemilla_hash_calls: 0, + }) + } GroveOp::DenseTreeInsert { value } => { // Cost of updating parent element in the Merk let item_cost = GroveDb::worst_case_merk_replace_tree( @@ -1344,6 +1447,31 @@ mod tests { assert_eq!(cost.sinsemilla_hash_calls, 0); } + #[test] + fn test_private_document_store_insert_worst_case_cost_direct() { + let grove_version = GroveVersion::latest(); + let op = GroveOp::PrivateDocumentStoreInsert { + entry: vec![0u8; 128], + }; + let key = KeyInfo::KnownKey(b"pds_key".to_vec()); + let cost = op + .worst_case_cost( + &key, + TreeType::NormalTree, + &MaxElementsNumber(100), + false, + grove_version, + ) + .cost_as_result() + .expect("expected worst case cost for private document store insert"); + assert!(cost.seek_count > 0); + assert!(cost.hash_node_calls > 0); + // The worst case includes the compaction blob bound plus the + // entry-size-parametrized per-append write. + assert!(cost.storage_cost.added_bytes >= 128 + 65536); + assert_eq!(cost.sinsemilla_hash_calls, 0); + } + #[test] fn test_dense_tree_insert_worst_case_cost_direct() { let grove_version = GroveVersion::latest(); diff --git a/grovedb/src/batch/indexed_tree/pre_state.rs b/grovedb/src/batch/indexed_tree/pre_state.rs index 2f342d8cd..849ef6f33 100644 --- a/grovedb/src/batch/indexed_tree/pre_state.rs +++ b/grovedb/src/batch/indexed_tree/pre_state.rs @@ -97,7 +97,8 @@ fn validate_indexed_child_ops( | GroveOp::CommitmentTreeInsert { .. } | GroveOp::MmrTreeAppend { .. } | GroveOp::BulkAppend { .. } - | GroveOp::DenseTreeInsert { .. } => continue, + | GroveOp::DenseTreeInsert { .. } + | GroveOp::PrivateDocumentStoreInsert { .. } => continue, }; // Child-type acceptance, delegated to merk's own rule rather than a // second copy of it: `get_feature_type` is what decides whether an diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index e01228bd3..a92d41e02 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -154,6 +154,16 @@ pub enum NonMerkTreeMeta { /// Fixed height of the dense Merkle tree. height: u8, }, + /// PrivateDocumentStore state: total_count plus the committed config + /// {entry_size, chunk_power}. + PrivateDocumentStore { + /// Total number of entries appended so far. + total_count: u64, + /// Committed byte length of every entry. + entry_size: u32, + /// Power-of-2 chunk size for epochs. + chunk_power: u8, + }, } impl NonMerkTreeMeta { @@ -170,6 +180,9 @@ impl NonMerkTreeMeta { NonMerkTreeMeta::DenseTree { height, .. } => { TreeType::DenseAppendOnlyFixedSizeTree(*height) } + NonMerkTreeMeta::PrivateDocumentStore { chunk_power, .. } => { + TreeType::PrivateDocumentStore(*chunk_power) + } } } @@ -188,6 +201,13 @@ impl NonMerkTreeMeta { NonMerkTreeMeta::DenseTree { count, height } => { Element::new_dense_tree(*count, *height, flags) } + NonMerkTreeMeta::PrivateDocumentStore { + total_count, + entry_size, + chunk_power, + } => { + Element::new_private_document_store(*total_count, *entry_size, *chunk_power, flags) + } } } @@ -198,6 +218,7 @@ impl NonMerkTreeMeta { NonMerkTreeMeta::MmrTree { mmr_size } => *mmr_size, NonMerkTreeMeta::BulkAppendTree { total_count, .. } => *total_count, NonMerkTreeMeta::DenseTree { count, .. } => *count as u64, + NonMerkTreeMeta::PrivateDocumentStore { total_count, .. } => *total_count, } } } @@ -484,6 +505,13 @@ pub enum GroveOp { /// Value to insert value: Vec, }, + /// Append a fixed-size entry to a PrivateDocumentStore. The entry's + /// length must equal the store's committed `entry_size`; the append is + /// rejected otherwise. + PrivateDocumentStoreInsert { + /// Entry to append (exactly `entry_size` bytes) + entry: Vec, + }, } impl GroveOp { @@ -513,6 +541,7 @@ impl GroveOp { GroveOp::InsertNonMerkTree { .. } => 16, GroveOp::ReplaceAggregateIndexedTreeRootKeys { .. } => 17, GroveOp::InsertAggregateIndexedTreeRootKeys { .. } => 18, + GroveOp::PrivateDocumentStoreInsert { .. } => 19, } } @@ -574,7 +603,8 @@ impl GroveOp { GroveOp::CommitmentTreeInsert { .. } | GroveOp::MmrTreeAppend { .. } | GroveOp::BulkAppend { .. } - | GroveOp::DenseTreeInsert { .. } => false, + | GroveOp::DenseTreeInsert { .. } + | GroveOp::PrivateDocumentStoreInsert { .. } => false, } } } @@ -865,6 +895,9 @@ impl fmt::Debug for QualifiedGroveDbOp { GroveOp::MmrTreeAppend { .. } => "MMR Tree Append".to_string(), GroveOp::BulkAppend { .. } => "Bulk Append".to_string(), GroveOp::DenseTreeInsert { .. } => "Dense Tree Insert".to_string(), + GroveOp::PrivateDocumentStoreInsert { .. } => { + "Private Document Store Insert".to_string() + } GroveOp::ReplaceAggregateIndexedTreeRootKeys { .. } => { "Replace CountIndexedTree primary+secondary roots".to_string() } @@ -1256,6 +1289,18 @@ impl QualifiedGroveDbOp { } } + /// A private document store insert op. `path` includes the store key as + /// its last segment. The entry must be exactly the store's committed + /// `entry_size` bytes; the batch preprocessor rejects any other length. + pub fn private_document_store_insert_op(path: Vec>, entry: Vec) -> Self { + let path = KeyInfoPath::from_known_owned_path(path); + Self { + path, + key: None, + op: GroveOp::PrivateDocumentStoreInsert { entry }, + } + } + /// Verify consistency of operations pub fn verify_consistency_of_operations( ops: &[QualifiedGroveDbOp], @@ -1990,9 +2035,10 @@ where | Element::DenseAppendOnlyFixedSizeTree(..) | Element::ProvableSumIndexedTree(..) | Element::ProvableCountIndexedTree(..) - | Element::ProvableCountProvableSumIndexedTree(..) => Err( - Error::InvalidBatchOperation("references can not point to trees being updated"), - ) + | Element::ProvableCountProvableSumIndexedTree(..) + | Element::PrivateDocumentStore(..) => Err(Error::InvalidBatchOperation( + "references can not point to trees being updated", + )) .wrap_with_cost(cost), // underlying() unwraps a single level; the constructor and // (de)serializer reject nested wrappers, so these are @@ -2059,7 +2105,8 @@ where | GroveOp::CommitmentTreeInsert { .. } | GroveOp::MmrTreeAppend { .. } | GroveOp::BulkAppend { .. } - | GroveOp::DenseTreeInsert { .. } => Err(Error::InvalidBatchOperation( + | GroveOp::DenseTreeInsert { .. } + | GroveOp::PrivateDocumentStoreInsert { .. } => Err(Error::InvalidBatchOperation( "references can not point to trees being updated", )) .wrap_with_cost(cost), @@ -2152,12 +2199,11 @@ where | Element::DenseAppendOnlyFixedSizeTree(..) | Element::ProvableSumIndexedTree(..) | Element::ProvableCountIndexedTree(..) - | Element::ProvableCountProvableSumIndexedTree(..) => { - Err(Error::InvalidBatchOperation( - "references can not point to trees being updated", - )) - .wrap_with_cost(cost) - } + | Element::ProvableCountProvableSumIndexedTree(..) + | Element::PrivateDocumentStore(..) => Err(Error::InvalidBatchOperation( + "references can not point to trees being updated", + )) + .wrap_with_cost(cost), // Wrappers are unwrapped via underlying() above. Element::NonCounted(_) | Element::NotSummed(_) @@ -2206,12 +2252,11 @@ where | Element::DenseAppendOnlyFixedSizeTree(..) | Element::ProvableSumIndexedTree(..) | Element::ProvableCountIndexedTree(..) - | Element::ProvableCountProvableSumIndexedTree(..) => { - Err(Error::InvalidBatchOperation( - "references can not point to trees being updated", - )) - .wrap_with_cost(cost) - } + | Element::ProvableCountProvableSumIndexedTree(..) + | Element::PrivateDocumentStore(..) => Err(Error::InvalidBatchOperation( + "references can not point to trees being updated", + )) + .wrap_with_cost(cost), // Wrappers are unwrapped via underlying() above. Element::NonCounted(_) | Element::NotSummed(_) @@ -2515,6 +2560,13 @@ where // Without these checks, batch users could persist // wrapped elements into the wrong tree types and silently // violate the wrapper invariant. + if matches!(in_tree_type, TreeType::PrivateDocumentStore(_)) { + return Err(Error::InvalidBatchOperation( + "private document stores cannot hold child elements; entries are \ + appended via the PrivateDocumentStoreInsert operation", + )) + .wrap_with_cost(cost); + } if element.is_non_counted() && !in_tree_type.accepts_non_counted_children() { return Err(Error::InvalidBatchOperation( "non-counted elements may only be inserted into non-provable \ @@ -2935,6 +2987,113 @@ where ) ); } + Element::PrivateDocumentStore(total_count, entry_size, chunk_power, _) => { + // Fail closed on protocol versions that predate + // the element type. + cost_return_on_error_no_add!( + cost, + crate::operations::private_document_store::check_pds_enabled( + "batch insert Element::PrivateDocumentStore", + grove_version + .grovedb_versions + .operations + .private_document_store + .element_creation, + ) + ); + // Only empty-creation is allowed: the child hash + // written below is the empty state root for this + // config, and entries can only be added through + // the typed append path. + if *total_count != 0 { + return Err(Error::InvalidBatchOperation( + "a PrivateDocumentStore must be empty at the moment of batch \ + insertion (total_count = 0); entries are appended via the \ + PrivateDocumentStoreInsert operation", + )) + .wrap_with_cost(cost); + } + // Same config validation the element constructors + // enforce — a caller-built element must not bypass + // it, since the config is committed into the state + // root. + if *entry_size == 0 + || *entry_size > u16::MAX as u32 + || !(1..=16).contains(chunk_power) + { + return Err(Error::InvalidBatchOperation( + "a PrivateDocumentStore requires entry_size in 1..=65535 and \ + chunk_power in 1..=16", + )) + .wrap_with_cost(cost); + } + // A private document store is write-once, so + // creating one over an existing element is + // ALWAYS rejected — not just under + // `InsertIfNotExists` or + // `validate_insertion_does_not_override`. + // + // Without this, a plain `InsertOrReplace` of a + // fresh (total_count = 0) store over a populated + // one is accepted with default options: the + // element resets to empty and re-binds the empty + // state root while the old chunk blobs, MMR nodes + // and buffer entries stay behind in the subtree's + // data namespace. That both leaks storage and + // breaks the type's central promise, since a + // wholesale overwrite is a delete of every entry. + // Replacing a store means deleting it first, + // which clears the data namespace. + let merk = self.merks.get_mut(path).expect("the Merk is cached"); + let existing = cost_return_on_error_into!( + &mut cost, + element.element_at_key_already_exists( + merk, + key_info.get_key_clone().as_slice(), + grove_version, + ) + ); + if existing { + if is_insert_if_not_exists && !error_if_exists { + // `InsertIfNotExists` semantics: not an + // error, just nothing to do. + continue; + } + return Err(Error::InvalidBatchOperation( + "a PrivateDocumentStore already exists at this key; it is \ + append-only and cannot be overwritten \u{2014} delete it \ + first, which clears its data namespace", + )) + .wrap_with_cost(cost); + } + let merk_feature_type = cost_return_on_error_into!( + &mut cost, + element + .get_feature_type(in_tree_type) + .wrap_with_cost(OperationCost::default()) + ); + // Deriving the empty root performs two blake3 + // calls — the committed-config hash and the + // composite `pds_state` hash — neither of which + // the helper can bill, since it returns a bare + // array. Charge them here so creating a store + // through a batch matches the direct path. + cost.hash_node_calls = cost.hash_node_calls.saturating_add(2); + cost_return_on_error_into!( + &mut cost, + element.insert_subtree_into_batch_operations( + key_info.get_key_clone(), + grovedb_private_document_store::empty_private_document_store_state_root( + *entry_size, + *chunk_power, + ), + false, + &mut batch_operations, + merk_feature_type, + grove_version, + ) + ); + } Element::Item(..) | Element::SumItem(..) | Element::ItemWithSumItem(..) => { let merk_feature_type = cost_return_on_error_into!( &mut cost, @@ -3140,6 +3299,13 @@ where // wrapper into a parent that doesn't accept it — // including any `Provable*` count tree where the // count is cryptographically committed. + if matches!(in_tree_type, TreeType::PrivateDocumentStore(_)) { + return Err(Error::InvalidBatchOperation( + "private document stores cannot hold child elements; entries are \ + appended via the PrivateDocumentStoreInsert operation", + )) + .wrap_with_cost(cost); + } if element.is_non_counted() && !in_tree_type.accepts_non_counted_children() { return Err(Error::InvalidBatchOperation( "RefreshReference with non_counted=true requires a non-provable \ @@ -3282,15 +3448,46 @@ where ); } GroveOp::ReplaceNonMerkTreeRoot { hash, meta } => { - // Read existing element to preserve flags + // Read existing element to preserve flags (and, for a + // PrivateDocumentStore, its NonCounted wrapper). let merk = self.merks.get(path).expect("the Merk is cached"); - let existing_flags = cost_return_on_error!( + let existing = cost_return_on_error!( &mut cost, GroveDb::get_element_from_subtree(merk, key_info.as_slice(), grove_version) - ) - .get_flags_owned(); + ); + let existing_non_counted = existing.is_non_counted(); + let existing_flags = existing.get_flags_owned(); let element = meta.to_element(existing_flags); + // `meta.to_element` always builds a BARE element, so a + // stored `NonCounted(tree)` would come back counted and + // change its parent count tree's aggregate — and with it + // the root hash. Restore the wrapper. + // + // Scoped to PrivateDocumentStore deliberately: the same + // latent defect exists for CommitmentTree / MmrTree / + // BulkAppendTree / DenseTree, but those are live on + // GROVE_V1..V3, so repairing them changes a released + // consensus outcome and belongs in its own version-gated + // change. PDS cannot exist before V4, so fixing it here + // alters nothing that has ever been committed. The + // element was already read above, so this costs nothing + // extra. + let element = if existing_non_counted + && matches!(meta, NonMerkTreeMeta::PrivateDocumentStore { .. }) + { + cost_return_on_error_no_add!( + cost, + element.into_non_counted().map_err(|_| { + Error::CorruptedCodeExecution( + "into_non_counted called on a wrapped element during \ + ReplaceNonMerkTreeRoot", + ) + }) + ) + } else { + element + }; let merk_feature_type = cost_return_on_error_into_no_add!( cost, element.get_feature_type(in_tree_type) @@ -3487,6 +3684,13 @@ where )) .wrap_with_cost(cost); } + GroveOp::PrivateDocumentStoreInsert { .. } => { + return Err(Error::InvalidBatchOperation( + "PrivateDocumentStoreInsert should have been preprocessed before batch \ + execution", + )) + .wrap_with_cost(cost); + } GroveOp::ReplaceAggregateIndexedTreeRootKeys { primary_hash, primary_root_key, @@ -3688,7 +3892,8 @@ where | Element::DenseAppendOnlyFixedSizeTree(..) | Element::ProvableSumIndexedTree(..) | Element::ProvableCountIndexedTree(..) - | Element::ProvableCountProvableSumIndexedTree(..) => { + | Element::ProvableCountProvableSumIndexedTree(..) + | Element::PrivateDocumentStore(..) => { let tree_type = new_element .tree_type() .expect("tree_type guaranteed by match arm"); @@ -4194,6 +4399,28 @@ impl GroveDb { meta, non_counted, } + } else if let Element::PrivateDocumentStore( + total_count, + entry_size, + chunk_power, + flags, + ) = element + { + let meta = + NonMerkTreeMeta::PrivateDocumentStore { + total_count: *total_count, + entry_size: *entry_size, + chunk_power: *chunk_power, + }; + *mutable_occupied_entry = + GroveOp::InsertNonMerkTree { + hash: root_hash, + root_key: calculated_root_key, + flags: flags.clone(), + aggregate_data, + meta, + non_counted, + } } else if let Element::MmrTree( mmr_size, flags, @@ -4344,6 +4571,13 @@ impl GroveDb { )) .wrap_with_cost(cost); } + GroveOp::PrivateDocumentStoreInsert { .. } => { + return Err(Error::InvalidBatchOperation( + "PrivateDocumentStoreInsert ops should \ + have been preprocessed", + )) + .wrap_with_cost(cost); + } } } } @@ -4786,6 +5020,26 @@ impl GroveDb { ) ); } + GroveOp::PrivateDocumentStoreInsert { entry } => { + let mut path_vec: Vec> = op.path.to_path(); + let key = cost_return_on_error_no_add!( + cost, + path_vec.pop().ok_or(Error::InvalidBatchOperation( + "append op path must include tree key" + )) + ); + let path_slices: Vec<&[u8]> = path_vec.iter().map(|p| p.as_slice()).collect(); + cost_return_on_error!( + &mut cost, + self.private_document_store_insert( + path_slices.as_slice(), + &key, + entry.clone(), + transaction, + grove_version, + ) + ); + } GroveOp::DenseTreeInsert { value } => { let mut path_vec: Vec> = op.path.to_path(); let key = cost_return_on_error_no_add!( @@ -5447,6 +5701,18 @@ impl GroveDb { self.preprocess_dense_tree_ops(ops, tx.as_ref(), &storage_batch, grove_version) ); + // Preprocess PrivateDocumentStoreInsert ops: execute size-validated + // appends then convert to ReplaceNonMerkTreeRoot ops + let ops = cost_return_on_error!( + &mut cost, + self.preprocess_private_document_store_ops( + ops, + tx.as_ref(), + &storage_batch, + grove_version + ) + ); + // Collect paths of subtrees being deleted (so their storage can be // cleaned up after apply_body) and run the pre-apply emptiness // checks / Skip filtering. On V1..V3 the cleanup lists are filled @@ -5841,6 +6107,18 @@ impl GroveDb { self.preprocess_dense_tree_ops(ops, tx.as_ref(), &storage_batch, grove_version) ); + // Preprocess PrivateDocumentStoreInsert ops: execute size-validated + // appends then convert to ReplaceNonMerkTreeRoot ops + let ops = cost_return_on_error!( + &mut cost, + self.preprocess_private_document_store_ops( + ops, + tx.as_ref(), + &storage_batch, + grove_version + ) + ); + let mut batch_apply_options = batch_apply_options.unwrap_or_default(); // Collect paths of subtrees being deleted (so their storage can be diff --git a/grovedb/src/debugger.rs b/grovedb/src/debugger.rs index c61d00075..32368a252 100644 --- a/grovedb/src/debugger.rs +++ b/grovedb/src/debugger.rs @@ -972,6 +972,12 @@ fn element_to_grovedbg(element: crate::Element) -> grovedbg_types::Element { element_flags, } } + crate::Element::PrivateDocumentStore(_, _, _, element_flags) => { + grovedbg_types::Element::Subtree { + root_key: None, + element_flags, + } + } // The visualizer wire format has no wrapper variants; render the // inner element. The wrapper is invisible at the debug-UI layer. crate::Element::NonCounted(inner) diff --git a/grovedb/src/error.rs b/grovedb/src/error.rs index 534b7b09f..6f616ac54 100644 --- a/grovedb/src/error.rs +++ b/grovedb/src/error.rs @@ -169,6 +169,10 @@ pub enum Error { #[error("commitment tree error: {0}")] /// Commitment tree operation error CommitmentTreeError(String), + + #[error("private document store error: {0}")] + /// Private document store operation error + PrivateDocumentStoreError(String), } impl Error { @@ -195,7 +199,8 @@ impl Error { | Self::ClientReturnedNonClientError(s) | Self::PathNotFoundInCacheForEstimatedCosts(s) | Self::NotSupported(s) - | Self::CommitmentTreeError(s) => { + | Self::CommitmentTreeError(s) + | Self::PrivateDocumentStoreError(s) => { s.push_str(", "); s.push_str(append.as_ref()); } diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index a035fd440..5f06040dc 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -1935,7 +1935,8 @@ impl GroveDb { | Element::CommitmentTree(..) | Element::MmrTree(..) | Element::BulkAppendTree(..) - | Element::DenseAppendOnlyFixedSizeTree(..) => { + | Element::DenseAppendOnlyFixedSizeTree(..) + | Element::PrivateDocumentStore(..) => { let (kv_value, element_value_hash) = merk .get_value_and_value_hash( &key, @@ -1971,6 +1972,7 @@ impl GroveDb { new_path_ref.clone(), transaction, merk_root_hash, + grove_version, ); let actual_value_hash = value_hash(&kv_value).unwrap(); @@ -1983,6 +1985,38 @@ impl GroveDb { ); } + // PrivateDocumentStore integrity: the state root + // authenticates whatever bytes were written, so a buggy + // or gate-bypassing writer could persist entries that + // violate the committed `entry_size` under a perfectly + // consistent hash chain. Report that as its own issue, + // carrying the specific failure, rather than folding it + // into the hash comparison above. + if let Some(label) = self.private_document_store_entry_size_issue( + &element, + new_path_ref.clone(), + transaction, + ) { + let expected_placeholder: CryptoHash = blake3::hash( + b"private document store entries match committed entry_size", + ) + .into(); + let actual_placeholder: CryptoHash = blake3::hash(label.as_bytes()).into(); + // Record under a dedicated sentinel child path, the + // way the indexed-tree integrity checks do. Keying it + // on `new_path` with `or_insert` would silently drop + // this violation whenever a child-hash mismatch was + // already recorded at the same path — i.e. exactly + // when both checks fail — which defeats the point of + // reporting it as its own diagnostic. + let mut issue_path = new_path.to_vec(); + issue_path.push(b"__pds_entry_size__".to_vec()); + issues.insert( + issue_path, + (root_hash, expected_placeholder, actual_placeholder), + ); + } + // Software-consistency check: the aggregate fields // stored in the parent's tree element (e.g. // `sum_value` in `ProvableSumTree(_, sum_value, _)`) @@ -2381,6 +2415,44 @@ impl GroveDb { Ok(issues) } + /// Run the PrivateDocumentStore entry-size integrity walk, returning a + /// human-readable description of the first violation, or `None` when the + /// element is not a store, is empty, or every entry is well-formed. + /// + /// Kept separate from `compute_non_merk_child_hash` so a violation is + /// reported with its actual message instead of being signalled by + /// returning a deliberately-wrong hash. + fn private_document_store_entry_size_issue<'b, B: AsRef<[u8]>>( + &self, + element: &Element, + subtree_path: SubtreePath<'b, B>, + transaction: &Transaction, + ) -> Option { + let Element::PrivateDocumentStore(total_count, entry_size, chunk_power, _) = + element.underlying() + else { + return None; + }; + if *total_count == 0 { + return None; + } + let storage_ctx = self + .db + .get_transactional_storage_context(subtree_path, None, transaction) + .unwrap(); + match grovedb_private_document_store::PrivateDocumentStore::from_state( + *total_count, + *entry_size, + *chunk_power, + storage_ctx, + ) + .unwrap() + { + Ok(store) => store.verify_entry_sizes().err().map(|e| e.to_string()), + Err(e) => Some(format!("cannot open private document store: {e}")), + } + } + /// Compute the child hash for a non-Merk tree element by reconstructing /// its tree from storage and computing the state root. /// Falls back to `merk_root_hash` on any error or for standard Merk trees. @@ -2390,6 +2462,7 @@ impl GroveDb { subtree_path: SubtreePath<'b, B>, transaction: &Transaction, merk_root_hash: [u8; 32], + grove_version: &GroveVersion, ) -> [u8; 32] { match element { Element::CommitmentTree(total_count, chunk_power, _) => { @@ -2438,7 +2511,7 @@ impl GroveDb { .unwrap(); let store = grovedb_merkle_mountain_range::MmrStore::new(&storage_ctx); let mmr = grovedb_merkle_mountain_range::MMR::new(*mmr_size, &store); - match mmr.get_root().value { + match mmr.get_root(grove_version).value { Ok(root) => root.hash(), Err(_) => merk_root_hash, } @@ -2460,6 +2533,40 @@ impl GroveDb { Err(_) => merk_root_hash, } } + Element::PrivateDocumentStore(total_count, entry_size, chunk_power, _) => { + // The state root binds the committed config even when the + // store is empty, so the empty case is the config-parametrized + // empty root rather than the (empty) Merk root. + if *total_count == 0 { + return grovedb_private_document_store::empty_private_document_store_state_root( + *entry_size, + *chunk_power, + ); + } + let storage_ctx = self + .db + .get_transactional_storage_context(subtree_path, None, transaction) + .unwrap(); + match grovedb_private_document_store::PrivateDocumentStore::from_state( + *total_count, + *entry_size, + *chunk_power, + storage_ctx, + ) + .unwrap() + { + // Report only the state root here. The entry-size + // integrity walk is a SEPARATE check reported by + // `private_document_store_entry_size_issue`, so a + // violation surfaces its real message ("entry at + // position N has size X, committed entry size is Y") + // instead of being laundered into an opaque hash + // mismatch — and a transient storage error during that + // walk no longer masquerades as corruption. + Ok(store) => store.compute_current_state_root().unwrap_or(merk_root_hash), + Err(_) => merk_root_hash, + } + } _ => merk_root_hash, } } @@ -2659,7 +2766,8 @@ fn aggregate_consistency_labels( (Element::CommitmentTree(..), _) | (Element::MmrTree(..), _) | (Element::BulkAppendTree(..), _) - | (Element::DenseAppendOnlyFixedSizeTree(..), _) => None, + | (Element::DenseAppendOnlyFixedSizeTree(..), _) + | (Element::PrivateDocumentStore(..), _) => None, // --- Anything else is a variant/aggregate-shape mismatch (e.g. // the inner Merk's tree-type has drifted from what the parent diff --git a/grovedb/src/operations/bulk_append_tree.rs b/grovedb/src/operations/bulk_append_tree.rs index 50d13cccb..0947cbede 100644 --- a/grovedb/src/operations/bulk_append_tree.rs +++ b/grovedb/src/operations/bulk_append_tree.rs @@ -68,7 +68,7 @@ impl GroveDb { // 2. Open transactional storage (write-through cache + MMR overlay // provide read-after-write visibility) - let subtree_path_vec = self.build_subtree_path_for_bulk(&path, key); + let subtree_path_vec = crate::util::subtree_path_with_key(&path, key); let subtree_path_refs: Vec<&[u8]> = subtree_path_vec.iter().map(|v| v.as_slice()).collect(); let subtree_path = SubtreePath::from(subtree_path_refs.as_slice()); @@ -84,7 +84,10 @@ impl GroveDb { BulkAppendTree::from_state(total_count, chunk_power, storage_ctx).map_err(map_bulk_err) ); - let result = cost_return_on_error_no_add!(cost, tree.append(&value).map_err(map_bulk_err)); + let result = cost_return_on_error_no_add!( + cost, + tree.append(&value, grove_version).map_err(map_bulk_err) + ); cost.hash_node_calls += result.hash_count; @@ -203,7 +206,7 @@ impl GroveDb { return Ok(None).wrap_with_cost(cost); } - let subtree_path_vec = self.build_subtree_path_for_bulk(&path, key); + let subtree_path_vec = crate::util::subtree_path_with_key(&path, key); let subtree_path_refs: Vec<&[u8]> = subtree_path_vec.iter().map(|v| v.as_slice()).collect(); let subtree_path = SubtreePath::from(subtree_path_refs.as_slice()); @@ -284,7 +287,7 @@ impl GroveDb { } }; - let subtree_path_vec = self.build_subtree_path_for_bulk(&path, key); + let subtree_path_vec = crate::util::subtree_path_with_key(&path, key); let subtree_path_refs: Vec<&[u8]> = subtree_path_vec.iter().map(|v| v.as_slice()).collect(); let subtree_path = SubtreePath::from(subtree_path_refs.as_slice()); @@ -338,7 +341,7 @@ impl GroveDb { } }; - let subtree_path_vec = self.build_subtree_path_for_bulk(&path, key); + let subtree_path_vec = crate::util::subtree_path_with_key(&path, key); let subtree_path_refs: Vec<&[u8]> = subtree_path_vec.iter().map(|v| v.as_slice()).collect(); let subtree_path = SubtreePath::from(subtree_path_refs.as_slice()); @@ -426,17 +429,6 @@ impl GroveDb { } } - /// Build subtree path for a BulkAppendTree at path/key. - fn build_subtree_path_for_bulk>( - &self, - path: &SubtreePath, - key: &[u8], - ) -> Vec> { - let mut v = path.to_vec(); - v.push(key.to_vec()); - v - } - /// Preprocess `BulkAppend` ops in a batch. /// /// Groups ops by (path, key), executes all appends (including compactions) @@ -535,8 +527,10 @@ impl GroveDb { // Process each value for value in values { - let result = - cost_return_on_error_no_add!(cost, tree.append(value).map_err(map_bulk_err)); + let result = cost_return_on_error_no_add!( + cost, + tree.append(value, grove_version).map_err(map_bulk_err) + ); cost.hash_node_calls += result.hash_count; } diff --git a/grovedb/src/operations/commitment_tree.rs b/grovedb/src/operations/commitment_tree.rs index b55a2f4e9..79433e540 100644 --- a/grovedb/src/operations/commitment_tree.rs +++ b/grovedb/src/operations/commitment_tree.rs @@ -136,7 +136,7 @@ impl GroveDb { // 2. Build subtree path and open transactional storage (write-through // cache + MMR overlay provide read-after-write visibility) - let ct_path_vec = self.build_ct_path(&path, key); + let ct_path_vec = crate::util::subtree_path_with_key(&path, key); let ct_path_refs: Vec<&[u8]> = ct_path_vec.iter().map(|v| v.as_slice()).collect(); let ct_path = SubtreePath::from(ct_path_refs.as_slice()); @@ -156,7 +156,7 @@ impl GroveDb { let append_result = cost_return_on_error!( &mut cost, - ct.append_raw(cmx, rho, cv_net, &payload) + ct.append_raw(cmx, rho, cv_net, &payload, grove_version) .map(|r| r.map_err(map_ct_err)) ); @@ -284,7 +284,7 @@ impl GroveDb { } }; - let ct_path_vec = self.build_ct_path(&path, key); + let ct_path_vec = crate::util::subtree_path_with_key(&path, key); let ct_path_refs: Vec<&[u8]> = ct_path_vec.iter().map(|v| v.as_slice()).collect(); let ct_path = SubtreePath::from(ct_path_refs.as_slice()); @@ -340,7 +340,7 @@ impl GroveDb { return Ok(None).wrap_with_cost(cost); } - let ct_path_vec = self.build_ct_path(&path, key); + let ct_path_vec = crate::util::subtree_path_with_key(&path, key); let ct_path_refs: Vec<&[u8]> = ct_path_vec.iter().map(|v| v.as_slice()).collect(); let ct_path = SubtreePath::from(ct_path_refs.as_slice()); @@ -415,13 +415,6 @@ impl GroveDb { } } - /// Build the subtree path for a commitment tree at path/key. - fn build_ct_path>(&self, path: &SubtreePath, key: &[u8]) -> Vec> { - let mut v = path.to_vec(); - v.push(key.to_vec()); - v - } - /// Preprocess `CommitmentTreeInsert` ops in a batch. /// /// For each group of insert ops targeting the same path: @@ -541,7 +534,7 @@ impl GroveDb { for (cmx, rho, cv_net, payload) in inserts { cost_return_on_error!( &mut cost, - ct.append_raw(*cmx, *rho, *cv_net, payload) + ct.append_raw(*cmx, *rho, *cv_net, payload, grove_version) .map(|r| r.map_err(map_ct_err)) ); } diff --git a/grovedb/src/operations/delete/mod.rs b/grovedb/src/operations/delete/mod.rs index 6f49f9f24..aaf56779e 100644 --- a/grovedb/src/operations/delete/mod.rs +++ b/grovedb/src/operations/delete/mod.rs @@ -976,12 +976,26 @@ impl GroveDb { .get_transactional_storage_context(path.clone(), Some(batch), transaction) .unwrap_add_cost(&mut cost); + // The merk reopened here is the PARENT merk (at `path`), but + // the historical code labels it with the DELETED CHILD's + // tree type. For a PrivateDocumentStore child that label + // would trip the merk-level "no ops on a PDS Merk" + // chokepoint (the delete below applies to the parent), so + // use the parent's actual tree type for PDS deletions. + // Existing types keep the historical label byte-for-byte to + // avoid any behavior change on released paths. + let reopen_tree_type = + if matches!(tree_type, grovedb_merk::TreeType::PrivateDocumentStore(_)) { + subtree_to_delete_from.tree_type + } else { + tree_type + }; let mut merk_to_delete_tree_from = cost_return_on_error!( &mut cost, Merk::open_layered_with_root_key( storage, subtree_to_delete_from.root_key(), - tree_type, + reopen_tree_type, Some(&Element::value_defined_cost_for_serialized_value), grove_version, ) diff --git a/grovedb/src/operations/get/mod.rs b/grovedb/src/operations/get/mod.rs index 40f4f0997..7fc9a8c8a 100644 --- a/grovedb/src/operations/get/mod.rs +++ b/grovedb/src/operations/get/mod.rs @@ -420,7 +420,8 @@ impl GroveDb { | Ok(Element::CommitmentTree(..)) | Ok(Element::MmrTree(..)) | Ok(Element::BulkAppendTree(..)) - | Ok(Element::DenseAppendOnlyFixedSizeTree(..)) => Ok(()).wrap_with_cost(cost), + | Ok(Element::DenseAppendOnlyFixedSizeTree(..)) + | Ok(Element::PrivateDocumentStore(..)) => Ok(()).wrap_with_cost(cost), Ok(_) | Err(Error::PathKeyNotFound(_)) => Err(error_fn()).wrap_with_cost(cost), Err(e) => Err(e).wrap_with_cost(cost), } diff --git a/grovedb/src/operations/get/query.rs b/grovedb/src/operations/get/query.rs index 4721197fb..3a4d446ff 100644 --- a/grovedb/src/operations/get/query.rs +++ b/grovedb/src/operations/get/query.rs @@ -280,6 +280,7 @@ where { | Element::MmrTree(..) | Element::BulkAppendTree(..) | Element::DenseAppendOnlyFixedSizeTree(..) + | Element::PrivateDocumentStore(..) | Element::ProvableSumIndexedTree(..) | Element::ProvableCountProvableSumIndexedTree(..) | Element::ProvableCountIndexedTree(..) => { @@ -426,6 +427,7 @@ where { | Element::MmrTree(..) | Element::BulkAppendTree(..) | Element::DenseAppendOnlyFixedSizeTree(..) + | Element::PrivateDocumentStore(..) | Element::ProvableSumIndexedTree(..) | Element::ProvableCountProvableSumIndexedTree(..) | Element::ProvableCountIndexedTree(..) => Err(Error::InvalidQuery( @@ -639,7 +641,8 @@ where { | Element::CommitmentTree(..) | Element::MmrTree(..) | Element::BulkAppendTree(..) - | Element::DenseAppendOnlyFixedSizeTree(..) => Err(Error::InvalidQuery( + | Element::DenseAppendOnlyFixedSizeTree(..) + | Element::PrivateDocumentStore(..) => Err(Error::InvalidQuery( "path_queries can only refer to items, sum items, references and sum \ trees", )), @@ -1116,6 +1119,7 @@ where { | Element::MmrTree(..) | Element::BulkAppendTree(..) | Element::DenseAppendOnlyFixedSizeTree(..) + | Element::PrivateDocumentStore(..) | Element::ProvableSumIndexedTree(..) | Element::ProvableCountProvableSumIndexedTree(..) | Element::ProvableCountIndexedTree(..) diff --git a/grovedb/src/operations/insert/add_element_on_transaction/v0.rs b/grovedb/src/operations/insert/add_element_on_transaction/v0.rs index 4d52613da..e4b5b0916 100644 --- a/grovedb/src/operations/insert/add_element_on_transaction/v0.rs +++ b/grovedb/src/operations/insert/add_element_on_transaction/v0.rs @@ -27,8 +27,9 @@ use grovedb_costs::{ use grovedb_element::reference_path::path_from_reference_path_type; use grovedb_merk::{ element::{ - costs::ElementCostExtensions, get::ElementFetchFromStorageExtensions, - insert::ElementInsertToStorageExtensions, ElementExt, + costs::ElementCostExtensions, exists::ElementExistsInStorageExtensions, + get::ElementFetchFromStorageExtensions, insert::ElementInsertToStorageExtensions, + ElementExt, }, tree::NULL_HASH, tree_type::TreeType, @@ -191,6 +192,62 @@ impl GroveDb { ) ); } + // PrivateDocumentStore: the initial child hash is the empty + // state root for the element's committed config (the state root + // binds {entry_size, chunk_power}), so V1 proof verification and + // verify_grovedb agree even before the first append. Creation is + // version-gated and only an empty store may be inserted; entries + // are appended via the typed private_document_store_insert path. + Element::PrivateDocumentStore(total_count, entry_size, chunk_power, _) => { + cost_return_on_error_no_add!( + cost, + crate::operations::private_document_store::validate_private_document_store_creation( + *total_count, + *entry_size, + *chunk_power, + grove_version, + ) + ); + // Write-once: creating a store over an existing element is + // rejected on the direct path too, matching the batch arm. + // A silent overwrite would reset the element to empty while + // leaving the old chunk blobs and MMR nodes in the data + // namespace. + let already_exists = cost_return_on_error_into!( + &mut cost, + element.element_at_key_already_exists( + &mut subtree_to_insert_into, + key, + grove_version, + ) + ); + if already_exists { + return Err(Error::InvalidInput( + "a private document store already exists at this key; it is append-only \ + and cannot be overwritten", + )) + .wrap_with_cost(cost); + } + // Deriving the empty root performs two blake3 calls — the + // committed-config hash and the composite `pds_state` hash — + // neither of which the helper can bill, since it returns a + // bare array. Charge them here so creating a store is not + // two hashes cheaper than it really is. + cost.hash_node_calls = cost.hash_node_calls.saturating_add(2); + cost_return_on_error_into!( + &mut cost, + element.insert_subtree( + &mut subtree_to_insert_into, + key, + grovedb_private_document_store::empty_private_document_store_state_root( + *entry_size, + *chunk_power, + ), + Some(options.as_merk_options()), + grove_version + ) + ); + } // MmrTree, BulkAppendTree, DenseAppendOnlyFixedSizeTree: initial // insert uses NULL_HASH since these trees start empty. Element::MmrTree(..) diff --git a/grovedb/src/operations/insert/add_element_on_transaction/v1.rs b/grovedb/src/operations/insert/add_element_on_transaction/v1.rs index 24abaedf4..e66ba3dc2 100644 --- a/grovedb/src/operations/insert/add_element_on_transaction/v1.rs +++ b/grovedb/src/operations/insert/add_element_on_transaction/v1.rs @@ -19,8 +19,9 @@ use grovedb_costs::{ use grovedb_element::reference_path::path_from_reference_path_type; use grovedb_merk::{ element::{ - costs::ElementCostExtensions, get::ElementFetchFromStorageExtensions, - insert::ElementInsertToStorageExtensions, ElementExt, + costs::ElementCostExtensions, exists::ElementExistsInStorageExtensions, + get::ElementFetchFromStorageExtensions, insert::ElementInsertToStorageExtensions, + ElementExt, }, tree::NULL_HASH, tree_type::TreeType, @@ -187,6 +188,62 @@ impl GroveDb { ) ); } + // PrivateDocumentStore: the initial child hash is the empty + // state root for the element's committed config (the state root + // binds {entry_size, chunk_power}), so V1 proof verification and + // verify_grovedb agree even before the first append. Creation is + // version-gated and only an empty store may be inserted; entries + // are appended via the typed private_document_store_insert path. + Element::PrivateDocumentStore(total_count, entry_size, chunk_power, _) => { + cost_return_on_error_no_add!( + cost, + crate::operations::private_document_store::validate_private_document_store_creation( + *total_count, + *entry_size, + *chunk_power, + grove_version, + ) + ); + // Write-once: creating a store over an existing element is + // rejected on the direct path too, matching the batch arm. + // A silent overwrite would reset the element to empty while + // leaving the old chunk blobs and MMR nodes in the data + // namespace. + let already_exists = cost_return_on_error_into!( + &mut cost, + element.element_at_key_already_exists( + &mut subtree_to_insert_into, + key, + grove_version, + ) + ); + if already_exists { + return Err(Error::InvalidInput( + "a private document store already exists at this key; it is append-only \ + and cannot be overwritten", + )) + .wrap_with_cost(cost); + } + // Deriving the empty root performs two blake3 calls — the + // committed-config hash and the composite `pds_state` hash — + // neither of which the helper can bill, since it returns a + // bare array. Charge them here so creating a store is not + // two hashes cheaper than it really is. + cost.hash_node_calls = cost.hash_node_calls.saturating_add(2); + cost_return_on_error_into!( + &mut cost, + element.insert_subtree( + &mut subtree_to_insert_into, + key, + grovedb_private_document_store::empty_private_document_store_state_root( + *entry_size, + *chunk_power, + ), + Some(options.as_merk_options()), + grove_version + ) + ); + } // MmrTree, BulkAppendTree, DenseAppendOnlyFixedSizeTree: initial // insert uses NULL_HASH since these trees start empty. Element::MmrTree(..) diff --git a/grovedb/src/operations/mmr_tree.rs b/grovedb/src/operations/mmr_tree.rs index 586c55b27..bd67cf78d 100644 --- a/grovedb/src/operations/mmr_tree.rs +++ b/grovedb/src/operations/mmr_tree.rs @@ -9,10 +9,12 @@ use std::collections::HashMap; -use grovedb_costs::{cost_return_on_error, CostResult, CostsExt, OperationCost}; +use grovedb_costs::{ + cost_return_on_error, cost_return_on_error_no_add, CostResult, CostsExt, OperationCost, +}; use grovedb_merk::element::insert::ElementInsertToStorageExtensions; use grovedb_merkle_mountain_range::{ - hash_count_for_push, mmr_size_to_leaf_count, MmrNode, MmrStore, MMR, + mmr_size_to_leaf_count, push_call_site_hashes, MmrNode, MmrStore, MMR, }; use grovedb_path::SubtreePath; use grovedb_storage::{rocksdb_storage::PrefixedRocksDbTransactionContext, Storage, StorageBatch}; @@ -84,21 +86,30 @@ impl GroveDb { let store = MmrStore::new(&storage_ctx); let leaf_count = mmr_size_to_leaf_count(mmr_size); - // Track Blake3 hash cost for this push - cost.hash_node_calls += hash_count_for_push(leaf_count); + // The leaf is hashed here, before `push`. How much of the push's work + // this call site still owes depends on the version — `push` bills its + // own merges from v1 on — so the split comes from + // `push_call_site_hashes` rather than being hard-coded. Charging + // `hash_count_for_push` unconditionally would double-count every + // merge once `push` started billing them. + cost.hash_node_calls += cost_return_on_error_no_add!( + cost, + push_call_site_hashes(leaf_count, grove_version) + .map_err(|e| Error::CorruptedData(format!("MMR push cost: {}", e))) + ); let leaf = MmrNode::leaf(value); let mut mmr = MMR::new(mmr_size, &store); cost_return_on_error!( &mut cost, - mmr.push(leaf) + mmr.push(leaf, grove_version) .map_err(|e| Error::CorruptedData(format!("MMR push failed: {}", e))) ); // Get root BEFORE commit — data is still in the MMRBatch overlay let new_root = cost_return_on_error!( &mut cost, - mmr.get_root() + mmr.get_root(grove_version) .map_err(|e| Error::CorruptedData(format!("MMR get_root failed: {}", e))) ); let new_mmr_root = new_root.hash(); @@ -229,7 +240,7 @@ impl GroveDb { let root = cost_return_on_error!( &mut cost, - mmr.get_root() + mmr.get_root(grove_version) .map_err(|e| Error::CorruptedData(format!("MMR get_root failed: {}", e))) ); @@ -446,13 +457,19 @@ impl GroveDb { // Push all values into a single MMR instance let mut mmr = MMR::new(mmr_size, &store); for value in values { + // Version-dependent split between this call site and `push`; + // see the note on the direct path above. let leaf_count = mmr_size_to_leaf_count(mmr.mmr_size); - cost.hash_node_calls += hash_count_for_push(leaf_count); + cost.hash_node_calls += cost_return_on_error_no_add!( + cost, + push_call_site_hashes(leaf_count, grove_version) + .map_err(|e| Error::CorruptedData(format!("MMR push cost: {}", e))) + ); let leaf = MmrNode::leaf(value.clone()); cost_return_on_error!( &mut cost, - mmr.push(leaf) + mmr.push(leaf, grove_version) .map_err(|e| Error::CorruptedData(format!("MMR push failed: {}", e))) ); } @@ -460,7 +477,7 @@ impl GroveDb { // Get root BEFORE commit — data is still in the MMRBatch overlay let new_root = cost_return_on_error!( &mut cost, - mmr.get_root() + mmr.get_root(grove_version) .map_err(|e| Error::CorruptedData(format!("MMR get_root failed: {}", e))) ); let new_mmr_root = new_root.hash(); diff --git a/grovedb/src/operations/mod.rs b/grovedb/src/operations/mod.rs index 8e556f3ad..ec79ad3a9 100644 --- a/grovedb/src/operations/mod.rs +++ b/grovedb/src/operations/mod.rs @@ -25,6 +25,9 @@ pub mod bulk_append_tree; #[cfg(feature = "minimal")] pub mod dense_tree; +#[cfg(feature = "minimal")] +/// Private document store operations +pub mod private_document_store; /// Caller-driven subtree-root replacement. Bypasses grovedb's normal /// "compute child hash from subtree state" invariant — see the module-level diff --git a/grovedb/src/operations/private_document_store.rs b/grovedb/src/operations/private_document_store.rs new file mode 100644 index 000000000..d644c94aa --- /dev/null +++ b/grovedb/src/operations/private_document_store.rs @@ -0,0 +1,602 @@ +//! PrivateDocumentStore operations for GroveDB. +//! +//! Thin bridge between GroveDB's storage/transaction/batch infrastructure and +//! the `grovedb-private-document-store` crate, which wraps a `BulkAppendTree` +//! with a committed `{entry_size, chunk_power}` configuration. Appends are +//! validated against the committed entry size and the state root binds the +//! config (`blake3("pds_state" || config_hash || bulk_state_root)`), so a +//! proof can never be reinterpreted under a different configuration. +//! +//! There is no per-entry delete or update — immutability is enforced by the +//! type. Every entry point here fails closed on protocol versions that +//! predate the element type (all slots are 0 before `GROVE_V4`). + +use std::collections::{BTreeMap, HashMap}; + +use grovedb_costs::{ + cost_return_on_error, cost_return_on_error_into, cost_return_on_error_no_add, CostResult, + CostsExt, OperationCost, +}; +use grovedb_merk::element::insert::ElementInsertToStorageExtensions; +use grovedb_path::SubtreePath; +use grovedb_private_document_store::PrivateDocumentStore; +use grovedb_storage::{Storage, StorageBatch}; +use grovedb_version::{error::GroveVersionError, version::GroveVersion}; + +use crate::{ + batch::{GroveOp, QualifiedGroveDbOp}, + util::TxRef, + Element, Error, GroveDb, Transaction, TransactionArg, +}; + +/// Map a `PrivateDocumentStoreError` to a GroveDB `Error`. +fn map_pds_err(e: grovedb_private_document_store::PrivateDocumentStoreError) -> Error { + Error::PrivateDocumentStoreError(format!("{}", e)) +} + +/// Fail-closed capability gate for the PrivateDocumentStore family. +/// +/// Slot `0` (every version before `GROVE_V4`) means the operation is +/// unavailable; slot `1` is the active v1 implementation. Unlike the +/// `check_grovedb_v0!` family this also rejects *older* versions — the +/// element type must not be creatable or operable under released protocol +/// versions. +/// +/// The comparison is an EXACT match, like every other version guard in the +/// codebase. Accepting `slot > 1` would be fail-open on a protocol +/// discriminator: a future GROVE_V5 that sets the slot to `2` to mean new +/// semantics would silently run v1 code on a node that only knows v1, which +/// is exactly the divergence this gate exists to prevent. +pub(crate) fn check_pds_enabled( + method: &str, + slot: grovedb_version::version::FeatureVersion, +) -> Result<(), Error> { + if slot != 1 { + return Err(GroveVersionError::UnknownVersionMismatch { + method: method.to_string(), + known_versions: vec![1], + received: slot, + } + .into()); + } + Ok(()) +} + +/// The rules a `PrivateDocumentStore` element must satisfy to be created, +/// shared by the direct insert paths (`add_element_on_transaction` v0 and +/// v1) and mirrored by the batch arm. +/// +/// Kept in one place because every one of these rules has to hold on all +/// three paths: the version gate, the empty-at-creation requirement, and the +/// committed-config bounds. They were previously written out per path, and +/// the `entry_size` cap had to be applied to each copy separately — exactly +/// the drift this prevents. +pub(crate) fn validate_private_document_store_creation( + total_count: u64, + entry_size: u32, + chunk_power: u8, + grove_version: &GroveVersion, +) -> Result<(), Error> { + check_pds_enabled( + "create Element::PrivateDocumentStore", + grove_version + .grovedb_versions + .operations + .private_document_store + .element_creation, + )?; + if total_count != 0 { + return Err(Error::InvalidCodeExecution( + "a private document store should be empty at the moment of insertion", + )); + } + // `entry_size` is capped at u16::MAX so the worst-case compaction blob + // (2^16 * entry_size) stays representable in the u32 storage-cost field. + if entry_size == 0 || entry_size > u16::MAX as u32 || !(1..=16).contains(&chunk_power) { + return Err(Error::InvalidInput( + "a PrivateDocumentStore requires entry_size in 1..=65535 and chunk_power in 1..=16", + )); + } + Ok(()) +} + +impl GroveDb { + /// Append an entry to a PrivateDocumentStore subtree. + /// + /// The entry's byte length must equal the store's committed `entry_size`; + /// any other length is rejected before mutation. The append: + /// 1. Opens the store (a `BulkAppendTree` reconstructed from the + /// element's `total_count` / `chunk_power`) + /// 2. Appends the entry (auto-compacting a full buffer into a chunk) + /// 3. Updates the `PrivateDocumentStore` element with the new + /// `total_count` and re-binds the composite state root as the Merk + /// child hash + /// 4. Propagates changes through the GroveDB Merk hierarchy + /// + /// `path` must point to the parent of the store's key, and `key` must + /// identify a `PrivateDocumentStore` element. + /// + /// Returns `(state_root, position)`: the new composite state root and the + /// 0-based global position of the appended entry. + pub fn private_document_store_insert<'b, B, P>( + &self, + path: P, + key: &[u8], + entry: Vec, + transaction: TransactionArg, + grove_version: &GroveVersion, + ) -> CostResult<([u8; 32], u64), Error> + where + B: AsRef<[u8]> + 'b, + P: Into>, + { + let path: SubtreePath = path.into(); + let mut cost = OperationCost::default(); + + cost_return_on_error_no_add!( + cost, + check_pds_enabled( + "private_document_store_insert", + grove_version + .grovedb_versions + .operations + .private_document_store + .insert, + ) + ); + + let tx = TxRef::new(&self.db, transaction); + + // 1. Validate the element at path/key is a PrivateDocumentStore. + let element = cost_return_on_error!( + &mut cost, + self.get_raw_caching_optional(path.clone(), key, true, transaction, grove_version) + ); + + // Look through NonCounted: a wrapped PrivateDocumentStore is still + // one. The wrapper must be RESTORED on the way out (below) — it is + // part of the stored element and controls the parent count tree's + // aggregate. + let was_non_counted = element.is_non_counted(); + let (total_count, entry_size, chunk_power, existing_flags) = match element.underlying() { + Element::PrivateDocumentStore(tc, es, cp, flags) => (*tc, *es, *cp, flags.clone()), + _ => { + return Err(Error::InvalidInput( + "element is not a private document store", + )) + .wrap_with_cost(cost); + } + }; + + // 2. Open transactional storage (write-through cache + MMR overlay + // provide read-after-write visibility). + let store_path_vec = crate::util::subtree_path_with_key(&path, key); + let store_path_refs: Vec<&[u8]> = store_path_vec.iter().map(|v| v.as_slice()).collect(); + let store_path = SubtreePath::from(store_path_refs.as_slice()); + + let data_batch = StorageBatch::new(); + let storage_ctx = self + .db + .get_transactional_storage_context(store_path, Some(&data_batch), tx.as_ref()) + .unwrap_add_cost(&mut cost); + + // 3. Open the store and append (validates the entry size). + let mut store = cost_return_on_error!( + &mut cost, + PrivateDocumentStore::from_state(total_count, entry_size, chunk_power, storage_ctx) + .map(|r| r.map_err(map_pds_err)) + ); + + let append_result = cost_return_on_error!( + &mut cost, + store + .append(&entry, grove_version) + .map(|r| r.map_err(map_pds_err)) + ); + + let new_state_root = append_result.state_root; + let position = append_result.global_position; + let new_total_count = store.total_count(); + + // Flush MMR overlay to storage (through the batch). + cost_return_on_error_no_add!(cost, store.commit_mmr().map_err(map_pds_err)); + + // Drop the store (and its storage context) before opening merk. + drop(store); + + // Commit data batch to make writes visible in the transaction. + // Note: this commits subtree data before the parent element update + // below. If the parent Merk update fails, the subtree data is orphaned + // in the transaction. This is the same pattern as other direct GroveDB + // operations — the caller is expected to rollback the tx on error. + // The batch path (preprocess_private_document_store_ops) avoids this + // by using a shared StorageBatch that commits atomically with all + // other ops. + cost_return_on_error!( + &mut cost, + self.db + .commit_multi_context_batch(data_batch, Some(tx.as_ref())) + .map_err(Into::into) + ); + + // 4. Update element in parent Merk. + let batch = StorageBatch::new(); + let mut parent_merk = cost_return_on_error!( + &mut cost, + self.open_transactional_merk_at_path( + path.clone(), + tx.as_ref(), + Some(&batch), + grove_version, + ) + ); + + let updated_element = Element::new_private_document_store( + new_total_count, + entry_size, + chunk_power, + existing_flags, + ); + // Re-wrap when the stored element was NonCounted. Dropping the + // wrapper here would flip the store's contribution to a CountTree / + // CountSumTree parent from 0 to 1 on its first append, changing the + // parent aggregate and the consensus root hash. + let updated_element = if was_non_counted { + cost_return_on_error_no_add!( + cost, + Element::new_non_counted(updated_element).map_err(|e| { + Error::CorruptedData(format!( + "failed to re-wrap private document store in NonCounted: {e}" + )) + }) + ) + } else { + updated_element + }; + + cost_return_on_error_into!( + &mut cost, + updated_element.insert_subtree( + &mut parent_merk, + key, + new_state_root, + None, + grove_version, + ) + ); + + // 5. Propagate changes from parent upward. + let mut merk_cache = HashMap::new(); + merk_cache.insert(path.clone(), parent_merk); + + cost_return_on_error!( + &mut cost, + self.propagate_changes_with_transaction( + merk_cache, + path, + tx.as_ref(), + &batch, + grove_version, + ) + ); + + // 6. Commit batch and transaction. + cost_return_on_error!( + &mut cost, + self.db + .commit_multi_context_batch(batch, Some(tx.as_ref())) + .map_err(Into::into) + ); + + tx.commit_local() + .map(|()| (new_state_root, position)) + .wrap_with_cost(cost) + } + + /// Get an entry from a PrivateDocumentStore by its global 0-based + /// position. + /// + /// Returns the raw fixed-size entry bytes, or `None` if the position is + /// out of range (`position >= total_count`). + pub fn private_document_store_get_value<'b, B, P>( + &self, + path: P, + key: &[u8], + global_position: u64, + transaction: TransactionArg, + grove_version: &GroveVersion, + ) -> CostResult>, Error> + where + B: AsRef<[u8]> + 'b, + P: Into>, + { + let path: SubtreePath = path.into(); + let mut cost = OperationCost::default(); + + cost_return_on_error_no_add!( + cost, + check_pds_enabled( + "private_document_store_get_value", + grove_version + .grovedb_versions + .operations + .private_document_store + .get_value, + ) + ); + + let tx = TxRef::new(&self.db, transaction); + + let element = cost_return_on_error!( + &mut cost, + self.get_raw_caching_optional(path.clone(), key, true, transaction, grove_version) + ); + + // Look through NonCounted: a wrapped PrivateDocumentStore is still one. + let (total_count, entry_size, chunk_power) = match element.underlying() { + Element::PrivateDocumentStore(tc, es, cp, _) => (*tc, *es, *cp), + _ => { + return Err(Error::InvalidInput( + "element is not a private document store", + )) + .wrap_with_cost(cost); + } + }; + + if global_position >= total_count { + return Ok(None).wrap_with_cost(cost); + } + + let store_path_vec = crate::util::subtree_path_with_key(&path, key); + let store_path_refs: Vec<&[u8]> = store_path_vec.iter().map(|v| v.as_slice()).collect(); + let store_path = SubtreePath::from(store_path_refs.as_slice()); + + let storage_ctx = self + .db + .get_transactional_storage_context(store_path, None, tx.as_ref()) + .unwrap_add_cost(&mut cost); + + let store = cost_return_on_error!( + &mut cost, + PrivateDocumentStore::from_state(total_count, entry_size, chunk_power, storage_ctx) + .map(|r| r.map_err(map_pds_err)) + ); + + // `get_value` is cost-bearing: the dense-tree / MMR reads that fetch + // the document must be billed, not served for free. + let value = cost_return_on_error!( + &mut cost, + store + .get_value(global_position) + .map(|r| r.map_err(map_pds_err)) + ); + + Ok(value).wrap_with_cost(cost) + } + + /// Get the total count of entries in a PrivateDocumentStore. + pub fn private_document_store_count<'b, B, P>( + &self, + path: P, + key: &[u8], + transaction: TransactionArg, + grove_version: &GroveVersion, + ) -> CostResult + where + B: AsRef<[u8]> + 'b, + P: Into>, + { + let path: SubtreePath = path.into(); + let mut cost = OperationCost::default(); + + cost_return_on_error_no_add!( + cost, + check_pds_enabled( + "private_document_store_count", + grove_version + .grovedb_versions + .operations + .private_document_store + .count, + ) + ); + + let element = cost_return_on_error!( + &mut cost, + self.get_raw_caching_optional(path, key, true, transaction, grove_version) + ); + + // Look through NonCounted: a wrapped PrivateDocumentStore is still one. + match element.into_underlying() { + Element::PrivateDocumentStore(total_count, ..) => Ok(total_count).wrap_with_cost(cost), + _ => Err(Error::InvalidInput( + "element is not a private document store", + )) + .wrap_with_cost(cost), + } + } + + /// Preprocess `PrivateDocumentStoreInsert` ops in a batch. + /// + /// For each group of insert ops targeting the same store: + /// 1. Opens the store (BulkAppendTree + committed config) + /// 2. Appends all entries in order (each validated against `entry_size`) + /// 3. Replaces the ops with a single `ReplaceNonMerkTreeRoot` carrying + /// the new composite state root and updated element metadata + /// + /// The returned ops list contains no `PrivateDocumentStoreInsert` + /// variants. + pub(crate) fn preprocess_private_document_store_ops( + &self, + ops: Vec, + transaction: &Transaction, + storage_batch: &StorageBatch, + grove_version: &GroveVersion, + ) -> CostResult, Error> { + let mut cost = OperationCost::default(); + + let has_pds_ops = ops + .iter() + .any(|op| matches!(op.op, GroveOp::PrivateDocumentStoreInsert { .. })); + if !has_pds_ops { + return Ok(ops).wrap_with_cost(cost); + } + + cost_return_on_error_no_add!( + cost, + check_pds_enabled( + "batch GroveOp::PrivateDocumentStoreInsert", + grove_version + .grovedb_versions + .operations + .private_document_store + .insert, + ) + ); + + /// Tree path identifying a store in a batch (includes tree key as + /// last segment). + type TreePath = Vec>; + + // Group insert ops by path (which includes tree key). + // + // A BTreeMap, not a HashMap: the loop below performs cost-bearing + // storage reads and entry-size validation per group, so with a + // randomized iteration order two processes replaying the same batch + // could fail on different groups, having accumulated different + // operation costs, and surface different errors. Costs feed fees and + // errors are consensus-visible, so the order must be canonical. + // Insertion order WITHIN a group is preserved by the Vec. + let mut pds_groups: BTreeMap>> = BTreeMap::new(); + for op in ops.iter() { + if let GroveOp::PrivateDocumentStoreInsert { entry } = &op.op { + let tree_path = op.path.to_path(); + pds_groups.entry(tree_path).or_default().push(entry.clone()); + } + } + + let mut replacements: BTreeMap = BTreeMap::new(); + + for (tree_path, entries) in pds_groups.iter() { + // Extract parent path and tree key from the full path. + let (path_vec, key_bytes) = { + let mut p = tree_path.clone(); + let k = match p.pop() { + Some(k) => k, + None => { + return Err(Error::InvalidBatchOperation( + "append op path must have at least one segment", + )) + .wrap_with_cost(cost); + } + }; + (p, k) + }; + + // Read the existing element to verify it's a PrivateDocumentStore. + let path_slices: Vec<&[u8]> = path_vec.iter().map(|v| v.as_slice()).collect(); + let subtree_path = SubtreePath::from(path_slices.as_slice()); + + let element = cost_return_on_error!( + &mut cost, + self.get_raw_caching_optional( + subtree_path.clone(), + key_bytes.as_slice(), + true, + Some(transaction), + grove_version + ) + ); + + // Look through NonCounted: a wrapped PrivateDocumentStore is + // still one. The wrapper is restored when the replacement op is + // applied (see the `ReplaceNonMerkTreeRoot` arm in + // `batch/mod.rs`, which re-reads the stored element anyway). + let (total_count, entry_size, chunk_power) = match element.underlying() { + Element::PrivateDocumentStore(tc, es, cp, _) => (*tc, *es, *cp), + _ => { + return Err(Error::InvalidInput( + "element is not a private document store", + )) + .wrap_with_cost(cost); + } + }; + + // Open transactional storage (write-through cache + MMR overlay + // provide read-after-write visibility). + let mut st_path_vec = path_vec.clone(); + st_path_vec.push(key_bytes.clone()); + let st_path_refs: Vec<&[u8]> = st_path_vec.iter().map(|v| v.as_slice()).collect(); + let st_path = SubtreePath::from(st_path_refs.as_slice()); + + let storage_ctx = self + .db + .get_transactional_storage_context(st_path, Some(storage_batch), transaction) + .unwrap_add_cost(&mut cost); + + let mut store = cost_return_on_error!( + &mut cost, + PrivateDocumentStore::from_state(total_count, entry_size, chunk_power, storage_ctx) + .map(|r| r.map_err(map_pds_err)) + ); + + // Execute all inserts in ONE pass. `append_many` is byte-for-byte + // equivalent to calling `append` per entry (same stored values, + // same final state root) but O(N) in hash calls instead of + // O(N^2): `append` recomputes the dense-buffer Merkle root on + // every insert, so a full epoch at chunk_power 16 would cost + // ~4.3 billion blake3 calls for 65,535 entries. Each entry is + // still size-validated before it is written. + let append_result = cost_return_on_error!( + &mut cost, + store + .append_many(entries.iter().map(|e| e.as_slice()), grove_version) + .map(|r| r.map_err(map_pds_err)) + ); + let new_state_root = append_result.state_root; + let current_total_count = store.total_count(); + + // Flush MMR overlay to storage (through the batch). + cost_return_on_error_no_add!(cost, store.commit_mmr().map_err(map_pds_err)); + + // Drop the store (and its storage context). + drop(store); + + // Create a ReplaceNonMerkTreeRoot carrying the new state root and + // element metadata. Key is restored for downstream (from_ops, + // execute_ops_on_path). + let replacement = QualifiedGroveDbOp { + path: crate::batch::KeyInfoPath::from_known_owned_path(path_vec), + key: Some(crate::batch::key_info::KeyInfo::KnownKey(key_bytes)), + op: GroveOp::ReplaceNonMerkTreeRoot { + hash: new_state_root, + meta: crate::batch::NonMerkTreeMeta::PrivateDocumentStore { + total_count: current_total_count, + entry_size, + chunk_power, + }, + }, + }; + replacements.insert(tree_path.clone(), replacement); + } + + // Build the new ops list: keep non-PDS ops, and emit each store's + // single replacement in place of that store's FIRST insert op. + // `replacements.remove` yields a given store exactly once, so it is + // its own "first seen" marker — subsequent inserts for the same + // store find nothing and are dropped. + let mut result = Vec::with_capacity(ops.len()); + + for op in ops.into_iter() { + if matches!(op.op, GroveOp::PrivateDocumentStoreInsert { .. }) { + if let Some(replacement) = replacements.remove(&op.path.to_path()) { + result.push(replacement); + } + } else { + result.push(op); + } + } + + Ok(result).wrap_with_cost(cost) + } +} diff --git a/grovedb/src/operations/proof/bind_terminal_non_merk_tree/mod.rs b/grovedb/src/operations/proof/bind_terminal_non_merk_tree/mod.rs index 33e584c02..9ad19ec23 100644 --- a/grovedb/src/operations/proof/bind_terminal_non_merk_tree/mod.rs +++ b/grovedb/src/operations/proof/bind_terminal_non_merk_tree/mod.rs @@ -70,7 +70,7 @@ impl GroveDb { .terminal_non_merk_tree_child_hash { 0 => self.bind_terminal_non_merk_tree_v0(node, element, parent_path, tx), - 1 => self.bind_terminal_non_merk_tree_v1(node, element, parent_path, tx), + 1 => self.bind_terminal_non_merk_tree_v1(node, element, parent_path, tx, grove_version), version => Err(Error::VersionError( grovedb_version::error::GroveVersionError::UnknownVersionMismatch { method: "bind_terminal_non_merk_tree".to_string(), diff --git a/grovedb/src/operations/proof/bind_terminal_non_merk_tree/v1.rs b/grovedb/src/operations/proof/bind_terminal_non_merk_tree/v1.rs index 664ccddc8..9507f614b 100644 --- a/grovedb/src/operations/proof/bind_terminal_non_merk_tree/v1.rs +++ b/grovedb/src/operations/proof/bind_terminal_non_merk_tree/v1.rs @@ -27,6 +27,7 @@ use grovedb_merk::{ CryptoHash, TreeFeatureType, }; use grovedb_storage::{Storage, StorageContext}; +use grovedb_version::version::GroveVersion; use crate::{Element, Error, GroveDb, Transaction}; @@ -38,6 +39,7 @@ impl GroveDb { element: &Element, parent_path: &[&[u8]], tx: &Transaction, + grove_version: &GroveVersion, ) -> CostResult<(), Error> { let mut cost = OperationCost::default(); @@ -64,7 +66,7 @@ impl GroveDb { let child_hash = cost_return_on_error!( &mut cost, - self.non_merk_tree_child_hash(element, &child_path, tx) + self.non_merk_tree_child_hash(element, &child_path, tx, grove_version) ); // Reuse the value_hash the node already carries — it is the one the @@ -136,6 +138,7 @@ impl GroveDb { element: &Element, subtree_path: &[&[u8]], tx: &Transaction, + grove_version: &GroveVersion, ) -> CostResult { let mut cost = OperationCost::default(); @@ -156,7 +159,7 @@ impl GroveDb { let mmr = grovedb_merkle_mountain_range::MMR::new(*mmr_size, &store); let root = cost_return_on_error!( &mut cost, - mmr.get_root() + mmr.get_root(grove_version) .map_err(|e| Error::CorruptedData(format!("MMR get_root failed: {}", e))) ); Ok(root.hash()).wrap_with_cost(cost) @@ -261,6 +264,54 @@ impl GroveDb { )) .wrap_with_cost(cost) } + Element::PrivateDocumentStore(total_count, entry_size, chunk_power, _) => { + // The state root binds the committed config even when the + // store is empty, so the empty case is the precomputed + // config-parametrized root rather than NULL_HASH. + if *total_count == 0 { + // Two blake3 calls: the committed-config hash and the + // composite pds_state root. + cost.hash_node_calls = cost.hash_node_calls.saturating_add(2); + return Ok( + grovedb_private_document_store::empty_private_document_store_state_root( + *entry_size, + *chunk_power, + ), + ) + .wrap_with_cost(cost); + } + let storage_ctx = self + .db + .get_transactional_storage_context(storage_path, None, tx) + .unwrap_add_cost(&mut cost); + let store = cost_return_on_error!( + &mut cost, + grovedb_private_document_store::PrivateDocumentStore::from_state( + *total_count, + *entry_size, + *chunk_power, + storage_ctx, + ) + .map(|r| r.map_err(|e| Error::CorruptedData(format!( + "failed to open PrivateDocumentStore: {}", + e + )))) + ); + let state_root = cost_return_on_error!( + &mut cost, + store + .compute_current_state_root_with_cost(grove_version) + .map(|r| { + r.map_err(|e| { + Error::CorruptedData(format!( + "private document store state root failed: {}", + e + )) + }) + }) + ); + Ok(state_root).wrap_with_cost(cost) + } _ => Err(Error::CorruptedCodeExecution( "non_merk_tree_child_hash called on an element that is not a non-Merk tree", )) diff --git a/grovedb/src/operations/proof/generate.rs b/grovedb/src/operations/proof/generate.rs index 9d176506d..df859d2a4 100644 --- a/grovedb/src/operations/proof/generate.rs +++ b/grovedb/src/operations/proof/generate.rs @@ -1004,13 +1004,14 @@ impl GroveDb { Ok(Element::MmrTree(..)) | Ok(Element::BulkAppendTree(..)) | Ok(Element::DenseAppendOnlyFixedSizeTree(..)) + | Ok(Element::PrivateDocumentStore(..)) if !done_with_results && query.has_subquery_or_matching_in_path_on_key(key) => { return Err(Error::NotSupported( "V0 proofs do not support subqueries into MmrTree, \ - BulkAppendTree, or DenseAppendOnlyFixedSizeTree elements; \ - use prove_query_v1 instead" + BulkAppendTree, DenseAppendOnlyFixedSizeTree, or \ + PrivateDocumentStore elements; use prove_query_v1 instead" .to_string(), )) .wrap_with_cost(cost); @@ -1036,6 +1037,7 @@ impl GroveDb { | Ok(Element::ProvableSumIndexedTree(..)) | Ok(Element::ProvableCountIndexedTree(..)) | Ok(Element::ProvableCountProvableSumIndexedTree(..)) + | Ok(Element::PrivateDocumentStore(..)) if !done_with_results => { #[cfg(feature = "proof_debug")] @@ -1079,7 +1081,8 @@ impl GroveDb { | Ok(Element::DenseAppendOnlyFixedSizeTree(..)) | Ok(Element::ProvableSumIndexedTree(..)) | Ok(Element::ProvableCountIndexedTree(..)) - | Ok(Element::ProvableCountProvableSumIndexedTree(..)) => continue, + | Ok(Element::ProvableCountProvableSumIndexedTree(..)) + | Ok(Element::PrivateDocumentStore(..)) => continue, // NonCounted is unwrapped above via into_underlying(). Ok(Element::NonCounted(_)) | Ok(Element::NotSummed(_)) @@ -2252,6 +2255,25 @@ impl GroveDb { lower_layers.insert(key.clone(), layer_proof); } + // PrivateDocumentStore range-read proofs are not + // implemented yet (planned at the BulkAppendTree + // layer so the anchored DataCommitmentTree can + // inherit them); reject subqueries instead of + // shipping an unverifiable proof. Terminal + // (no-subquery) queries still bind the store's + // state root via the child-hash arm below. + Ok(Element::PrivateDocumentStore(..)) + if !done_with_results + && query.has_subquery_or_matching_in_path_on_key(key) => + { + return Err(Error::NotSupported( + "V1 proofs do not yet support subqueries into \ + PrivateDocumentStore elements" + .to_string(), + )) + .wrap_with_cost(cost); + } + // Axis-ordered read of an indexed tree: the // query node governing this element carries // ReadMode::Axis, so instead of descending the @@ -2742,6 +2764,7 @@ impl GroveDb { ref non_merk_elem @ Element::DenseAppendOnlyFixedSizeTree(..), ) | Ok(ref non_merk_elem @ Element::CommitmentTree(..)) + | Ok(ref non_merk_elem @ Element::PrivateDocumentStore(..)) if !done_with_results => { cost_return_on_error!( @@ -3038,7 +3061,8 @@ impl GroveDb { | Ok(Element::DenseAppendOnlyFixedSizeTree(..)) | Ok(Element::ProvableSumIndexedTree(..)) | Ok(Element::ProvableCountIndexedTree(..)) - | Ok(Element::ProvableCountProvableSumIndexedTree(..)) => continue, + | Ok(Element::ProvableCountProvableSumIndexedTree(..)) + | Ok(Element::PrivateDocumentStore(..)) => continue, // NonCounted is unwrapped above via into_underlying(). Ok(Element::NonCounted(_)) | Ok(Element::NotSummed(_)) diff --git a/grovedb/src/operations/proof/verify.rs b/grovedb/src/operations/proof/verify.rs index 7510b3cdf..af7f767ae 100644 --- a/grovedb/src/operations/proof/verify.rs +++ b/grovedb/src/operations/proof/verify.rs @@ -1724,6 +1724,18 @@ impl GroveDb { } } } + // The V1 prover rejects subqueries into a + // PrivateDocumentStore (range-read proofs are not + // implemented yet), so an honest proof never + // carries a lower layer for one — refuse to + // fabricate a chain for it. + Element::PrivateDocumentStore(..) => { + return Err(Error::NotSupported( + "V1 proofs do not yet support lower layers for \ + PrivateDocumentStore elements" + .to_string(), + )); + } Element::Tree(Some(_), _) | Element::SumTree(Some(_), ..) | Element::BigSumTree(Some(_), ..) @@ -3154,6 +3166,7 @@ impl GroveDb { | Element::MmrTree(..) | Element::BulkAppendTree(..) | Element::DenseAppendOnlyFixedSizeTree(..) + | Element::PrivateDocumentStore(..) | Element::SumItem(..) | Element::Item(..) | Element::ItemWithSumItem(..) diff --git a/grovedb/src/tests/mmr_tree_tests.rs b/grovedb/src/tests/mmr_tree_tests.rs index 5c5d75d77..78503319c 100644 --- a/grovedb/src/tests/mmr_tree_tests.rs +++ b/grovedb/src/tests/mmr_tree_tests.rs @@ -56,12 +56,12 @@ fn expected_mmr_root(values: &[Vec]) -> [u8; 32] { let store = MemStore::new(); let mut mmr = MMR::new(0, &store); for v in values { - mmr.push(MmrNode::leaf(v.clone())) + mmr.push(MmrNode::leaf(v.clone()), GroveVersion::latest()) .unwrap() .expect("push should succeed"); } mmr.commit().unwrap().expect("commit should succeed"); - mmr.get_root() + mmr.get_root(GroveVersion::latest()) .unwrap() .expect("root hash should succeed") .hash() @@ -1992,3 +1992,66 @@ fn test_mmr_successful_batch_after_failed_batch() { .expect("leaf count"); assert_eq!(count, 1, "only the second batch's leaf should be present"); } + +/// `MMR::push` bills one hash per peak it collapses, and the ops layer hashes +/// the leaf eagerly before calling it. Charging `hash_count_for_push` here as +/// well — that helper is the leaf hash PLUS the same merges — double-counted +/// every merge. +/// +/// Asserted as deltas against the first append: each append does identical +/// Merk work (it replaces the same element with a same-sized value), so the +/// difference between appends isolates the MMR hashing and the test does not +/// break when unrelated Merk costs shift. +#[test] +fn test_mmr_tree_append_does_not_double_charge_merges() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + db.insert( + EMPTY_PATH, + b"log", + Element::empty_mmr_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert mmr tree"); + + let mut hashes = Vec::new(); + for i in 0..4u8 { + let ctx = db.mmr_tree_append(EMPTY_PATH, b"log", vec![i; 8], None, grove_version); + ctx.value.expect("append"); + hashes.push(ctx.cost.hash_node_calls); + } + + // MMR hashing per append is the eager leaf hash, plus the merges `push` + // performs, plus the peak bagging the trailing `get_root` performs: + // + // leaf 0: 1 leaf + 0 push + 0 bag = 1 (mmr_size 1, single-element root) + // leaf 1: 1 leaf + 1 push + 0 bag = 2 (mmr_size 3, one peak) + // leaf 2: 1 leaf + 0 push + 1 bag = 2 (mmr_size 4, two peaks to fold) + // leaf 3: 1 leaf + 2 push + 0 bag = 3 (mmr_size 7, one peak) + // + // so relative to the first append the deltas are 0, +1, +1, +2. Before the + // fix each `push` merge was charged twice, which showed up as 3 hashes on + // the second leaf for the 2 it performs. + let base = hashes[0]; + assert_eq!( + hashes[1].checked_sub(base), + Some(1), + "second leaf: one push merge, no bagging, got {:?}", + hashes + ); + assert_eq!( + hashes[2].checked_sub(base), + Some(1), + "third leaf: no push merge, but two peaks to bag, got {:?}", + hashes + ); + assert_eq!( + hashes[3].checked_sub(base), + Some(2), + "fourth leaf: two push merges, one peak so no bagging, got {:?}", + hashes + ); +} diff --git a/grovedb/src/tests/mod.rs b/grovedb/src/tests/mod.rs index bafe8cfa7..ec54c73c5 100644 --- a/grovedb/src/tests/mod.rs +++ b/grovedb/src/tests/mod.rs @@ -25,6 +25,7 @@ mod chunk_branch_proof_tests; mod commitment_tree_cost_bound_tests; mod commitment_tree_tests; mod coverage_round7_tests; +mod private_document_store_tests; // NOTE: the former `count_indexed_tree_tests` (~12.3k LOC) was written // against the now-removed non-provable `Element::CountIndexedTree` and was // carried here behind a `#[cfg(any())]` gate that made it permanently dead — diff --git a/grovedb/src/tests/private_document_store_tests.rs b/grovedb/src/tests/private_document_store_tests.rs new file mode 100644 index 000000000..226fd9c84 --- /dev/null +++ b/grovedb/src/tests/private_document_store_tests.rs @@ -0,0 +1,1715 @@ +//! PrivateDocumentStore integration tests +//! +//! Tests for PrivateDocumentStore as a GroveDB subtree type: an append-only +//! store of fixed-size opaque entries over a BulkAppendTree, with the +//! committed `{entry_size, chunk_power}` configuration bound into the state +//! root. Includes the fail-closed version-gating tests: every operation and +//! element creation must be rejected under GROVE_V3 and earlier. + +use grovedb_merk::proofs::{query::SubqueryBranch, Query}; +use grovedb_version::version::{v3::GROVE_V3, v4::GROVE_V4, GroveVersion}; + +use crate::{ + batch::QualifiedGroveDbOp, + operations::delete::DeleteOptions, + query_result_type::QueryResultType, + tests::{common::EMPTY_PATH, make_empty_grovedb}, + Element, Error, GroveDb, PathQuery, SizedQuery, +}; + +/// Small chunk power for tests — epoch size = 2^2 = 4, triggers compaction +/// after 4 appends. +const TEST_CHUNK_POWER: u8 = 2; +/// Committed entry size for tests. +const TEST_ENTRY_SIZE: u32 = 16; + +fn entry(byte: u8) -> Vec { + vec![byte; TEST_ENTRY_SIZE as usize] +} + +// =========================================================================== +// Element tests +// =========================================================================== + +#[test] +fn test_insert_private_document_store_at_root() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + db.insert( + EMPTY_PATH, + b"pds", + Element::empty_private_document_store(TEST_ENTRY_SIZE, TEST_CHUNK_POWER) + .expect("valid config"), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert private document store at root"); + + let element = db + .get(EMPTY_PATH, b"pds", None, grove_version) + .unwrap() + .expect("should retrieve private document store"); + assert!(element.is_private_document_store()); + assert!(element.is_any_tree()); + assert!(element.uses_non_merk_data_storage()); + assert_eq!(element.non_merk_entry_count(), Some(0)); + + // The whole database must pass the integrity walk. + let issues = db + .verify_grovedb(None, true, false, grove_version) + .expect("verify_grovedb"); + assert!(issues.is_empty(), "issues: {:?}", issues); +} + +#[test] +fn test_private_document_store_constructor_validation() { + // entry_size 0 is rejected. + assert!(Element::empty_private_document_store(0, TEST_CHUNK_POWER).is_err()); + // chunk_power outside 1..=16 is rejected. + assert!(Element::empty_private_document_store(TEST_ENTRY_SIZE, 0).is_err()); + assert!(Element::empty_private_document_store(TEST_ENTRY_SIZE, 17).is_err()); + // entry_size is capped at u16::MAX so the worst-case compaction blob + // (2^16 * entry_size) stays representable in the u32 added_bytes field, + // keeping the worst-case storage estimate a real upper bound. + assert!(Element::empty_private_document_store(u16::MAX as u32 + 1, 4).is_err()); + assert!(Element::empty_private_document_store(u32::MAX, 4).is_err()); + // Boundary values are accepted. + assert!(Element::empty_private_document_store(1, 1).is_ok()); + assert!(Element::empty_private_document_store(u16::MAX as u32, 16).is_ok()); +} + +#[test] +fn test_private_document_store_serialization_roundtrip() { + let grove_version = GroveVersion::latest(); + let original = Element::new_private_document_store(100, 216, 12, Some(vec![7, 8, 9])); + let bytes = original.serialize(grove_version).expect("serialize"); + // Discriminant 24 is the wire byte for PrivateDocumentStore. + assert_eq!(bytes[0], 24, "bincode discriminant must be 24"); + let deserialized = Element::deserialize(&bytes, grove_version).expect("deserialize"); + assert_eq!(deserialized, original); + + // NonCounted wrapper round-trips as [15, 24, ...]. + let wrapped = Element::new_non_counted(original.clone()).expect("wrap"); + let wrapped_bytes = wrapped.serialize(grove_version).expect("serialize wrapped"); + assert_eq!(&wrapped_bytes[0..2], &[15, 24]); + let unwrapped = Element::deserialize(&wrapped_bytes, grove_version).expect("deserialize"); + assert_eq!(unwrapped, wrapped); + assert!(unwrapped.is_private_document_store()); +} + +// =========================================================================== +// Operation tests: insert (append), get_value, count +// =========================================================================== + +/// Build a db with a parent tree at "root" and an empty store at +/// root/"docs". +fn make_db_with_store() -> crate::tests::TempGroveDb { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + db.insert( + EMPTY_PATH, + b"root", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert root tree"); + + db.insert( + &[b"root"], + b"docs", + Element::empty_private_document_store(TEST_ENTRY_SIZE, TEST_CHUNK_POWER) + .expect("valid config"), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert private document store"); + + db +} + +#[test] +fn test_private_document_store_insert_get_count_roundtrip() { + let grove_version = GroveVersion::latest(); + let db = make_db_with_store(); + + // 10 appends span two completed chunks (epoch size 4) plus a partial + // buffer, exercising both storage tiers. + let mut roots = Vec::new(); + for i in 0..10u8 { + let (state_root, position) = db + .private_document_store_insert(&[b"root"], b"docs", entry(i), None, grove_version) + .unwrap() + .expect("append entry"); + assert_eq!(position, i as u64); + roots.push(state_root); + } + // Every append must move the state root. + for w in roots.windows(2) { + assert_ne!(w[0], w[1]); + } + + assert_eq!( + db.private_document_store_count(&[b"root"], b"docs", None, grove_version) + .unwrap() + .expect("count"), + 10 + ); + + for i in 0..10u8 { + let value = db + .private_document_store_get_value(&[b"root"], b"docs", i as u64, None, grove_version) + .unwrap() + .expect("get value"); + assert_eq!(value, Some(entry(i)), "position {}", i); + } + // Out-of-range read returns None. + assert_eq!( + db.private_document_store_get_value(&[b"root"], b"docs", 10, None, grove_version) + .unwrap() + .expect("get out of range"), + None + ); + + // The whole database (including the value-hash binding of the store's + // state root and the entry-size integrity walk) must verify. + let issues = db + .verify_grovedb(None, true, false, grove_version) + .expect("verify_grovedb"); + assert!(issues.is_empty(), "issues: {:?}", issues); +} + +#[test] +fn test_private_document_store_insert_rejects_wrong_entry_size() { + let grove_version = GroveVersion::latest(); + let db = make_db_with_store(); + let root_before = db.root_hash(None, grove_version).unwrap().unwrap(); + + for bad in [ + vec![0u8; TEST_ENTRY_SIZE as usize - 1], + vec![0u8; TEST_ENTRY_SIZE as usize + 1], + Vec::new(), + ] { + let result = db + .private_document_store_insert(&[b"root"], b"docs", bad, None, grove_version) + .unwrap(); + assert!( + matches!(result, Err(Error::PrivateDocumentStoreError(_))), + "expected entry-size rejection, got {:?}", + result + ); + } + + // Nothing was appended and the root hash is unchanged by rejections. + assert_eq!( + root_before, + db.root_hash(None, grove_version).unwrap().unwrap(), + "rejected appends must not change the grove root hash" + ); + assert_eq!( + db.private_document_store_count(&[b"root"], b"docs", None, grove_version) + .unwrap() + .expect("count"), + 0 + ); +} + +#[test] +fn test_private_document_store_ops_reject_wrong_element_type() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + db.insert( + EMPTY_PATH, + b"plain", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert plain tree"); + + assert!(matches!( + db.private_document_store_insert(EMPTY_PATH, b"plain", entry(0), None, grove_version) + .unwrap(), + Err(Error::InvalidInput(_)) + )); + assert!(matches!( + db.private_document_store_get_value(EMPTY_PATH, b"plain", 0, None, grove_version) + .unwrap(), + Err(Error::InvalidInput(_)) + )); + assert!(matches!( + db.private_document_store_count(EMPTY_PATH, b"plain", None, grove_version) + .unwrap(), + Err(Error::InvalidInput(_)) + )); +} + +#[test] +fn test_private_document_store_root_hash_changes_and_persists() { + let grove_version = GroveVersion::latest(); + let db = make_db_with_store(); + + let root_before = db.root_hash(None, grove_version).unwrap().unwrap(); + db.private_document_store_insert(&[b"root"], b"docs", entry(1), None, grove_version) + .unwrap() + .expect("append"); + let root_after = db.root_hash(None, grove_version).unwrap().unwrap(); + assert_ne!( + root_before, root_after, + "append must change the grove root hash" + ); + + // The updated element reflects the new count. + let element = db + .get(&[b"root"], b"docs", None, grove_version) + .unwrap() + .expect("get element"); + assert_eq!(element.non_merk_entry_count(), Some(1)); +} + +// =========================================================================== +// Batch tests +// =========================================================================== + +#[test] +fn test_private_document_store_batch_create_and_append() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + // Create the parent and the empty store in one batch. + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op(vec![], b"root".to_vec(), Element::empty_tree()), + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"root".to_vec()], + b"docs".to_vec(), + Element::empty_private_document_store(TEST_ENTRY_SIZE, TEST_CHUNK_POWER) + .expect("valid config"), + ), + ]; + db.apply_batch(ops, None, None, grove_version) + .unwrap() + .expect("apply creation batch"); + + // Append 6 entries via batch ops (spans one compaction at epoch size 4). + let ops = (0..6u8) + .map(|i| { + QualifiedGroveDbOp::private_document_store_insert_op( + vec![b"root".to_vec(), b"docs".to_vec()], + entry(i), + ) + }) + .collect(); + db.apply_batch(ops, None, None, grove_version) + .unwrap() + .expect("apply append batch"); + + assert_eq!( + db.private_document_store_count(&[b"root"], b"docs", None, grove_version) + .unwrap() + .expect("count"), + 6 + ); + for i in 0..6u8 { + assert_eq!( + db.private_document_store_get_value(&[b"root"], b"docs", i as u64, None, grove_version) + .unwrap() + .expect("get"), + Some(entry(i)) + ); + } + + let issues = db + .verify_grovedb(None, true, false, grove_version) + .expect("verify_grovedb"); + assert!(issues.is_empty(), "issues: {:?}", issues); + + // Batch appends must produce the same root as the equivalent direct + // appends. + let direct_db = make_db_with_store(); + for i in 0..6u8 { + direct_db + .private_document_store_insert(&[b"root"], b"docs", entry(i), None, grove_version) + .unwrap() + .expect("direct append"); + } + assert_eq!( + db.root_hash(None, grove_version).unwrap().unwrap(), + direct_db.root_hash(None, grove_version).unwrap().unwrap(), + "batch and direct appends must converge to the same root hash" + ); +} + +#[test] +fn test_private_document_store_batch_rejects_wrong_entry_size() { + let grove_version = GroveVersion::latest(); + let db = make_db_with_store(); + + let ops = vec![QualifiedGroveDbOp::private_document_store_insert_op( + vec![b"root".to_vec(), b"docs".to_vec()], + vec![0u8; TEST_ENTRY_SIZE as usize + 1], + )]; + let result = db.apply_batch(ops, None, None, grove_version).unwrap(); + assert!( + matches!(result, Err(Error::PrivateDocumentStoreError(_))), + "expected entry-size rejection, got {:?}", + result + ); + assert_eq!( + db.private_document_store_count(&[b"root"], b"docs", None, grove_version) + .unwrap() + .expect("count"), + 0 + ); +} + +#[test] +fn test_private_document_store_batch_rejects_non_empty_element_insert() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + // Inserting a PrivateDocumentStore element claiming a non-zero count is + // rejected: the batch write binds the empty state root, so a non-zero + // claim would corrupt the root binding. + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"docs".to_vec(), + Element::new_private_document_store(5, TEST_ENTRY_SIZE, TEST_CHUNK_POWER, None), + )]; + let result = db.apply_batch(ops, None, None, grove_version).unwrap(); + assert!( + matches!(result, Err(Error::InvalidBatchOperation(_))), + "expected non-empty rejection, got {:?}", + result + ); + + // Same for an invalid config built without the checked constructors. + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"docs".to_vec(), + Element::new_private_document_store(0, 0, TEST_CHUNK_POWER, None), + )]; + let result = db.apply_batch(ops, None, None, grove_version).unwrap(); + assert!( + matches!(result, Err(Error::InvalidBatchOperation(_))), + "expected config rejection, got {:?}", + result + ); +} + +#[test] +fn test_private_document_store_direct_insert_rejects_non_empty_element() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + let result = db + .insert( + EMPTY_PATH, + b"docs", + Element::new_private_document_store(5, TEST_ENTRY_SIZE, TEST_CHUNK_POWER, None), + None, + None, + grove_version, + ) + .unwrap(); + assert!( + matches!(result, Err(Error::InvalidCodeExecution(_))), + "expected non-empty rejection, got {:?}", + result + ); + + // A caller-built element with an invalid config (bypassing the checked + // constructors) is rejected by the insert path as well. + for bad in [ + Element::new_private_document_store(0, 0, TEST_CHUNK_POWER, None), + Element::new_private_document_store(0, TEST_ENTRY_SIZE, 0, None), + Element::new_private_document_store(0, TEST_ENTRY_SIZE, 17, None), + ] { + let result = db + .insert(EMPTY_PATH, b"docs", bad, None, None, grove_version) + .unwrap(); + assert!( + matches!(result, Err(Error::InvalidInput(_))), + "expected config rejection, got {:?}", + result + ); + } +} + +#[test] +fn test_private_document_store_rejects_child_element_inserts() { + // Immutability is enforced by the type: the store's (always-empty) Merk + // may never hold child elements. Both the generic direct insert and the + // batch insert into the store's path must be rejected at the merk layer. + let grove_version = GroveVersion::latest(); + let db = make_db_with_store(); + + let result = db + .insert( + &[b"root".as_slice(), b"docs".as_slice()], + b"child", + Element::new_item(b"data".to_vec()), + None, + None, + grove_version, + ) + .unwrap(); + match &result { + Err(e) => assert!( + e.to_string() + .contains("private document stores cannot hold child elements"), + "unexpected rejection error: {}", + e + ), + Ok(_) => panic!("direct child insert into a store must be rejected"), + } + + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( + vec![b"root".to_vec(), b"docs".to_vec()], + b"child".to_vec(), + Element::new_item(b"data".to_vec()), + )]; + let result = db.apply_batch(ops, None, None, grove_version).unwrap(); + assert!( + result.is_err(), + "batch child insert into a store must be rejected, got {:?}", + result + ); + + // is_empty_tree on the store path still works (reads the element count + // from the parent). + assert!(db + .is_empty_tree( + &[b"root".as_slice(), b"docs".as_slice()], + None, + grove_version + ) + .unwrap() + .expect("is_empty_tree")); +} + +// =========================================================================== +// Delete tests +// =========================================================================== + +#[test] +fn test_private_document_store_delete() { + let grove_version = GroveVersion::latest(); + let db = make_db_with_store(); + + for i in 0..5u8 { + db.private_document_store_insert(&[b"root"], b"docs", entry(i), None, grove_version) + .unwrap() + .expect("append"); + } + + // A populated store requires allow_deleting_non_empty_trees. + let result = db + .delete( + &[b"root"], + b"docs", + Some(DeleteOptions { + allow_deleting_non_empty_trees: false, + deleting_non_empty_trees_returns_error: true, + ..Default::default() + }), + None, + grove_version, + ) + .unwrap(); + assert!(result.is_err(), "deleting populated store must error"); + + db.delete( + &[b"root"], + b"docs", + Some(DeleteOptions { + allow_deleting_non_empty_trees: true, + deleting_non_empty_trees_returns_error: false, + ..Default::default() + }), + None, + grove_version, + ) + .unwrap() + .expect("delete populated store"); + + assert!(matches!( + db.get(&[b"root"], b"docs", None, grove_version).unwrap(), + Err(Error::PathKeyNotFound(_)) + )); + + // The store's non-Merk data namespace (buffer entries, chunk blobs, MMR + // nodes) must be reclaimed by the delete — not just the parent element. + { + use grovedb_storage::{RawIterator, Storage, StorageContext}; + let tx = db.start_transaction(); + let ctx = db + .db + .get_transactional_storage_context( + grovedb_path::SubtreePath::from([b"root".as_slice(), b"docs".as_slice()].as_ref()), + None, + &tx, + ) + .unwrap(); + let mut iter = ctx.raw_iter(); + iter.seek_to_first().unwrap(); + assert!( + !iter.valid().unwrap(), + "deleted store left orphaned rows in its data namespace" + ); + } + + // A store recreated at the same path starts from scratch: appends begin + // at position 0 and nothing from the deleted store is visible. + db.insert( + &[b"root"], + b"docs", + Element::empty_private_document_store(TEST_ENTRY_SIZE, TEST_CHUNK_POWER) + .expect("valid config"), + None, + None, + grove_version, + ) + .unwrap() + .expect("recreate store"); + let (_, position) = db + .private_document_store_insert(&[b"root"], b"docs", entry(99), None, grove_version) + .unwrap() + .expect("append to recreated store"); + assert_eq!(position, 0); + assert_eq!( + db.private_document_store_get_value(&[b"root"], b"docs", 0, None, grove_version) + .unwrap() + .expect("get"), + Some(entry(99)) + ); + + let issues = db + .verify_grovedb(None, true, false, grove_version) + .expect("verify_grovedb"); + assert!(issues.is_empty(), "issues: {:?}", issues); +} + +// =========================================================================== +// Proof tests (terminal binding only — range reads are a follow-up) +// =========================================================================== + +/// Query the store's key itself (no subquery) and verify the V1 proof. This +/// exercises the terminal non-Merk binding: the store's config-bound state +/// root is carried as the node's child hash and checked against the parent +/// commit. +fn prove_and_verify_store_element(db: &GroveDb, expected_count: u64) { + let grove_version = GroveVersion::latest(); + let path_query = PathQuery { + path: vec![b"root".to_vec()], + query: SizedQuery { + query: Query { + items: vec![grovedb_merk::proofs::query::QueryItem::Key( + b"docs".to_vec(), + )], + default_subquery_branch: SubqueryBranch { + subquery_path: None, + subquery: None, + }, + left_to_right: true, + conditional_subquery_branches: None, + add_parent_tree_on_subquery: false, + read_mode: None, + }, + limit: None, + offset: None, + }, + }; + + let proof_bytes = db + .prove_query(&path_query, None, grove_version) + .unwrap() + .expect("generate V1 proof for store element"); + + let (root_hash, result_set) = GroveDb::verify_query_with_options( + &proof_bytes, + &path_query, + grovedb_merk::proofs::query::VerifyOptions { + absence_proofs_for_non_existing_searched_keys: false, + verify_proof_succinctness: false, + include_empty_trees_in_result: true, + }, + grove_version, + ) + .expect("verify V1 proof for store element"); + + let expected_root = db.root_hash(None, grove_version).unwrap().unwrap(); + assert_eq!(root_hash, expected_root, "root hash should match"); + assert_eq!(result_set.len(), 1, "store element should be in results"); + let element = result_set[0] + .2 + .clone() + .expect("proved element should be present"); + match element { + Element::PrivateDocumentStore(total_count, entry_size, chunk_power, _) => { + assert_eq!(total_count, expected_count); + assert_eq!(entry_size, TEST_ENTRY_SIZE); + assert_eq!(chunk_power, TEST_CHUNK_POWER); + } + other => panic!("expected PrivateDocumentStore, got {:?}", other.type_str()), + } +} + +#[test] +fn test_private_document_store_prove_element_empty_and_populated() { + let grove_version = GroveVersion::latest(); + let db = make_db_with_store(); + + // Empty store: the terminal binding uses the config-parametrized empty + // state root. + prove_and_verify_store_element(&db, 0); + + // Populated store (across a compaction boundary). + for i in 0..5u8 { + db.private_document_store_insert(&[b"root"], b"docs", entry(i), None, grove_version) + .unwrap() + .expect("append"); + } + prove_and_verify_store_element(&db, 5); +} + +#[test] +fn test_private_document_store_subquery_proofs_not_supported() { + let grove_version = GroveVersion::latest(); + let db = make_db_with_store(); + db.private_document_store_insert(&[b"root"], b"docs", entry(1), None, grove_version) + .unwrap() + .expect("append"); + + let mut inner_query = Query::new(); + inner_query.insert_all(); + let path_query = PathQuery { + path: vec![b"root".to_vec()], + query: SizedQuery { + query: Query { + items: vec![grovedb_merk::proofs::query::QueryItem::Key( + b"docs".to_vec(), + )], + default_subquery_branch: SubqueryBranch { + subquery_path: None, + subquery: Some(inner_query.into()), + }, + left_to_right: true, + conditional_subquery_branches: None, + add_parent_tree_on_subquery: false, + read_mode: None, + }, + limit: None, + offset: None, + }, + }; + + let result = db.prove_query(&path_query, None, grove_version).unwrap(); + assert!( + matches!(result, Err(Error::NotSupported(_))), + "subquery proofs into a PrivateDocumentStore are not implemented yet, got {:?}", + result + ); +} + +// =========================================================================== +// Query tests (non-proof reads) +// =========================================================================== + +#[test] +fn test_private_document_store_path_query_rejects_tree_result() { + let grove_version = GroveVersion::latest(); + let db = make_db_with_store(); + + // Item-value queries cannot return trees. + let path_query = PathQuery::new_unsized( + vec![b"root".to_vec()], + Query::new_single_key(b"docs".to_vec()), + ); + let result = db + .query_item_value(&path_query, true, true, true, None, grove_version) + .unwrap(); + assert!(matches!(result, Err(Error::InvalidQuery(_)))); + + // The item-or-sum variant rejects stores too. + let result = db + .query_item_value_or_sum(&path_query, true, true, true, None, grove_version) + .unwrap(); + assert!(matches!(result, Err(Error::InvalidQuery(_)))); + + // Raw element queries return the store element itself. + let (elements, _) = db + .query_raw( + &path_query, + true, + true, + true, + QueryResultType::QueryElementResultType, + None, + grove_version, + ) + .unwrap() + .expect("raw query"); + assert_eq!(elements.len(), 1); +} + +// =========================================================================== +// Version gating: fail closed on GROVE_V3 and earlier +// =========================================================================== + +#[test] +fn test_private_document_store_version_slots_pinned() { + // The gate itself: all slots 0 on V3 (fail closed), 1 on V4. + let v3 = &GROVE_V3.grovedb_versions.operations.private_document_store; + assert_eq!(v3.element_creation, 0); + assert_eq!(v3.insert, 0); + assert_eq!(v3.get_value, 0); + assert_eq!(v3.count, 0); + let v4 = &GROVE_V4.grovedb_versions.operations.private_document_store; + assert_eq!(v4.element_creation, 1); + assert_eq!(v4.insert, 1); + assert_eq!(v4.get_value, 1); + assert_eq!(v4.count, 1); +} + +#[test] +fn test_private_document_store_element_creation_rejected_under_v3() { + let db = make_empty_grovedb(); + + // Direct insert of the element fails closed under V3. + let result = db + .insert( + EMPTY_PATH, + b"docs", + Element::empty_private_document_store(TEST_ENTRY_SIZE, TEST_CHUNK_POWER) + .expect("valid config"), + None, + None, + &GROVE_V3, + ) + .unwrap(); + assert!( + matches!(result, Err(Error::VersionError(_))), + "V3 direct insert must fail closed, got {:?}", + result + ); + + // Batch insert of the element fails closed under V3. + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"docs".to_vec(), + Element::empty_private_document_store(TEST_ENTRY_SIZE, TEST_CHUNK_POWER) + .expect("valid config"), + )]; + let result = db.apply_batch(ops, None, None, &GROVE_V3).unwrap(); + assert!( + matches!(result, Err(Error::VersionError(_))), + "V3 batch insert must fail closed, got {:?}", + result + ); + + // Under V4 the same element inserts fine (registered latest is V4). + db.insert( + EMPTY_PATH, + b"docs", + Element::empty_private_document_store(TEST_ENTRY_SIZE, TEST_CHUNK_POWER) + .expect("valid config"), + None, + None, + &GROVE_V4, + ) + .unwrap() + .expect("V4 insert works"); +} + +#[test] +fn test_private_document_store_operations_rejected_under_v3() { + // Build the store under V4 (latest), then attempt V3 operations on it. + let db = make_db_with_store(); + + let result = db + .private_document_store_insert(&[b"root"], b"docs", entry(0), None, &GROVE_V3) + .unwrap(); + assert!( + matches!(result, Err(Error::VersionError(_))), + "V3 insert op must fail closed, got {:?}", + result + ); + + let result = db + .private_document_store_get_value(&[b"root"], b"docs", 0, None, &GROVE_V3) + .unwrap(); + assert!( + matches!(result, Err(Error::VersionError(_))), + "V3 get_value op must fail closed, got {:?}", + result + ); + + let result = db + .private_document_store_count(&[b"root"], b"docs", None, &GROVE_V3) + .unwrap(); + assert!( + matches!(result, Err(Error::VersionError(_))), + "V3 count op must fail closed, got {:?}", + result + ); + + // Batch append op fails closed under V3. + let ops = vec![QualifiedGroveDbOp::private_document_store_insert_op( + vec![b"root".to_vec(), b"docs".to_vec()], + entry(0), + )]; + let result = db.apply_batch(ops, None, None, &GROVE_V3).unwrap(); + assert!( + matches!(result, Err(Error::VersionError(_))), + "V3 batch append must fail closed, got {:?}", + result + ); + + // Nothing was appended by the rejected calls. + assert_eq!( + db.private_document_store_count(&[b"root"], b"docs", None, GroveVersion::latest()) + .unwrap() + .expect("count under latest"), + 0 + ); +} + +// =========================================================================== +// State-root binding tests +// =========================================================================== + +#[test] +fn test_private_document_store_config_binds_root_hash() { + // Two stores with identical entries but different committed configs must + // produce different grove root hashes (the config is bound into the + // state root even for empty stores). + let grove_version = GroveVersion::latest(); + + let build = |entry_size: u32, chunk_power: u8| { + let db = make_empty_grovedb(); + db.insert( + EMPTY_PATH, + b"docs", + Element::empty_private_document_store(entry_size, chunk_power).expect("valid config"), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert store"); + db.root_hash(None, grove_version).unwrap().unwrap() + }; + + let base = build(TEST_ENTRY_SIZE, TEST_CHUNK_POWER); + // NOTE: entry_size and chunk_power are serialized in the element bytes, + // so the value hash differs too — the stronger claim (the *child hash* + // alone differs) is covered by the crate-level state-root tests. Here we + // pin that config changes are visible at the grove root. + assert_ne!(base, build(TEST_ENTRY_SIZE + 1, TEST_CHUNK_POWER)); + assert_ne!(base, build(TEST_ENTRY_SIZE, TEST_CHUNK_POWER + 1)); +} + +#[test] +fn test_private_document_store_empty_root_constant_matches_insert_binding() { + // The empty-root helper must agree with what the insert path binds: + // verify_grovedb recomputes the binding from the helper, so a clean + // verify after a fresh insert proves the two paths agree. + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + db.insert( + EMPTY_PATH, + b"docs", + Element::empty_private_document_store(TEST_ENTRY_SIZE, TEST_CHUNK_POWER) + .expect("valid config"), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert store"); + let issues = db + .verify_grovedb(None, true, false, grove_version) + .expect("verify_grovedb"); + assert!(issues.is_empty(), "issues: {:?}", issues); +} + +// =========================================================================== +// Coverage: v0 insert path, batch policy branches, op metadata +// =========================================================================== + +/// Exercise the v0 `add_element_on_transaction` arm for +/// PrivateDocumentStore. No registered version pairs the v0 insert +/// implementation with an enabled PDS family (V1..V3 fail closed, V4 uses +/// v1), so drive it with a custom version: V4 with the insert +/// implementation slot dialed back to 0. +#[test] +fn test_private_document_store_insert_v0_element_path() { + let mut custom = GROVE_V4.clone(); + custom + .grovedb_versions + .operations + .insert + .add_element_on_transaction = 0; + let db = make_empty_grovedb(); + + db.insert( + EMPTY_PATH, + b"docs", + Element::empty_private_document_store(TEST_ENTRY_SIZE, TEST_CHUNK_POWER) + .expect("valid config"), + None, + None, + &custom, + ) + .unwrap() + .expect("v0 insert of empty store works"); + + // The v0 arm enforces the same emptiness and config validation. + let result = db + .insert( + EMPTY_PATH, + b"docs2", + Element::new_private_document_store(5, TEST_ENTRY_SIZE, TEST_CHUNK_POWER, None), + None, + None, + &custom, + ) + .unwrap(); + assert!(matches!(result, Err(Error::InvalidCodeExecution(_)))); + + let result = db + .insert( + EMPTY_PATH, + b"docs3", + Element::new_private_document_store(0, 0, TEST_CHUNK_POWER, None), + None, + None, + &custom, + ) + .unwrap(); + assert!(matches!(result, Err(Error::InvalidInput(_)))); + + // The binding written by the v0 arm matches what verify_grovedb + // recomputes (same config-parametrized empty root as the v1 arm). + let issues = db + .verify_grovedb(None, true, false, GroveVersion::latest()) + .expect("verify_grovedb"); + assert!(issues.is_empty(), "issues: {:?}", issues); +} + +#[test] +fn test_private_document_store_batch_insert_if_not_exists() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + let store_element = || { + Element::empty_private_document_store(TEST_ENTRY_SIZE, TEST_CHUNK_POWER) + .expect("valid config") + }; + + // First InsertIfNotExists creates the store. + let ops = vec![QualifiedGroveDbOp::insert_if_not_exists_op( + vec![], + b"docs".to_vec(), + store_element(), + )]; + db.apply_batch(ops, None, None, grove_version) + .unwrap() + .expect("first insert_if_not_exists creates the store"); + + // A second InsertIfNotExists over the same key hits the existence check + // and errors (validate_insertion_does_not_override defaults to erroring + // for InsertIfNotExists in batches). + let ops = vec![QualifiedGroveDbOp::insert_if_not_exists_op( + vec![], + b"docs".to_vec(), + store_element(), + )]; + let result = db.apply_batch(ops, None, None, grove_version).unwrap(); + assert!( + matches!(result, Err(Error::InvalidBatchOperation(_))), + "duplicate insert_if_not_exists must be rejected, got {:?}", + result + ); + + // The store is intact. + assert_eq!( + db.private_document_store_count(EMPTY_PATH, b"docs", None, grove_version) + .unwrap() + .expect("count"), + 0 + ); +} + +#[test] +fn test_private_document_store_reference_to_updated_store_rejected() { + let grove_version = GroveVersion::latest(); + let db = make_db_with_store(); + + // A reference in the same batch pointing at a store that receives + // appends must be rejected: references cannot point to trees being + // updated. + let ops = vec![ + QualifiedGroveDbOp::private_document_store_insert_op( + vec![b"root".to_vec(), b"docs".to_vec()], + entry(1), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"root".to_vec()], + b"link".to_vec(), + Element::new_reference( + crate::reference_path::ReferencePathType::AbsolutePathReference(vec![ + b"root".to_vec(), + b"docs".to_vec(), + ]), + ), + ), + ]; + let result = db.apply_batch(ops, None, None, grove_version).unwrap(); + assert!( + matches!(result, Err(Error::InvalidBatchOperation(_))), + "reference to a store being appended to must be rejected, got {:?}", + result + ); +} + +#[test] +fn test_private_document_store_op_metadata() { + // The sort tag is consensus-relevant for batch op ordering; pin it. + let op = crate::batch::GroveOp::PrivateDocumentStoreInsert { entry: entry(0) }; + assert_eq!(op.to_u8(), 19); + assert!(!op.can_mutate_child_count()); + + // NonMerkTreeMeta round-trips the element state. + let meta = crate::batch::NonMerkTreeMeta::PrivateDocumentStore { + total_count: 7, + entry_size: TEST_ENTRY_SIZE, + chunk_power: TEST_CHUNK_POWER, + }; + assert_eq!( + meta.to_tree_type(), + grovedb_merk::tree_type::TreeType::PrivateDocumentStore(TEST_CHUNK_POWER) + ); + assert_eq!(meta.count(), 7); + assert_eq!( + meta.to_element(Some(vec![1])), + Element::new_private_document_store(7, TEST_ENTRY_SIZE, TEST_CHUNK_POWER, Some(vec![1])) + ); +} + +#[test] +fn test_private_document_store_apply_without_batching() { + // The non-batch fallback path dispatches PrivateDocumentStoreInsert ops + // to the direct typed insert. + let grove_version = GroveVersion::latest(); + let db = make_db_with_store(); + + let ops = vec![ + QualifiedGroveDbOp::private_document_store_insert_op( + vec![b"root".to_vec(), b"docs".to_vec()], + entry(1), + ), + QualifiedGroveDbOp::private_document_store_insert_op( + vec![b"root".to_vec(), b"docs".to_vec()], + entry(2), + ), + ]; + db.apply_operations_without_batching(ops, None, None, grove_version) + .unwrap() + .expect("apply without batching"); + + assert_eq!( + db.private_document_store_count(&[b"root"], b"docs", None, grove_version) + .unwrap() + .expect("count"), + 2 + ); + assert_eq!( + db.private_document_store_get_value(&[b"root"], b"docs", 1, None, grove_version) + .unwrap() + .expect("get"), + Some(entry(2)) + ); + + let issues = db + .verify_grovedb(None, true, false, grove_version) + .expect("verify_grovedb"); + assert!(issues.is_empty(), "issues: {:?}", issues); +} + +#[test] +fn test_private_document_store_element_display_and_type_str() { + let element = Element::new_private_document_store(3, TEST_ENTRY_SIZE, TEST_CHUNK_POWER, None); + let display = format!("{}", element); + assert!( + display.contains("PrivateDocumentStore") + && display.contains("entry_size: 16") + && display.contains("chunk_power: 2"), + "unexpected display: {}", + display + ); + assert_eq!(element.type_str(), "private_document_store"); + assert_eq!( + Element::new_non_counted(element).expect("wrap").type_str(), + "non_counted private_document_store" + ); +} + +#[test] +fn test_private_document_store_v0_prover_rejects_subqueries() { + use grovedb_version::version::v2::GROVE_V2; + + // Build the store under the latest version, then generate proofs with + // GROVE_V2 (whose prover is the locked V0 wire format). Subqueries into + // the store must be rejected by the V0 dispatch... + let grove_version = GroveVersion::latest(); + let db = make_db_with_store(); + db.private_document_store_insert(&[b"root"], b"docs", entry(1), None, grove_version) + .unwrap() + .expect("append"); + + let mut inner_query = Query::new(); + inner_query.insert_all(); + let subquery = PathQuery { + path: vec![b"root".to_vec()], + query: SizedQuery { + query: Query { + items: vec![grovedb_merk::proofs::query::QueryItem::Key( + b"docs".to_vec(), + )], + default_subquery_branch: SubqueryBranch { + subquery_path: None, + subquery: Some(inner_query.into()), + }, + left_to_right: true, + conditional_subquery_branches: None, + add_parent_tree_on_subquery: false, + read_mode: None, + }, + limit: None, + offset: None, + }, + }; + let result = db.prove_query(&subquery, None, &GROVE_V2).unwrap(); + assert!( + matches!(result, Err(Error::NotSupported(_))), + "V0 subquery into a store must be rejected, got {:?}", + result + ); + + // ...while a terminal query for the element itself still proves and + // verifies (the node passes through as a result, V0 shape unchanged). + let terminal = PathQuery::new_unsized( + vec![b"root".to_vec()], + Query::new_single_key(b"docs".to_vec()), + ); + let proof_bytes = db + .prove_query(&terminal, None, &GROVE_V2) + .unwrap() + .expect("V0 terminal proof over a store element generates"); + let (root_hash, result_set) = GroveDb::verify_query_with_options( + &proof_bytes, + &terminal, + grovedb_merk::proofs::query::VerifyOptions { + absence_proofs_for_non_existing_searched_keys: false, + verify_proof_succinctness: false, + include_empty_trees_in_result: true, + }, + &GROVE_V2, + ) + .expect("verify V0 terminal proof"); + assert_eq!( + root_hash, + db.root_hash(None, grove_version).unwrap().unwrap(), + "V0 proof must bind the current grove root" + ); + assert_eq!(result_set.len(), 1); +} + +// =========================================================================== +// Regression: wrapper preservation, write-once enforcement, guard exactness +// =========================================================================== + +/// Read a `CountTree`'s recorded aggregate count. +fn count_tree_value(db: &crate::tests::TempGroveDb, key: &[u8]) -> u64 { + let grove_version = GroveVersion::latest(); + match db + .get(EMPTY_PATH, key, None, grove_version) + .unwrap() + .expect("get count tree") + { + Element::CountTree(_, count, _) => count, + other => panic!("expected CountTree, got {}", other.type_str()), + } +} + +/// Build a `CountTree` at `counted` holding a `NonCounted`-wrapped store at +/// `counted/docs`, and return the parent's count before any append. +fn make_non_counted_store_under_count_tree() -> (crate::tests::TempGroveDb, u64) { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + db.insert( + EMPTY_PATH, + b"counted", + Element::empty_count_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert count tree"); + + let wrapped = Element::new_non_counted( + Element::empty_private_document_store(TEST_ENTRY_SIZE, TEST_CHUNK_POWER) + .expect("valid config"), + ) + .expect("wrap in NonCounted"); + db.insert(&[b"counted"], b"docs", wrapped, None, None, grove_version) + .unwrap() + .expect("insert NonCounted store"); + + let count = count_tree_value(&db, b"counted"); + assert_eq!(count, 0, "a NonCounted child must not be counted"); + (db, count) +} + +#[test] +fn test_direct_append_preserves_non_counted_wrapper() { + let grove_version = GroveVersion::latest(); + let (db, count_before) = make_non_counted_store_under_count_tree(); + + db.private_document_store_insert(&[b"counted"], b"docs", entry(1), None, grove_version) + .unwrap() + .expect("append"); + + // `get` resolves through wrappers by design, so read the RAW stored + // element to see whether the wrapper survived. + let stored = db + .get_raw( + [b"counted".as_slice()].as_ref().into(), + b"docs", + None, + grove_version, + ) + .unwrap() + .expect("get raw store"); + assert!( + stored.is_non_counted(), + "a direct append must not strip the NonCounted wrapper" + ); + assert!(stored.is_private_document_store()); + assert_eq!( + count_tree_value(&db, b"counted"), + count_before, + "appending to a NonCounted store must not change the parent's count" + ); + + let issues = db + .verify_grovedb(None, true, false, grove_version) + .expect("verify_grovedb"); + assert!(issues.is_empty(), "issues: {:?}", issues); +} + +#[test] +fn test_batch_append_preserves_non_counted_wrapper() { + let grove_version = GroveVersion::latest(); + let (db, count_before) = make_non_counted_store_under_count_tree(); + + // Several entries so the batch path goes through append_many and the + // ReplaceNonMerkTreeRoot rebuild. + let ops = (0..6u8) + .map(|i| { + QualifiedGroveDbOp::private_document_store_insert_op( + vec![b"counted".to_vec(), b"docs".to_vec()], + entry(i), + ) + }) + .collect(); + db.apply_batch(ops, None, None, grove_version) + .unwrap() + .expect("batch append"); + + let stored = db + .get_raw( + [b"counted".as_slice()].as_ref().into(), + b"docs", + None, + grove_version, + ) + .unwrap() + .expect("get raw store"); + assert!( + stored.is_non_counted(), + "a batch append must not strip the NonCounted wrapper" + ); + assert_eq!(stored.non_merk_entry_count(), Some(6)); + assert_eq!( + count_tree_value(&db, b"counted"), + count_before, + "batch appending to a NonCounted store must not change the parent's count" + ); + + let issues = db + .verify_grovedb(None, true, false, grove_version) + .expect("verify_grovedb"); + assert!(issues.is_empty(), "issues: {:?}", issues); +} + +#[test] +fn test_private_document_store_is_write_once() { + let grove_version = GroveVersion::latest(); + let db = make_db_with_store(); + for i in 0..5u8 { + db.private_document_store_insert(&[b"root"], b"docs", entry(i), None, grove_version) + .unwrap() + .expect("append"); + } + let root_before = db.root_hash(None, grove_version).unwrap().unwrap(); + + // Direct overwrite with a fresh empty store must be rejected, not + // silently accepted (which would orphan the existing chunk data). + let result = db + .insert( + &[b"root"], + b"docs", + Element::empty_private_document_store(TEST_ENTRY_SIZE, TEST_CHUNK_POWER) + .expect("valid config"), + None, + None, + grove_version, + ) + .unwrap(); + assert!( + matches!( + result, + Err(Error::InvalidInput(_)) | Err(Error::OverrideNotAllowed(_)) + ), + "direct overwrite must be rejected, got {:?}", + result + ); + + // Batch overwrite likewise. + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( + vec![b"root".to_vec()], + b"docs".to_vec(), + Element::empty_private_document_store(TEST_ENTRY_SIZE, TEST_CHUNK_POWER) + .expect("valid config"), + )]; + let result = db.apply_batch(ops, None, None, grove_version).unwrap(); + assert!( + matches!(result, Err(Error::InvalidBatchOperation(_))), + "batch overwrite must be rejected, got {:?}", + result + ); + + // The store is untouched by the rejected overwrites. + assert_eq!( + db.private_document_store_count(&[b"root"], b"docs", None, grove_version) + .unwrap() + .expect("count"), + 5 + ); + assert_eq!( + root_before, + db.root_hash(None, grove_version).unwrap().unwrap(), + "rejected overwrites must not change the root hash" + ); +} + +#[test] +fn test_version_guard_rejects_unknown_future_slot() { + // The guard is an exact match: a future slot value must NOT silently run + // v1 semantics, or an un-upgraded node diverges from an upgraded one. + let mut future = GROVE_V4.clone(); + future + .grovedb_versions + .operations + .private_document_store + .insert = 2; + let db = make_db_with_store(); + + let result = db + .private_document_store_insert(&[b"root"], b"docs", entry(0), None, &future) + .unwrap(); + assert!( + matches!(result, Err(Error::VersionError(_))), + "an unknown future slot must fail closed, got {:?}", + result + ); + + let ops = vec![QualifiedGroveDbOp::private_document_store_insert_op( + vec![b"root".to_vec(), b"docs".to_vec()], + entry(0), + )]; + let result = db.apply_batch(ops, None, None, &future).unwrap(); + assert!( + matches!(result, Err(Error::VersionError(_))), + "an unknown future slot must fail closed in batch too, got {:?}", + result + ); +} + +#[test] +fn test_private_document_store_reads_are_billed() { + // An in-range read must charge for actually fetching the document, not + // just for reading the parent element. + let grove_version = GroveVersion::latest(); + let db = make_db_with_store(); + // Enough entries to span a completed chunk and the live buffer. + for i in 0..6u8 { + db.private_document_store_insert(&[b"root"], b"docs", entry(i), None, grove_version) + .unwrap() + .expect("append"); + } + + let out_of_range = db + .private_document_store_get_value(&[b"root"], b"docs", 99, None, grove_version) + .cost; + // Position 0 lives in a completed chunk (MMR read), position 5 in the + // buffer (dense-tree read); both must cost more than the element lookup. + for pos in [0u64, 5] { + let cost = db + .private_document_store_get_value(&[b"root"], b"docs", pos, None, grove_version) + .cost; + assert!( + cost.storage_loaded_bytes > out_of_range.storage_loaded_bytes, + "reading position {} must load the document bytes (got {:?} vs out-of-range {:?})", + pos, + cost.storage_loaded_bytes, + out_of_range.storage_loaded_bytes + ); + } +} + +#[test] +fn test_private_document_store_delete_empty_and_via_batch() { + let grove_version = GroveVersion::latest(); + + // Deleting an EMPTY store takes the is_empty branch, which the populated + // delete test never reaches. + let db = make_db_with_store(); + db.delete(&[b"root"], b"docs", None, None, grove_version) + .unwrap() + .expect("delete empty store"); + assert!(matches!( + db.get(&[b"root"], b"docs", None, grove_version).unwrap(), + Err(Error::PathKeyNotFound(_)) + )); + let issues = db + .verify_grovedb(None, true, false, grove_version) + .expect("verify_grovedb after empty-store delete"); + assert!(issues.is_empty(), "issues: {:?}", issues); + + // Batch DeleteTree is a wholly different path (scan_delete_tree_ops / + // non_merk_delete_paths) from the direct delete. + let db = make_db_with_store(); + for i in 0..6u8 { + db.private_document_store_insert(&[b"root"], b"docs", entry(i), None, grove_version) + .unwrap() + .expect("append"); + } + let ops = vec![QualifiedGroveDbOp::delete_tree_op( + vec![b"root".to_vec()], + b"docs".to_vec(), + grovedb_merk::tree_type::TreeType::PrivateDocumentStore(TEST_CHUNK_POWER), + crate::batch::SubelementsDeletionBehavior::DeleteChildren, + )]; + db.apply_batch(ops, None, None, grove_version) + .unwrap() + .expect("batch delete populated store"); + assert!(matches!( + db.get(&[b"root"], b"docs", None, grove_version).unwrap(), + Err(Error::PathKeyNotFound(_)) + )); + + let issues = db + .verify_grovedb(None, true, false, grove_version) + .expect("verify_grovedb"); + assert!(issues.is_empty(), "issues: {:?}", issues); +} + +#[test] +fn test_private_document_store_reopened_reads_are_billed() { + // A store reopened from persisted state resolves its MMR root through + // the lazy (uncached) path — the path proof binding and the integrity + // walk take. Against real storage that read must be billed. + let grove_version = GroveVersion::latest(); + let db = make_db_with_store(); + for i in 0..6u8 { + db.private_document_store_insert(&[b"root"], b"docs", entry(i), None, grove_version) + .unwrap() + .expect("append"); + } + + // A terminal proof over the store derives its state root from a freshly + // opened (cache-cold) store. + let path_query = PathQuery::new_unsized( + vec![b"root".to_vec()], + Query::new_single_key(b"docs".to_vec()), + ); + let ctx = db.prove_query(&path_query, None, grove_version); + let cost = ctx.cost.clone(); + ctx.value.expect("prove terminal store element"); + assert!( + cost.storage_loaded_bytes > 0 && cost.seek_count > 0, + "deriving a reopened store's state root during proof binding must bill \ + its storage reads, got {:?}", + cost + ); +} + +/// Creating a store derives its empty state root, which is two blake3 calls: +/// the committed-config hash and the composite `pds_state` hash. Neither is +/// visible to `insert_subtree`, which receives an already-computed array, so +/// both have to be charged at the creation site or the operation is billed two +/// hashes short. +/// +/// Asserted as a difference against a `BulkAppendTree` creation rather than as +/// an absolute figure: that type creates from `NULL_HASH` and so performs +/// neither hash, while every other part of the insert — the parent Merk write, +/// the element value hash — is the same work. The gap is therefore exactly the +/// two hashes under test, and the test does not break when unrelated Merk +/// costs shift. +#[test] +fn test_private_document_store_creation_bills_its_two_hashes() { + let grove_version = GroveVersion::latest(); + + let pds_cost = { + let db = make_empty_grovedb(); + db.insert( + EMPTY_PATH, + b"root", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert root tree"); + db.insert( + &[b"root"], + b"store", + Element::empty_private_document_store(TEST_ENTRY_SIZE, TEST_CHUNK_POWER) + .expect("valid config"), + None, + None, + grove_version, + ) + .cost + }; + + let bulk_cost = { + let db = make_empty_grovedb(); + db.insert( + EMPTY_PATH, + b"root", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert root tree"); + db.insert( + &[b"root"], + b"store", + Element::empty_bulk_append_tree(TEST_CHUNK_POWER).expect("valid config"), + None, + None, + grove_version, + ) + .cost + }; + + assert_eq!( + pds_cost.hash_node_calls, + bulk_cost.hash_node_calls + 2, + "store creation must bill the config hash and the composite root \ + hash (pds {:?} vs bulk {:?})", + pds_cost, + bulk_cost + ); +} + +/// A store element that CLAIMS entries it has no data for must be refused at +/// creation, on every path. +/// +/// `Element::PrivateDocumentStore` is a public variant with public fields, so +/// `total_count` is not guaranteed to come from +/// `Element::empty_private_document_store` — a caller can construct one +/// directly, and one can arrive by deserialization. Accepting it would commit +/// an element whose committed count has no backing chunks or buffer entries: +/// the state root would be derived as if empty, and reads of the claimed +/// positions would fail against a tree that verifies as intact. +#[test] +fn test_private_document_store_rejects_nonempty_total_count_at_creation() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + db.insert( + EMPTY_PATH, + b"root", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert root tree"); + + let claims_five = Element::PrivateDocumentStore(5, TEST_ENTRY_SIZE, TEST_CHUNK_POWER, None); + + // Direct insert. + let direct = db + .insert( + &[b"root"], + b"store", + claims_five.clone(), + None, + None, + grove_version, + ) + .unwrap(); + assert!( + direct.is_err(), + "direct insert must refuse a store claiming entries it has no data \ + for, got {:?}", + direct + ); + + // Batch insert. + let batch = vec![QualifiedGroveDbOp::insert_or_replace_op( + vec![b"root".to_vec()], + b"store".to_vec(), + claims_five, + )]; + let batched = db.apply_batch(batch, None, None, grove_version).unwrap(); + assert!( + batched.is_err(), + "batch insert must refuse it too, got {:?}", + batched + ); + + // Nothing was created either way. + assert!( + db.get_raw( + [b"root".as_slice()].as_ref().into(), + b"store", + None, + grove_version + ) + .unwrap() + .is_err(), + "no element should exist at the key" + ); +} diff --git a/grovedb/src/util.rs b/grovedb/src/util.rs index c91205a2c..ad346fa9f 100644 --- a/grovedb/src/util.rs +++ b/grovedb/src/util.rs @@ -37,3 +37,18 @@ impl<'db> AsRef> for TxRef<'_, 'db> { } } } + +/// Build the storage path of a subtree living at `path`/`key`. +/// +/// Every non-Merk tree type (commitment tree, bulk-append tree, private +/// document store) needs its own subtree path to open the data namespace, +/// and each had grown a private copy of this three-line helper under a +/// different name. One shared function keeps them provably identical. +pub(crate) fn subtree_path_with_key>( + path: &grovedb_path::SubtreePath, + key: &[u8], +) -> Vec> { + let mut v = path.to_vec(); + v.push(key.to_vec()); + v +} diff --git a/merk/src/element/costs.rs b/merk/src/element/costs.rs index 2779c4413..1fc33bb12 100644 --- a/merk/src/element/costs.rs +++ b/merk/src/element/costs.rs @@ -12,7 +12,7 @@ use crate::{ tree_type::{ BIG_SUM_TREE_COST_SIZE, BULK_APPEND_TREE_COST_SIZE, COMMITMENT_TREE_COST_SIZE, COUNT_INDEXED_TREE_COST_SIZE, COUNT_SUM_TREE_COST_SIZE, COUNT_TREE_COST_SIZE, - DENSE_TREE_COST_SIZE, MMR_TREE_COST_SIZE, + DENSE_TREE_COST_SIZE, MMR_TREE_COST_SIZE, PRIVATE_DOCUMENT_STORE_COST_SIZE, PROVABLE_COUNT_PROVABLE_SUM_INDEXED_TREE_COST_SIZE, SUM_ITEM_COST_SIZE, SUM_TREE_COST_SIZE, TREE_COST_SIZE, }, @@ -99,6 +99,7 @@ impl ElementCostPrivateExtensions for Element { Element::ProvableCountProvableSumIndexedTree(..) => { Ok(PROVABLE_COUNT_PROVABLE_SUM_INDEXED_TREE_COST_SIZE) } + Element::PrivateDocumentStore(..) => Ok(PRIVATE_DOCUMENT_STORE_COST_SIZE), Element::NonCounted(inner) | Element::NotSummed(inner) | Element::NotCountedOrSummed(inner) => { @@ -313,6 +314,17 @@ impl ElementCostExtensions for Element { key_len, value_len, node_type, ) } + Element::PrivateDocumentStore(_, _, _, flags) => { + let flags_len = flags.map_or(0, |flags| { + let flags_len = flags.len() as u32; + flags_len + flags_len.required_space() as u32 + }); + let value_len = PRIVATE_DOCUMENT_STORE_COST_SIZE + flags_len + wrapper_overhead; + let key_len = key.len() as u32; + KV::layered_value_byte_cost_size_for_key_and_value_lengths( + key_len, value_len, node_type, + ) + } Element::SumItem(.., flags) => { let flags_len = flags.map_or(0, |flags| { let flags_len = flags.len() as u32; @@ -391,7 +403,8 @@ impl ElementCostExtensions for Element { | Element::DenseAppendOnlyFixedSizeTree(..) | Element::ProvableSumIndexedTree(..) | Element::ProvableCountIndexedTree(..) - | Element::ProvableCountProvableSumIndexedTree(..) => Some(cost), + | Element::ProvableCountProvableSumIndexedTree(..) + | Element::PrivateDocumentStore(..) => Some(cost), _ => None, } } @@ -411,7 +424,8 @@ impl ElementCostExtensions for Element { | Element::CommitmentTree(..) | Element::MmrTree(..) | Element::BulkAppendTree(..) - | Element::DenseAppendOnlyFixedSizeTree(..) => Some(LayeredValueDefinedCost(cost)), + | Element::DenseAppendOnlyFixedSizeTree(..) + | Element::PrivateDocumentStore(..) => Some(LayeredValueDefinedCost(cost)), Element::SumTree(..) => Some(LayeredValueDefinedCost(cost)), Element::BigSumTree(..) => Some(LayeredValueDefinedCost(cost)), Element::CountTree(..) => Some(LayeredValueDefinedCost(cost)), @@ -443,3 +457,55 @@ impl ElementCostExtensions for Element { element.value_defined_cost(grove_version) } } + +#[cfg(test)] +mod tests { + use grovedb_version::version::GroveVersion; + + use super::*; + + #[test] + fn private_document_store_cost_paths() { + let grove_version = GroveVersion::latest(); + let element = Element::new_private_document_store(5, 64, 4, Some(vec![1, 2])); + + // Specialized (layered) cost uses the entry-size-carrying constant. + assert_eq!( + element.get_specialized_cost(grove_version).expect("cost"), + crate::tree_type::PRIVATE_DOCUMENT_STORE_COST_SIZE + ); + + // Layered value defined cost = specialized cost + flags overhead. + let layered = element + .layered_value_defined_cost(grove_version) + .expect("layered cost"); + assert_eq!( + layered, + crate::tree_type::PRIVATE_DOCUMENT_STORE_COST_SIZE + 3 + ); + assert!(matches!( + element.value_defined_cost(grove_version), + Some(LayeredValueDefinedCost(c)) if c == layered + )); + // Not an item-like element: no specialized (sum-item) value cost. + assert!(element + .specialized_value_defined_cost(grove_version) + .is_none()); + + // The serialized-value cost path takes the PDS block in + // specialized_costs_for_key_value. + let serialized = element.serialize(grove_version).expect("serialize"); + let cost = Element::specialized_costs_for_key_value( + b"key", + &serialized, + NodeType::NormalNode, + grove_version, + ) + .expect("cost for key value"); + assert!(cost > 0); + assert_eq!( + Element::value_defined_cost_for_serialized_value(&serialized, grove_version), + Some(LayeredValueDefinedCost(layered)) + ); + } +} diff --git a/merk/src/element/delete.rs b/merk/src/element/delete.rs index e017b6ea3..f43bf8166 100644 --- a/merk/src/element/delete.rs +++ b/merk/src/element/delete.rs @@ -76,6 +76,7 @@ impl ElementDeleteFromStorageExtensions for Element { | (TreeType::DenseAppendOnlyFixedSizeTree(_), true) | (TreeType::ProvableSumIndexedTree, true) | (TreeType::ProvableCountProvableSumIndexedTree, true) + | (TreeType::PrivateDocumentStore(_), true) | (TreeType::ProvableCountIndexedTree, true) => Op::DeleteLayeredMaybeSpecialized, (TreeType::SumTree, false) | (TreeType::BigSumTree, false) @@ -91,6 +92,7 @@ impl ElementDeleteFromStorageExtensions for Element { | (TreeType::DenseAppendOnlyFixedSizeTree(_), false) | (TreeType::ProvableSumIndexedTree, false) | (TreeType::ProvableCountProvableSumIndexedTree, false) + | (TreeType::PrivateDocumentStore(_), false) | (TreeType::ProvableCountIndexedTree, false) => Op::DeleteMaybeSpecialized, }; let batch = [(key, op)]; @@ -154,6 +156,7 @@ impl ElementDeleteFromStorageExtensions for Element { | (TreeType::DenseAppendOnlyFixedSizeTree(_), true) | (TreeType::ProvableSumIndexedTree, true) | (TreeType::ProvableCountProvableSumIndexedTree, true) + | (TreeType::PrivateDocumentStore(_), true) | (TreeType::ProvableCountIndexedTree, true) => Op::DeleteLayeredMaybeSpecialized, (TreeType::SumTree, false) | (TreeType::BigSumTree, false) @@ -169,6 +172,7 @@ impl ElementDeleteFromStorageExtensions for Element { | (TreeType::DenseAppendOnlyFixedSizeTree(_), false) | (TreeType::ProvableSumIndexedTree, false) | (TreeType::ProvableCountProvableSumIndexedTree, false) + | (TreeType::PrivateDocumentStore(_), false) | (TreeType::ProvableCountIndexedTree, false) => Op::DeleteMaybeSpecialized, }; let batch = [(key, op)]; @@ -228,6 +232,7 @@ impl ElementDeleteFromStorageExtensions for Element { | (TreeType::DenseAppendOnlyFixedSizeTree(_), true) | (TreeType::ProvableSumIndexedTree, true) | (TreeType::ProvableCountProvableSumIndexedTree, true) + | (TreeType::PrivateDocumentStore(_), true) | (TreeType::ProvableCountIndexedTree, true) => Op::DeleteLayeredMaybeSpecialized, (TreeType::SumTree, false) | (TreeType::BigSumTree, false) @@ -243,6 +248,7 @@ impl ElementDeleteFromStorageExtensions for Element { | (TreeType::DenseAppendOnlyFixedSizeTree(_), false) | (TreeType::ProvableSumIndexedTree, false) | (TreeType::ProvableCountProvableSumIndexedTree, false) + | (TreeType::PrivateDocumentStore(_), false) | (TreeType::ProvableCountIndexedTree, false) => Op::DeleteMaybeSpecialized, }; let entry = (key, op); diff --git a/merk/src/element/get.rs b/merk/src/element/get.rs index 766b8fd35..d76649e52 100644 --- a/merk/src/element/get.rs +++ b/merk/src/element/get.rs @@ -468,7 +468,8 @@ impl ElementFetchFromStoragePrivateExtensions for Element { | Some(Element::DenseAppendOnlyFixedSizeTree(.., flags)) | Some(Element::ProvableSumIndexedTree(.., flags)) | Some(Element::ProvableCountIndexedTree(.., flags)) - | Some(Element::ProvableCountProvableSumIndexedTree(_, _, _, _, flags)) => { + | Some(Element::ProvableCountProvableSumIndexedTree(_, _, _, _, flags)) + | Some(Element::PrivateDocumentStore(.., flags)) => { // tree_type() looks through NonCounted, so this works for both // a bare tree and a NonCounted(tree). let tree_cost_size = element.as_ref().unwrap().tree_type().unwrap().cost_size(); @@ -589,7 +590,8 @@ impl ElementFetchFromStoragePrivateExtensions for Element { | Element::DenseAppendOnlyFixedSizeTree(.., flags) | Element::ProvableSumIndexedTree(.., flags) | Element::ProvableCountIndexedTree(.., flags) - | Element::ProvableCountProvableSumIndexedTree(_, _, _, _, flags) => { + | Element::ProvableCountProvableSumIndexedTree(_, _, _, _, flags) + | Element::PrivateDocumentStore(.., flags) => { // tree_type() looks through NonCounted. let tree_cost_size = element.tree_type().unwrap().cost_size(); let flags_len = flags.as_ref().map_or(0, |flags| { diff --git a/merk/src/element/insert.rs b/merk/src/element/insert.rs index 39303767f..7871b273c 100644 --- a/merk/src/element/insert.rs +++ b/merk/src/element/insert.rs @@ -204,6 +204,12 @@ impl ElementInsertToStorageExtensions for Element { /// If transaction is passed, the operation will be committed on the /// transaction commit. fn validate_insertable_into(&self, tree_type: TreeType) -> Result<(), Error> { + if matches!(tree_type, TreeType::PrivateDocumentStore(_)) { + return Err(Error::InvalidInputError( + "private document stores cannot hold child elements; entries are appended \ + via the private_document_store_insert API", + )); + } if self.is_non_counted() && !tree_type.accepts_non_counted_children() { return Err(Error::InvalidInputError( "non-counted elements may only be inserted into non-provable count-bearing \ @@ -503,6 +509,14 @@ impl ElementInsertToStorageExtensions for Element { grove_version.grovedb_versions.element.insert_reference ); + if matches!(merk.tree_type, TreeType::PrivateDocumentStore(_)) { + return Err(Error::InvalidInputError( + "private document stores cannot hold child elements; entries are appended \ + via the private_document_store_insert API", + )) + .wrap_with_cost(Default::default()); + } + if self.is_non_counted() && !merk.tree_type.accepts_non_counted_children() { return Err(Error::InvalidInputError( "non-counted elements may only be inserted into non-provable count-bearing \ @@ -615,6 +629,14 @@ impl ElementInsertToStorageExtensions for Element { grove_version.grovedb_versions.element.insert_subtree ); + if matches!(merk.tree_type, TreeType::PrivateDocumentStore(_)) { + return Err(Error::InvalidInputError( + "private document stores cannot hold child elements; entries are appended \ + via the private_document_store_insert API", + )) + .wrap_with_cost(Default::default()); + } + if self.is_non_counted() && !merk.tree_type.accepts_non_counted_children() { return Err(Error::InvalidInputError( "non-counted elements may only be inserted into non-provable count-bearing \ @@ -756,6 +778,14 @@ impl ElementInsertToStorageExtensions for Element { .wrap_with_cost(Default::default()); } + if matches!(merk.tree_type, TreeType::PrivateDocumentStore(_)) { + return Err(Error::InvalidInputError( + "private document stores cannot hold child elements; entries are appended \ + via the private_document_store_insert API", + )) + .wrap_with_cost(Default::default()); + } + if self.is_non_counted() && !merk.tree_type.accepts_non_counted_children() { return Err(Error::InvalidInputError( "non-counted elements may only be inserted into non-provable count-bearing trees", @@ -909,6 +939,110 @@ mod tests { TreeType, }; + #[test] + fn private_document_store_merk_rejects_all_child_element_inserts() { + // A PrivateDocumentStore's Merk must stay empty forever: every + // element-insert entry point rejects when the destination Merk is + // PDS-typed — validate_insertable_into (used by insert / + // insert_if_not_exists), the inline guards in insert_reference and + // insert_subtree, and insert_count_indexed_subtree. + let grove_version = GroveVersion::latest(); + let mut merk = + TempMerk::new_with_tree_type(grove_version, TreeType::PrivateDocumentStore(4)); + + let item = Element::new_item(b"value".to_vec()); + assert!(item + .validate_insertable_into(TreeType::PrivateDocumentStore(4)) + .is_err()); + assert!(item + .insert(&mut merk, b"k", None, grove_version) + .unwrap() + .is_err()); + assert!(item + .insert_if_not_exists(&mut merk, b"k", None, grove_version) + .unwrap() + .is_err()); + + let reference = Element::new_reference( + grovedb_element::reference_path::ReferencePathType::AbsolutePathReference(vec![ + b"a".to_vec() + ]), + ); + assert!(reference + .insert_reference(&mut merk, b"k", [0u8; 32], None, grove_version) + .unwrap() + .is_err()); + + let subtree = Element::empty_tree(); + assert!(subtree + .insert_subtree(&mut merk, b"k", [0u8; 32], None, grove_version) + .unwrap() + .is_err()); + + let cidx = Element::empty_provable_count_indexed_tree(); + assert!(cidx + .insert_count_indexed_subtree( + &mut merk, + b"k", + [0u8; 32], + [0u8; 32], + None, + grove_version + ) + .unwrap() + .is_err()); + + // The `*_into_batch_operations` builders cannot see the destination + // Merk, so ops CAN be queued — but the apply chokepoint + // (`Merk::apply_unchecked`, which every apply variant funnels + // through) rejects any non-empty batch aimed at a PDS Merk, so the + // queued ops can never take effect. + let mut ops: Vec>> = Vec::new(); + item.insert_into_batch_operations( + b"k".to_vec(), + &mut ops, + crate::TreeFeatureType::BasicMerkNode, + grove_version, + ) + .unwrap() + .expect("builder itself queues"); + subtree + .insert_subtree_into_batch_operations( + b"k2".to_vec(), + [0u8; 32], + true, + &mut ops, + crate::TreeFeatureType::BasicMerkNode, + grove_version, + ) + .unwrap() + .expect("builder itself queues"); + reference + .insert_reference_into_batch_operations( + b"k3".to_vec(), + [0u8; 32], + &mut ops, + crate::TreeFeatureType::BasicMerkNode, + grove_version, + ) + .unwrap() + .expect("builder itself queues"); + let apply_result = merk + .apply_with_specialized_costs::<_, Vec>( + &ops, + &[], + None, + &|_, _| Ok(0), + Some(&Element::value_defined_cost_for_serialized_value), + grove_version, + ) + .unwrap(); + assert!( + apply_result.is_err(), + "applying queued ops to a PDS Merk must be rejected" + ); + } + #[test] fn test_success_insert() { let grove_version = GroveVersion::latest(); diff --git a/merk/src/element/reconstruct.rs b/merk/src/element/reconstruct.rs index 332dfaad2..26587dcfe 100644 --- a/merk/src/element/reconstruct.rs +++ b/merk/src/element/reconstruct.rs @@ -116,6 +116,12 @@ impl ElementReconstructExtensions for Element { Element::DenseAppendOnlyFixedSizeTree(c, h, f) => { Some(Element::DenseAppendOnlyFixedSizeTree(*c, *h, f.clone())) } + // Like the other non-Merk data trees, the private document store + // has no root key and no Merk aggregate — the element is + // reconstructed verbatim. + Element::PrivateDocumentStore(tc, es, cp, f) => { + Some(Element::PrivateDocumentStore(*tc, *es, *cp, f.clone())) + } // Recurse on the inner element and re-wrap. Without this, a // batch that mutates a subtree under a wrapped tree would lose // the wrapper on the parent's stored element when its root key @@ -197,6 +203,18 @@ mod tests { use super::ElementReconstructExtensions; use crate::tree::AggregateData; + #[test] + fn reconstruct_private_document_store_is_verbatim() { + // Like the other non-Merk data trees, a PrivateDocumentStore has no + // root key and no Merk aggregate: reconstruction ignores both inputs + // and returns the element verbatim. + let element = Element::new_private_document_store(9, 64, 4, Some(vec![1, 2])); + let reconstructed = element + .reconstruct_with_root_key(Some(b"ignored".to_vec()), AggregateData::NoAggregateData) + .expect("reconstruct ok"); + assert_eq!(reconstructed, element); + } + #[test] fn reconstruct_preserves_non_counted_wrapper() { // A NonCounted-wrapped tree must come back wrapped after a root-key diff --git a/merk/src/element/tree_type.rs b/merk/src/element/tree_type.rs index 45c4baf04..c911cc701 100644 --- a/merk/src/element/tree_type.rs +++ b/merk/src/element/tree_type.rs @@ -79,6 +79,9 @@ impl ElementTreeTypeExtensions for Element { primary_root_key, TreeType::ProvableCountProvableSumIndexedTree, )), + Element::PrivateDocumentStore(_, _, chunk_power, _) => { + Some((None, TreeType::PrivateDocumentStore(chunk_power))) + } Element::NonCounted(inner) | Element::NotSummed(inner) | Element::NotCountedOrSummed(inner) => inner.root_key_and_tree_type_owned(), @@ -128,6 +131,9 @@ impl ElementTreeTypeExtensions for Element { primary_root_key, TreeType::ProvableCountProvableSumIndexedTree, )), + Element::PrivateDocumentStore(_, _, chunk_power, _) => { + Some((&NONE_ROOT_KEY, TreeType::PrivateDocumentStore(*chunk_power))) + } Element::NonCounted(inner) | Element::NotSummed(inner) | Element::NotCountedOrSummed(inner) => inner.root_key_and_tree_type(), @@ -172,6 +178,9 @@ impl ElementTreeTypeExtensions for Element { Element::ProvableCountProvableSumIndexedTree(_, _, _, _, flags) => { Some((flags, TreeType::ProvableCountProvableSumIndexedTree)) } + Element::PrivateDocumentStore(_, _, chunk_power, flags) => { + Some((flags, TreeType::PrivateDocumentStore(*chunk_power))) + } Element::NonCounted(inner) | Element::NotSummed(inner) | Element::NotCountedOrSummed(inner) => inner.tree_flags_and_type(), @@ -209,6 +218,9 @@ impl ElementTreeTypeExtensions for Element { Element::ProvableCountProvableSumIndexedTree(..) => { Some(TreeType::ProvableCountProvableSumIndexedTree) } + Element::PrivateDocumentStore(_, _, chunk_power, _) => { + Some(TreeType::PrivateDocumentStore(*chunk_power)) + } Element::NonCounted(inner) | Element::NotSummed(inner) | Element::NotCountedOrSummed(inner) => inner.tree_type(), @@ -255,6 +267,7 @@ impl ElementTreeTypeExtensions for Element { Element::ProvableCountProvableSumIndexedTree(_, count_value, sum_value, _, _) => Some( TreeFeatureType::ProvableCountedAndProvableSummedMerkNode(*count_value, *sum_value), ), + Element::PrivateDocumentStore(..) => Some(BasicMerkNode), Element::NonCounted(inner) | Element::NotSummed(inner) | Element::NotCountedOrSummed(inner) => inner.tree_feature_type(), @@ -296,6 +309,9 @@ impl ElementTreeTypeExtensions for Element { Element::ProvableCountProvableSumIndexedTree(..) => { MaybeTree::Tree(TreeType::ProvableCountProvableSumIndexedTree) } + Element::PrivateDocumentStore(_, _, chunk_power, _) => { + MaybeTree::Tree(TreeType::PrivateDocumentStore(*chunk_power)) + } Element::NonCounted(inner) | Element::NotSummed(inner) | Element::NotCountedOrSummed(inner) => inner.maybe_tree_type(), @@ -334,6 +350,7 @@ impl ElementTreeTypeExtensions for Element { TreeType::MmrTree => Ok(BasicMerkNode), TreeType::BulkAppendTree(_) => Ok(BasicMerkNode), TreeType::DenseAppendOnlyFixedSizeTree(_) => Ok(BasicMerkNode), + TreeType::PrivateDocumentStore(_) => Ok(BasicMerkNode), // ProvableSumTree aggregates an i64 sum (same arithmetic // shape as plain SumTree) but carries it via // `ProvableSummedMerkNode` so the sum is baked into every @@ -657,6 +674,46 @@ mod tests { assert_eq!(e.tree_feature_type(), Some(BasicMerkNode)); } + #[test] + fn private_document_store_extension_arms_direct() { + // PrivateDocumentStore carries {entry_size, chunk_power}; only the + // chunk_power flows into the TreeType. Drive every dispatch arm. + let entry_size = 64u32; + let chunk_power = 4u8; + let e = Element::PrivateDocumentStore(3, entry_size, chunk_power, Some(vec![1])); + + let (rk, tt) = e.root_key_and_tree_type().expect("Some"); + assert!(rk.is_none()); + assert_eq!(tt, TreeType::PrivateDocumentStore(chunk_power)); + + let (rk, tt) = e.clone().root_key_and_tree_type_owned().expect("Some"); + assert!(rk.is_none()); + assert_eq!(tt, TreeType::PrivateDocumentStore(chunk_power)); + + assert_eq!( + e.tree_type(), + Some(TreeType::PrivateDocumentStore(chunk_power)) + ); + assert_eq!( + e.maybe_tree_type(), + MaybeTree::Tree(TreeType::PrivateDocumentStore(chunk_power)) + ); + + let (flags, tt) = e.tree_flags_and_type().expect("Some"); + assert!(flags.is_some()); + assert_eq!(tt, TreeType::PrivateDocumentStore(chunk_power)); + assert_eq!(e.tree_feature_type(), Some(BasicMerkNode)); + + // Children of a PDS parent (unreachable in practice — inserts are + // rejected) still resolve to BasicMerkNode for exhaustiveness. + assert_eq!( + Element::new_item(b"x".to_vec()) + .get_feature_type(TreeType::PrivateDocumentStore(chunk_power)) + .expect("feature type"), + BasicMerkNode + ); + } + #[test] fn bulk_append_tree_extension_arms_direct() { let chunk_power = 8u8; diff --git a/merk/src/merk/apply.rs b/merk/src/merk/apply.rs index 4609e2932..9a045b6cb 100644 --- a/merk/src/merk/apply.rs +++ b/merk/src/merk/apply.rs @@ -16,7 +16,7 @@ use crate::{ kv::{ValueDefinedCostType, KV}, AuxMerkBatch, OldValueDisposition, Walker, }, - Error, Merk, MerkBatch, MerkOptions, + Error, Merk, MerkBatch, MerkOptions, TreeType, }; const MAX_KEY_LENGTH: usize = u8::MAX as usize; @@ -404,6 +404,21 @@ where R: FnMut(&Vec, u32, u32) -> Result<(StorageRemovedBytes, StorageRemovedBytes), Error>, O: FnMut(&[u8], &[u8], OldValueDisposition), { + // A PrivateDocumentStore's Merk must stay empty forever — its + // entries live in the non-Merk data namespace and immutability is + // enforced by the type. The element-insert entry points already + // reject PDS destinations; this chokepoint additionally covers ops + // assembled through the `*_into_batch_operations` builders (which + // cannot see the destination), since queued ops can only take + // effect through an apply on the destination Merk. Every public + // apply variant funnels through here. + if !batch.is_empty() && matches!(self.tree_type, TreeType::PrivateDocumentStore(_)) { + return Err(Error::InvalidInputError( + "private document stores cannot hold child elements; entries are appended \ + via the private_document_store_insert API", + )) + .wrap_with_cost(Default::default()); + } for (key, ..) in batch.iter() { if key.as_ref().len() > MAX_KEY_LENGTH { return Err(Error::InvalidInputError( diff --git a/merk/src/tree_type/costs.rs b/merk/src/tree_type/costs.rs index fdb2bdeec..d3aaabb4f 100644 --- a/merk/src/tree_type/costs.rs +++ b/merk/src/tree_type/costs.rs @@ -45,6 +45,12 @@ pub const BULK_APPEND_TREE_COST_SIZE: u32 = 9 + 1 + 2; // 12 /// height (u8) + 2 bytes overhead) pub const DENSE_TREE_COST_SIZE: u32 = 3 + 1 + 2; // 6 +/// The cost of a private document store (9 bytes total_count (u64 varint +/// worst case) + 5 bytes entry_size (u32 varint worst case) + 1 byte +/// chunk_power (u8) + 2 bytes overhead). Same shape as +/// `BULK_APPEND_TREE_COST_SIZE` plus the committed entry size field. +pub const PRIVATE_DOCUMENT_STORE_COST_SIZE: u32 = 9 + 5 + 1 + 2; // 17 + /// The cost of a count-indexed tree element. Same shape as `COUNT_TREE_COST_SIZE` /// but with one extra byte of overhead for the second `Option` field /// (the secondary root key). @@ -97,6 +103,7 @@ impl CostSize for TreeType { TreeType::ProvableCountProvableSumIndexedTree => { PROVABLE_COUNT_PROVABLE_SUM_INDEXED_TREE_COST_SIZE } + TreeType::PrivateDocumentStore(_) => PRIVATE_DOCUMENT_STORE_COST_SIZE, } } } diff --git a/merk/src/tree_type/mod.rs b/merk/src/tree_type/mod.rs index f069f7a5a..9e25a40fd 100644 --- a/merk/src/tree_type/mod.rs +++ b/merk/src/tree_type/mod.rs @@ -93,6 +93,13 @@ pub enum TreeType { /// offset pagination, and the sum/avg axes additionally support /// sum-on-range proofs. ProvableCountProvableSumIndexedTree, + /// A private document store: an append-only store of fixed-size opaque + /// entries over a `BulkAppendTree`, with the `{entry_size, chunk_power}` + /// configuration bound into the state root. Carries the chunk power + /// (log2 of the epoch size) like `CommitmentTree` / `BulkAppendTree`; + /// the entry size lives only in the Element (it does not affect Merk + /// node layout). + PrivateDocumentStore(u8), } impl TreeType { @@ -117,6 +124,7 @@ impl TreeType { TreeType::ProvableSumIndexedTree => 13, TreeType::ProvableCountIndexedTree => 14, TreeType::ProvableCountProvableSumIndexedTree => 15, + TreeType::PrivateDocumentStore(_) => 16, } } } @@ -142,7 +150,8 @@ impl TryFrom for TreeType { 13 => Ok(TreeType::ProvableSumIndexedTree), 14 => Ok(TreeType::ProvableCountIndexedTree), 15 => Ok(TreeType::ProvableCountProvableSumIndexedTree), - n => Err(Error::UnknownTreeType(format!("got {}, max is 15", n))), + 16 => Ok(TreeType::PrivateDocumentStore(0)), + n => Err(Error::UnknownTreeType(format!("got {}, max is 16", n))), } } } @@ -168,6 +177,7 @@ impl fmt::Display for TreeType { TreeType::ProvableCountProvableSumIndexedTree => { "Provable Count Provable Sum Indexed Tree" } + TreeType::PrivateDocumentStore(_) => "Private Document Store", }; write!(f, "{}", s) } @@ -186,6 +196,7 @@ impl TreeType { | TreeType::MmrTree | TreeType::BulkAppendTree(_) | TreeType::DenseAppendOnlyFixedSizeTree(_) + | TreeType::PrivateDocumentStore(_) ) } @@ -320,6 +331,7 @@ impl TreeType { TreeType::ProvableSumIndexedTree => true, TreeType::ProvableCountIndexedTree => false, TreeType::ProvableCountProvableSumIndexedTree => true, + TreeType::PrivateDocumentStore(_) => false, } } @@ -349,6 +361,7 @@ impl TreeType { // The primary of a ProvableCountProvableSumIndexedTree mirrors // a ProvableCountProvableSumTree. TreeType::ProvableCountProvableSumIndexedTree => NodeType::ProvableCountProvableSumNode, + TreeType::PrivateDocumentStore(_) => NodeType::NormalNode, } } @@ -381,6 +394,7 @@ impl TreeType { TreeType::ProvableCountProvableSumIndexedTree => { TreeFeatureType::ProvableCountedAndProvableSummedMerkNode(0, 0) } + TreeType::PrivateDocumentStore(_) => TreeFeatureType::BasicMerkNode, } } @@ -414,6 +428,7 @@ impl TreeType { TreeType::ProvableCountProvableSumIndexedTree => { Some(ElementType::ProvableCountProvableSumIndexedTree) } + TreeType::PrivateDocumentStore(_) => Some(ElementType::PrivateDocumentStore), } } } @@ -438,6 +453,7 @@ mod tests { TreeType::DenseAppendOnlyFixedSizeTree(8), TreeType::ProvableSumTree, TreeType::ProvableCountProvableSumTree, + TreeType::PrivateDocumentStore(4), ]; for v in &variants { let d = v.discriminant(); @@ -449,10 +465,35 @@ mod tests { #[test] fn tree_type_try_from_invalid() { - assert!(TreeType::try_from(16u8).is_err()); + assert!(TreeType::try_from(17u8).is_err()); assert!(TreeType::try_from(255u8).is_err()); } + #[test] + fn private_document_store_tree_type_basics() { + // Discriminant round-trip (chunk_power defaults to 0 through the byte). + assert_eq!(TreeType::PrivateDocumentStore(4).discriminant(), 16); + assert_eq!( + TreeType::try_from(16u8).unwrap(), + TreeType::PrivateDocumentStore(0) + ); + let t = TreeType::PrivateDocumentStore(4); + assert_eq!(format!("{}", t), "Private Document Store"); + assert!(t.uses_non_merk_data_storage()); + assert!(!t.is_count_bearing()); + assert!(!t.is_sum_bearing()); + assert!(!t.is_count_and_sum_bearing()); + assert!(!t.is_indexed_primary()); + assert!(!t.is_count_indexed_primary()); + assert!(!t.accepts_non_counted_children()); + assert!(!t.accepts_not_counted_or_summed_children()); + assert!(!t.allows_sum_item()); + assert_eq!(t.inner_node_type(), NodeType::NormalNode); + assert_eq!(t.empty_tree_feature_type(), TreeFeatureType::BasicMerkNode); + assert_eq!(t.to_element_type(), Some(ElementType::PrivateDocumentStore)); + assert_eq!(t.cost_size(), PRIVATE_DOCUMENT_STORE_COST_SIZE); + } + #[test] fn indexed_tree_types_round_trip_through_discriminant() { for v in [