From 3e6f29b0ed285790abe7e55b173cc0257f02b30b Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Mon, 3 Aug 2026 07:32:45 +0700 Subject: [PATCH 01/19] feat(private-document-store): add grovedb-private-document-store crate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A thin wrapper over BulkAppendTree for append-only storage of fixed-size opaque entries — the same relationship CommitmentTree has to it, minus the Sinsemilla frontier (phase-one private documents are write-once and never proven against later, so no anchor is needed). The committed config {entry_size, chunk_power} is bound into the state root: pds_state_root = blake3("pds_state" || config_hash || bulk_state_root) config_hash = blake3("pds_config" || entry_size_be(4) || chunk_power(1)) so the declared entry size is consensus-visible and a proof can never be reinterpreted under a different configuration. Because the root binds the config, the empty root is a function of the config rather than a single constant; the config-independent inner EMPTY_BULK_APPEND_TREE_STATE_ROOT is precomputed with a runtime-equivalence test (mirroring EMPTY_COMMITMENT_TREE_STATE_ROOT), plus a pinned test vector for the full composite. The store validates every append against the committed entry size, offers get-by-position across the buffer/chunk tiers, and exposes a verify_entry_sizes integrity walk for verify_grovedb. Part of #784. Co-Authored-By: Claude Fable 5 --- Cargo.toml | 1 + grovedb-private-document-store/Cargo.toml | 22 + grovedb-private-document-store/src/error.rs | 27 + grovedb-private-document-store/src/lib.rs | 160 ++++++ grovedb-private-document-store/src/store.rs | 484 ++++++++++++++++++ .../src/test_utils.rs | 227 ++++++++ 6 files changed, 921 insertions(+) create mode 100644 grovedb-private-document-store/Cargo.toml create mode 100644 grovedb-private-document-store/src/error.rs create mode 100644 grovedb-private-document-store/src/lib.rs create mode 100644 grovedb-private-document-store/src/store.rs create mode 100644 grovedb-private-document-store/src/test_utils.rs diff --git a/Cargo.toml b/Cargo.toml index 7616adb84..93df3316f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ members = [ "grovedb-bulk-append-tree", "grovedb-dense-fixed-sized-merkle-tree", "grovedb-query", + "grovedb-private-document-store", ] [workspace.dependencies] diff --git a/grovedb-private-document-store/Cargo.toml b/grovedb-private-document-store/Cargo.toml new file mode 100644 index 000000000..f1eeb34f3 --- /dev/null +++ b/grovedb-private-document-store/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "grovedb-private-document-store" +version = "5.0.1" +authors = ["Samuel Westrich "] +edition = "2024" +license = "MIT" +description = "PrivateDocumentStore: append-only store of fixed-size opaque entries with committed config for GroveDB" +homepage = "https://www.grovedb.org" +repository = "https://github.com/dashpay/grovedb" +readme = "../README.md" +documentation = "https://docs.rs/grovedb" + +[features] +default = ["storage"] +storage = ["grovedb-storage", "grovedb-bulk-append-tree/storage"] + +[dependencies] +grovedb-bulk-append-tree = { version = "5.0.1", path = "../grovedb-bulk-append-tree", default-features = false } +grovedb-costs = { version = "5.0.1", path = "../costs" } +grovedb-storage = { version = "5.0.1", path = "../storage", optional = true } +blake3 = { workspace = true } +thiserror = { workspace = true } diff --git a/grovedb-private-document-store/src/error.rs b/grovedb-private-document-store/src/error.rs new file mode 100644 index 000000000..d7e1565b2 --- /dev/null +++ b/grovedb-private-document-store/src/error.rs @@ -0,0 +1,27 @@ +//! Error type for the private document store. + +/// Errors returned by [`PrivateDocumentStore`](crate::PrivateDocumentStore) +/// operations. +#[derive(Debug, thiserror::Error)] +pub enum PrivateDocumentStoreError { + /// The store configuration is invalid (zero entry size, bad chunk power). + #[error("invalid private document store config: {0}")] + InvalidConfig(String), + + /// An entry's byte length does not match the committed `entry_size`. + #[error("invalid entry size: expected {expected} bytes, got {actual}")] + InvalidEntrySize { + /// The committed entry size of the store. + expected: u32, + /// The actual length of the offered entry. + actual: usize, + }, + + /// Underlying data is missing or inconsistent. + #[error("corrupted private document store data: {0}")] + CorruptedData(String), + + /// Wrapped storage / bulk tree failure. + #[error("private document store data error: {0}")] + InvalidData(String), +} diff --git a/grovedb-private-document-store/src/lib.rs b/grovedb-private-document-store/src/lib.rs new file mode 100644 index 000000000..88c055939 --- /dev/null +++ b/grovedb-private-document-store/src/lib.rs @@ -0,0 +1,160 @@ +#![deny(missing_docs)] +//! PrivateDocumentStore: an append-only store of fixed-size opaque entries +//! for GroveDB. +//! +//! This crate is a thin wrapper over [`BulkAppendTree`] — the same +//! relationship `grovedb-commitment-tree` has to it, but with **no +//! Sinsemilla frontier**: entries here are write-once and never proven +//! against later, so no anchor is needed. +//! +//! # Committed configuration +//! +//! The store's configuration `{entry_size, chunk_power}` is bound into the +//! state root: +//! +//! ```text +//! pds_state_root = blake3("pds_state" || config_hash || bulk_state_root) +//! config_hash = blake3("pds_config" || entry_size_be(4) || chunk_power(1)) +//! ``` +//! +//! so the declared entry size is consensus-visible and a proof can never be +//! reinterpreted under a different configuration. +//! +//! # Platform use +//! +//! Dash Platform stores each private document as a hiding commitment plus a +//! ciphertext of uniform, contract-declared size. GroveDB never interprets a +//! "document" — behaviorally this is a fully generic append-only log of +//! fixed-size opaque entries; the name simply describes its Platform use. + +mod error; +#[cfg(feature = "storage")] +mod store; +#[cfg(all(test, feature = "storage"))] +pub(crate) mod test_utils; + +pub use error::PrivateDocumentStoreError; +pub use grovedb_bulk_append_tree::{ + deserialize_chunk_blob, serialize_chunk_blob, BulkAppendError, BulkAppendTree, +}; +#[cfg(feature = "storage")] +pub use store::{PrivateDocumentStore, PrivateDocumentStoreAppendResult}; + +/// Pre-computed state root of an empty [`BulkAppendTree`]: +/// `blake3("bulk_state" || [0; 32] || [0; 32])`. +/// +/// The empty bulk root is independent of `chunk_power` (an empty MMR and an +/// empty dense tree both contribute a zero hash regardless of height), so it +/// can be a true constant. The full empty *store* root additionally binds the +/// configuration and is therefore a function — +/// [`empty_private_document_store_state_root`]. +/// +/// The `test_empty_bulk_append_tree_state_root_constant` test pins this +/// constant to the runtime computation. +pub const EMPTY_BULK_APPEND_TREE_STATE_ROOT: [u8; 32] = [ + 0x41, 0xe0, 0x80, 0xa7, 0xfc, 0x26, 0x32, 0x3a, 0x1a, 0x44, 0x90, 0x5d, 0xa2, 0x0d, 0x6d, 0x59, + 0x85, 0x11, 0xf8, 0x39, 0xef, 0xd7, 0x03, 0x42, 0xe2, 0x1e, 0x7e, 0xdc, 0xd5, 0xc3, 0xff, 0x61, +]; + +/// Compute the committed-configuration hash for a private document store. +/// +/// `config_hash = blake3("pds_config" || entry_size.to_be_bytes() || [chunk_power])` +/// +/// The fixed-width big-endian encoding (4 bytes for `entry_size`, 1 byte for +/// `chunk_power`) makes the preimage unambiguous. +pub fn private_document_store_config_hash(entry_size: u32, chunk_power: u8) -> [u8; 32] { + let mut hasher = blake3::Hasher::new(); + hasher.update(b"pds_config"); + hasher.update(&entry_size.to_be_bytes()); + hasher.update(&[chunk_power]); + *hasher.finalize().as_bytes() +} + +/// Compute the combined PrivateDocumentStore state root that binds the +/// committed configuration to the [`BulkAppendTree`] data root. +/// +/// `pds_state_root = blake3("pds_state" || config_hash || bulk_state_root)` +/// +/// This is the value that flows as the Merk child hash, ensuring both the +/// configuration (entry size, chunk power) and the appended data are +/// authenticated by the GroveDB root hash. +pub fn compute_private_document_store_state_root( + config_hash: &[u8; 32], + bulk_state_root: &[u8; 32], +) -> [u8; 32] { + let mut hasher = blake3::Hasher::new(); + hasher.update(b"pds_state"); + hasher.update(config_hash); + hasher.update(bulk_state_root); + *hasher.finalize().as_bytes() +} + +/// The state root of an empty private document store with the given +/// configuration. +/// +/// Unlike `EMPTY_COMMITMENT_TREE_STATE_ROOT` (a constant — the commitment +/// tree root does not bind its configuration), the empty PDS root depends on +/// `{entry_size, chunk_power}`, so it is a function built from the +/// pre-computed [`EMPTY_BULK_APPEND_TREE_STATE_ROOT`] constant: two blake3 +/// calls instead of opening the tree. +pub fn empty_private_document_store_state_root(entry_size: u32, chunk_power: u8) -> [u8; 32] { + compute_private_document_store_state_root( + &private_document_store_config_hash(entry_size, chunk_power), + &EMPTY_BULK_APPEND_TREE_STATE_ROOT, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_empty_bulk_append_tree_state_root_constant() { + let null = [0u8; 32]; + let computed = grovedb_bulk_append_tree::compute_state_root(&null, &null); + assert_eq!( + computed, EMPTY_BULK_APPEND_TREE_STATE_ROOT, + "EMPTY_BULK_APPEND_TREE_STATE_ROOT constant does not match runtime computation" + ); + } + + /// Pin the full empty-store root for one canonical configuration so any + /// accidental change to a domain tag or preimage encoding is caught. + #[test] + fn test_empty_private_document_store_state_root_pinned_vector() { + let expected: [u8; 32] = [ + 0x56, 0x45, 0xf9, 0x7a, 0xe8, 0x5d, 0xba, 0xec, 0x29, 0x55, 0x47, 0x7a, 0x61, 0x65, + 0xdc, 0xbd, 0x17, 0xa6, 0x40, 0x71, 0xd5, 0x5a, 0xed, 0x9f, 0x0f, 0xf7, 0xe5, 0xff, + 0x41, 0xe0, 0x7f, 0xd9, + ]; + assert_eq!(empty_private_document_store_state_root(64, 4), expected); + } + + /// The state root must change when either configuration parameter + /// changes — that is the whole point of binding the config. + #[test] + fn test_state_root_binds_configuration() { + let base = empty_private_document_store_state_root(64, 4); + assert_ne!(base, empty_private_document_store_state_root(65, 4)); + assert_ne!(base, empty_private_document_store_state_root(64, 5)); + // Field boundaries are unambiguous: (entry_size, chunk_power) + // pairs that would collide under a length-prefix-free encoding + // must still differ thanks to the fixed-width layout. + assert_ne!( + private_document_store_config_hash(0x0102, 0x03), + private_document_store_config_hash(0x01, 0x02), + ); + } + + /// The PDS domain tags must not collide with the commitment tree's + /// `"ct_state"` or the bulk tree's `"bulk_state"` domains for identical + /// 64-byte payloads. + #[test] + fn test_domain_separation() { + let a = [7u8; 32]; + let b = [9u8; 32]; + let pds = compute_private_document_store_state_root(&a, &b); + let bulk = grovedb_bulk_append_tree::compute_state_root(&a, &b); + assert_ne!(pds, bulk); + } +} diff --git a/grovedb-private-document-store/src/store.rs b/grovedb-private-document-store/src/store.rs new file mode 100644 index 000000000..2a73d37bc --- /dev/null +++ b/grovedb-private-document-store/src/store.rs @@ -0,0 +1,484 @@ +//! Storage adapter bridging GroveDB's `StorageContext` to the private +//! document store. +//! +//! Provides [`PrivateDocumentStore`], a thin wrapper owning a +//! [`BulkAppendTree`] plus the committed configuration `{entry_size, +//! chunk_power}`. Unlike the commitment tree there is no Sinsemilla +//! frontier and no extra persisted state: everything lives in the bulk +//! tree; the wrapper adds entry-size validation and the config-binding +//! state root. + +use grovedb_bulk_append_tree::BulkAppendTree; +use grovedb_costs::{CostResult, CostsExt, OperationCost}; +use grovedb_storage::StorageContext; + +use crate::{ + compute_private_document_store_state_root, private_document_store_config_hash, + PrivateDocumentStoreError, +}; + +/// Result of appending to a [`PrivateDocumentStore`]. +#[derive(Debug, Clone)] +pub struct PrivateDocumentStoreAppendResult { + /// The new composite state root + /// (`blake3("pds_state" || config_hash || bulk_state_root)`). + /// This flows as the Merk child hash via `insert_subtree`. + pub state_root: [u8; 32], + /// The underlying BulkAppendTree state root. + pub bulk_state_root: [u8; 32], + /// The 0-based global position of the appended entry. + pub global_position: u64, + /// Number of blake3 hash calls performed during the bulk append. + pub hash_count: u32, + /// Whether compaction (epoch flush) occurred during this append. + pub compacted: bool, +} + +/// An append-only store of fixed-size opaque entries. +/// +/// Thin wrapper over [`BulkAppendTree`]: appends are validated against the +/// committed `entry_size` and the state root binds the configuration, so a +/// proof can never be reinterpreted under a different config. There is no +/// per-entry delete or update — immutability is enforced by the type. +pub struct PrivateDocumentStore { + entry_size: u32, + config_hash: [u8; 32], + pub(crate) bulk_tree: BulkAppendTree, +} + +impl std::fmt::Debug for PrivateDocumentStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PrivateDocumentStore") + .field("entry_size", &self.entry_size) + .field("chunk_power", &self.bulk_tree.height()) + .field("total_count", &self.bulk_tree.total_count) + .finish_non_exhaustive() + } +} + +impl<'db, S: StorageContext<'db>> PrivateDocumentStore { + /// Create a new empty private document store. + /// + /// `entry_size` is the committed byte length of every entry (must be + /// non-zero). `chunk_power` is the log2 of the epoch size for the + /// underlying [`BulkAppendTree`] (its dense-buffer height, 1–16). + pub fn new( + entry_size: u32, + chunk_power: u8, + storage: S, + ) -> Result { + Self::from_state(0, entry_size, chunk_power, storage) + } + + /// Restore a private document store from persisted state. + /// + /// Purely in-memory reconstruction: the bulk tree derives its chunk and + /// buffer counts from `total_count` and `chunk_power`; no storage reads + /// happen until an entry is accessed or appended. + pub fn from_state( + total_count: u64, + entry_size: u32, + chunk_power: u8, + storage: S, + ) -> Result { + if entry_size == 0 { + return Err(PrivateDocumentStoreError::InvalidConfig( + "entry_size must be non-zero".to_string(), + )); + } + let bulk_tree = BulkAppendTree::from_state(total_count, chunk_power, storage) + .map_err(|e| PrivateDocumentStoreError::InvalidData(format!("bulk tree: {}", e)))?; + Ok(Self { + entry_size, + config_hash: private_document_store_config_hash(entry_size, chunk_power), + bulk_tree, + }) + } + + /// Append an entry to the store. + /// + /// Validates that `entry.len()` equals the committed `entry_size` before + /// any mutation, then appends to the underlying [`BulkAppendTree`] and + /// returns the new composite state root. + pub fn append( + &mut self, + entry: &[u8], + ) -> CostResult { + let mut cost = OperationCost::default(); + + if entry.len() != self.entry_size as usize { + return Err(PrivateDocumentStoreError::InvalidEntrySize { + expected: self.entry_size, + actual: entry.len(), + }) + .wrap_with_cost(cost); + } + + let bulk_result = match self.bulk_tree.append(entry) { + Ok(r) => r, + Err(e) => { + return Err(PrivateDocumentStoreError::InvalidData(format!( + "bulk append: {}", + e + ))) + .wrap_with_cost(cost); + } + }; + cost.hash_node_calls += bulk_result.hash_count; + + let state_root = + compute_private_document_store_state_root(&self.config_hash, &bulk_result.state_root); + + Ok(PrivateDocumentStoreAppendResult { + state_root, + bulk_state_root: bulk_result.state_root, + global_position: bulk_result.global_position, + hash_count: bulk_result.hash_count, + compacted: bulk_result.compacted, + }) + .wrap_with_cost(cost) + } + + /// Get an entry by its global 0-based position. + /// + /// Returns `None` when `global_position >= total_count`. Dispatches to + /// the current dense-tree buffer or a completed chunk blob as + /// appropriate. + pub fn get_value( + &self, + global_position: u64, + ) -> Result>, PrivateDocumentStoreError> { + if global_position >= self.bulk_tree.total_count { + return Ok(None); + } + + let epoch_size = self.bulk_tree.epoch_size(); + let chunk_count = self.bulk_tree.chunk_count(); + let buffer_start = chunk_count * epoch_size; + + let value = if global_position >= buffer_start { + // Entry is in the current buffer. + let buffer_pos = (global_position - buffer_start) as u16; + self.bulk_tree + .get_buffer_value(buffer_pos) + .map_err(|e| PrivateDocumentStoreError::InvalidData(format!("{}", e)))? + } else { + // Entry is in a completed chunk. + let chunk_idx = global_position / epoch_size; + let pos_in_chunk = (global_position % epoch_size) as usize; + let blob = self + .bulk_tree + .get_chunk_value(chunk_idx) + .map_err(|e| PrivateDocumentStoreError::InvalidData(format!("{}", e)))? + .ok_or_else(|| { + PrivateDocumentStoreError::CorruptedData(format!( + "missing chunk blob for index {}", + chunk_idx + )) + })?; + let entries = grovedb_bulk_append_tree::deserialize_chunk_blob(&blob) + .map_err(|e| PrivateDocumentStoreError::CorruptedData(format!("{}", e)))?; + entries.get(pos_in_chunk).cloned() + }; + + // Defensive: a stored entry that violates the committed entry size + // indicates corruption (the append path rejects such entries). + if let Some(v) = &value + && v.len() != self.entry_size as usize + { + return Err(PrivateDocumentStoreError::CorruptedData(format!( + "entry at position {} has size {}, committed entry size is {}", + global_position, + v.len(), + self.entry_size + ))); + } + + Ok(value) + } + + /// Verify that all stored entries respect the committed `entry_size`. + /// + /// Walks the current buffer and every completed chunk blob; returns an + /// error naming the first violating position. Used by GroveDB's + /// `verify_grovedb` integrity walk. O(total_count) reads — intended for + /// integrity audits, not hot paths. + pub fn verify_entry_sizes(&self) -> Result<(), PrivateDocumentStoreError> { + let expected = self.entry_size as usize; + + // Completed chunks: each blob must deserialize to exactly + // `epoch_size` entries of `entry_size` bytes. + let epoch_size = self.bulk_tree.epoch_size(); + for chunk_idx in 0..self.bulk_tree.chunk_count() { + let blob = self + .bulk_tree + .get_chunk_value(chunk_idx) + .map_err(|e| PrivateDocumentStoreError::InvalidData(format!("{}", e)))? + .ok_or_else(|| { + PrivateDocumentStoreError::CorruptedData(format!( + "missing chunk blob for index {}", + chunk_idx + )) + })?; + let entries = grovedb_bulk_append_tree::deserialize_chunk_blob(&blob) + .map_err(|e| PrivateDocumentStoreError::CorruptedData(format!("{}", e)))?; + if entries.len() as u64 != epoch_size { + return Err(PrivateDocumentStoreError::CorruptedData(format!( + "chunk {} has {} entries, expected {}", + chunk_idx, + entries.len(), + epoch_size + ))); + } + for (i, entry) in entries.iter().enumerate() { + if entry.len() != expected { + return Err(PrivateDocumentStoreError::CorruptedData(format!( + "entry at position {} has size {}, committed entry size is {}", + chunk_idx * epoch_size + i as u64, + entry.len(), + expected + ))); + } + } + } + + // Current buffer. + let buffer_start = self.bulk_tree.chunk_count() * epoch_size; + for pos in 0..self.bulk_tree.buffer_count() { + let entry = self + .bulk_tree + .get_buffer_value(pos) + .map_err(|e| PrivateDocumentStoreError::InvalidData(format!("{}", e)))? + .ok_or_else(|| { + PrivateDocumentStoreError::CorruptedData(format!( + "missing buffer entry at position {}", + pos + )) + })?; + if entry.len() != expected { + return Err(PrivateDocumentStoreError::CorruptedData(format!( + "entry at position {} has size {}, committed entry size is {}", + buffer_start + pos as u64, + entry.len(), + expected + ))); + } + } + + Ok(()) + } + + /// Compute the composite state root + /// (`blake3("pds_state" || config_hash || bulk_state_root)`) without + /// modifying the store. + pub fn compute_current_state_root(&self) -> Result<[u8; 32], PrivateDocumentStoreError> { + let bulk_root = self + .bulk_tree + .compute_current_state_root() + .map_err(|e| PrivateDocumentStoreError::InvalidData(format!("state root: {}", e)))?; + Ok(compute_private_document_store_state_root( + &self.config_hash, + &bulk_root, + )) + } + + /// Flush the MMR overlay to storage. + /// + /// Delegates to [`BulkAppendTree::commit_mmr`]. Call this at the end of + /// a session to persist MMR nodes buffered during compaction cycles. + pub fn commit_mmr(&mut self) -> Result<(), PrivateDocumentStoreError> { + self.bulk_tree + .commit_mmr() + .map_err(|e| PrivateDocumentStoreError::InvalidData(format!("MMR commit: {}", e))) + } + + // ── Accessors ───────────────────────────────────────────────────── + + /// The committed entry size in bytes. + pub fn entry_size(&self) -> u32 { + self.entry_size + } + + /// The chunk power (dense-buffer height) of the underlying bulk tree. + pub fn chunk_power(&self) -> u8 { + self.bulk_tree.height() + } + + /// Total number of entries appended so far. + pub fn total_count(&self) -> u64 { + self.bulk_tree.total_count + } + + /// The number of entries per completed chunk (epoch). + pub fn epoch_size(&self) -> u64 { + self.bulk_tree.epoch_size() + } + + /// Number of completed chunks in the MMR. + pub fn chunk_count(&self) -> u64 { + self.bulk_tree.chunk_count() + } +} + +#[cfg(test)] +impl PrivateDocumentStore { + /// Test helper: tear down the store and recover its storage context so a + /// reopen can be simulated against the same in-memory backing. + pub(crate) fn into_storage_for_test(store: Self) -> S { + store.bulk_tree.dense_tree.storage + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + empty_private_document_store_state_root, test_utils::MemStorageContext, + EMPTY_BULK_APPEND_TREE_STATE_ROOT, + }; + + #[test] + fn test_empty_store_state_root_matches_helper() { + let store = PrivateDocumentStore::new(64, 4, MemStorageContext::new()).expect("new store"); + assert_eq!( + store.compute_current_state_root().expect("state root"), + empty_private_document_store_state_root(64, 4), + ); + // And the inner bulk root of an empty store matches the constant. + assert_eq!( + store.bulk_tree.compute_current_state_root().expect("bulk"), + EMPTY_BULK_APPEND_TREE_STATE_ROOT, + ); + } + + #[test] + fn test_zero_entry_size_rejected() { + assert!(matches!( + PrivateDocumentStore::new(0, 4, MemStorageContext::new()), + Err(PrivateDocumentStoreError::InvalidConfig(_)) + )); + } + + #[test] + fn test_invalid_chunk_power_rejected() { + assert!(PrivateDocumentStore::new(64, 0, MemStorageContext::new()).is_err()); + assert!(PrivateDocumentStore::new(64, 17, MemStorageContext::new()).is_err()); + } + + #[test] + fn test_append_validates_entry_size() { + let mut store = + PrivateDocumentStore::new(8, 2, MemStorageContext::new()).expect("new store"); + assert!(matches!( + store.append(&[0u8; 7]).unwrap(), + Err(PrivateDocumentStoreError::InvalidEntrySize { + expected: 8, + actual: 7 + }) + )); + assert!(matches!( + store.append(&[0u8; 9]).unwrap(), + Err(PrivateDocumentStoreError::InvalidEntrySize { + expected: 8, + actual: 9 + }) + )); + // A rejected append must not mutate the store. + assert_eq!(store.total_count(), 0); + let ok = store.append(&[1u8; 8]).unwrap().expect("valid append"); + assert_eq!(ok.global_position, 0); + assert_eq!(store.total_count(), 1); + } + + #[test] + fn test_append_get_roundtrip_across_compaction() { + // chunk_power 2 → capacity 3, epoch size 4: 10 appends span two + // completed chunks plus a partial buffer. + let mut store = + PrivateDocumentStore::new(8, 2, MemStorageContext::new()).expect("new store"); + let mut roots = Vec::new(); + for i in 0..10u8 { + let entry = [i; 8]; + let r = store.append(&entry).unwrap().expect("append"); + assert_eq!(r.global_position, i as u64); + roots.push(r.state_root); + } + // Every append must move the state root. + for w in roots.windows(2) { + assert_ne!(w[0], w[1]); + } + assert_eq!(store.total_count(), 10); + assert_eq!(store.chunk_count(), 2); + + for i in 0..10u8 { + let v = store.get_value(i as u64).expect("get"); + assert_eq!(v, Some(vec![i; 8]), "position {}", i); + } + assert_eq!(store.get_value(10).expect("get"), None); + + // The append-path state root matches a fresh computation. + assert_eq!( + store.compute_current_state_root().expect("state root"), + *roots.last().unwrap() + ); + + // And the whole store passes the entry-size integrity walk. + store.verify_entry_sizes().expect("sizes ok"); + } + + #[test] + fn test_state_root_differs_from_raw_bulk_root() { + // The composite root must bind the config: it can never equal the + // raw bulk root, and two stores with identical data but different + // configs must have different roots. + let mut a = PrivateDocumentStore::new(8, 2, MemStorageContext::new()).expect("a"); + let mut b = PrivateDocumentStore::new(8, 3, MemStorageContext::new()).expect("b"); + let ra = a.append(&[7u8; 8]).unwrap().expect("append a"); + let rb = b.append(&[7u8; 8]).unwrap().expect("append b"); + assert_ne!(ra.state_root, ra.bulk_state_root); + assert_ne!(ra.state_root, rb.state_root); + } + + #[test] + fn test_reopen_from_state() { + let storage = MemStorageContext::new(); + let mut store = PrivateDocumentStore::new(8, 2, storage).expect("new store"); + for i in 0..6u8 { + store.append(&[i; 8]).unwrap().expect("append"); + } + store.commit_mmr().expect("commit mmr"); + let root_before = store.compute_current_state_root().expect("root"); + let storage = PrivateDocumentStore::into_storage_for_test(store); + + let reopened = PrivateDocumentStore::from_state(6, 8, 2, storage).expect("reopen"); + assert_eq!( + reopened.compute_current_state_root().expect("root"), + root_before + ); + for i in 0..6u8 { + assert_eq!(reopened.get_value(i as u64).expect("get"), Some(vec![i; 8])); + } + reopened.verify_entry_sizes().expect("sizes ok"); + } + + #[test] + fn test_verify_entry_sizes_detects_wrong_config() { + // Write entries of size 8, then reopen claiming entry_size 16 — the + // integrity walk must flag the first entry. + let storage = MemStorageContext::new(); + let mut store = PrivateDocumentStore::new(8, 2, storage).expect("new store"); + for i in 0..6u8 { + store.append(&[i; 8]).unwrap().expect("append"); + } + store.commit_mmr().expect("commit mmr"); + let storage = PrivateDocumentStore::into_storage_for_test(store); + + let reopened = PrivateDocumentStore::from_state(6, 16, 2, storage).expect("reopen"); + assert!(matches!( + reopened.verify_entry_sizes(), + Err(PrivateDocumentStoreError::CorruptedData(_)) + )); + // get_value performs the same defensive check. + assert!(reopened.get_value(0).is_err()); + } +} diff --git a/grovedb-private-document-store/src/test_utils.rs b/grovedb-private-document-store/src/test_utils.rs new file mode 100644 index 000000000..67978db5f --- /dev/null +++ b/grovedb-private-document-store/src/test_utils.rs @@ -0,0 +1,227 @@ +//! Test utilities: in-memory StorageContext for PrivateDocumentStore tests. + +use std::{cell::RefCell, collections::HashMap}; + +use grovedb_costs::{ + storage_cost::key_value_cost::KeyValueStorageCost, ChildrenSizesWithIsSumTree, CostContext, + CostResult, CostsExt, OperationCost, +}; +use grovedb_storage::{Batch, RawIterator, StorageContext}; + +/// In-memory storage context for testing. +/// +/// Immediate reads and writes backed by a `HashMap`. Only `get` and `put` +/// (data storage) have real implementations; all other `StorageContext` +/// methods panic if called. +#[derive(Default)] +pub(crate) struct MemStorageContext { + pub data: RefCell, Vec>>, +} + +impl MemStorageContext { + pub fn new() -> Self { + Self::default() + } +} + +impl<'db> StorageContext<'db> for MemStorageContext { + type Batch = MemBatch; + type RawIterator = MemRawIterator; + + fn get>(&self, key: K) -> CostResult>, grovedb_storage::Error> { + Ok(self.data.borrow().get(key.as_ref()).cloned()).wrap_with_cost(OperationCost::default()) + } + + fn put>( + &self, + key: K, + value: &[u8], + _children_sizes: ChildrenSizesWithIsSumTree, + _cost_info: Option, + ) -> CostResult<(), grovedb_storage::Error> { + self.data + .borrow_mut() + .insert(key.as_ref().to_vec(), value.to_vec()); + Ok(()).wrap_with_cost(OperationCost::default()) + } + + fn put_aux>( + &self, + _key: K, + _value: &[u8], + _cost_info: Option, + ) -> CostResult<(), grovedb_storage::Error> { + unimplemented!("MemStorageContext::put_aux") + } + + fn put_root>( + &self, + _key: K, + _value: &[u8], + _cost_info: Option, + ) -> CostResult<(), grovedb_storage::Error> { + unimplemented!("MemStorageContext::put_root") + } + + fn put_meta>( + &self, + _key: K, + _value: &[u8], + _cost_info: Option, + ) -> CostResult<(), grovedb_storage::Error> { + unimplemented!("MemStorageContext::put_meta") + } + + fn delete>( + &self, + _key: K, + _cost_info: Option, + ) -> CostResult<(), grovedb_storage::Error> { + unimplemented!("MemStorageContext::delete") + } + + fn delete_aux>( + &self, + _key: K, + _cost_info: Option, + ) -> CostResult<(), grovedb_storage::Error> { + unimplemented!("MemStorageContext::delete_aux") + } + + fn delete_root>( + &self, + _key: K, + _cost_info: Option, + ) -> CostResult<(), grovedb_storage::Error> { + unimplemented!("MemStorageContext::delete_root") + } + + fn delete_meta>( + &self, + _key: K, + _cost_info: Option, + ) -> CostResult<(), grovedb_storage::Error> { + unimplemented!("MemStorageContext::delete_meta") + } + + fn get_aux>( + &self, + _key: K, + ) -> CostResult>, grovedb_storage::Error> { + unimplemented!("MemStorageContext::get_aux") + } + + fn get_root>( + &self, + _key: K, + ) -> CostResult>, grovedb_storage::Error> { + unimplemented!("MemStorageContext::get_root") + } + + fn get_meta>( + &self, + _key: K, + ) -> CostResult>, grovedb_storage::Error> { + unimplemented!("MemStorageContext::get_meta") + } + + fn new_batch(&self) -> Self::Batch { + MemBatch + } + + fn commit_batch(&self, _batch: Self::Batch) -> CostResult<(), grovedb_storage::Error> { + Ok(()).wrap_with_cost(OperationCost::default()) + } + + fn raw_iter(&self) -> Self::RawIterator { + unimplemented!("MemStorageContext::raw_iter") + } +} + +// ── Batch and RawIterator stubs ─────────────────────────────────────── + +/// No-op batch (never used — MemStorageContext does immediate writes). +pub(crate) struct MemBatch; + +impl Batch for MemBatch { + fn put>( + &mut self, + _key: K, + _value: &[u8], + _children_sizes: ChildrenSizesWithIsSumTree, + _cost_info: Option, + ) -> Result<(), grovedb_costs::error::Error> { + unimplemented!("MemBatch::put") + } + + fn put_aux>( + &mut self, + _key: K, + _value: &[u8], + _cost_info: Option, + ) -> Result<(), grovedb_costs::error::Error> { + unimplemented!("MemBatch::put_aux") + } + + fn put_root>( + &mut self, + _key: K, + _value: &[u8], + _cost_info: Option, + ) -> Result<(), grovedb_costs::error::Error> { + unimplemented!("MemBatch::put_root") + } + + fn delete>(&mut self, _key: K, _cost_info: Option) { + unimplemented!("MemBatch::delete") + } + + fn delete_aux>(&mut self, _key: K, _cost_info: Option) { + unimplemented!("MemBatch::delete_aux") + } + + fn delete_root>(&mut self, _key: K, _cost_info: Option) { + unimplemented!("MemBatch::delete_root") + } +} + +/// Stub iterator (never used by the bulk append tree). +pub(crate) struct MemRawIterator; + +impl RawIterator for MemRawIterator { + fn seek_to_first(&mut self) -> CostContext<()> { + unimplemented!() + } + + fn seek_to_last(&mut self) -> CostContext<()> { + unimplemented!() + } + + fn seek>(&mut self, _key: K) -> CostContext<()> { + unimplemented!() + } + + fn seek_for_prev>(&mut self, _key: K) -> CostContext<()> { + unimplemented!() + } + + fn next(&mut self) -> CostContext<()> { + unimplemented!() + } + + fn prev(&mut self) -> CostContext<()> { + unimplemented!() + } + + fn value(&self) -> CostContext> { + unimplemented!() + } + + fn key(&self) -> CostContext> { + unimplemented!() + } + + fn valid(&self) -> CostContext { + unimplemented!() + } +} From 26a804be71225ba738972b2f597ccba60625a26f Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Mon, 3 Aug 2026 07:33:15 +0700 Subject: [PATCH 02/19] feat: add PrivateDocumentStore element type, gated to GROVE_V4 (#784) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New non-Merk tree element for Platform's phase-one private documents: an append-only store of fixed-size opaque entries whose committed config {entry_size, chunk_power} is bound into the state root. GroveDB never interprets a "document" — behaviorally the type is fully generic. Discriminant allocation. Issue #784 proposed 15/143, but 15 is the Element::NonCounted wrapper byte and 143 (= 0x80|15) is rejected as wrapper-on-wrapper; 21-23 / 149-151 and TreeType 13-15 are taken by the indexed trees on develop. The next free pair following the +128 convention is therefore: - ElementType::PrivateDocumentStore = 24, NonCountedPrivateDocumentStore = 152 (0x80|24) - Element::PrivateDocumentStore(total_count, entry_size, chunk_power, flags) — bincode variant index 24 (appended, wire format of every existing variant unchanged) - TreeType::PrivateDocumentStore(chunk_power) = 16 - GroveOp::PrivateDocumentStoreInsert { entry } — sort tag 19 Operations (grovedb/src/operations/private_document_store.rs, modeled on the commitment-tree/bulk-append ops): size-validated append, get by global position (buffer + chunk tiers), count; plus the PrivateDocumentStoreInsert batch op with a preprocess pass that folds a group of appends into one ReplaceNonMerkTreeRoot (NonMerkTreeMeta::PrivateDocumentStore). Batch and direct appends converge to the same root hash (tested). Fail-closed versioning. Unlike earlier element types, a new GroveDBOperationsPrivateDocumentStoreVersions family acts as a capability gate: every slot is 0 on GROVE_V1..V3 — element creation (direct and batch) and all operations return a version-mismatch error — and 1 on GROVE_V4. Element::deserialize intentionally stays protocol-independent per the append-only codec contract; slot values are pinned by tests and V3 rejection is covered end to end. Immutability. No per-entry delete or update exists, and — stricter than the other non-Merk trees — the store's always-empty Merk rejects ALL child-element inserts at the merk chokepoints (validate_insertable_into, insert_reference/insert_subtree, insert_count_indexed_subtree) and in batch execute_ops_on_path. Empty-root binding. Insert (v0/v1), batch insert, the V1 terminal non-Merk proof binding, and the verify_grovedb walk all derive the child hash from empty_private_document_store_state_root(entry_size, chunk_power) when the store is empty and from the reconstructed store otherwise; verify_grovedb additionally runs the entry-size integrity walk over every chunk blob and buffer entry. Proof policy. V0 (locked wire format) rejects subqueries into the type, like the other non-Merk trees; V1 rejects subqueries too for now — range-read proofs are a follow-up planned at the BulkAppendTree layer so the anchored DataCommitmentTree (#783) inherits them — while terminal queries bind the config-carrying state root via bind_terminal_non_merk_tree (round-trip tested for empty and populated stores). Costs are entry-size-parametrized: PRIVATE_DOCUMENT_STORE_COST_SIZE (9 + 5 + 1 + 2) in merk, and average/worst-case arms for the batch op mirroring BulkAppend plus the composite-root blake3. No behavior change to any existing element type: the full workspace suite (43 binaries, including all CommitmentTree tests) passes unchanged, and clippy reports nothing in any touched file. Co-Authored-By: Claude Fable 5 --- grovedb-element/src/element/constructor.rs | 51 ++ grovedb-element/src/element/helpers.rs | 14 + grovedb-element/src/element/mod.rs | 43 + grovedb-element/src/element/visualize.rs | 15 + grovedb-element/src/element_type.rs | 53 +- grovedb-version/src/tests.rs | 21 + .../src/version/grovedb_versions.rs | 38 +- grovedb-version/src/version/v1.rs | 12 +- grovedb-version/src/version/v2.rs | 12 +- grovedb-version/src/version/v3.rs | 12 +- grovedb-version/src/version/v4.rs | 11 +- grovedb/Cargo.toml | 3 + grovedb/src/batch/batch_structure.rs | 1 + .../estimated_costs/average_case_costs.rs | 33 + .../batch/estimated_costs/worst_case_costs.rs | 38 + grovedb/src/batch/indexed_tree/pre_state.rs | 3 +- grovedb/src/batch/mod.rs | 259 +++++- grovedb/src/debugger.rs | 6 + grovedb/src/error.rs | 7 +- grovedb/src/lib.rs | 42 +- grovedb/src/operations/get/mod.rs | 3 +- grovedb/src/operations/get/query.rs | 6 +- .../insert/add_element_on_transaction/v0.rs | 44 + .../insert/add_element_on_transaction/v1.rs | 44 + grovedb/src/operations/mod.rs | 3 + .../src/operations/private_document_store.rs | 536 +++++++++++ .../proof/bind_terminal_non_merk_tree/v1.rs | 41 + grovedb/src/operations/proof/generate.rs | 32 +- grovedb/src/operations/proof/verify.rs | 13 + grovedb/src/tests/mod.rs | 1 + .../src/tests/private_document_store_tests.rs | 863 ++++++++++++++++++ merk/src/element/costs.rs | 20 +- merk/src/element/delete.rs | 6 + merk/src/element/get.rs | 6 +- merk/src/element/insert.rs | 30 + merk/src/element/reconstruct.rs | 6 + merk/src/element/tree_type.rs | 17 + merk/src/tree_type/costs.rs | 7 + merk/src/tree_type/mod.rs | 45 +- 39 files changed, 2339 insertions(+), 58 deletions(-) create mode 100644 grovedb/src/operations/private_document_store.rs create mode 100644 grovedb/src/tests/private_document_store_tests.rs diff --git a/grovedb-element/src/element/constructor.rs b/grovedb-element/src/element/constructor.rs index 8949ca1aa..cbc41d626 100644 --- a/grovedb-element/src/element/constructor.rs +++ b/grovedb-element/src/element/constructor.rs @@ -500,6 +500,57 @@ impl Element { Element::DenseAppendOnlyFixedSizeTree(count, height, flags) } + /// Set element to an empty private document store. + /// + /// Returns `InvalidInput` unless `entry_size >= 1` and `chunk_power` is + /// in `1..=16` (the underlying `BulkAppendTree` dense-buffer height + /// range). Unlike `empty_commitment_tree` / `empty_bulk_append_tree`, + /// the constraints are enforced eagerly here: the configuration is + /// committed into the state root, so an unusable config must never be + /// constructible. + pub fn empty_private_document_store( + entry_size: u32, + chunk_power: u8, + ) -> Result { + Self::empty_private_document_store_with_flags(entry_size, chunk_power, None) + } + + /// Set element to an empty private document store with flags. + /// + /// Same validation as [`Element::empty_private_document_store`]. + pub fn empty_private_document_store_with_flags( + entry_size: u32, + chunk_power: u8, + flags: Option, + ) -> Result { + if entry_size == 0 { + return Err(ElementError::InvalidInput( + "private document store entry_size must be non-zero", + )); + } + if !(1..=16).contains(&chunk_power) { + return Err(ElementError::InvalidInput( + "private document store chunk_power must be between 1 and 16", + )); + } + Ok(Element::PrivateDocumentStore( + 0, + entry_size, + chunk_power, + flags, + )) + } + + /// Set element to a private document store with all fields + pub fn new_private_document_store( + total_count: u64, + entry_size: u32, + chunk_power: u8, + flags: Option, + ) -> Self { + Element::PrivateDocumentStore(total_count, entry_size, chunk_power, flags) + } + /// Set element to an empty provable sum-indexed tree without flags. pub fn empty_provable_sum_indexed_tree() -> Self { Element::ProvableSumIndexedTree(None, None, 0, None) diff --git a/grovedb-element/src/element/helpers.rs b/grovedb-element/src/element/helpers.rs index 623726c1d..f17be7612 100644 --- a/grovedb-element/src/element/helpers.rs +++ b/grovedb-element/src/element/helpers.rs @@ -358,6 +358,7 @@ impl Element { | Element::ProvableSumIndexedTree(..) | Element::ProvableCountIndexedTree(..) | Element::ProvableCountProvableSumIndexedTree(..) + | Element::PrivateDocumentStore(..) ) } @@ -414,6 +415,12 @@ impl Element { matches!(self.underlying(), Element::DenseAppendOnlyFixedSizeTree(..)) } + /// Check if the element is a private document store. Looks through + /// `NonCounted`. + pub fn is_private_document_store(&self) -> bool { + matches!(self.underlying(), Element::PrivateDocumentStore(..)) + } + /// Check if the element is a tree type that stores data in the data /// namespace as non-Merk entries. These tree types have an always-empty /// Merk (root_key = None) and never contain child subtrees. The data @@ -430,6 +437,7 @@ impl Element { | Element::MmrTree(..) | Element::BulkAppendTree(..) | Element::DenseAppendOnlyFixedSizeTree(..) + | Element::PrivateDocumentStore(..) ) } @@ -443,6 +451,7 @@ impl Element { Element::MmrTree(mmr_size, _) => Some(*mmr_size), Element::BulkAppendTree(count, ..) => Some(*count), Element::DenseAppendOnlyFixedSizeTree(count, ..) => Some(*count as u64), + Element::PrivateDocumentStore(count, ..) => Some(*count), _ => None, } } @@ -470,6 +479,7 @@ impl Element { | Element::MmrTree(..) | Element::BulkAppendTree(..) | Element::DenseAppendOnlyFixedSizeTree(..) + | Element::PrivateDocumentStore(..) | Element::ProvableSumIndexedTree(Some(_), ..) | Element::ProvableSumIndexedTree(_, Some(_), ..) | Element::ProvableCountIndexedTree(Some(_), ..) @@ -663,6 +673,7 @@ impl Element { | Element::MmrTree(.., flags) | Element::BulkAppendTree(.., flags) | Element::DenseAppendOnlyFixedSizeTree(.., flags) + | Element::PrivateDocumentStore(.., flags) | Element::ProvableSumIndexedTree(.., flags) | Element::ProvableCountIndexedTree(.., flags) | Element::ReferenceWithSumItem(.., flags) => flags, @@ -695,6 +706,7 @@ impl Element { | Element::MmrTree(.., flags) | Element::BulkAppendTree(.., flags) | Element::DenseAppendOnlyFixedSizeTree(.., flags) + | Element::PrivateDocumentStore(.., flags) | Element::ProvableSumIndexedTree(.., flags) | Element::ProvableCountIndexedTree(.., flags) | Element::ReferenceWithSumItem(.., flags) => flags, @@ -727,6 +739,7 @@ impl Element { | Element::MmrTree(.., flags) | Element::BulkAppendTree(.., flags) | Element::DenseAppendOnlyFixedSizeTree(.., flags) + | Element::PrivateDocumentStore(.., flags) | Element::ProvableSumIndexedTree(.., flags) | Element::ProvableCountIndexedTree(.., flags) | Element::ReferenceWithSumItem(.., flags) => flags, @@ -758,6 +771,7 @@ impl Element { | Element::MmrTree(.., flags) | Element::BulkAppendTree(.., flags) | Element::DenseAppendOnlyFixedSizeTree(.., flags) + | Element::PrivateDocumentStore(.., flags) | Element::ProvableSumIndexedTree(.., flags) | Element::ProvableCountIndexedTree(.., flags) | Element::ReferenceWithSumItem(.., flags) => *flags = new_flags, diff --git a/grovedb-element/src/element/mod.rs b/grovedb-element/src/element/mod.rs index b46a9e039..a71d6d70a 100644 --- a/grovedb-element/src/element/mod.rs +++ b/grovedb-element/src/element/mod.rs @@ -316,6 +316,31 @@ pub enum Element { Vec<(u8, Option>)>, Option, ), + /// Private document store: an append-only store of fixed-size opaque + /// entries, a thin wrapper over a `BulkAppendTree` (the same + /// relationship `CommitmentTree` has to it, minus the Sinsemilla + /// frontier). Entries are write-once — there is no per-entry delete or + /// update; immutability is enforced by the type. + /// + /// Fields: `(total_count, entry_size, chunk_power, flags)` + /// - `total_count`: Number of entries appended so far. + /// - `entry_size`: Committed byte length of every entry; appends of any + /// other length are rejected. + /// - `chunk_power`: Log2 of the chunk size (actual size = `1 << + /// chunk_power`). + /// - `flags`: Optional per-element metadata. + /// + /// The state root + /// (`blake3("pds_state" || config_hash || bulk_state_root)`, where + /// `config_hash` commits to `{entry_size, chunk_power}`) flows through + /// the Merk child hash mechanism (`insert_subtree`'s + /// `subtree_root_hash` parameter), so the declared configuration is + /// consensus-visible and a proof can never be reinterpreted under a + /// different config. + /// + /// Variant order in this enum determines bincode's variant-index + /// encoding on disk. This variant gets index 24. + PrivateDocumentStore(u64, u32, u8, Option), } pub fn hex_to_ascii(hex_value: &[u8]) -> String { @@ -583,6 +608,18 @@ impl fmt::Display for Element { .map_or(String::new(), |f| format!(", flags: {:?}", f)) ) } + Element::PrivateDocumentStore(total_count, entry_size, chunk_power, flags) => { + write!( + f, + "PrivateDocumentStore(count: {}, entry_size: {}, chunk_power: {}{})", + total_count, + entry_size, + chunk_power, + flags + .as_ref() + .map_or(String::new(), |f| format!(", flags: {:?}", f)) + ) + } Element::NotSummed(inner) => { write!(f, "NotSummed({})", inner) } @@ -660,6 +697,7 @@ impl Element { Element::ProvableCountProvableSumIndexedTree(..) => { ElementType::ProvableCountProvableSumIndexedTree } + Element::PrivateDocumentStore(..) => ElementType::PrivateDocumentStore, Element::NonCounted(inner) => match inner.element_type() { ElementType::Item => ElementType::NonCountedItem, ElementType::Reference => ElementType::NonCountedReference, @@ -692,6 +730,7 @@ impl Element { ElementType::ProvableCountProvableSumIndexedTree => { ElementType::NonCountedProvableCountProvableSumIndexedTree } + ElementType::PrivateDocumentStore => ElementType::NonCountedPrivateDocumentStore, // Inner is always a base type — nested wrappers are // forbidden at construction and (de)serialization. already_non_counted => already_non_counted, @@ -885,6 +924,7 @@ mod serde_impl { Vec<(u8, Option>)>, Option, ), + PrivateDocumentStore(u64, u32, u8, Option), } impl From for Element { @@ -934,6 +974,9 @@ mod serde_impl { ElementShadow::ProvableCountProvableSumIndexedTree(pk, c, s, axes, f) => { Element::ProvableCountProvableSumIndexedTree(pk, c, s, axes, f) } + ElementShadow::PrivateDocumentStore(c, e, p, f) => { + Element::PrivateDocumentStore(c, e, p, f) + } } } } diff --git a/grovedb-element/src/element/visualize.rs b/grovedb-element/src/element/visualize.rs index b2e26d83a..652706072 100644 --- a/grovedb-element/src/element/visualize.rs +++ b/grovedb-element/src/element/visualize.rs @@ -176,6 +176,21 @@ impl Visualize for Element { drawer = f.visualize(drawer)?; } } + Element::PrivateDocumentStore(total_count, entry_size, chunk_power, flags) => { + drawer.write( + format!( + "private_document_store: count: {total_count} entry_size: {entry_size} \ + chunk_power: {chunk_power}", + ) + .as_bytes(), + )?; + + if let Some(f) = flags + && !f.is_empty() + { + drawer = f.visualize(drawer)?; + } + } Element::NonCounted(inner) => { drawer.write(b"non_counted(")?; drawer = inner.visualize(drawer)?; diff --git a/grovedb-element/src/element_type.rs b/grovedb-element/src/element_type.rs index a799366cd..3b19b9d4d 100644 --- a/grovedb-element/src/element_type.rs +++ b/grovedb-element/src/element_type.rs @@ -293,6 +293,10 @@ pub enum ElementType { /// `ProvableCountProvableSumTree` + 1..=3 secondaries keyed by axis /// sort-prefix) - discriminant 23. The new dual/triple-axis variant. ProvableCountProvableSumIndexedTree = 23, + /// Private document store (append-only fixed-size opaque entries over a + /// `BulkAppendTree`, with the `{entry_size, chunk_power}` config bound + /// into the state root) - discriminant 24. + PrivateDocumentStore = 24, /// Non-counted wrapper around `Item` - discriminant 128 NonCountedItem = 128, /// Non-counted wrapper around `Reference` - discriminant 129 @@ -339,6 +343,9 @@ pub enum ElementType { /// Non-counted wrapper around `ProvableCountProvableSumIndexedTree` - /// discriminant 151 (`0x80 | 23`) NonCountedProvableCountProvableSumIndexedTree = 151, + /// Non-counted wrapper around `PrivateDocumentStore` - discriminant 152 + /// (`0x80 | 24`) + NonCountedPrivateDocumentStore = 152, /// Not-summed wrapper around `SumTree` - discriminant 180 (`0xB4`) NotSummedSumTree = 180, /// Not-summed wrapper around `BigSumTree` - discriminant 181 (`0xB5`) @@ -407,10 +414,11 @@ impl ElementType { // those are `0..=14`, `18` (ReferenceWithSumItem), `19` // (ProvableSumTree), `20` (ProvableCountProvableSumTree), // `21` (ProvableSumIndexedTree), `22` (ProvableCountIndexedTree), - // and `23` (ProvableCountProvableSumIndexedTree). + // `23` (ProvableCountProvableSumIndexedTree), and `24` + // (PrivateDocumentStore). // Bytes 15, 16, and 17 are the wrapper bytes themselves - // (nested wrappers forbidden in either direction); 24..=127 - // are unallocated; 128..=151 are the synthetic NonCountedXxx + // (nested wrappers forbidden in either direction); 25..=127 + // are unallocated; 128..=152 are the synthetic NonCountedXxx // twins which never appear on disk. // Without this check, the bitwise OR below would collapse // `0x80 | inner_byte` into `inner_byte` and a payload like @@ -419,10 +427,10 @@ impl ElementType { // Use an explicit allowlist so the next base-variant addition // is a one-line edit here and the check stays robust to new // variants landing without updating the guard. - if !matches!(inner_byte, 0..=14 | 18 | 19 | 20 | 21 | 22 | 23) { + if !matches!(inner_byte, 0..=14 | 18 | 19 | 20 | 21 | 22 | 23 | 24) { return Err(ElementError::CorruptedData(format!( "NonCounted inner discriminant must be a base type \ - (0..=14, 18, 19, 20, 21, 22, or 23), got {}", + (0..=14, 18, 19, 20, 21, 22, 23, or 24), got {}", inner_byte ))); } @@ -728,6 +736,7 @@ impl ElementType { | ElementType::ProvableSumIndexedTree | ElementType::ProvableCountIndexedTree | ElementType::ProvableCountProvableSumIndexedTree + | ElementType::PrivateDocumentStore ) } @@ -804,6 +813,7 @@ impl ElementType { ElementType::ProvableCountProvableSumIndexedTree => { "provable count provable sum indexed tree" } + ElementType::PrivateDocumentStore => "private_document_store", ElementType::NonCountedItem => "non_counted item", ElementType::NonCountedReference => "non_counted reference", ElementType::NonCountedTree => "non_counted tree", @@ -833,6 +843,7 @@ impl ElementType { ElementType::NonCountedProvableCountProvableSumIndexedTree => { "non_counted provable count provable sum indexed tree" } + ElementType::NonCountedPrivateDocumentStore => "non_counted private_document_store", ElementType::NotSummedSumTree => "not_summed sum tree", ElementType::NotSummedBigSumTree => "not_summed big sum tree", ElementType::NotSummedCountSumTree => "not_summed count sum tree", @@ -892,6 +903,7 @@ impl TryFrom for ElementType { 21 => Ok(ElementType::ProvableSumIndexedTree), 22 => Ok(ElementType::ProvableCountIndexedTree), 23 => Ok(ElementType::ProvableCountProvableSumIndexedTree), + 24 => Ok(ElementType::PrivateDocumentStore), 128 => Ok(ElementType::NonCountedItem), 129 => Ok(ElementType::NonCountedReference), 130 => Ok(ElementType::NonCountedTree), @@ -913,6 +925,7 @@ impl TryFrom for ElementType { 149 => Ok(ElementType::NonCountedProvableSumIndexedTree), 150 => Ok(ElementType::NonCountedProvableCountIndexedTree), 151 => Ok(ElementType::NonCountedProvableCountProvableSumIndexedTree), + 152 => Ok(ElementType::NonCountedPrivateDocumentStore), // NotSummed twins occupy the 0xB0..=0xBF family range; slots // are assigned explicitly per variant. 177 => Ok(ElementType::NotSummedProvableSumTree), @@ -1022,8 +1035,12 @@ mod tests { ElementType::try_from(23).unwrap(), ElementType::ProvableCountProvableSumIndexedTree ); - // 24..=127 are unallocated and invalid. - assert!(ElementType::try_from(24).is_err()); + assert_eq!( + ElementType::try_from(24).unwrap(), + ElementType::PrivateDocumentStore + ); + // 25..=127 are unallocated and invalid. + assert!(ElementType::try_from(25).is_err()); assert!(ElementType::try_from(100).is_err()); // NonCounted twins (0x80 | base): 128..142, plus 146 (= 0x80|18 = @@ -1065,6 +1082,10 @@ mod tests { ElementType::try_from(151).unwrap(), ElementType::NonCountedProvableCountProvableSumIndexedTree ); + assert_eq!( + ElementType::try_from(152).unwrap(), + ElementType::NonCountedPrivateDocumentStore + ); // Bytes between the base and NonCounted-twin ranges are invalid. assert!(ElementType::try_from(127).is_err()); // 143 (= 0x80|15), 144 (= 0x80|16), 145 (= 0x80|17): wrapper bytes @@ -1072,10 +1093,10 @@ mod tests { assert!(ElementType::try_from(143).is_err()); assert!(ElementType::try_from(144).is_err()); assert!(ElementType::try_from(145).is_err()); - // 152..=176 (between NonCounted-twin and NotSummed-twin ranges) are + // 153..=176 (between NonCounted-twin and NotSummed-twin ranges) are // invalid (with the exception of 177 = NotSummedProvableSumTree and // 178 = NotSummedProvableCountProvableSumTree). - assert!(ElementType::try_from(152).is_err()); + assert!(ElementType::try_from(153).is_err()); assert!(ElementType::try_from(176).is_err()); // NotSummed twins live in 0xB0..=0xBF with explicit per-variant @@ -1716,16 +1737,16 @@ mod tests { assert!(ElementType::from_serialized_value(&[15, 200]).is_err()); // Wrapper whose inner byte is itself a synthetic twin discriminant // (high bit set) is rejected — only base discriminants 0..=14, 18, - // 19, 20, 21, 22, 23 are legal on-disk inner bytes. Without this + // 19, 20, 21, 22, 23, 24 are legal on-disk inner bytes. Without this // guard, `0x80 | 128 == 128` would silently parse as `NonCountedItem`. assert!(ElementType::from_serialized_value(&[15, 128]).is_err()); assert!(ElementType::from_serialized_value(&[15, 142]).is_err()); // Wrapper with an unallocated mid-range inner byte (16, 17, - // 24..=127) is also rejected, even though it has no high bit + // 25..=127) is also rejected, even though it has no high bit // set. assert!(ElementType::from_serialized_value(&[15, 16]).is_err()); assert!(ElementType::from_serialized_value(&[15, 17]).is_err()); - assert!(ElementType::from_serialized_value(&[15, 24]).is_err()); + assert!(ElementType::from_serialized_value(&[15, 25]).is_err()); assert!(ElementType::from_serialized_value(&[15, 100]).is_err()); // Inner byte 18 (ReferenceWithSumItem) IS a legal base; resolves to @@ -1762,6 +1783,12 @@ mod tests { ElementType::from_serialized_value(&[15, 23]).unwrap(), ElementType::NonCountedProvableCountProvableSumIndexedTree ); + // Inner byte 24 (PrivateDocumentStore) resolves to its NonCounted + // twin at slot 152. + assert_eq!( + ElementType::from_serialized_value(&[15, 24]).unwrap(), + ElementType::NonCountedPrivateDocumentStore + ); } #[test] @@ -1786,6 +1813,8 @@ mod tests { assert!(ElementType::ProvableSumIndexedTree.is_tree()); assert!(ElementType::ProvableCountIndexedTree.is_tree()); assert!(ElementType::ProvableCountProvableSumIndexedTree.is_tree()); + assert!(ElementType::PrivateDocumentStore.is_tree()); + assert!(ElementType::NonCountedPrivateDocumentStore.is_tree()); // ReferenceWithSumItem is a reference, not a tree and not an item. assert!(!ElementType::ReferenceWithSumItem.is_tree()); assert!(ElementType::ReferenceWithSumItem.is_reference()); diff --git a/grovedb-version/src/tests.rs b/grovedb-version/src/tests.rs index e58a85c6d..dcd718d8c 100644 --- a/grovedb-version/src/tests.rs +++ b/grovedb-version/src/tests.rs @@ -3,6 +3,7 @@ use crate::version::grovedb_versions::*; use crate::version::merk_versions::*; use crate::version::v1::GROVE_V1; use crate::version::v2::GROVE_V2; +use crate::version::v3::GROVE_V3; use crate::version::v4::GROVE_V4; use crate::version::{GroveVersion, GROVE_VERSIONS}; use crate::{TryFromVersioned, TryIntoVersioned}; @@ -74,6 +75,26 @@ fn grove_versions_count() { assert_eq!(GROVE_VERSIONS.len(), 4); } +#[test] +fn private_document_store_slots_are_gated_to_v4() { + // The PrivateDocumentStore family fails closed on every released + // version: all slots must be 0 on V1..V3 and 1 on V4. Changing a V1..V3 + // value would retroactively enable (or a V4 value disable) the element + // type on a live protocol version — a consensus break. + for v in [&GROVE_V1, &GROVE_V2, &GROVE_V3] { + let pds = &v.grovedb_versions.operations.private_document_store; + assert_eq!(pds.element_creation, 0, "v{}", v.protocol_version); + assert_eq!(pds.insert, 0, "v{}", v.protocol_version); + assert_eq!(pds.get_value, 0, "v{}", v.protocol_version); + assert_eq!(pds.count, 0, "v{}", v.protocol_version); + } + let pds = &GROVE_V4.grovedb_versions.operations.private_document_store; + assert_eq!(pds.element_creation, 1); + assert_eq!(pds.insert, 1); + assert_eq!(pds.get_value, 1); + assert_eq!(pds.count, 1); +} + #[test] fn grove_versions_ordered_by_protocol_version() { for window in GROVE_VERSIONS.windows(2) { diff --git a/grovedb-version/src/version/grovedb_versions.rs b/grovedb-version/src/version/grovedb_versions.rs index b63af3694..8edc6c17f 100644 --- a/grovedb-version/src/version/grovedb_versions.rs +++ b/grovedb-version/src/version/grovedb_versions.rs @@ -96,6 +96,36 @@ pub struct GroveDBOperationsVersions { pub proof: GroveDBOperationsProofVersions, pub average_case: GroveDBOperationsAverageCaseVersions, pub worst_case: GroveDBOperationsWorstCaseVersions, + pub private_document_store: GroveDBOperationsPrivateDocumentStoreVersions, +} + +/// Version slots for the PrivateDocumentStore operation family. +/// +/// Unlike most families, these slots act as **capability gates**, not +/// implementation selectors: slot value `0` means the operation (and the +/// element type itself, via `element_creation`) is unavailable and fails +/// closed with a version-mismatch error; `1` means the v1 implementation is +/// active. `GROVE_V1`..`GROVE_V3` hold every slot at `0` — the +/// `PrivateDocumentStore` element (discriminant 24) cannot be created or +/// operated on under released protocol versions. `GROVE_V4` flips them to +/// `1`. +/// +/// Note this does NOT gate `Element::deserialize` — the element bincode +/// codec stays protocol-independent (append-only discriminants, see the +/// doc on `GroveDBElementMethodVersions::serialize`). The gate lives at +/// the write/read operation entry points and at element insertion. +#[derive(Clone, Debug, Default)] +pub struct GroveDBOperationsPrivateDocumentStoreVersions { + /// Creating a `PrivateDocumentStore` element (direct or batch insert of + /// the element itself). + pub element_creation: FeatureVersion, + /// Appending an entry (`private_document_store_insert`, the + /// `PrivateDocumentStoreInsert` batch op). + pub insert: FeatureVersion, + /// Reading an entry by position (`private_document_store_get_value`). + pub get_value: FeatureVersion, + /// Reading the entry count (`private_document_store_count`). + pub count: FeatureVersion, } #[derive(Clone, Debug, Default)] @@ -141,9 +171,11 @@ pub struct GroveDBOperationsProofVersions { pub verify_query_get_parent_tree_info_with_options: FeatureVersion, /// Whether a V1 proof binds the element bytes of a **terminally-reported /// non-Merk tree** — `CommitmentTree`, `MmrTree`, `BulkAppendTree`, - /// `DenseAppendOnlyFixedSizeTree` — to the `value_hash` its parent Merk - /// commits to. "Terminal" means the query targets the tree element itself - /// and the prover emits no lower layer. + /// `DenseAppendOnlyFixedSizeTree`, `PrivateDocumentStore` — to the + /// `value_hash` its parent Merk commits to. "Terminal" means the query + /// targets the tree element itself and the prover emits no lower layer. + /// (`PrivateDocumentStore` cannot exist before V4, so it is always + /// bound.) /// /// - `0` (V1..V3): the prover emits a bare `KVValueHash` node and the /// verifier does not require a child hash. That node hashes only diff --git a/grovedb-version/src/version/v1.rs b/grovedb-version/src/version/v1.rs index be2b1d472..da01ef992 100644 --- a/grovedb-version/src/version/v1.rs +++ b/grovedb-version/src/version/v1.rs @@ -4,8 +4,8 @@ use crate::version::{ GroveDBApplyBatchVersions, GroveDBElementMethodVersions, GroveDBOperationsAverageCaseVersions, GroveDBOperationsDeleteUpTreeVersions, GroveDBOperationsDeleteVersions, GroveDBOperationsGetVersions, - GroveDBOperationsInsertVersions, GroveDBOperationsProofVersions, - GroveDBOperationsQueryVersions, GroveDBOperationsVersions, + GroveDBOperationsInsertVersions, GroveDBOperationsPrivateDocumentStoreVersions, + GroveDBOperationsProofVersions, GroveDBOperationsQueryVersions, GroveDBOperationsVersions, GroveDBOperationsWorstCaseVersions, GroveDBPathQueryMethodVersions, GroveDBQueryLimits, GroveDBReplicationVersions, GroveDBVersions, }, @@ -197,6 +197,14 @@ pub const GROVE_V1: GroveVersion = GroveVersion { add_worst_case_get_raw_cost: 0, add_worst_case_get_cost: 0, }, + // PrivateDocumentStore is unavailable before GROVE_V4: every + // slot is 0 and the operations fail closed. + private_document_store: GroveDBOperationsPrivateDocumentStoreVersions { + element_creation: 0, + insert: 0, + get_value: 0, + count: 0, + }, }, aggregate_sum_path_query_methods: GroveDBAggregateSumPathQueryMethodVersions { merge: 0 }, path_query_methods: GroveDBPathQueryMethodVersions { diff --git a/grovedb-version/src/version/v2.rs b/grovedb-version/src/version/v2.rs index 14f8d0936..c32b7b9ae 100644 --- a/grovedb-version/src/version/v2.rs +++ b/grovedb-version/src/version/v2.rs @@ -4,8 +4,8 @@ use crate::version::{ GroveDBApplyBatchVersions, GroveDBElementMethodVersions, GroveDBOperationsAverageCaseVersions, GroveDBOperationsDeleteUpTreeVersions, GroveDBOperationsDeleteVersions, GroveDBOperationsGetVersions, - GroveDBOperationsInsertVersions, GroveDBOperationsProofVersions, - GroveDBOperationsQueryVersions, GroveDBOperationsVersions, + GroveDBOperationsInsertVersions, GroveDBOperationsPrivateDocumentStoreVersions, + GroveDBOperationsProofVersions, GroveDBOperationsQueryVersions, GroveDBOperationsVersions, GroveDBOperationsWorstCaseVersions, GroveDBPathQueryMethodVersions, GroveDBQueryLimits, GroveDBReplicationVersions, GroveDBVersions, }, @@ -197,6 +197,14 @@ pub const GROVE_V2: GroveVersion = GroveVersion { add_worst_case_get_raw_cost: 0, add_worst_case_get_cost: 0, }, + // PrivateDocumentStore is unavailable before GROVE_V4: every + // slot is 0 and the operations fail closed. + private_document_store: GroveDBOperationsPrivateDocumentStoreVersions { + element_creation: 0, + insert: 0, + get_value: 0, + count: 0, + }, }, aggregate_sum_path_query_methods: GroveDBAggregateSumPathQueryMethodVersions { merge: 0 }, path_query_methods: GroveDBPathQueryMethodVersions { diff --git a/grovedb-version/src/version/v3.rs b/grovedb-version/src/version/v3.rs index c65b8518f..cf0135248 100644 --- a/grovedb-version/src/version/v3.rs +++ b/grovedb-version/src/version/v3.rs @@ -4,8 +4,8 @@ use crate::version::{ GroveDBApplyBatchVersions, GroveDBElementMethodVersions, GroveDBOperationsAverageCaseVersions, GroveDBOperationsDeleteUpTreeVersions, GroveDBOperationsDeleteVersions, GroveDBOperationsGetVersions, - GroveDBOperationsInsertVersions, GroveDBOperationsProofVersions, - GroveDBOperationsQueryVersions, GroveDBOperationsVersions, + GroveDBOperationsInsertVersions, GroveDBOperationsPrivateDocumentStoreVersions, + GroveDBOperationsProofVersions, GroveDBOperationsQueryVersions, GroveDBOperationsVersions, GroveDBOperationsWorstCaseVersions, GroveDBPathQueryMethodVersions, GroveDBQueryLimits, GroveDBReplicationVersions, GroveDBVersions, }, @@ -201,6 +201,14 @@ pub const GROVE_V3: GroveVersion = GroveVersion { add_worst_case_get_raw_cost: 0, add_worst_case_get_cost: 0, }, + // PrivateDocumentStore is unavailable before GROVE_V4: every + // slot is 0 and the operations fail closed. + private_document_store: GroveDBOperationsPrivateDocumentStoreVersions { + element_creation: 0, + insert: 0, + get_value: 0, + count: 0, + }, }, aggregate_sum_path_query_methods: GroveDBAggregateSumPathQueryMethodVersions { merge: 0 }, path_query_methods: GroveDBPathQueryMethodVersions { diff --git a/grovedb-version/src/version/v4.rs b/grovedb-version/src/version/v4.rs index 7b53a2033..f6a1d214e 100644 --- a/grovedb-version/src/version/v4.rs +++ b/grovedb-version/src/version/v4.rs @@ -50,8 +50,8 @@ use crate::version::{ GroveDBApplyBatchVersions, GroveDBElementMethodVersions, GroveDBOperationsAverageCaseVersions, GroveDBOperationsDeleteUpTreeVersions, GroveDBOperationsDeleteVersions, GroveDBOperationsGetVersions, - GroveDBOperationsInsertVersions, GroveDBOperationsProofVersions, - GroveDBOperationsQueryVersions, GroveDBOperationsVersions, + GroveDBOperationsInsertVersions, GroveDBOperationsPrivateDocumentStoreVersions, + GroveDBOperationsProofVersions, GroveDBOperationsQueryVersions, GroveDBOperationsVersions, GroveDBOperationsWorstCaseVersions, GroveDBPathQueryMethodVersions, GroveDBQueryLimits, GroveDBReplicationVersions, GroveDBVersions, }, @@ -247,6 +247,13 @@ pub const GROVE_V4: GroveVersion = GroveVersion { add_worst_case_get_raw_cost: 0, add_worst_case_get_cost: 0, }, + // PrivateDocumentStore activates in GROVE_V4. + private_document_store: GroveDBOperationsPrivateDocumentStoreVersions { + element_creation: 1, + insert: 1, + get_value: 1, + count: 1, + }, }, aggregate_sum_path_query_methods: GroveDBAggregateSumPathQueryMethodVersions { merge: 0 }, path_query_methods: GroveDBPathQueryMethodVersions { diff --git a/grovedb/Cargo.toml b/grovedb/Cargo.toml index 6358fbba8..a3b527f45 100644 --- a/grovedb/Cargo.toml +++ b/grovedb/Cargo.toml @@ -26,6 +26,7 @@ grovedb-element = { version = "5.0.1", path = "../grovedb-element" } grovedb-commitment-tree = { version = "5.0.1", path = "../grovedb-commitment-tree", optional = true } grovedb-merkle-mountain-range = { version = "5.0.1", path = "../grovedb-merkle-mountain-range", optional = true, default-features = false } grovedb-bulk-append-tree = { version = "5.0.1", path = "../grovedb-bulk-append-tree", optional = true, default-features = false } +grovedb-private-document-store = { version = "5.0.1", path = "../grovedb-private-document-store", optional = true, default-features = false } grovedb-dense-fixed-sized-merkle-tree = { version = "5.0.1", path = "../grovedb-dense-fixed-sized-merkle-tree", optional = true, default-features = false } grovedb-query = { version = "5.0.1", path = "../grovedb-query" } @@ -93,6 +94,8 @@ minimal = [ "grovedb-merkle-mountain-range/storage", "grovedb-bulk-append-tree", "grovedb-bulk-append-tree/storage", + "grovedb-private-document-store", + "grovedb-private-document-store/storage", "grovedb-dense-fixed-sized-merkle-tree", "grovedb-dense-fixed-sized-merkle-tree/storage", "thiserror", diff --git a/grovedb/src/batch/batch_structure.rs b/grovedb/src/batch/batch_structure.rs index 65132a301..9abd43cbc 100644 --- a/grovedb/src/batch/batch_structure.rs +++ b/grovedb/src/batch/batch_structure.rs @@ -168,6 +168,7 @@ where | GroveOp::MmrTreeAppend { .. } | GroveOp::BulkAppend { .. } | GroveOp::DenseTreeInsert { .. } + | GroveOp::PrivateDocumentStoreInsert { .. } | GroveOp::ReplaceNonMerkTreeRoot { .. } => { // User-facing tree ops are preprocessed before batch // execution into ReplaceNonMerkTreeRoot ops, which must diff --git a/grovedb/src/batch/estimated_costs/average_case_costs.rs b/grovedb/src/batch/estimated_costs/average_case_costs.rs index b79da061a..1ba34302e 100644 --- a/grovedb/src/batch/estimated_costs/average_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/average_case_costs.rs @@ -307,6 +307,39 @@ impl GroveOp { sinsemilla_hash_calls: 0, }) } + GroveOp::PrivateDocumentStoreInsert { entry } => { + // Cost of updating parent element in the Merk. The entry + // length is the store's committed entry_size, so the added + // storage bytes are entry-size-parametrized. + let item_cost = GroveDb::average_case_merk_replace_tree( + key, + layer_element_estimates, + TreeType::PrivateDocumentStore(0), + propagate, + grove_version, + ); + // Additional cost: buffer write + running hash, identical to + // the underlying BulkAppend (the composite pds_state blake3 + // is folded into the same single-hash average as the bulk + // state-root hash it replaces per append). Most appends only + // write to the buffer (O(1)); compaction happens once per + // epoch_size appends and is amortized. + use grovedb_costs::storage_cost::{removal::StorageRemovedBytes, StorageCost}; + let entry_size = entry.len() as u32; + // 1 blake3 hash for the running buffer hash chain + const AVG_HASH_CALLS: u32 = 1; + item_cost.add_cost(OperationCost { + seek_count: 1, // 1 buffer entry write + storage_cost: StorageCost { + added_bytes: entry_size, + replaced_bytes: 0, + removed_bytes: StorageRemovedBytes::NoStorageRemoval, + }, + storage_loaded_bytes: 0, + hash_node_calls: AVG_HASH_CALLS, + sinsemilla_hash_calls: 0, + }) + } GroveOp::DenseTreeInsert { value } => { // Cost of updating parent element in the Merk let item_cost = GroveDb::average_case_merk_replace_tree( diff --git a/grovedb/src/batch/estimated_costs/worst_case_costs.rs b/grovedb/src/batch/estimated_costs/worst_case_costs.rs index 0689de98e..8236aa256 100644 --- a/grovedb/src/batch/estimated_costs/worst_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/worst_case_costs.rs @@ -303,6 +303,44 @@ impl GroveOp { sinsemilla_hash_calls: 0, }) } + GroveOp::PrivateDocumentStoreInsert { entry } => { + // Cost of updating parent element in the Merk. + let item_cost = GroveDb::worst_case_merk_replace_tree( + key, + TreeType::PrivateDocumentStore(0), + in_parent_tree_type, + worst_case_layer_element_estimates, + propagate, + grove_version, + ); + // Worst case mirrors the underlying BulkAppend: compaction + // trigger (buffer fills -> serialize chunk blob -> dense + // Merkle root -> MMR push), plus one blake3 for the + // config-binding composite pds_state root. The per-append + // write is entry-size-parametrized (entry.len() is the + // store's committed entry_size). + use grovedb_costs::storage_cost::{removal::StorageRemovedBytes, StorageCost}; + let entry_size = entry.len() as u32; + // Max compaction overhead: 64KB safe bound for chunk blob + const MAX_COMPACTION_BLOB: u32 = 65536; + // Dense Merkle root: epoch_size hashes. Buffer hash: 1. + // MMR push: up to 64 merges. Composite pds_state root: 1. + const MAX_HASH_CALLS: u32 = 1024 + 1 + 65 + 1; + // Writes: buffer entry + chunk blob + MMR nodes + const MAX_WRITES: u32 = 1 + 1 + 65; + const MAX_READS: u32 = 64; // MMR sibling reads + item_cost.add_cost(OperationCost { + seek_count: MAX_WRITES + MAX_READS, + storage_cost: StorageCost { + added_bytes: entry_size + MAX_COMPACTION_BLOB, + replaced_bytes: 0, + removed_bytes: StorageRemovedBytes::NoStorageRemoval, + }, + storage_loaded_bytes: (33 * MAX_READS) as u64, + hash_node_calls: MAX_HASH_CALLS, + sinsemilla_hash_calls: 0, + }) + } GroveOp::DenseTreeInsert { value } => { // Cost of updating parent element in the Merk let item_cost = GroveDb::worst_case_merk_replace_tree( diff --git a/grovedb/src/batch/indexed_tree/pre_state.rs b/grovedb/src/batch/indexed_tree/pre_state.rs index 2f342d8cd..849ef6f33 100644 --- a/grovedb/src/batch/indexed_tree/pre_state.rs +++ b/grovedb/src/batch/indexed_tree/pre_state.rs @@ -97,7 +97,8 @@ fn validate_indexed_child_ops( | GroveOp::CommitmentTreeInsert { .. } | GroveOp::MmrTreeAppend { .. } | GroveOp::BulkAppend { .. } - | GroveOp::DenseTreeInsert { .. } => continue, + | GroveOp::DenseTreeInsert { .. } + | GroveOp::PrivateDocumentStoreInsert { .. } => continue, }; // Child-type acceptance, delegated to merk's own rule rather than a // second copy of it: `get_feature_type` is what decides whether an diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index 1467d1383..eaabdd117 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -153,6 +153,16 @@ pub enum NonMerkTreeMeta { /// Fixed height of the dense Merkle tree. height: u8, }, + /// PrivateDocumentStore state: total_count plus the committed config + /// {entry_size, chunk_power}. + PrivateDocumentStore { + /// Total number of entries appended so far. + total_count: u64, + /// Committed byte length of every entry. + entry_size: u32, + /// Power-of-2 chunk size for epochs. + chunk_power: u8, + }, } impl NonMerkTreeMeta { @@ -169,6 +179,9 @@ impl NonMerkTreeMeta { NonMerkTreeMeta::DenseTree { height, .. } => { TreeType::DenseAppendOnlyFixedSizeTree(*height) } + NonMerkTreeMeta::PrivateDocumentStore { chunk_power, .. } => { + TreeType::PrivateDocumentStore(*chunk_power) + } } } @@ -187,6 +200,13 @@ impl NonMerkTreeMeta { NonMerkTreeMeta::DenseTree { count, height } => { Element::new_dense_tree(*count, *height, flags) } + NonMerkTreeMeta::PrivateDocumentStore { + total_count, + entry_size, + chunk_power, + } => { + Element::new_private_document_store(*total_count, *entry_size, *chunk_power, flags) + } } } @@ -197,6 +217,7 @@ impl NonMerkTreeMeta { NonMerkTreeMeta::MmrTree { mmr_size } => *mmr_size, NonMerkTreeMeta::BulkAppendTree { total_count, .. } => *total_count, NonMerkTreeMeta::DenseTree { count, .. } => *count as u64, + NonMerkTreeMeta::PrivateDocumentStore { total_count, .. } => *total_count, } } } @@ -483,6 +504,13 @@ pub enum GroveOp { /// Value to insert value: Vec, }, + /// Append a fixed-size entry to a PrivateDocumentStore. The entry's + /// length must equal the store's committed `entry_size`; the append is + /// rejected otherwise. + PrivateDocumentStoreInsert { + /// Entry to append (exactly `entry_size` bytes) + entry: Vec, + }, } impl GroveOp { @@ -512,6 +540,7 @@ impl GroveOp { GroveOp::InsertNonMerkTree { .. } => 16, GroveOp::ReplaceAggregateIndexedTreeRootKeys { .. } => 17, GroveOp::InsertAggregateIndexedTreeRootKeys { .. } => 18, + GroveOp::PrivateDocumentStoreInsert { .. } => 19, } } @@ -573,7 +602,8 @@ impl GroveOp { GroveOp::CommitmentTreeInsert { .. } | GroveOp::MmrTreeAppend { .. } | GroveOp::BulkAppend { .. } - | GroveOp::DenseTreeInsert { .. } => false, + | GroveOp::DenseTreeInsert { .. } + | GroveOp::PrivateDocumentStoreInsert { .. } => false, } } } @@ -864,6 +894,9 @@ impl fmt::Debug for QualifiedGroveDbOp { GroveOp::MmrTreeAppend { .. } => "MMR Tree Append".to_string(), GroveOp::BulkAppend { .. } => "Bulk Append".to_string(), GroveOp::DenseTreeInsert { .. } => "Dense Tree Insert".to_string(), + GroveOp::PrivateDocumentStoreInsert { .. } => { + "Private Document Store Insert".to_string() + } GroveOp::ReplaceAggregateIndexedTreeRootKeys { .. } => { "Replace CountIndexedTree primary+secondary roots".to_string() } @@ -1255,6 +1288,18 @@ impl QualifiedGroveDbOp { } } + /// A private document store insert op. `path` includes the store key as + /// its last segment. The entry must be exactly the store's committed + /// `entry_size` bytes; the batch preprocessor rejects any other length. + pub fn private_document_store_insert_op(path: Vec>, entry: Vec) -> Self { + let path = KeyInfoPath::from_known_owned_path(path); + Self { + path, + key: None, + op: GroveOp::PrivateDocumentStoreInsert { entry }, + } + } + /// Verify consistency of operations pub fn verify_consistency_of_operations( ops: &[QualifiedGroveDbOp], @@ -1884,9 +1929,10 @@ where | Element::DenseAppendOnlyFixedSizeTree(..) | Element::ProvableSumIndexedTree(..) | Element::ProvableCountIndexedTree(..) - | Element::ProvableCountProvableSumIndexedTree(..) => Err( - Error::InvalidBatchOperation("references can not point to trees being updated"), - ) + | Element::ProvableCountProvableSumIndexedTree(..) + | Element::PrivateDocumentStore(..) => Err(Error::InvalidBatchOperation( + "references can not point to trees being updated", + )) .wrap_with_cost(cost), // underlying() unwraps a single level; the constructor and // (de)serializer reject nested wrappers, so these are @@ -1953,7 +1999,8 @@ where | GroveOp::CommitmentTreeInsert { .. } | GroveOp::MmrTreeAppend { .. } | GroveOp::BulkAppend { .. } - | GroveOp::DenseTreeInsert { .. } => Err(Error::InvalidBatchOperation( + | GroveOp::DenseTreeInsert { .. } + | GroveOp::PrivateDocumentStoreInsert { .. } => Err(Error::InvalidBatchOperation( "references can not point to trees being updated", )) .wrap_with_cost(cost), @@ -2046,12 +2093,11 @@ where | Element::DenseAppendOnlyFixedSizeTree(..) | Element::ProvableSumIndexedTree(..) | Element::ProvableCountIndexedTree(..) - | Element::ProvableCountProvableSumIndexedTree(..) => { - Err(Error::InvalidBatchOperation( - "references can not point to trees being updated", - )) - .wrap_with_cost(cost) - } + | Element::ProvableCountProvableSumIndexedTree(..) + | Element::PrivateDocumentStore(..) => Err(Error::InvalidBatchOperation( + "references can not point to trees being updated", + )) + .wrap_with_cost(cost), // Wrappers are unwrapped via underlying() above. Element::NonCounted(_) | Element::NotSummed(_) @@ -2100,12 +2146,11 @@ where | Element::DenseAppendOnlyFixedSizeTree(..) | Element::ProvableSumIndexedTree(..) | Element::ProvableCountIndexedTree(..) - | Element::ProvableCountProvableSumIndexedTree(..) => { - Err(Error::InvalidBatchOperation( - "references can not point to trees being updated", - )) - .wrap_with_cost(cost) - } + | Element::ProvableCountProvableSumIndexedTree(..) + | Element::PrivateDocumentStore(..) => Err(Error::InvalidBatchOperation( + "references can not point to trees being updated", + )) + .wrap_with_cost(cost), // Wrappers are unwrapped via underlying() above. Element::NonCounted(_) | Element::NotSummed(_) @@ -2427,6 +2472,13 @@ where // Without these checks, batch users could persist // wrapped elements into the wrong tree types and silently // violate the wrapper invariant. + if matches!(in_tree_type, TreeType::PrivateDocumentStore(_)) { + return Err(Error::InvalidBatchOperation( + "private document stores cannot hold child elements; entries are \ + appended via the PrivateDocumentStoreInsert operation", + )) + .wrap_with_cost(cost); + } if element.is_non_counted() && !in_tree_type.accepts_non_counted_children() { return Err(Error::InvalidBatchOperation( "non-counted elements may only be inserted into non-provable \ @@ -2847,6 +2899,89 @@ where ) ); } + Element::PrivateDocumentStore(total_count, entry_size, chunk_power, _) => { + // Fail closed on protocol versions that predate + // the element type. + cost_return_on_error_no_add!( + cost, + crate::operations::private_document_store::check_pds_enabled( + "batch insert Element::PrivateDocumentStore", + grove_version + .grovedb_versions + .operations + .private_document_store + .element_creation, + ) + ); + // Only empty-creation is allowed: the child hash + // written below is the empty state root for this + // config, and entries can only be added through + // the typed append path. + if *total_count != 0 { + return Err(Error::InvalidBatchOperation( + "a PrivateDocumentStore must be empty at the moment of batch \ + insertion (total_count = 0); entries are appended via the \ + PrivateDocumentStoreInsert operation", + )) + .wrap_with_cost(cost); + } + // Same config validation the element constructors + // enforce — a caller-built element must not bypass + // it, since the config is committed into the state + // root. + if *entry_size == 0 || !(1..=16).contains(chunk_power) { + return Err(Error::InvalidBatchOperation( + "a PrivateDocumentStore requires entry_size >= 1 and \ + chunk_power in 1..=16", + )) + .wrap_with_cost(cost); + } + if is_insert_if_not_exists + || batch_apply_options.validate_insertion_does_not_override + { + let merk = self.merks.get_mut(path).expect("the Merk is cached"); + let existing = cost_return_on_error_into!( + &mut cost, + element.element_at_key_already_exists( + merk, + key_info.get_key_clone().as_slice(), + grove_version, + ) + ); + if existing { + if error_if_exists + || batch_apply_options.validate_insertion_does_not_override + { + return Err(Error::InvalidBatchOperation( + "attempting to insert PrivateDocumentStore element \ + that already exists", + )) + .wrap_with_cost(cost); + } + continue; + } + } + let merk_feature_type = cost_return_on_error_into!( + &mut cost, + element + .get_feature_type(in_tree_type) + .wrap_with_cost(OperationCost::default()) + ); + cost_return_on_error_into!( + &mut cost, + element.insert_subtree_into_batch_operations( + key_info.get_key_clone(), + grovedb_private_document_store::empty_private_document_store_state_root( + *entry_size, + *chunk_power, + ), + false, + &mut batch_operations, + merk_feature_type, + grove_version, + ) + ); + } Element::Item(..) | Element::SumItem(..) | Element::ItemWithSumItem(..) => { let merk_feature_type = cost_return_on_error_into!( &mut cost, @@ -3052,6 +3187,13 @@ where // wrapper into a parent that doesn't accept it — // including any `Provable*` count tree where the // count is cryptographically committed. + if matches!(in_tree_type, TreeType::PrivateDocumentStore(_)) { + return Err(Error::InvalidBatchOperation( + "private document stores cannot hold child elements; entries are \ + appended via the PrivateDocumentStoreInsert operation", + )) + .wrap_with_cost(cost); + } if element.is_non_counted() && !in_tree_type.accepts_non_counted_children() { return Err(Error::InvalidBatchOperation( "RefreshReference with non_counted=true requires a non-provable \ @@ -3384,6 +3526,13 @@ where )) .wrap_with_cost(cost); } + GroveOp::PrivateDocumentStoreInsert { .. } => { + return Err(Error::InvalidBatchOperation( + "PrivateDocumentStoreInsert should have been preprocessed before batch \ + execution", + )) + .wrap_with_cost(cost); + } GroveOp::ReplaceAggregateIndexedTreeRootKeys { primary_hash, primary_root_key, @@ -3528,7 +3677,8 @@ where | Element::DenseAppendOnlyFixedSizeTree(..) | Element::ProvableSumIndexedTree(..) | Element::ProvableCountIndexedTree(..) - | Element::ProvableCountProvableSumIndexedTree(..) => { + | Element::ProvableCountProvableSumIndexedTree(..) + | Element::PrivateDocumentStore(..) => { let tree_type = new_element .tree_type() .expect("tree_type guaranteed by match arm"); @@ -4013,6 +4163,28 @@ impl GroveDb { meta, non_counted, } + } else if let Element::PrivateDocumentStore( + total_count, + entry_size, + chunk_power, + flags, + ) = element + { + let meta = + NonMerkTreeMeta::PrivateDocumentStore { + total_count: *total_count, + entry_size: *entry_size, + chunk_power: *chunk_power, + }; + *mutable_occupied_entry = + GroveOp::InsertNonMerkTree { + hash: root_hash, + root_key: calculated_root_key, + flags: flags.clone(), + aggregate_data, + meta, + non_counted, + } } else if let Element::MmrTree( mmr_size, flags, @@ -4163,6 +4335,13 @@ impl GroveDb { )) .wrap_with_cost(cost); } + GroveOp::PrivateDocumentStoreInsert { .. } => { + return Err(Error::InvalidBatchOperation( + "PrivateDocumentStoreInsert ops should \ + have been preprocessed", + )) + .wrap_with_cost(cost); + } } } } @@ -4596,6 +4775,26 @@ impl GroveDb { ) ); } + GroveOp::PrivateDocumentStoreInsert { entry } => { + let mut path_vec: Vec> = op.path.to_path(); + let key = cost_return_on_error_no_add!( + cost, + path_vec.pop().ok_or(Error::InvalidBatchOperation( + "append op path must include tree key" + )) + ); + let path_slices: Vec<&[u8]> = path_vec.iter().map(|p| p.as_slice()).collect(); + cost_return_on_error!( + &mut cost, + self.private_document_store_insert( + path_slices.as_slice(), + &key, + entry.clone(), + transaction, + grove_version, + ) + ); + } GroveOp::DenseTreeInsert { value } => { let mut path_vec: Vec> = op.path.to_path(); let key = cost_return_on_error_no_add!( @@ -4890,6 +5089,18 @@ impl GroveDb { self.preprocess_dense_tree_ops(ops, tx.as_ref(), &storage_batch, grove_version) ); + // Preprocess PrivateDocumentStoreInsert ops: execute size-validated + // appends then convert to ReplaceNonMerkTreeRoot ops + let ops = cost_return_on_error!( + &mut cost, + self.preprocess_private_document_store_ops( + ops, + tx.as_ref(), + &storage_batch, + grove_version + ) + ); + // Collect paths of subtrees being deleted, separated by type. // // Non-Merk trees (MmrTree, BulkAppendTree, DenseTree, CommitmentTree) @@ -5465,6 +5676,18 @@ impl GroveDb { self.preprocess_dense_tree_ops(ops, tx.as_ref(), &storage_batch, grove_version) ); + // Preprocess PrivateDocumentStoreInsert ops: execute size-validated + // appends then convert to ReplaceNonMerkTreeRoot ops + let ops = cost_return_on_error!( + &mut cost, + self.preprocess_private_document_store_ops( + ops, + tx.as_ref(), + &storage_batch, + grove_version + ) + ); + // See comment in apply_batch_with_element_flags_update for why // deleted tree subtrees need explicit storage cleanup, and why // emptiness checks are needed (H2). diff --git a/grovedb/src/debugger.rs b/grovedb/src/debugger.rs index 6b683205f..6ed7f587c 100644 --- a/grovedb/src/debugger.rs +++ b/grovedb/src/debugger.rs @@ -967,6 +967,12 @@ fn element_to_grovedbg(element: crate::Element) -> grovedbg_types::Element { element_flags, } } + crate::Element::PrivateDocumentStore(_, _, _, element_flags) => { + grovedbg_types::Element::Subtree { + root_key: None, + element_flags, + } + } // The visualizer wire format has no wrapper variants; render the // inner element. The wrapper is invisible at the debug-UI layer. crate::Element::NonCounted(inner) diff --git a/grovedb/src/error.rs b/grovedb/src/error.rs index 534b7b09f..6f616ac54 100644 --- a/grovedb/src/error.rs +++ b/grovedb/src/error.rs @@ -169,6 +169,10 @@ pub enum Error { #[error("commitment tree error: {0}")] /// Commitment tree operation error CommitmentTreeError(String), + + #[error("private document store error: {0}")] + /// Private document store operation error + PrivateDocumentStoreError(String), } impl Error { @@ -195,7 +199,8 @@ impl Error { | Self::ClientReturnedNonClientError(s) | Self::PathNotFoundInCacheForEstimatedCosts(s) | Self::NotSupported(s) - | Self::CommitmentTreeError(s) => { + | Self::CommitmentTreeError(s) + | Self::PrivateDocumentStoreError(s) => { s.push_str(", "); s.push_str(append.as_ref()); } diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index a3f509a8a..f1fd1ec52 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -1928,7 +1928,8 @@ impl GroveDb { | Element::CommitmentTree(..) | Element::MmrTree(..) | Element::BulkAppendTree(..) - | Element::DenseAppendOnlyFixedSizeTree(..) => { + | Element::DenseAppendOnlyFixedSizeTree(..) + | Element::PrivateDocumentStore(..) => { let (kv_value, element_value_hash) = merk .get_value_and_value_hash( &key, @@ -2452,6 +2453,42 @@ impl GroveDb { Err(_) => merk_root_hash, } } + Element::PrivateDocumentStore(total_count, entry_size, chunk_power, _) => { + // The state root binds the committed config even when the + // store is empty, so the empty case is the config-parametrized + // empty root rather than the (empty) Merk root. + if *total_count == 0 { + return grovedb_private_document_store::empty_private_document_store_state_root( + *entry_size, + *chunk_power, + ); + } + let storage_ctx = self + .db + .get_transactional_storage_context(subtree_path, None, transaction) + .unwrap(); + match grovedb_private_document_store::PrivateDocumentStore::from_state( + *total_count, + *entry_size, + *chunk_power, + storage_ctx, + ) { + Ok(store) => { + // Integrity walk: every stored entry must respect the + // committed entry size (the state root authenticates + // whatever bytes were written, so a buggy or bypassing + // writer could persist wrong-size entries under a + // consistent root). On violation, fall back to + // `merk_root_hash` — the caller's chain check then + // reports the path as an issue. + if store.verify_entry_sizes().is_err() { + return merk_root_hash; + } + store.compute_current_state_root().unwrap_or(merk_root_hash) + } + Err(_) => merk_root_hash, + } + } _ => merk_root_hash, } } @@ -2651,7 +2688,8 @@ fn aggregate_consistency_labels( (Element::CommitmentTree(..), _) | (Element::MmrTree(..), _) | (Element::BulkAppendTree(..), _) - | (Element::DenseAppendOnlyFixedSizeTree(..), _) => None, + | (Element::DenseAppendOnlyFixedSizeTree(..), _) + | (Element::PrivateDocumentStore(..), _) => None, // --- Anything else is a variant/aggregate-shape mismatch (e.g. // the inner Merk's tree-type has drifted from what the parent diff --git a/grovedb/src/operations/get/mod.rs b/grovedb/src/operations/get/mod.rs index dbfc9331c..9e27153bb 100644 --- a/grovedb/src/operations/get/mod.rs +++ b/grovedb/src/operations/get/mod.rs @@ -417,7 +417,8 @@ impl GroveDb { | Ok(Element::CommitmentTree(..)) | Ok(Element::MmrTree(..)) | Ok(Element::BulkAppendTree(..)) - | Ok(Element::DenseAppendOnlyFixedSizeTree(..)) => Ok(()).wrap_with_cost(cost), + | Ok(Element::DenseAppendOnlyFixedSizeTree(..)) + | Ok(Element::PrivateDocumentStore(..)) => Ok(()).wrap_with_cost(cost), Ok(_) | Err(Error::PathKeyNotFound(_)) => Err(error_fn()).wrap_with_cost(cost), Err(e) => Err(e).wrap_with_cost(cost), } diff --git a/grovedb/src/operations/get/query.rs b/grovedb/src/operations/get/query.rs index 11faf2773..6657ce35e 100644 --- a/grovedb/src/operations/get/query.rs +++ b/grovedb/src/operations/get/query.rs @@ -280,6 +280,7 @@ where { | Element::MmrTree(..) | Element::BulkAppendTree(..) | Element::DenseAppendOnlyFixedSizeTree(..) + | Element::PrivateDocumentStore(..) | Element::ProvableSumIndexedTree(..) | Element::ProvableCountProvableSumIndexedTree(..) | Element::ProvableCountIndexedTree(..) => { @@ -426,6 +427,7 @@ where { | Element::MmrTree(..) | Element::BulkAppendTree(..) | Element::DenseAppendOnlyFixedSizeTree(..) + | Element::PrivateDocumentStore(..) | Element::ProvableSumIndexedTree(..) | Element::ProvableCountProvableSumIndexedTree(..) | Element::ProvableCountIndexedTree(..) => Err(Error::InvalidQuery( @@ -639,7 +641,8 @@ where { | Element::CommitmentTree(..) | Element::MmrTree(..) | Element::BulkAppendTree(..) - | Element::DenseAppendOnlyFixedSizeTree(..) => Err(Error::InvalidQuery( + | Element::DenseAppendOnlyFixedSizeTree(..) + | Element::PrivateDocumentStore(..) => Err(Error::InvalidQuery( "path_queries can only refer to items, sum items, references and sum \ trees", )), @@ -1285,6 +1288,7 @@ where { | Element::MmrTree(..) | Element::BulkAppendTree(..) | Element::DenseAppendOnlyFixedSizeTree(..) + | Element::PrivateDocumentStore(..) | Element::ProvableSumIndexedTree(..) | Element::ProvableCountProvableSumIndexedTree(..) | Element::ProvableCountIndexedTree(..) diff --git a/grovedb/src/operations/insert/add_element_on_transaction/v0.rs b/grovedb/src/operations/insert/add_element_on_transaction/v0.rs index 4d52613da..82aeee237 100644 --- a/grovedb/src/operations/insert/add_element_on_transaction/v0.rs +++ b/grovedb/src/operations/insert/add_element_on_transaction/v0.rs @@ -191,6 +191,50 @@ impl GroveDb { ) ); } + // PrivateDocumentStore: the initial child hash is the empty + // state root for the element's committed config (the state root + // binds {entry_size, chunk_power}), so V1 proof verification and + // verify_grovedb agree even before the first append. Creation is + // version-gated and only an empty store may be inserted; entries + // are appended via the typed private_document_store_insert path. + Element::PrivateDocumentStore(total_count, entry_size, chunk_power, _) => { + cost_return_on_error_no_add!( + cost, + crate::operations::private_document_store::check_pds_enabled( + "insert Element::PrivateDocumentStore", + grove_version + .grovedb_versions + .operations + .private_document_store + .element_creation, + ) + ); + if *total_count != 0 { + return Err(Error::InvalidCodeExecution( + "a private document store should be empty at the moment of insertion", + )) + .wrap_with_cost(cost); + } + if *entry_size == 0 || !(1..=16).contains(chunk_power) { + return Err(Error::InvalidInput( + "a PrivateDocumentStore requires entry_size >= 1 and chunk_power in 1..=16", + )) + .wrap_with_cost(cost); + } + cost_return_on_error_into!( + &mut cost, + element.insert_subtree( + &mut subtree_to_insert_into, + key, + grovedb_private_document_store::empty_private_document_store_state_root( + *entry_size, + *chunk_power, + ), + Some(options.as_merk_options()), + grove_version + ) + ); + } // MmrTree, BulkAppendTree, DenseAppendOnlyFixedSizeTree: initial // insert uses NULL_HASH since these trees start empty. Element::MmrTree(..) diff --git a/grovedb/src/operations/insert/add_element_on_transaction/v1.rs b/grovedb/src/operations/insert/add_element_on_transaction/v1.rs index 24abaedf4..e2e0a5799 100644 --- a/grovedb/src/operations/insert/add_element_on_transaction/v1.rs +++ b/grovedb/src/operations/insert/add_element_on_transaction/v1.rs @@ -187,6 +187,50 @@ impl GroveDb { ) ); } + // PrivateDocumentStore: the initial child hash is the empty + // state root for the element's committed config (the state root + // binds {entry_size, chunk_power}), so V1 proof verification and + // verify_grovedb agree even before the first append. Creation is + // version-gated and only an empty store may be inserted; entries + // are appended via the typed private_document_store_insert path. + Element::PrivateDocumentStore(total_count, entry_size, chunk_power, _) => { + cost_return_on_error_no_add!( + cost, + crate::operations::private_document_store::check_pds_enabled( + "insert Element::PrivateDocumentStore", + grove_version + .grovedb_versions + .operations + .private_document_store + .element_creation, + ) + ); + if *total_count != 0 { + return Err(Error::InvalidCodeExecution( + "a private document store should be empty at the moment of insertion", + )) + .wrap_with_cost(cost); + } + if *entry_size == 0 || !(1..=16).contains(chunk_power) { + return Err(Error::InvalidInput( + "a PrivateDocumentStore requires entry_size >= 1 and chunk_power in 1..=16", + )) + .wrap_with_cost(cost); + } + cost_return_on_error_into!( + &mut cost, + element.insert_subtree( + &mut subtree_to_insert_into, + key, + grovedb_private_document_store::empty_private_document_store_state_root( + *entry_size, + *chunk_power, + ), + Some(options.as_merk_options()), + grove_version + ) + ); + } // MmrTree, BulkAppendTree, DenseAppendOnlyFixedSizeTree: initial // insert uses NULL_HASH since these trees start empty. Element::MmrTree(..) diff --git a/grovedb/src/operations/mod.rs b/grovedb/src/operations/mod.rs index 8e556f3ad..ec79ad3a9 100644 --- a/grovedb/src/operations/mod.rs +++ b/grovedb/src/operations/mod.rs @@ -25,6 +25,9 @@ pub mod bulk_append_tree; #[cfg(feature = "minimal")] pub mod dense_tree; +#[cfg(feature = "minimal")] +/// Private document store operations +pub mod private_document_store; /// Caller-driven subtree-root replacement. Bypasses grovedb's normal /// "compute child hash from subtree state" invariant — see the module-level diff --git a/grovedb/src/operations/private_document_store.rs b/grovedb/src/operations/private_document_store.rs new file mode 100644 index 000000000..631afb416 --- /dev/null +++ b/grovedb/src/operations/private_document_store.rs @@ -0,0 +1,536 @@ +//! PrivateDocumentStore operations for GroveDB. +//! +//! Thin bridge between GroveDB's storage/transaction/batch infrastructure and +//! the `grovedb-private-document-store` crate, which wraps a `BulkAppendTree` +//! with a committed `{entry_size, chunk_power}` configuration. Appends are +//! validated against the committed entry size and the state root binds the +//! config (`blake3("pds_state" || config_hash || bulk_state_root)`), so a +//! proof can never be reinterpreted under a different configuration. +//! +//! There is no per-entry delete or update — immutability is enforced by the +//! type. Every entry point here fails closed on protocol versions that +//! predate the element type (all slots are 0 before `GROVE_V4`). + +use std::collections::HashMap; + +use grovedb_costs::{ + cost_return_on_error, cost_return_on_error_into, cost_return_on_error_no_add, CostResult, + CostsExt, OperationCost, +}; +use grovedb_merk::element::insert::ElementInsertToStorageExtensions; +use grovedb_path::SubtreePath; +use grovedb_private_document_store::PrivateDocumentStore; +use grovedb_storage::{Storage, StorageBatch}; +use grovedb_version::{error::GroveVersionError, version::GroveVersion}; + +use crate::{ + batch::{GroveOp, QualifiedGroveDbOp}, + util::TxRef, + Element, Error, GroveDb, Transaction, TransactionArg, +}; + +/// Map a `PrivateDocumentStoreError` to a GroveDB `Error`. +fn map_pds_err(e: grovedb_private_document_store::PrivateDocumentStoreError) -> Error { + Error::PrivateDocumentStoreError(format!("{}", e)) +} + +/// Fail-closed capability gate for the PrivateDocumentStore family. +/// +/// Slot `0` (every version before `GROVE_V4`) means the operation is +/// unavailable and returns a version-mismatch error; slot `1` is the active +/// v1 implementation. Unlike the `check_grovedb_v0!` family this rejects +/// *older* versions rather than newer ones — the element type must not be +/// creatable or operable under released protocol versions. +pub(crate) fn check_pds_enabled( + method: &str, + slot: grovedb_version::version::FeatureVersion, +) -> Result<(), Error> { + if slot < 1 { + return Err(GroveVersionError::UnknownVersionMismatch { + method: method.to_string(), + known_versions: vec![1], + received: slot, + } + .into()); + } + Ok(()) +} + +impl GroveDb { + /// Append an entry to a PrivateDocumentStore subtree. + /// + /// The entry's byte length must equal the store's committed `entry_size`; + /// any other length is rejected before mutation. The append: + /// 1. Opens the store (a `BulkAppendTree` reconstructed from the + /// element's `total_count` / `chunk_power`) + /// 2. Appends the entry (auto-compacting a full buffer into a chunk) + /// 3. Updates the `PrivateDocumentStore` element with the new + /// `total_count` and re-binds the composite state root as the Merk + /// child hash + /// 4. Propagates changes through the GroveDB Merk hierarchy + /// + /// `path` must point to the parent of the store's key, and `key` must + /// identify a `PrivateDocumentStore` element. + /// + /// Returns `(state_root, position)`: the new composite state root and the + /// 0-based global position of the appended entry. + pub fn private_document_store_insert<'b, B, P>( + &self, + path: P, + key: &[u8], + entry: Vec, + transaction: TransactionArg, + grove_version: &GroveVersion, + ) -> CostResult<([u8; 32], u64), Error> + where + B: AsRef<[u8]> + 'b, + P: Into>, + { + let path: SubtreePath = path.into(); + let mut cost = OperationCost::default(); + + cost_return_on_error_no_add!( + cost, + check_pds_enabled( + "private_document_store_insert", + grove_version + .grovedb_versions + .operations + .private_document_store + .insert, + ) + ); + + let tx = TxRef::new(&self.db, transaction); + + // 1. Validate the element at path/key is a PrivateDocumentStore. + let element = cost_return_on_error!( + &mut cost, + self.get_raw_caching_optional(path.clone(), key, true, transaction, grove_version) + ); + + // Look through NonCounted: a wrapped PrivateDocumentStore is still one. + let (total_count, entry_size, chunk_power, existing_flags) = match element.underlying() { + Element::PrivateDocumentStore(tc, es, cp, flags) => (*tc, *es, *cp, flags.clone()), + _ => { + return Err(Error::InvalidInput( + "element is not a private document store", + )) + .wrap_with_cost(cost); + } + }; + + // 2. Open transactional storage (write-through cache + MMR overlay + // provide read-after-write visibility). + let store_path_vec = self.build_pds_path(&path, key); + let store_path_refs: Vec<&[u8]> = store_path_vec.iter().map(|v| v.as_slice()).collect(); + let store_path = SubtreePath::from(store_path_refs.as_slice()); + + let data_batch = StorageBatch::new(); + let storage_ctx = self + .db + .get_transactional_storage_context(store_path, Some(&data_batch), tx.as_ref()) + .unwrap_add_cost(&mut cost); + + // 3. Open the store and append (validates the entry size). + let mut store = cost_return_on_error_no_add!( + cost, + PrivateDocumentStore::from_state(total_count, entry_size, chunk_power, storage_ctx) + .map_err(map_pds_err) + ); + + let append_result = cost_return_on_error!( + &mut cost, + store.append(&entry).map(|r| r.map_err(map_pds_err)) + ); + + let new_state_root = append_result.state_root; + let position = append_result.global_position; + let new_total_count = store.total_count(); + + // Flush MMR overlay to storage (through the batch). + cost_return_on_error_no_add!(cost, store.commit_mmr().map_err(map_pds_err)); + + // Drop the store (and its storage context) before opening merk. + drop(store); + + // Commit data batch to make writes visible in the transaction. + // Note: this commits subtree data before the parent element update + // below. If the parent Merk update fails, the subtree data is orphaned + // in the transaction. This is the same pattern as other direct GroveDB + // operations — the caller is expected to rollback the tx on error. + // The batch path (preprocess_private_document_store_ops) avoids this + // by using a shared StorageBatch that commits atomically with all + // other ops. + cost_return_on_error!( + &mut cost, + self.db + .commit_multi_context_batch(data_batch, Some(tx.as_ref())) + .map_err(Into::into) + ); + + // 4. Update element in parent Merk. + let batch = StorageBatch::new(); + let mut parent_merk = cost_return_on_error!( + &mut cost, + self.open_transactional_merk_at_path( + path.clone(), + tx.as_ref(), + Some(&batch), + grove_version, + ) + ); + + let updated_element = Element::new_private_document_store( + new_total_count, + entry_size, + chunk_power, + existing_flags, + ); + + cost_return_on_error_into!( + &mut cost, + updated_element.insert_subtree( + &mut parent_merk, + key, + new_state_root, + None, + grove_version, + ) + ); + + // 5. Propagate changes from parent upward. + let mut merk_cache = HashMap::new(); + merk_cache.insert(path.clone(), parent_merk); + + cost_return_on_error!( + &mut cost, + self.propagate_changes_with_transaction( + merk_cache, + path, + tx.as_ref(), + &batch, + grove_version, + ) + ); + + // 6. Commit batch and transaction. + cost_return_on_error!( + &mut cost, + self.db + .commit_multi_context_batch(batch, Some(tx.as_ref())) + .map_err(Into::into) + ); + + tx.commit_local() + .map(|()| (new_state_root, position)) + .wrap_with_cost(cost) + } + + /// Get an entry from a PrivateDocumentStore by its global 0-based + /// position. + /// + /// Returns the raw fixed-size entry bytes, or `None` if the position is + /// out of range (`position >= total_count`). + pub fn private_document_store_get_value<'b, B, P>( + &self, + path: P, + key: &[u8], + global_position: u64, + transaction: TransactionArg, + grove_version: &GroveVersion, + ) -> CostResult>, Error> + where + B: AsRef<[u8]> + 'b, + P: Into>, + { + let path: SubtreePath = path.into(); + let mut cost = OperationCost::default(); + + cost_return_on_error_no_add!( + cost, + check_pds_enabled( + "private_document_store_get_value", + grove_version + .grovedb_versions + .operations + .private_document_store + .get_value, + ) + ); + + let tx = TxRef::new(&self.db, transaction); + + let element = cost_return_on_error!( + &mut cost, + self.get_raw_caching_optional(path.clone(), key, true, transaction, grove_version) + ); + + // Look through NonCounted: a wrapped PrivateDocumentStore is still one. + let (total_count, entry_size, chunk_power) = match element.underlying() { + Element::PrivateDocumentStore(tc, es, cp, _) => (*tc, *es, *cp), + _ => { + return Err(Error::InvalidInput( + "element is not a private document store", + )) + .wrap_with_cost(cost); + } + }; + + if global_position >= total_count { + return Ok(None).wrap_with_cost(cost); + } + + let store_path_vec = self.build_pds_path(&path, key); + let store_path_refs: Vec<&[u8]> = store_path_vec.iter().map(|v| v.as_slice()).collect(); + let store_path = SubtreePath::from(store_path_refs.as_slice()); + + let storage_ctx = self + .db + .get_transactional_storage_context(store_path, None, tx.as_ref()) + .unwrap_add_cost(&mut cost); + + let store = cost_return_on_error_no_add!( + cost, + PrivateDocumentStore::from_state(total_count, entry_size, chunk_power, storage_ctx) + .map_err(map_pds_err) + ); + + let value = cost_return_on_error_no_add!( + cost, + store.get_value(global_position).map_err(map_pds_err) + ); + + Ok(value).wrap_with_cost(cost) + } + + /// Get the total count of entries in a PrivateDocumentStore. + pub fn private_document_store_count<'b, B, P>( + &self, + path: P, + key: &[u8], + transaction: TransactionArg, + grove_version: &GroveVersion, + ) -> CostResult + where + B: AsRef<[u8]> + 'b, + P: Into>, + { + let path: SubtreePath = path.into(); + let mut cost = OperationCost::default(); + + cost_return_on_error_no_add!( + cost, + check_pds_enabled( + "private_document_store_count", + grove_version + .grovedb_versions + .operations + .private_document_store + .count, + ) + ); + + let element = cost_return_on_error!( + &mut cost, + self.get_raw_caching_optional(path, key, true, transaction, grove_version) + ); + + // Look through NonCounted: a wrapped PrivateDocumentStore is still one. + match element.into_underlying() { + Element::PrivateDocumentStore(total_count, ..) => Ok(total_count).wrap_with_cost(cost), + _ => Err(Error::InvalidInput( + "element is not a private document store", + )) + .wrap_with_cost(cost), + } + } + + /// Build the subtree path for a private document store at path/key. + fn build_pds_path>(&self, path: &SubtreePath, key: &[u8]) -> Vec> { + let mut v = path.to_vec(); + v.push(key.to_vec()); + v + } + + /// Preprocess `PrivateDocumentStoreInsert` ops in a batch. + /// + /// For each group of insert ops targeting the same store: + /// 1. Opens the store (BulkAppendTree + committed config) + /// 2. Appends all entries in order (each validated against `entry_size`) + /// 3. Replaces the ops with a single `ReplaceNonMerkTreeRoot` carrying + /// the new composite state root and updated element metadata + /// + /// The returned ops list contains no `PrivateDocumentStoreInsert` + /// variants. + pub(crate) fn preprocess_private_document_store_ops( + &self, + ops: Vec, + transaction: &Transaction, + storage_batch: &StorageBatch, + grove_version: &GroveVersion, + ) -> CostResult, Error> { + let mut cost = OperationCost::default(); + + let has_pds_ops = ops + .iter() + .any(|op| matches!(op.op, GroveOp::PrivateDocumentStoreInsert { .. })); + if !has_pds_ops { + return Ok(ops).wrap_with_cost(cost); + } + + cost_return_on_error_no_add!( + cost, + check_pds_enabled( + "batch GroveOp::PrivateDocumentStoreInsert", + grove_version + .grovedb_versions + .operations + .private_document_store + .insert, + ) + ); + + /// Tree path identifying a store in a batch (includes tree key as + /// last segment). + type TreePath = Vec>; + + // Group insert ops by path (which includes tree key). + let mut pds_groups: HashMap>> = HashMap::new(); + for op in ops.iter() { + if let GroveOp::PrivateDocumentStoreInsert { entry } = &op.op { + let tree_path = op.path.to_path(); + pds_groups.entry(tree_path).or_default().push(entry.clone()); + } + } + + let mut replacements: HashMap = HashMap::new(); + + for (tree_path, entries) in pds_groups.iter() { + // Extract parent path and tree key from the full path. + let (path_vec, key_bytes) = { + let mut p = tree_path.clone(); + let k = match p.pop() { + Some(k) => k, + None => { + return Err(Error::InvalidBatchOperation( + "append op path must have at least one segment", + )) + .wrap_with_cost(cost); + } + }; + (p, k) + }; + + // Read the existing element to verify it's a PrivateDocumentStore. + let path_slices: Vec<&[u8]> = path_vec.iter().map(|v| v.as_slice()).collect(); + let subtree_path = SubtreePath::from(path_slices.as_slice()); + + let element = cost_return_on_error!( + &mut cost, + self.get_raw_caching_optional( + subtree_path.clone(), + key_bytes.as_slice(), + true, + Some(transaction), + grove_version + ) + ); + + // Look through NonCounted: a wrapped PrivateDocumentStore is + // still one. + let (total_count, entry_size, chunk_power) = match element.underlying() { + Element::PrivateDocumentStore(tc, es, cp, _) => (*tc, *es, *cp), + _ => { + return Err(Error::InvalidInput( + "element is not a private document store", + )) + .wrap_with_cost(cost); + } + }; + + // Open transactional storage (write-through cache + MMR overlay + // provide read-after-write visibility). + let mut st_path_vec = path_vec.clone(); + st_path_vec.push(key_bytes.clone()); + let st_path_refs: Vec<&[u8]> = st_path_vec.iter().map(|v| v.as_slice()).collect(); + let st_path = SubtreePath::from(st_path_refs.as_slice()); + + let storage_ctx = self + .db + .get_transactional_storage_context(st_path, Some(storage_batch), transaction) + .unwrap_add_cost(&mut cost); + + let mut store = cost_return_on_error_no_add!( + cost, + PrivateDocumentStore::from_state(total_count, entry_size, chunk_power, storage_ctx) + .map_err(map_pds_err) + ); + + // Execute all inserts in order; each validates the entry size. + let mut last_state_root = None; + for entry in entries { + let r = cost_return_on_error!( + &mut cost, + store.append(entry).map(|r| r.map_err(map_pds_err)) + ); + last_state_root = Some(r.state_root); + } + + let new_state_root = match last_state_root { + Some(root) => root, + // Unreachable: groups only exist for at least one op. + None => cost_return_on_error_no_add!( + cost, + store.compute_current_state_root().map_err(map_pds_err) + ), + }; + let current_total_count = store.total_count(); + + // Flush MMR overlay to storage (through the batch). + cost_return_on_error_no_add!(cost, store.commit_mmr().map_err(map_pds_err)); + + // Drop the store (and its storage context). + drop(store); + + // Create a ReplaceNonMerkTreeRoot carrying the new state root and + // element metadata. Key is restored for downstream (from_ops, + // execute_ops_on_path). + let replacement = QualifiedGroveDbOp { + path: crate::batch::KeyInfoPath::from_known_owned_path(path_vec), + key: Some(crate::batch::key_info::KeyInfo::KnownKey(key_bytes)), + op: GroveOp::ReplaceNonMerkTreeRoot { + hash: new_state_root, + meta: crate::batch::NonMerkTreeMeta::PrivateDocumentStore { + total_count: current_total_count, + entry_size, + chunk_power, + }, + }, + }; + replacements.insert(tree_path.clone(), replacement); + } + + // Build the new ops list: keep non-PDS ops, replace the first PDS + // insert op per group with the replacement, skip the rest. + let mut first_seen: HashMap = HashMap::new(); + let mut result = Vec::with_capacity(ops.len()); + + for op in ops.into_iter() { + if matches!(op.op, GroveOp::PrivateDocumentStoreInsert { .. }) { + let tree_path = op.path.to_path(); + if !first_seen.contains_key(&tree_path) { + first_seen.insert(tree_path.clone(), true); + if let Some(replacement) = replacements.remove(&tree_path) { + result.push(replacement); + } + } + // Skip subsequent PDS ops for the same store. + } else { + result.push(op); + } + } + + Ok(result).wrap_with_cost(cost) + } +} diff --git a/grovedb/src/operations/proof/bind_terminal_non_merk_tree/v1.rs b/grovedb/src/operations/proof/bind_terminal_non_merk_tree/v1.rs index 664ccddc8..45b017f27 100644 --- a/grovedb/src/operations/proof/bind_terminal_non_merk_tree/v1.rs +++ b/grovedb/src/operations/proof/bind_terminal_non_merk_tree/v1.rs @@ -261,6 +261,47 @@ impl GroveDb { )) .wrap_with_cost(cost) } + Element::PrivateDocumentStore(total_count, entry_size, chunk_power, _) => { + // The state root binds the committed config even when the + // store is empty, so the empty case is the precomputed + // config-parametrized root rather than NULL_HASH. + if *total_count == 0 { + return Ok( + grovedb_private_document_store::empty_private_document_store_state_root( + *entry_size, + *chunk_power, + ), + ) + .wrap_with_cost(cost); + } + let storage_ctx = self + .db + .get_transactional_storage_context(storage_path, None, tx) + .unwrap_add_cost(&mut cost); + let store = cost_return_on_error_no_add!( + cost, + grovedb_private_document_store::PrivateDocumentStore::from_state( + *total_count, + *entry_size, + *chunk_power, + storage_ctx, + ) + .map_err(|e| Error::CorruptedData(format!( + "failed to open PrivateDocumentStore: {}", + e + ))) + ); + let state_root = cost_return_on_error_no_add!( + cost, + store.compute_current_state_root().map_err(|e| { + Error::CorruptedData(format!( + "private document store state root failed: {}", + e + )) + }) + ); + Ok(state_root).wrap_with_cost(cost) + } _ => Err(Error::CorruptedCodeExecution( "non_merk_tree_child_hash called on an element that is not a non-Merk tree", )) diff --git a/grovedb/src/operations/proof/generate.rs b/grovedb/src/operations/proof/generate.rs index b095ee0f5..c9274cbdd 100644 --- a/grovedb/src/operations/proof/generate.rs +++ b/grovedb/src/operations/proof/generate.rs @@ -815,13 +815,14 @@ impl GroveDb { Ok(Element::MmrTree(..)) | Ok(Element::BulkAppendTree(..)) | Ok(Element::DenseAppendOnlyFixedSizeTree(..)) + | Ok(Element::PrivateDocumentStore(..)) if !done_with_results && query.has_subquery_or_matching_in_path_on_key(key) => { return Err(Error::NotSupported( "V0 proofs do not support subqueries into MmrTree, \ - BulkAppendTree, or DenseAppendOnlyFixedSizeTree elements; \ - use prove_query_v1 instead" + BulkAppendTree, DenseAppendOnlyFixedSizeTree, or \ + PrivateDocumentStore elements; use prove_query_v1 instead" .to_string(), )) .wrap_with_cost(cost); @@ -847,6 +848,7 @@ impl GroveDb { | Ok(Element::ProvableSumIndexedTree(..)) | Ok(Element::ProvableCountIndexedTree(..)) | Ok(Element::ProvableCountProvableSumIndexedTree(..)) + | Ok(Element::PrivateDocumentStore(..)) if !done_with_results => { #[cfg(feature = "proof_debug")] @@ -890,7 +892,8 @@ impl GroveDb { | Ok(Element::DenseAppendOnlyFixedSizeTree(..)) | Ok(Element::ProvableSumIndexedTree(..)) | Ok(Element::ProvableCountIndexedTree(..)) - | Ok(Element::ProvableCountProvableSumIndexedTree(..)) => continue, + | Ok(Element::ProvableCountProvableSumIndexedTree(..)) + | Ok(Element::PrivateDocumentStore(..)) => continue, // NonCounted is unwrapped above via into_underlying(). Ok(Element::NonCounted(_)) | Ok(Element::NotSummed(_)) @@ -1952,6 +1955,25 @@ impl GroveDb { lower_layers.insert(key.clone(), layer_proof); } + // PrivateDocumentStore range-read proofs are not + // implemented yet (planned at the BulkAppendTree + // layer so the anchored DataCommitmentTree can + // inherit them); reject subqueries instead of + // shipping an unverifiable proof. Terminal + // (no-subquery) queries still bind the store's + // state root via the child-hash arm below. + Ok(Element::PrivateDocumentStore(..)) + if !done_with_results + && query.has_subquery_or_matching_in_path_on_key(key) => + { + return Err(Error::NotSupported( + "V1 proofs do not yet support subqueries into \ + PrivateDocumentStore elements" + .to_string(), + )) + .wrap_with_cost(cost); + } + // Subquery into CountIndexedTree: descend into // the primary like a regular tree, then wrap // the resulting Merk proof bytes with a 32-byte @@ -2367,6 +2389,7 @@ impl GroveDb { ref non_merk_elem @ Element::DenseAppendOnlyFixedSizeTree(..), ) | Ok(ref non_merk_elem @ Element::CommitmentTree(..)) + | Ok(ref non_merk_elem @ Element::PrivateDocumentStore(..)) if !done_with_results => { cost_return_on_error!( @@ -2663,7 +2686,8 @@ impl GroveDb { | Ok(Element::DenseAppendOnlyFixedSizeTree(..)) | Ok(Element::ProvableSumIndexedTree(..)) | Ok(Element::ProvableCountIndexedTree(..)) - | Ok(Element::ProvableCountProvableSumIndexedTree(..)) => continue, + | Ok(Element::ProvableCountProvableSumIndexedTree(..)) + | Ok(Element::PrivateDocumentStore(..)) => continue, // NonCounted is unwrapped above via into_underlying(). Ok(Element::NonCounted(_)) | Ok(Element::NotSummed(_)) diff --git a/grovedb/src/operations/proof/verify.rs b/grovedb/src/operations/proof/verify.rs index ab3db34cd..114e1c58a 100644 --- a/grovedb/src/operations/proof/verify.rs +++ b/grovedb/src/operations/proof/verify.rs @@ -1028,6 +1028,18 @@ impl GroveDb { } } } + // The V1 prover rejects subqueries into a + // PrivateDocumentStore (range-read proofs are not + // implemented yet), so an honest proof never + // carries a lower layer for one — refuse to + // fabricate a chain for it. + Element::PrivateDocumentStore(..) => { + return Err(Error::NotSupported( + "V1 proofs do not yet support lower layers for \ + PrivateDocumentStore elements" + .to_string(), + )); + } Element::Tree(Some(_), _) | Element::SumTree(Some(_), ..) | Element::BigSumTree(Some(_), ..) @@ -2425,6 +2437,7 @@ impl GroveDb { | Element::MmrTree(..) | Element::BulkAppendTree(..) | Element::DenseAppendOnlyFixedSizeTree(..) + | Element::PrivateDocumentStore(..) | Element::SumItem(..) | Element::Item(..) | Element::ItemWithSumItem(..) diff --git a/grovedb/src/tests/mod.rs b/grovedb/src/tests/mod.rs index b835bedc6..1898c8863 100644 --- a/grovedb/src/tests/mod.rs +++ b/grovedb/src/tests/mod.rs @@ -24,6 +24,7 @@ mod checkpoint_tests; mod chunk_branch_proof_tests; mod commitment_tree_tests; mod coverage_round7_tests; +mod private_document_store_tests; // NOTE: the former `count_indexed_tree_tests` (~12.3k LOC) was written // against the now-removed non-provable `Element::CountIndexedTree` and was // carried here behind a `#[cfg(any())]` gate that made it permanently dead — diff --git a/grovedb/src/tests/private_document_store_tests.rs b/grovedb/src/tests/private_document_store_tests.rs new file mode 100644 index 000000000..b7ddc01a2 --- /dev/null +++ b/grovedb/src/tests/private_document_store_tests.rs @@ -0,0 +1,863 @@ +//! PrivateDocumentStore integration tests +//! +//! Tests for PrivateDocumentStore as a GroveDB subtree type: an append-only +//! store of fixed-size opaque entries over a BulkAppendTree, with the +//! committed `{entry_size, chunk_power}` configuration bound into the state +//! root. Includes the fail-closed version-gating tests: every operation and +//! element creation must be rejected under GROVE_V3 and earlier. + +use grovedb_merk::proofs::{query::SubqueryBranch, Query}; +use grovedb_version::version::{v3::GROVE_V3, v4::GROVE_V4, GroveVersion}; + +use crate::{ + batch::QualifiedGroveDbOp, + operations::delete::DeleteOptions, + query_result_type::QueryResultType, + tests::{common::EMPTY_PATH, make_empty_grovedb}, + Element, Error, GroveDb, PathQuery, SizedQuery, +}; + +/// Small chunk power for tests — epoch size = 2^2 = 4, triggers compaction +/// after 4 appends. +const TEST_CHUNK_POWER: u8 = 2; +/// Committed entry size for tests. +const TEST_ENTRY_SIZE: u32 = 16; + +fn entry(byte: u8) -> Vec { + vec![byte; TEST_ENTRY_SIZE as usize] +} + +// =========================================================================== +// Element tests +// =========================================================================== + +#[test] +fn test_insert_private_document_store_at_root() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + db.insert( + EMPTY_PATH, + b"pds", + Element::empty_private_document_store(TEST_ENTRY_SIZE, TEST_CHUNK_POWER) + .expect("valid config"), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert private document store at root"); + + let element = db + .get(EMPTY_PATH, b"pds", None, grove_version) + .unwrap() + .expect("should retrieve private document store"); + assert!(element.is_private_document_store()); + assert!(element.is_any_tree()); + assert!(element.uses_non_merk_data_storage()); + assert_eq!(element.non_merk_entry_count(), Some(0)); + + // The whole database must pass the integrity walk. + let issues = db + .verify_grovedb(None, true, false, grove_version) + .expect("verify_grovedb"); + assert!(issues.is_empty(), "issues: {:?}", issues); +} + +#[test] +fn test_private_document_store_constructor_validation() { + // entry_size 0 is rejected. + assert!(Element::empty_private_document_store(0, TEST_CHUNK_POWER).is_err()); + // chunk_power outside 1..=16 is rejected. + assert!(Element::empty_private_document_store(TEST_ENTRY_SIZE, 0).is_err()); + assert!(Element::empty_private_document_store(TEST_ENTRY_SIZE, 17).is_err()); + // Boundary values are accepted. + assert!(Element::empty_private_document_store(1, 1).is_ok()); + assert!(Element::empty_private_document_store(u32::MAX, 16).is_ok()); +} + +#[test] +fn test_private_document_store_serialization_roundtrip() { + let grove_version = GroveVersion::latest(); + let original = Element::new_private_document_store(100, 216, 12, Some(vec![7, 8, 9])); + let bytes = original.serialize(grove_version).expect("serialize"); + // Discriminant 24 is the wire byte for PrivateDocumentStore. + assert_eq!(bytes[0], 24, "bincode discriminant must be 24"); + let deserialized = Element::deserialize(&bytes, grove_version).expect("deserialize"); + assert_eq!(deserialized, original); + + // NonCounted wrapper round-trips as [15, 24, ...]. + let wrapped = Element::new_non_counted(original.clone()).expect("wrap"); + let wrapped_bytes = wrapped.serialize(grove_version).expect("serialize wrapped"); + assert_eq!(&wrapped_bytes[0..2], &[15, 24]); + let unwrapped = Element::deserialize(&wrapped_bytes, grove_version).expect("deserialize"); + assert_eq!(unwrapped, wrapped); + assert!(unwrapped.is_private_document_store()); +} + +// =========================================================================== +// Operation tests: insert (append), get_value, count +// =========================================================================== + +/// Build a db with a parent tree at "root" and an empty store at +/// root/"docs". +fn make_db_with_store() -> crate::tests::TempGroveDb { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + db.insert( + EMPTY_PATH, + b"root", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert root tree"); + + db.insert( + &[b"root"], + b"docs", + Element::empty_private_document_store(TEST_ENTRY_SIZE, TEST_CHUNK_POWER) + .expect("valid config"), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert private document store"); + + db +} + +#[test] +fn test_private_document_store_insert_get_count_roundtrip() { + let grove_version = GroveVersion::latest(); + let db = make_db_with_store(); + + // 10 appends span two completed chunks (epoch size 4) plus a partial + // buffer, exercising both storage tiers. + let mut roots = Vec::new(); + for i in 0..10u8 { + let (state_root, position) = db + .private_document_store_insert(&[b"root"], b"docs", entry(i), None, grove_version) + .unwrap() + .expect("append entry"); + assert_eq!(position, i as u64); + roots.push(state_root); + } + // Every append must move the state root. + for w in roots.windows(2) { + assert_ne!(w[0], w[1]); + } + + assert_eq!( + db.private_document_store_count(&[b"root"], b"docs", None, grove_version) + .unwrap() + .expect("count"), + 10 + ); + + for i in 0..10u8 { + let value = db + .private_document_store_get_value(&[b"root"], b"docs", i as u64, None, grove_version) + .unwrap() + .expect("get value"); + assert_eq!(value, Some(entry(i)), "position {}", i); + } + // Out-of-range read returns None. + assert_eq!( + db.private_document_store_get_value(&[b"root"], b"docs", 10, None, grove_version) + .unwrap() + .expect("get out of range"), + None + ); + + // The whole database (including the value-hash binding of the store's + // state root and the entry-size integrity walk) must verify. + let issues = db + .verify_grovedb(None, true, false, grove_version) + .expect("verify_grovedb"); + assert!(issues.is_empty(), "issues: {:?}", issues); +} + +#[test] +fn test_private_document_store_insert_rejects_wrong_entry_size() { + let grove_version = GroveVersion::latest(); + let db = make_db_with_store(); + + for bad in [ + vec![0u8; TEST_ENTRY_SIZE as usize - 1], + vec![0u8; TEST_ENTRY_SIZE as usize + 1], + Vec::new(), + ] { + let result = db + .private_document_store_insert(&[b"root"], b"docs", bad, None, grove_version) + .unwrap(); + assert!( + matches!(result, Err(Error::PrivateDocumentStoreError(_))), + "expected entry-size rejection, got {:?}", + result + ); + } + + // Nothing was appended and the root hash is unchanged by rejections. + assert_eq!( + db.private_document_store_count(&[b"root"], b"docs", None, grove_version) + .unwrap() + .expect("count"), + 0 + ); +} + +#[test] +fn test_private_document_store_ops_reject_wrong_element_type() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + db.insert( + EMPTY_PATH, + b"plain", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert plain tree"); + + assert!(matches!( + db.private_document_store_insert(EMPTY_PATH, b"plain", entry(0), None, grove_version) + .unwrap(), + Err(Error::InvalidInput(_)) + )); + assert!(matches!( + db.private_document_store_get_value(EMPTY_PATH, b"plain", 0, None, grove_version) + .unwrap(), + Err(Error::InvalidInput(_)) + )); + assert!(matches!( + db.private_document_store_count(EMPTY_PATH, b"plain", None, grove_version) + .unwrap(), + Err(Error::InvalidInput(_)) + )); +} + +#[test] +fn test_private_document_store_root_hash_changes_and_persists() { + let grove_version = GroveVersion::latest(); + let db = make_db_with_store(); + + let root_before = db.root_hash(None, grove_version).unwrap().unwrap(); + db.private_document_store_insert(&[b"root"], b"docs", entry(1), None, grove_version) + .unwrap() + .expect("append"); + let root_after = db.root_hash(None, grove_version).unwrap().unwrap(); + assert_ne!( + root_before, root_after, + "append must change the grove root hash" + ); + + // The updated element reflects the new count. + let element = db + .get(&[b"root"], b"docs", None, grove_version) + .unwrap() + .expect("get element"); + assert_eq!(element.non_merk_entry_count(), Some(1)); +} + +// =========================================================================== +// Batch tests +// =========================================================================== + +#[test] +fn test_private_document_store_batch_create_and_append() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + // Create the parent and the empty store in one batch. + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op(vec![], b"root".to_vec(), Element::empty_tree()), + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"root".to_vec()], + b"docs".to_vec(), + Element::empty_private_document_store(TEST_ENTRY_SIZE, TEST_CHUNK_POWER) + .expect("valid config"), + ), + ]; + db.apply_batch(ops, None, None, grove_version) + .unwrap() + .expect("apply creation batch"); + + // Append 6 entries via batch ops (spans one compaction at epoch size 4). + let ops = (0..6u8) + .map(|i| { + QualifiedGroveDbOp::private_document_store_insert_op( + vec![b"root".to_vec(), b"docs".to_vec()], + entry(i), + ) + }) + .collect(); + db.apply_batch(ops, None, None, grove_version) + .unwrap() + .expect("apply append batch"); + + assert_eq!( + db.private_document_store_count(&[b"root"], b"docs", None, grove_version) + .unwrap() + .expect("count"), + 6 + ); + for i in 0..6u8 { + assert_eq!( + db.private_document_store_get_value(&[b"root"], b"docs", i as u64, None, grove_version) + .unwrap() + .expect("get"), + Some(entry(i)) + ); + } + + let issues = db + .verify_grovedb(None, true, false, grove_version) + .expect("verify_grovedb"); + assert!(issues.is_empty(), "issues: {:?}", issues); + + // Batch appends must produce the same root as the equivalent direct + // appends. + let direct_db = make_db_with_store(); + for i in 0..6u8 { + direct_db + .private_document_store_insert(&[b"root"], b"docs", entry(i), None, grove_version) + .unwrap() + .expect("direct append"); + } + assert_eq!( + db.root_hash(None, grove_version).unwrap().unwrap(), + direct_db.root_hash(None, grove_version).unwrap().unwrap(), + "batch and direct appends must converge to the same root hash" + ); +} + +#[test] +fn test_private_document_store_batch_rejects_wrong_entry_size() { + let grove_version = GroveVersion::latest(); + let db = make_db_with_store(); + + let ops = vec![QualifiedGroveDbOp::private_document_store_insert_op( + vec![b"root".to_vec(), b"docs".to_vec()], + vec![0u8; TEST_ENTRY_SIZE as usize + 1], + )]; + let result = db.apply_batch(ops, None, None, grove_version).unwrap(); + assert!( + matches!(result, Err(Error::PrivateDocumentStoreError(_))), + "expected entry-size rejection, got {:?}", + result + ); + assert_eq!( + db.private_document_store_count(&[b"root"], b"docs", None, grove_version) + .unwrap() + .expect("count"), + 0 + ); +} + +#[test] +fn test_private_document_store_batch_rejects_non_empty_element_insert() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + // Inserting a PrivateDocumentStore element claiming a non-zero count is + // rejected: the batch write binds the empty state root, so a non-zero + // claim would corrupt the root binding. + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"docs".to_vec(), + Element::new_private_document_store(5, TEST_ENTRY_SIZE, TEST_CHUNK_POWER, None), + )]; + let result = db.apply_batch(ops, None, None, grove_version).unwrap(); + assert!( + matches!(result, Err(Error::InvalidBatchOperation(_))), + "expected non-empty rejection, got {:?}", + result + ); + + // Same for an invalid config built without the checked constructors. + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"docs".to_vec(), + Element::new_private_document_store(0, 0, TEST_CHUNK_POWER, None), + )]; + let result = db.apply_batch(ops, None, None, grove_version).unwrap(); + assert!( + matches!(result, Err(Error::InvalidBatchOperation(_))), + "expected config rejection, got {:?}", + result + ); +} + +#[test] +fn test_private_document_store_direct_insert_rejects_non_empty_element() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + let result = db + .insert( + EMPTY_PATH, + b"docs", + Element::new_private_document_store(5, TEST_ENTRY_SIZE, TEST_CHUNK_POWER, None), + None, + None, + grove_version, + ) + .unwrap(); + assert!( + matches!(result, Err(Error::InvalidCodeExecution(_))), + "expected non-empty rejection, got {:?}", + result + ); +} + +#[test] +fn test_private_document_store_rejects_child_element_inserts() { + // Immutability is enforced by the type: the store's (always-empty) Merk + // may never hold child elements. Both the generic direct insert and the + // batch insert into the store's path must be rejected at the merk layer. + let grove_version = GroveVersion::latest(); + let db = make_db_with_store(); + + let result = db + .insert( + &[b"root".as_slice(), b"docs".as_slice()], + b"child", + Element::new_item(b"data".to_vec()), + None, + None, + grove_version, + ) + .unwrap(); + match &result { + Err(e) => assert!( + e.to_string() + .contains("private document stores cannot hold child elements"), + "unexpected rejection error: {}", + e + ), + Ok(_) => panic!("direct child insert into a store must be rejected"), + } + + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( + vec![b"root".to_vec(), b"docs".to_vec()], + b"child".to_vec(), + Element::new_item(b"data".to_vec()), + )]; + let result = db.apply_batch(ops, None, None, grove_version).unwrap(); + assert!( + result.is_err(), + "batch child insert into a store must be rejected, got {:?}", + result + ); + + // is_empty_tree on the store path still works (reads the element count + // from the parent). + assert!(db + .is_empty_tree( + &[b"root".as_slice(), b"docs".as_slice()], + None, + grove_version + ) + .unwrap() + .expect("is_empty_tree")); +} + +// =========================================================================== +// Delete tests +// =========================================================================== + +#[test] +fn test_private_document_store_delete() { + let grove_version = GroveVersion::latest(); + let db = make_db_with_store(); + + for i in 0..5u8 { + db.private_document_store_insert(&[b"root"], b"docs", entry(i), None, grove_version) + .unwrap() + .expect("append"); + } + + // A populated store requires allow_deleting_non_empty_trees. + let result = db + .delete( + &[b"root"], + b"docs", + Some(DeleteOptions { + allow_deleting_non_empty_trees: false, + deleting_non_empty_trees_returns_error: true, + ..Default::default() + }), + None, + grove_version, + ) + .unwrap(); + assert!(result.is_err(), "deleting populated store must error"); + + db.delete( + &[b"root"], + b"docs", + Some(DeleteOptions { + allow_deleting_non_empty_trees: true, + deleting_non_empty_trees_returns_error: false, + ..Default::default() + }), + None, + grove_version, + ) + .unwrap() + .expect("delete populated store"); + + assert!(matches!( + db.get(&[b"root"], b"docs", None, grove_version).unwrap(), + Err(Error::PathKeyNotFound(_)) + )); + + let issues = db + .verify_grovedb(None, true, false, grove_version) + .expect("verify_grovedb"); + assert!(issues.is_empty(), "issues: {:?}", issues); +} + +// =========================================================================== +// Proof tests (terminal binding only — range reads are a follow-up) +// =========================================================================== + +/// Query the store's key itself (no subquery) and verify the V1 proof. This +/// exercises the terminal non-Merk binding: the store's config-bound state +/// root is carried as the node's child hash and checked against the parent +/// commit. +fn prove_and_verify_store_element(db: &GroveDb, expected_count: u64) { + let grove_version = GroveVersion::latest(); + let path_query = PathQuery { + path: vec![b"root".to_vec()], + query: SizedQuery { + query: Query { + items: vec![grovedb_merk::proofs::query::QueryItem::Key( + b"docs".to_vec(), + )], + default_subquery_branch: SubqueryBranch { + subquery_path: None, + subquery: None, + }, + left_to_right: true, + conditional_subquery_branches: None, + add_parent_tree_on_subquery: false, + }, + limit: None, + offset: None, + }, + }; + + let proof_bytes = db + .prove_query(&path_query, None, grove_version) + .unwrap() + .expect("generate V1 proof for store element"); + + let (root_hash, result_set) = GroveDb::verify_query_with_options( + &proof_bytes, + &path_query, + grovedb_merk::proofs::query::VerifyOptions { + absence_proofs_for_non_existing_searched_keys: false, + verify_proof_succinctness: false, + include_empty_trees_in_result: true, + }, + grove_version, + ) + .expect("verify V1 proof for store element"); + + let expected_root = db.root_hash(None, grove_version).unwrap().unwrap(); + assert_eq!(root_hash, expected_root, "root hash should match"); + assert_eq!(result_set.len(), 1, "store element should be in results"); + let element = result_set[0] + .2 + .clone() + .expect("proved element should be present"); + match element { + Element::PrivateDocumentStore(total_count, entry_size, chunk_power, _) => { + assert_eq!(total_count, expected_count); + assert_eq!(entry_size, TEST_ENTRY_SIZE); + assert_eq!(chunk_power, TEST_CHUNK_POWER); + } + other => panic!("expected PrivateDocumentStore, got {:?}", other.type_str()), + } +} + +#[test] +fn test_private_document_store_prove_element_empty_and_populated() { + let grove_version = GroveVersion::latest(); + let db = make_db_with_store(); + + // Empty store: the terminal binding uses the config-parametrized empty + // state root. + prove_and_verify_store_element(&db, 0); + + // Populated store (across a compaction boundary). + for i in 0..5u8 { + db.private_document_store_insert(&[b"root"], b"docs", entry(i), None, grove_version) + .unwrap() + .expect("append"); + } + prove_and_verify_store_element(&db, 5); +} + +#[test] +fn test_private_document_store_subquery_proofs_not_supported() { + let grove_version = GroveVersion::latest(); + let db = make_db_with_store(); + db.private_document_store_insert(&[b"root"], b"docs", entry(1), None, grove_version) + .unwrap() + .expect("append"); + + let mut inner_query = Query::new(); + inner_query.insert_all(); + let path_query = PathQuery { + path: vec![b"root".to_vec()], + query: SizedQuery { + query: Query { + items: vec![grovedb_merk::proofs::query::QueryItem::Key( + b"docs".to_vec(), + )], + default_subquery_branch: SubqueryBranch { + subquery_path: None, + subquery: Some(inner_query.into()), + }, + left_to_right: true, + conditional_subquery_branches: None, + add_parent_tree_on_subquery: false, + }, + limit: None, + offset: None, + }, + }; + + let result = db.prove_query(&path_query, None, grove_version).unwrap(); + assert!( + matches!(result, Err(Error::NotSupported(_))), + "subquery proofs into a PrivateDocumentStore are not implemented yet, got {:?}", + result + ); +} + +// =========================================================================== +// Query tests (non-proof reads) +// =========================================================================== + +#[test] +fn test_private_document_store_path_query_rejects_tree_result() { + let grove_version = GroveVersion::latest(); + let db = make_db_with_store(); + + // Item-value queries cannot return trees. + let path_query = PathQuery::new_unsized( + vec![b"root".to_vec()], + Query::new_single_key(b"docs".to_vec()), + ); + let result = db + .query_item_value(&path_query, true, true, true, None, grove_version) + .unwrap(); + assert!(matches!(result, Err(Error::InvalidQuery(_)))); + + // Raw element queries return the store element itself. + let (elements, _) = db + .query_raw( + &path_query, + true, + true, + true, + QueryResultType::QueryElementResultType, + None, + grove_version, + ) + .unwrap() + .expect("raw query"); + assert_eq!(elements.len(), 1); +} + +// =========================================================================== +// Version gating: fail closed on GROVE_V3 and earlier +// =========================================================================== + +#[test] +fn test_private_document_store_version_slots_pinned() { + // The gate itself: all slots 0 on V3 (fail closed), 1 on V4. + let v3 = &GROVE_V3.grovedb_versions.operations.private_document_store; + assert_eq!(v3.element_creation, 0); + assert_eq!(v3.insert, 0); + assert_eq!(v3.get_value, 0); + assert_eq!(v3.count, 0); + let v4 = &GROVE_V4.grovedb_versions.operations.private_document_store; + assert_eq!(v4.element_creation, 1); + assert_eq!(v4.insert, 1); + assert_eq!(v4.get_value, 1); + assert_eq!(v4.count, 1); +} + +#[test] +fn test_private_document_store_element_creation_rejected_under_v3() { + let db = make_empty_grovedb(); + + // Direct insert of the element fails closed under V3. + let result = db + .insert( + EMPTY_PATH, + b"docs", + Element::empty_private_document_store(TEST_ENTRY_SIZE, TEST_CHUNK_POWER) + .expect("valid config"), + None, + None, + &GROVE_V3, + ) + .unwrap(); + assert!( + matches!(result, Err(Error::VersionError(_))), + "V3 direct insert must fail closed, got {:?}", + result + ); + + // Batch insert of the element fails closed under V3. + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"docs".to_vec(), + Element::empty_private_document_store(TEST_ENTRY_SIZE, TEST_CHUNK_POWER) + .expect("valid config"), + )]; + let result = db.apply_batch(ops, None, None, &GROVE_V3).unwrap(); + assert!( + matches!(result, Err(Error::VersionError(_))), + "V3 batch insert must fail closed, got {:?}", + result + ); + + // Under V4 the same element inserts fine (registered latest is V4). + db.insert( + EMPTY_PATH, + b"docs", + Element::empty_private_document_store(TEST_ENTRY_SIZE, TEST_CHUNK_POWER) + .expect("valid config"), + None, + None, + &GROVE_V4, + ) + .unwrap() + .expect("V4 insert works"); +} + +#[test] +fn test_private_document_store_operations_rejected_under_v3() { + // Build the store under V4 (latest), then attempt V3 operations on it. + let db = make_db_with_store(); + + let result = db + .private_document_store_insert(&[b"root"], b"docs", entry(0), None, &GROVE_V3) + .unwrap(); + assert!( + matches!(result, Err(Error::VersionError(_))), + "V3 insert op must fail closed, got {:?}", + result + ); + + let result = db + .private_document_store_get_value(&[b"root"], b"docs", 0, None, &GROVE_V3) + .unwrap(); + assert!( + matches!(result, Err(Error::VersionError(_))), + "V3 get_value op must fail closed, got {:?}", + result + ); + + let result = db + .private_document_store_count(&[b"root"], b"docs", None, &GROVE_V3) + .unwrap(); + assert!( + matches!(result, Err(Error::VersionError(_))), + "V3 count op must fail closed, got {:?}", + result + ); + + // Batch append op fails closed under V3. + let ops = vec![QualifiedGroveDbOp::private_document_store_insert_op( + vec![b"root".to_vec(), b"docs".to_vec()], + entry(0), + )]; + let result = db.apply_batch(ops, None, None, &GROVE_V3).unwrap(); + assert!( + matches!(result, Err(Error::VersionError(_))), + "V3 batch append must fail closed, got {:?}", + result + ); + + // Nothing was appended by the rejected calls. + assert_eq!( + db.private_document_store_count(&[b"root"], b"docs", None, GroveVersion::latest()) + .unwrap() + .expect("count under latest"), + 0 + ); +} + +// =========================================================================== +// State-root binding tests +// =========================================================================== + +#[test] +fn test_private_document_store_config_binds_root_hash() { + // Two stores with identical entries but different committed configs must + // produce different grove root hashes (the config is bound into the + // state root even for empty stores). + let grove_version = GroveVersion::latest(); + + let build = |entry_size: u32, chunk_power: u8| { + let db = make_empty_grovedb(); + db.insert( + EMPTY_PATH, + b"docs", + Element::empty_private_document_store(entry_size, chunk_power).expect("valid config"), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert store"); + db.root_hash(None, grove_version).unwrap().unwrap() + }; + + let base = build(TEST_ENTRY_SIZE, TEST_CHUNK_POWER); + // NOTE: entry_size and chunk_power are serialized in the element bytes, + // so the value hash differs too — the stronger claim (the *child hash* + // alone differs) is covered by the crate-level state-root tests. Here we + // pin that config changes are visible at the grove root. + assert_ne!(base, build(TEST_ENTRY_SIZE + 1, TEST_CHUNK_POWER)); + assert_ne!(base, build(TEST_ENTRY_SIZE, TEST_CHUNK_POWER + 1)); +} + +#[test] +fn test_private_document_store_empty_root_constant_matches_insert_binding() { + // The empty-root helper must agree with what the insert path binds: + // verify_grovedb recomputes the binding from the helper, so a clean + // verify after a fresh insert proves the two paths agree. + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + db.insert( + EMPTY_PATH, + b"docs", + Element::empty_private_document_store(TEST_ENTRY_SIZE, TEST_CHUNK_POWER) + .expect("valid config"), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert store"); + let issues = db + .verify_grovedb(None, true, false, grove_version) + .expect("verify_grovedb"); + assert!(issues.is_empty(), "issues: {:?}", issues); +} diff --git a/merk/src/element/costs.rs b/merk/src/element/costs.rs index 2779c4413..bd6ece380 100644 --- a/merk/src/element/costs.rs +++ b/merk/src/element/costs.rs @@ -12,7 +12,7 @@ use crate::{ tree_type::{ BIG_SUM_TREE_COST_SIZE, BULK_APPEND_TREE_COST_SIZE, COMMITMENT_TREE_COST_SIZE, COUNT_INDEXED_TREE_COST_SIZE, COUNT_SUM_TREE_COST_SIZE, COUNT_TREE_COST_SIZE, - DENSE_TREE_COST_SIZE, MMR_TREE_COST_SIZE, + DENSE_TREE_COST_SIZE, MMR_TREE_COST_SIZE, PRIVATE_DOCUMENT_STORE_COST_SIZE, PROVABLE_COUNT_PROVABLE_SUM_INDEXED_TREE_COST_SIZE, SUM_ITEM_COST_SIZE, SUM_TREE_COST_SIZE, TREE_COST_SIZE, }, @@ -99,6 +99,7 @@ impl ElementCostPrivateExtensions for Element { Element::ProvableCountProvableSumIndexedTree(..) => { Ok(PROVABLE_COUNT_PROVABLE_SUM_INDEXED_TREE_COST_SIZE) } + Element::PrivateDocumentStore(..) => Ok(PRIVATE_DOCUMENT_STORE_COST_SIZE), Element::NonCounted(inner) | Element::NotSummed(inner) | Element::NotCountedOrSummed(inner) => { @@ -313,6 +314,17 @@ impl ElementCostExtensions for Element { key_len, value_len, node_type, ) } + Element::PrivateDocumentStore(_, _, _, flags) => { + let flags_len = flags.map_or(0, |flags| { + let flags_len = flags.len() as u32; + flags_len + flags_len.required_space() as u32 + }); + let value_len = PRIVATE_DOCUMENT_STORE_COST_SIZE + flags_len + wrapper_overhead; + let key_len = key.len() as u32; + KV::layered_value_byte_cost_size_for_key_and_value_lengths( + key_len, value_len, node_type, + ) + } Element::SumItem(.., flags) => { let flags_len = flags.map_or(0, |flags| { let flags_len = flags.len() as u32; @@ -391,7 +403,8 @@ impl ElementCostExtensions for Element { | Element::DenseAppendOnlyFixedSizeTree(..) | Element::ProvableSumIndexedTree(..) | Element::ProvableCountIndexedTree(..) - | Element::ProvableCountProvableSumIndexedTree(..) => Some(cost), + | Element::ProvableCountProvableSumIndexedTree(..) + | Element::PrivateDocumentStore(..) => Some(cost), _ => None, } } @@ -411,7 +424,8 @@ impl ElementCostExtensions for Element { | Element::CommitmentTree(..) | Element::MmrTree(..) | Element::BulkAppendTree(..) - | Element::DenseAppendOnlyFixedSizeTree(..) => Some(LayeredValueDefinedCost(cost)), + | Element::DenseAppendOnlyFixedSizeTree(..) + | Element::PrivateDocumentStore(..) => Some(LayeredValueDefinedCost(cost)), Element::SumTree(..) => Some(LayeredValueDefinedCost(cost)), Element::BigSumTree(..) => Some(LayeredValueDefinedCost(cost)), Element::CountTree(..) => Some(LayeredValueDefinedCost(cost)), diff --git a/merk/src/element/delete.rs b/merk/src/element/delete.rs index e017b6ea3..f43bf8166 100644 --- a/merk/src/element/delete.rs +++ b/merk/src/element/delete.rs @@ -76,6 +76,7 @@ impl ElementDeleteFromStorageExtensions for Element { | (TreeType::DenseAppendOnlyFixedSizeTree(_), true) | (TreeType::ProvableSumIndexedTree, true) | (TreeType::ProvableCountProvableSumIndexedTree, true) + | (TreeType::PrivateDocumentStore(_), true) | (TreeType::ProvableCountIndexedTree, true) => Op::DeleteLayeredMaybeSpecialized, (TreeType::SumTree, false) | (TreeType::BigSumTree, false) @@ -91,6 +92,7 @@ impl ElementDeleteFromStorageExtensions for Element { | (TreeType::DenseAppendOnlyFixedSizeTree(_), false) | (TreeType::ProvableSumIndexedTree, false) | (TreeType::ProvableCountProvableSumIndexedTree, false) + | (TreeType::PrivateDocumentStore(_), false) | (TreeType::ProvableCountIndexedTree, false) => Op::DeleteMaybeSpecialized, }; let batch = [(key, op)]; @@ -154,6 +156,7 @@ impl ElementDeleteFromStorageExtensions for Element { | (TreeType::DenseAppendOnlyFixedSizeTree(_), true) | (TreeType::ProvableSumIndexedTree, true) | (TreeType::ProvableCountProvableSumIndexedTree, true) + | (TreeType::PrivateDocumentStore(_), true) | (TreeType::ProvableCountIndexedTree, true) => Op::DeleteLayeredMaybeSpecialized, (TreeType::SumTree, false) | (TreeType::BigSumTree, false) @@ -169,6 +172,7 @@ impl ElementDeleteFromStorageExtensions for Element { | (TreeType::DenseAppendOnlyFixedSizeTree(_), false) | (TreeType::ProvableSumIndexedTree, false) | (TreeType::ProvableCountProvableSumIndexedTree, false) + | (TreeType::PrivateDocumentStore(_), false) | (TreeType::ProvableCountIndexedTree, false) => Op::DeleteMaybeSpecialized, }; let batch = [(key, op)]; @@ -228,6 +232,7 @@ impl ElementDeleteFromStorageExtensions for Element { | (TreeType::DenseAppendOnlyFixedSizeTree(_), true) | (TreeType::ProvableSumIndexedTree, true) | (TreeType::ProvableCountProvableSumIndexedTree, true) + | (TreeType::PrivateDocumentStore(_), true) | (TreeType::ProvableCountIndexedTree, true) => Op::DeleteLayeredMaybeSpecialized, (TreeType::SumTree, false) | (TreeType::BigSumTree, false) @@ -243,6 +248,7 @@ impl ElementDeleteFromStorageExtensions for Element { | (TreeType::DenseAppendOnlyFixedSizeTree(_), false) | (TreeType::ProvableSumIndexedTree, false) | (TreeType::ProvableCountProvableSumIndexedTree, false) + | (TreeType::PrivateDocumentStore(_), false) | (TreeType::ProvableCountIndexedTree, false) => Op::DeleteMaybeSpecialized, }; let entry = (key, op); diff --git a/merk/src/element/get.rs b/merk/src/element/get.rs index 766b8fd35..d76649e52 100644 --- a/merk/src/element/get.rs +++ b/merk/src/element/get.rs @@ -468,7 +468,8 @@ impl ElementFetchFromStoragePrivateExtensions for Element { | Some(Element::DenseAppendOnlyFixedSizeTree(.., flags)) | Some(Element::ProvableSumIndexedTree(.., flags)) | Some(Element::ProvableCountIndexedTree(.., flags)) - | Some(Element::ProvableCountProvableSumIndexedTree(_, _, _, _, flags)) => { + | Some(Element::ProvableCountProvableSumIndexedTree(_, _, _, _, flags)) + | Some(Element::PrivateDocumentStore(.., flags)) => { // tree_type() looks through NonCounted, so this works for both // a bare tree and a NonCounted(tree). let tree_cost_size = element.as_ref().unwrap().tree_type().unwrap().cost_size(); @@ -589,7 +590,8 @@ impl ElementFetchFromStoragePrivateExtensions for Element { | Element::DenseAppendOnlyFixedSizeTree(.., flags) | Element::ProvableSumIndexedTree(.., flags) | Element::ProvableCountIndexedTree(.., flags) - | Element::ProvableCountProvableSumIndexedTree(_, _, _, _, flags) => { + | Element::ProvableCountProvableSumIndexedTree(_, _, _, _, flags) + | Element::PrivateDocumentStore(.., flags) => { // tree_type() looks through NonCounted. let tree_cost_size = element.tree_type().unwrap().cost_size(); let flags_len = flags.as_ref().map_or(0, |flags| { diff --git a/merk/src/element/insert.rs b/merk/src/element/insert.rs index 39303767f..1d24cde65 100644 --- a/merk/src/element/insert.rs +++ b/merk/src/element/insert.rs @@ -204,6 +204,12 @@ impl ElementInsertToStorageExtensions for Element { /// If transaction is passed, the operation will be committed on the /// transaction commit. fn validate_insertable_into(&self, tree_type: TreeType) -> Result<(), Error> { + if matches!(tree_type, TreeType::PrivateDocumentStore(_)) { + return Err(Error::InvalidInputError( + "private document stores cannot hold child elements; entries are appended \ + via the private_document_store_insert API", + )); + } if self.is_non_counted() && !tree_type.accepts_non_counted_children() { return Err(Error::InvalidInputError( "non-counted elements may only be inserted into non-provable count-bearing \ @@ -503,6 +509,14 @@ impl ElementInsertToStorageExtensions for Element { grove_version.grovedb_versions.element.insert_reference ); + if matches!(merk.tree_type, TreeType::PrivateDocumentStore(_)) { + return Err(Error::InvalidInputError( + "private document stores cannot hold child elements; entries are appended \ + via the private_document_store_insert API", + )) + .wrap_with_cost(Default::default()); + } + if self.is_non_counted() && !merk.tree_type.accepts_non_counted_children() { return Err(Error::InvalidInputError( "non-counted elements may only be inserted into non-provable count-bearing \ @@ -615,6 +629,14 @@ impl ElementInsertToStorageExtensions for Element { grove_version.grovedb_versions.element.insert_subtree ); + if matches!(merk.tree_type, TreeType::PrivateDocumentStore(_)) { + return Err(Error::InvalidInputError( + "private document stores cannot hold child elements; entries are appended \ + via the private_document_store_insert API", + )) + .wrap_with_cost(Default::default()); + } + if self.is_non_counted() && !merk.tree_type.accepts_non_counted_children() { return Err(Error::InvalidInputError( "non-counted elements may only be inserted into non-provable count-bearing \ @@ -756,6 +778,14 @@ impl ElementInsertToStorageExtensions for Element { .wrap_with_cost(Default::default()); } + if matches!(merk.tree_type, TreeType::PrivateDocumentStore(_)) { + return Err(Error::InvalidInputError( + "private document stores cannot hold child elements; entries are appended \ + via the private_document_store_insert API", + )) + .wrap_with_cost(Default::default()); + } + if self.is_non_counted() && !merk.tree_type.accepts_non_counted_children() { return Err(Error::InvalidInputError( "non-counted elements may only be inserted into non-provable count-bearing trees", diff --git a/merk/src/element/reconstruct.rs b/merk/src/element/reconstruct.rs index 332dfaad2..c943a3ab8 100644 --- a/merk/src/element/reconstruct.rs +++ b/merk/src/element/reconstruct.rs @@ -116,6 +116,12 @@ impl ElementReconstructExtensions for Element { Element::DenseAppendOnlyFixedSizeTree(c, h, f) => { Some(Element::DenseAppendOnlyFixedSizeTree(*c, *h, f.clone())) } + // Like the other non-Merk data trees, the private document store + // has no root key and no Merk aggregate — the element is + // reconstructed verbatim. + Element::PrivateDocumentStore(tc, es, cp, f) => { + Some(Element::PrivateDocumentStore(*tc, *es, *cp, f.clone())) + } // Recurse on the inner element and re-wrap. Without this, a // batch that mutates a subtree under a wrapped tree would lose // the wrapper on the parent's stored element when its root key diff --git a/merk/src/element/tree_type.rs b/merk/src/element/tree_type.rs index 45c4baf04..7518e0914 100644 --- a/merk/src/element/tree_type.rs +++ b/merk/src/element/tree_type.rs @@ -79,6 +79,9 @@ impl ElementTreeTypeExtensions for Element { primary_root_key, TreeType::ProvableCountProvableSumIndexedTree, )), + Element::PrivateDocumentStore(_, _, chunk_power, _) => { + Some((None, TreeType::PrivateDocumentStore(chunk_power))) + } Element::NonCounted(inner) | Element::NotSummed(inner) | Element::NotCountedOrSummed(inner) => inner.root_key_and_tree_type_owned(), @@ -128,6 +131,9 @@ impl ElementTreeTypeExtensions for Element { primary_root_key, TreeType::ProvableCountProvableSumIndexedTree, )), + Element::PrivateDocumentStore(_, _, chunk_power, _) => { + Some((&NONE_ROOT_KEY, TreeType::PrivateDocumentStore(*chunk_power))) + } Element::NonCounted(inner) | Element::NotSummed(inner) | Element::NotCountedOrSummed(inner) => inner.root_key_and_tree_type(), @@ -172,6 +178,9 @@ impl ElementTreeTypeExtensions for Element { Element::ProvableCountProvableSumIndexedTree(_, _, _, _, flags) => { Some((flags, TreeType::ProvableCountProvableSumIndexedTree)) } + Element::PrivateDocumentStore(_, _, chunk_power, flags) => { + Some((flags, TreeType::PrivateDocumentStore(*chunk_power))) + } Element::NonCounted(inner) | Element::NotSummed(inner) | Element::NotCountedOrSummed(inner) => inner.tree_flags_and_type(), @@ -209,6 +218,9 @@ impl ElementTreeTypeExtensions for Element { Element::ProvableCountProvableSumIndexedTree(..) => { Some(TreeType::ProvableCountProvableSumIndexedTree) } + Element::PrivateDocumentStore(_, _, chunk_power, _) => { + Some(TreeType::PrivateDocumentStore(*chunk_power)) + } Element::NonCounted(inner) | Element::NotSummed(inner) | Element::NotCountedOrSummed(inner) => inner.tree_type(), @@ -255,6 +267,7 @@ impl ElementTreeTypeExtensions for Element { Element::ProvableCountProvableSumIndexedTree(_, count_value, sum_value, _, _) => Some( TreeFeatureType::ProvableCountedAndProvableSummedMerkNode(*count_value, *sum_value), ), + Element::PrivateDocumentStore(..) => Some(BasicMerkNode), Element::NonCounted(inner) | Element::NotSummed(inner) | Element::NotCountedOrSummed(inner) => inner.tree_feature_type(), @@ -296,6 +309,9 @@ impl ElementTreeTypeExtensions for Element { Element::ProvableCountProvableSumIndexedTree(..) => { MaybeTree::Tree(TreeType::ProvableCountProvableSumIndexedTree) } + Element::PrivateDocumentStore(_, _, chunk_power, _) => { + MaybeTree::Tree(TreeType::PrivateDocumentStore(*chunk_power)) + } Element::NonCounted(inner) | Element::NotSummed(inner) | Element::NotCountedOrSummed(inner) => inner.maybe_tree_type(), @@ -334,6 +350,7 @@ impl ElementTreeTypeExtensions for Element { TreeType::MmrTree => Ok(BasicMerkNode), TreeType::BulkAppendTree(_) => Ok(BasicMerkNode), TreeType::DenseAppendOnlyFixedSizeTree(_) => Ok(BasicMerkNode), + TreeType::PrivateDocumentStore(_) => Ok(BasicMerkNode), // ProvableSumTree aggregates an i64 sum (same arithmetic // shape as plain SumTree) but carries it via // `ProvableSummedMerkNode` so the sum is baked into every diff --git a/merk/src/tree_type/costs.rs b/merk/src/tree_type/costs.rs index fdb2bdeec..d3aaabb4f 100644 --- a/merk/src/tree_type/costs.rs +++ b/merk/src/tree_type/costs.rs @@ -45,6 +45,12 @@ pub const BULK_APPEND_TREE_COST_SIZE: u32 = 9 + 1 + 2; // 12 /// height (u8) + 2 bytes overhead) pub const DENSE_TREE_COST_SIZE: u32 = 3 + 1 + 2; // 6 +/// The cost of a private document store (9 bytes total_count (u64 varint +/// worst case) + 5 bytes entry_size (u32 varint worst case) + 1 byte +/// chunk_power (u8) + 2 bytes overhead). Same shape as +/// `BULK_APPEND_TREE_COST_SIZE` plus the committed entry size field. +pub const PRIVATE_DOCUMENT_STORE_COST_SIZE: u32 = 9 + 5 + 1 + 2; // 17 + /// The cost of a count-indexed tree element. Same shape as `COUNT_TREE_COST_SIZE` /// but with one extra byte of overhead for the second `Option` field /// (the secondary root key). @@ -97,6 +103,7 @@ impl CostSize for TreeType { TreeType::ProvableCountProvableSumIndexedTree => { PROVABLE_COUNT_PROVABLE_SUM_INDEXED_TREE_COST_SIZE } + TreeType::PrivateDocumentStore(_) => PRIVATE_DOCUMENT_STORE_COST_SIZE, } } } diff --git a/merk/src/tree_type/mod.rs b/merk/src/tree_type/mod.rs index db60b7d94..2107c0736 100644 --- a/merk/src/tree_type/mod.rs +++ b/merk/src/tree_type/mod.rs @@ -82,6 +82,13 @@ pub enum TreeType { /// derived storage prefix and is itself a `ProvableCountProvableSumTree` /// so any axis can produce both count-on-range and sum-on-range proofs. ProvableCountProvableSumIndexedTree, + /// A private document store: an append-only store of fixed-size opaque + /// entries over a `BulkAppendTree`, with the `{entry_size, chunk_power}` + /// configuration bound into the state root. Carries the chunk power + /// (log2 of the epoch size) like `CommitmentTree` / `BulkAppendTree`; + /// the entry size lives only in the Element (it does not affect Merk + /// node layout). + PrivateDocumentStore(u8), } impl TreeType { @@ -106,6 +113,7 @@ impl TreeType { TreeType::ProvableSumIndexedTree => 13, TreeType::ProvableCountIndexedTree => 14, TreeType::ProvableCountProvableSumIndexedTree => 15, + TreeType::PrivateDocumentStore(_) => 16, } } } @@ -131,7 +139,8 @@ impl TryFrom for TreeType { 13 => Ok(TreeType::ProvableSumIndexedTree), 14 => Ok(TreeType::ProvableCountIndexedTree), 15 => Ok(TreeType::ProvableCountProvableSumIndexedTree), - n => Err(Error::UnknownTreeType(format!("got {}, max is 15", n))), + 16 => Ok(TreeType::PrivateDocumentStore(0)), + n => Err(Error::UnknownTreeType(format!("got {}, max is 16", n))), } } } @@ -157,6 +166,7 @@ impl fmt::Display for TreeType { TreeType::ProvableCountProvableSumIndexedTree => { "Provable Count Provable Sum Indexed Tree" } + TreeType::PrivateDocumentStore(_) => "Private Document Store", }; write!(f, "{}", s) } @@ -175,6 +185,7 @@ impl TreeType { | TreeType::MmrTree | TreeType::BulkAppendTree(_) | TreeType::DenseAppendOnlyFixedSizeTree(_) + | TreeType::PrivateDocumentStore(_) ) } @@ -309,6 +320,7 @@ impl TreeType { TreeType::ProvableSumIndexedTree => true, TreeType::ProvableCountIndexedTree => false, TreeType::ProvableCountProvableSumIndexedTree => true, + TreeType::PrivateDocumentStore(_) => false, } } @@ -338,6 +350,7 @@ impl TreeType { // The primary of a ProvableCountProvableSumIndexedTree mirrors // a ProvableCountProvableSumTree. TreeType::ProvableCountProvableSumIndexedTree => NodeType::ProvableCountProvableSumNode, + TreeType::PrivateDocumentStore(_) => NodeType::NormalNode, } } @@ -370,6 +383,7 @@ impl TreeType { TreeType::ProvableCountProvableSumIndexedTree => { TreeFeatureType::ProvableCountedAndProvableSummedMerkNode(0, 0) } + TreeType::PrivateDocumentStore(_) => TreeFeatureType::BasicMerkNode, } } @@ -403,6 +417,7 @@ impl TreeType { TreeType::ProvableCountProvableSumIndexedTree => { Some(ElementType::ProvableCountProvableSumIndexedTree) } + TreeType::PrivateDocumentStore(_) => Some(ElementType::PrivateDocumentStore), } } } @@ -427,6 +442,7 @@ mod tests { TreeType::DenseAppendOnlyFixedSizeTree(8), TreeType::ProvableSumTree, TreeType::ProvableCountProvableSumTree, + TreeType::PrivateDocumentStore(4), ]; for v in &variants { let d = v.discriminant(); @@ -438,10 +454,35 @@ mod tests { #[test] fn tree_type_try_from_invalid() { - assert!(TreeType::try_from(16u8).is_err()); + assert!(TreeType::try_from(17u8).is_err()); assert!(TreeType::try_from(255u8).is_err()); } + #[test] + fn private_document_store_tree_type_basics() { + // Discriminant round-trip (chunk_power defaults to 0 through the byte). + assert_eq!(TreeType::PrivateDocumentStore(4).discriminant(), 16); + assert_eq!( + TreeType::try_from(16u8).unwrap(), + TreeType::PrivateDocumentStore(0) + ); + let t = TreeType::PrivateDocumentStore(4); + assert_eq!(format!("{}", t), "Private Document Store"); + assert!(t.uses_non_merk_data_storage()); + assert!(!t.is_count_bearing()); + assert!(!t.is_sum_bearing()); + assert!(!t.is_count_and_sum_bearing()); + assert!(!t.is_indexed_primary()); + assert!(!t.is_count_indexed_primary()); + assert!(!t.accepts_non_counted_children()); + assert!(!t.accepts_not_counted_or_summed_children()); + assert!(!t.allows_sum_item()); + assert_eq!(t.inner_node_type(), NodeType::NormalNode); + assert_eq!(t.empty_tree_feature_type(), TreeFeatureType::BasicMerkNode); + assert_eq!(t.to_element_type(), Some(ElementType::PrivateDocumentStore)); + assert_eq!(t.cost_size(), PRIVATE_DOCUMENT_STORE_COST_SIZE); + } + #[test] fn indexed_tree_types_round_trip_through_discriminant() { for v in [ From 7df52c3f0ace3994ed809766aa0c61f62621ffc4 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Mon, 3 Aug 2026 08:41:57 +0700 Subject: [PATCH 03/19] test: raise PrivateDocumentStore patch coverage above the 90% codecov bar codecov/patch reported 76.7% on the initial push. Locally-measured single-run patch coverage is now 90.5% (1004/1109 diff lines); the gap to the earlier number was part genuine and part the known under-reporting when codecov merges the three nextest coverage shards. New targeted tests, each pinned to previously-unexercised patch lines: - estimated-costs: direct average/worst-case cost tests for GroveOp::PrivateDocumentStoreInsert, including the entry-size parametrization contract (doubling the entry length grows added_bytes by exactly the difference) and the compaction-blob worst-case bound. - v0 insert arm: no registered version pairs the v0 add_element_on_transaction implementation with an enabled PDS family, so a custom version (V4 with the slot dialed to 0) drives the arm: happy path, non-empty rejection, invalid-config rejection, plus a verify_grovedb pass proving v0 and v1 bind the identical config-parametrized empty root. - merk chokepoints: a PDS-typed Merk rejects every element-insert entry point (validate_insertable_into, insert, insert_if_not_exists, insert_reference, insert_subtree, insert_count_indexed_subtree). - merk dispatch: PrivateDocumentStore arms of every ElementTreeTypeExtensions method, reconstruct_with_root_key passthrough, and all four element cost paths against PRIVATE_DOCUMENT_STORE_COST_SIZE. - batch policy: duplicate InsertIfNotExists rejection, reference-to- updated-store rejection, the apply_operations_without_batching fallback, and op metadata pins (sort tag 19, can_mutate_child_count, NonMerkTreeMeta round-trip). - V0 prover: subqueries into a store rejected under GROVE_V2's locked V0 wire format while terminal element proofs still generate. - store crate: Debug/Display, error Display variants, and wiped-storage error paths (missing chunk reads, failing integrity walk). - element crate: serde shadow round-trip, flag-accessor round-trip, Display/type_str strings. - direct v1 insert config rejection and query_item_value_or_sum tree rejection. Remaining uncovered patch lines are defensive arms that need storage faults or forged proofs to reach (verify.rs PDS lower-layer rejection, compute_non_merk_child_hash fallbacks) or are unreachable by design (the batch propagation else-if for a type whose children are rejected). Co-Authored-By: Claude Fable 5 --- grovedb-element/src/element/helpers.rs | 6 + grovedb-element/src/element/mod.rs | 1 + grovedb-private-document-store/src/store.rs | 57 ++++ .../estimated_costs/average_case_costs.rs | 50 +++ .../batch/estimated_costs/worst_case_costs.rs | 25 ++ .../src/tests/private_document_store_tests.rs | 297 ++++++++++++++++++ merk/src/element/costs.rs | 52 +++ merk/src/element/insert.rs | 54 ++++ merk/src/element/reconstruct.rs | 12 + merk/src/element/tree_type.rs | 40 +++ 10 files changed, 594 insertions(+) diff --git a/grovedb-element/src/element/helpers.rs b/grovedb-element/src/element/helpers.rs index f17be7612..a4774c7da 100644 --- a/grovedb-element/src/element/helpers.rs +++ b/grovedb-element/src/element/helpers.rs @@ -1334,6 +1334,12 @@ mod flag_accessor_tests { check_accessors_round_trip(e); } + #[test] + fn private_document_store_flag_accessors() { + let e = Element::PrivateDocumentStore(0, 64, 4, flags()); + check_accessors_round_trip(e); + } + #[test] fn dense_append_only_tree_flag_accessors() { let e = Element::DenseAppendOnlyFixedSizeTree(0, 0, flags()); diff --git a/grovedb-element/src/element/mod.rs b/grovedb-element/src/element/mod.rs index a71d6d70a..372976148 100644 --- a/grovedb-element/src/element/mod.rs +++ b/grovedb-element/src/element/mod.rs @@ -1026,6 +1026,7 @@ mod serde_impl { let cases = vec![ Element::Item(b"abc".to_vec(), None), Element::SumTree(Some(b"r".to_vec()), 42, None), + Element::PrivateDocumentStore(9, 64, 4, Some(vec![1])), Element::new_non_counted(Element::Item(b"x".to_vec(), None)).unwrap(), Element::new_not_summed(Element::SumTree(None, 100, None)).unwrap(), Element::new_not_counted_or_summed(Element::CountSumTree(None, 3, 100, None)) diff --git a/grovedb-private-document-store/src/store.rs b/grovedb-private-document-store/src/store.rs index 2a73d37bc..25efd8349 100644 --- a/grovedb-private-document-store/src/store.rs +++ b/grovedb-private-document-store/src/store.rs @@ -320,6 +320,63 @@ impl<'db, S: StorageContext<'db>> PrivateDocumentStore { } } +#[cfg(test)] +mod error_path_tests { + use super::*; + use crate::test_utils::MemStorageContext; + + #[test] + fn test_debug_and_error_display() { + let store = PrivateDocumentStore::new(64, 4, MemStorageContext::new()).expect("new"); + let dbg = format!("{:?}", store); + assert!(dbg.contains("PrivateDocumentStore") && dbg.contains("entry_size: 64")); + assert_eq!(store.entry_size(), 64); + assert_eq!(store.chunk_power(), 4); + assert_eq!(store.epoch_size(), 16); + + assert!(format!( + "{}", + PrivateDocumentStoreError::InvalidEntrySize { + expected: 8, + actual: 9 + } + ) + .contains("expected 8 bytes, got 9")); + assert!( + format!("{}", PrivateDocumentStoreError::InvalidConfig("x".into())).contains("config") + ); + assert!( + format!("{}", PrivateDocumentStoreError::CorruptedData("x".into())) + .contains("corrupted") + ); + assert!(format!("{}", PrivateDocumentStoreError::InvalidData("x".into())).contains("data")); + } + + #[test] + fn test_reads_error_on_wiped_storage() { + // Populate past a compaction so both a chunk and the buffer exist, + // then wipe the backing storage and reopen with the same claimed + // state: chunk reads and the integrity walk must surface errors + // rather than fabricate data. + let mut store = PrivateDocumentStore::new(8, 2, MemStorageContext::new()).expect("new"); + for i in 0..6u8 { + store.append(&[i; 8]).unwrap().expect("append"); + } + store.commit_mmr().expect("commit mmr"); + let storage = PrivateDocumentStore::into_storage_for_test(store); + storage.data.borrow_mut().clear(); + + let broken = PrivateDocumentStore::from_state(6, 8, 2, storage).expect("reopen"); + // Position 0 lives in the (now missing) completed chunk. + assert!(broken.get_value(0).is_err()); + // The integrity walk fails on the missing chunk too. + assert!(broken.verify_entry_sizes().is_err()); + // Buffer positions read as missing entries in the walk; direct + // get_value returns the underlying error or None consistently. + assert!(broken.compute_current_state_root().is_err() || broken.get_value(5).is_err()); + } +} + #[cfg(test)] impl PrivateDocumentStore { /// Test helper: tear down the store and recover its storage context so a diff --git a/grovedb/src/batch/estimated_costs/average_case_costs.rs b/grovedb/src/batch/estimated_costs/average_case_costs.rs index 1ba34302e..d37517e85 100644 --- a/grovedb/src/batch/estimated_costs/average_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/average_case_costs.rs @@ -1788,6 +1788,56 @@ mod tests { ); } + #[test] + fn test_private_document_store_insert_average_case_cost_direct() { + let grove_version = GroveVersion::latest(); + let op = GroveOp::PrivateDocumentStoreInsert { + entry: vec![42u8; 64], + }; + let key = KeyInfo::KnownKey(b"pds_key".to_vec()); + let layer_info = EstimatedLayerInformation { + tree_type: TreeType::NormalTree, + estimated_layer_count: ApproximateElements(5), + estimated_layer_sizes: AllSubtrees(4, NoSumTrees, None), + }; + let cost = op + .average_case_cost(&key, &layer_info, false, grove_version) + .cost_as_result() + .expect("expected cost for private document store insert"); + // PrivateDocumentStoreInsert mirrors BulkAppend: parent replace cost + // plus buffer write + running hash; added bytes are entry-size + // parametrized (the entry length is the committed entry_size). + assert!( + cost.seek_count > 0, + "expected seek_count > 0, got {}", + cost.seek_count + ); + assert!( + cost.hash_node_calls > 0, + "expected hash_node_calls > 0, got {}", + cost.hash_node_calls + ); + assert!( + cost.storage_cost.added_bytes >= 64, + "expected added_bytes >= entry size, got {}", + cost.storage_cost.added_bytes + ); + + // Entry-size parametrization: doubling the entry length grows the + // added bytes by exactly the difference. + let op_large = GroveOp::PrivateDocumentStoreInsert { + entry: vec![42u8; 128], + }; + let cost_large = op_large + .average_case_cost(&key, &layer_info, false, grove_version) + .cost_as_result() + .expect("expected cost for larger entry"); + assert_eq!( + cost_large.storage_cost.added_bytes - cost.storage_cost.added_bytes, + 64 + ); + } + #[test] fn test_dense_tree_insert_average_case_cost_direct() { let grove_version = GroveVersion::latest(); diff --git a/grovedb/src/batch/estimated_costs/worst_case_costs.rs b/grovedb/src/batch/estimated_costs/worst_case_costs.rs index 8236aa256..4d40925a7 100644 --- a/grovedb/src/batch/estimated_costs/worst_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/worst_case_costs.rs @@ -1282,6 +1282,31 @@ mod tests { assert_eq!(cost.sinsemilla_hash_calls, 0); } + #[test] + fn test_private_document_store_insert_worst_case_cost_direct() { + let grove_version = GroveVersion::latest(); + let op = GroveOp::PrivateDocumentStoreInsert { + entry: vec![0u8; 128], + }; + let key = KeyInfo::KnownKey(b"pds_key".to_vec()); + let cost = op + .worst_case_cost( + &key, + TreeType::NormalTree, + &MaxElementsNumber(100), + false, + grove_version, + ) + .cost_as_result() + .expect("expected worst case cost for private document store insert"); + assert!(cost.seek_count > 0); + assert!(cost.hash_node_calls > 0); + // The worst case includes the compaction blob bound plus the + // entry-size-parametrized per-append write. + assert!(cost.storage_cost.added_bytes >= 128 + 65536); + assert_eq!(cost.sinsemilla_hash_calls, 0); + } + #[test] fn test_dense_tree_insert_worst_case_cost_direct() { let grove_version = GroveVersion::latest(); diff --git a/grovedb/src/tests/private_document_store_tests.rs b/grovedb/src/tests/private_document_store_tests.rs index b7ddc01a2..6441fcaab 100644 --- a/grovedb/src/tests/private_document_store_tests.rs +++ b/grovedb/src/tests/private_document_store_tests.rs @@ -416,6 +416,23 @@ fn test_private_document_store_direct_insert_rejects_non_empty_element() { "expected non-empty rejection, got {:?}", result ); + + // A caller-built element with an invalid config (bypassing the checked + // constructors) is rejected by the insert path as well. + for bad in [ + Element::new_private_document_store(0, 0, TEST_CHUNK_POWER, None), + Element::new_private_document_store(0, TEST_ENTRY_SIZE, 0, None), + Element::new_private_document_store(0, TEST_ENTRY_SIZE, 17, None), + ] { + let result = db + .insert(EMPTY_PATH, b"docs", bad, None, None, grove_version) + .unwrap(); + assert!( + matches!(result, Err(Error::InvalidInput(_))), + "expected config rejection, got {:?}", + result + ); + } } #[test] @@ -665,6 +682,12 @@ fn test_private_document_store_path_query_rejects_tree_result() { .unwrap(); assert!(matches!(result, Err(Error::InvalidQuery(_)))); + // The item-or-sum variant rejects stores too. + let result = db + .query_item_value_or_sum(&path_query, true, true, true, None, grove_version) + .unwrap(); + assert!(matches!(result, Err(Error::InvalidQuery(_)))); + // Raw element queries return the store element itself. let (elements, _) = db .query_raw( @@ -861,3 +884,277 @@ fn test_private_document_store_empty_root_constant_matches_insert_binding() { .expect("verify_grovedb"); assert!(issues.is_empty(), "issues: {:?}", issues); } + +// =========================================================================== +// Coverage: v0 insert path, batch policy branches, op metadata +// =========================================================================== + +/// Exercise the v0 `add_element_on_transaction` arm for +/// PrivateDocumentStore. No registered version pairs the v0 insert +/// implementation with an enabled PDS family (V1..V3 fail closed, V4 uses +/// v1), so drive it with a custom version: V4 with the insert +/// implementation slot dialed back to 0. +#[test] +fn test_private_document_store_insert_v0_element_path() { + let mut custom = GROVE_V4.clone(); + custom + .grovedb_versions + .operations + .insert + .add_element_on_transaction = 0; + let db = make_empty_grovedb(); + + db.insert( + EMPTY_PATH, + b"docs", + Element::empty_private_document_store(TEST_ENTRY_SIZE, TEST_CHUNK_POWER) + .expect("valid config"), + None, + None, + &custom, + ) + .unwrap() + .expect("v0 insert of empty store works"); + + // The v0 arm enforces the same emptiness and config validation. + let result = db + .insert( + EMPTY_PATH, + b"docs2", + Element::new_private_document_store(5, TEST_ENTRY_SIZE, TEST_CHUNK_POWER, None), + None, + None, + &custom, + ) + .unwrap(); + assert!(matches!(result, Err(Error::InvalidCodeExecution(_)))); + + let result = db + .insert( + EMPTY_PATH, + b"docs3", + Element::new_private_document_store(0, 0, TEST_CHUNK_POWER, None), + None, + None, + &custom, + ) + .unwrap(); + assert!(matches!(result, Err(Error::InvalidInput(_)))); + + // The binding written by the v0 arm matches what verify_grovedb + // recomputes (same config-parametrized empty root as the v1 arm). + let issues = db + .verify_grovedb(None, true, false, GroveVersion::latest()) + .expect("verify_grovedb"); + assert!(issues.is_empty(), "issues: {:?}", issues); +} + +#[test] +fn test_private_document_store_batch_insert_if_not_exists() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + let store_element = || { + Element::empty_private_document_store(TEST_ENTRY_SIZE, TEST_CHUNK_POWER) + .expect("valid config") + }; + + // First InsertIfNotExists creates the store. + let ops = vec![QualifiedGroveDbOp::insert_if_not_exists_op( + vec![], + b"docs".to_vec(), + store_element(), + )]; + db.apply_batch(ops, None, None, grove_version) + .unwrap() + .expect("first insert_if_not_exists creates the store"); + + // A second InsertIfNotExists over the same key hits the existence check + // and errors (validate_insertion_does_not_override defaults to erroring + // for InsertIfNotExists in batches). + let ops = vec![QualifiedGroveDbOp::insert_if_not_exists_op( + vec![], + b"docs".to_vec(), + store_element(), + )]; + let result = db.apply_batch(ops, None, None, grove_version).unwrap(); + assert!( + matches!(result, Err(Error::InvalidBatchOperation(_))), + "duplicate insert_if_not_exists must be rejected, got {:?}", + result + ); + + // The store is intact. + assert_eq!( + db.private_document_store_count(EMPTY_PATH, b"docs", None, grove_version) + .unwrap() + .expect("count"), + 0 + ); +} + +#[test] +fn test_private_document_store_reference_to_updated_store_rejected() { + let grove_version = GroveVersion::latest(); + let db = make_db_with_store(); + + // A reference in the same batch pointing at a store that receives + // appends must be rejected: references cannot point to trees being + // updated. + let ops = vec![ + QualifiedGroveDbOp::private_document_store_insert_op( + vec![b"root".to_vec(), b"docs".to_vec()], + entry(1), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"root".to_vec()], + b"link".to_vec(), + Element::new_reference( + crate::reference_path::ReferencePathType::AbsolutePathReference(vec![ + b"root".to_vec(), + b"docs".to_vec(), + ]), + ), + ), + ]; + let result = db.apply_batch(ops, None, None, grove_version).unwrap(); + assert!( + matches!(result, Err(Error::InvalidBatchOperation(_))), + "reference to a store being appended to must be rejected, got {:?}", + result + ); +} + +#[test] +fn test_private_document_store_op_metadata() { + // The sort tag is consensus-relevant for batch op ordering; pin it. + let op = crate::batch::GroveOp::PrivateDocumentStoreInsert { entry: entry(0) }; + assert_eq!(op.to_u8(), 19); + assert!(!op.can_mutate_child_count()); + + // NonMerkTreeMeta round-trips the element state. + let meta = crate::batch::NonMerkTreeMeta::PrivateDocumentStore { + total_count: 7, + entry_size: TEST_ENTRY_SIZE, + chunk_power: TEST_CHUNK_POWER, + }; + assert_eq!( + meta.to_tree_type(), + grovedb_merk::tree_type::TreeType::PrivateDocumentStore(TEST_CHUNK_POWER) + ); + assert_eq!(meta.count(), 7); + assert_eq!( + meta.to_element(Some(vec![1])), + Element::new_private_document_store(7, TEST_ENTRY_SIZE, TEST_CHUNK_POWER, Some(vec![1])) + ); +} + +#[test] +fn test_private_document_store_apply_without_batching() { + // The non-batch fallback path dispatches PrivateDocumentStoreInsert ops + // to the direct typed insert. + let grove_version = GroveVersion::latest(); + let db = make_db_with_store(); + + let ops = vec![ + QualifiedGroveDbOp::private_document_store_insert_op( + vec![b"root".to_vec(), b"docs".to_vec()], + entry(1), + ), + QualifiedGroveDbOp::private_document_store_insert_op( + vec![b"root".to_vec(), b"docs".to_vec()], + entry(2), + ), + ]; + db.apply_operations_without_batching(ops, None, None, grove_version) + .unwrap() + .expect("apply without batching"); + + assert_eq!( + db.private_document_store_count(&[b"root"], b"docs", None, grove_version) + .unwrap() + .expect("count"), + 2 + ); + assert_eq!( + db.private_document_store_get_value(&[b"root"], b"docs", 1, None, grove_version) + .unwrap() + .expect("get"), + Some(entry(2)) + ); + + let issues = db + .verify_grovedb(None, true, false, grove_version) + .expect("verify_grovedb"); + assert!(issues.is_empty(), "issues: {:?}", issues); +} + +#[test] +fn test_private_document_store_element_display_and_visualize() { + let element = Element::new_private_document_store(3, TEST_ENTRY_SIZE, TEST_CHUNK_POWER, None); + let display = format!("{}", element); + assert!( + display.contains("PrivateDocumentStore") + && display.contains("entry_size: 16") + && display.contains("chunk_power: 2"), + "unexpected display: {}", + display + ); + assert_eq!(element.type_str(), "private_document_store"); + assert_eq!( + Element::new_non_counted(element).expect("wrap").type_str(), + "non_counted private_document_store" + ); +} + +#[test] +fn test_private_document_store_v0_prover_rejects_subqueries() { + use grovedb_version::version::v2::GROVE_V2; + + // Build the store under the latest version, then generate proofs with + // GROVE_V2 (whose prover is the locked V0 wire format). Subqueries into + // the store must be rejected by the V0 dispatch... + let grove_version = GroveVersion::latest(); + let db = make_db_with_store(); + db.private_document_store_insert(&[b"root"], b"docs", entry(1), None, grove_version) + .unwrap() + .expect("append"); + + let mut inner_query = Query::new(); + inner_query.insert_all(); + let subquery = PathQuery { + path: vec![b"root".to_vec()], + query: SizedQuery { + query: Query { + items: vec![grovedb_merk::proofs::query::QueryItem::Key( + b"docs".to_vec(), + )], + default_subquery_branch: SubqueryBranch { + subquery_path: None, + subquery: Some(inner_query.into()), + }, + left_to_right: true, + conditional_subquery_branches: None, + add_parent_tree_on_subquery: false, + }, + limit: None, + offset: None, + }, + }; + let result = db.prove_query(&subquery, None, &GROVE_V2).unwrap(); + assert!( + matches!(result, Err(Error::NotSupported(_))), + "V0 subquery into a store must be rejected, got {:?}", + result + ); + + // ...while a terminal query for the element itself still proves (the + // node passes through as a result, V0 shape unchanged). + let terminal = PathQuery::new_unsized( + vec![b"root".to_vec()], + Query::new_single_key(b"docs".to_vec()), + ); + db.prove_query(&terminal, None, &GROVE_V2) + .unwrap() + .expect("V0 terminal proof over a store element generates"); +} diff --git a/merk/src/element/costs.rs b/merk/src/element/costs.rs index bd6ece380..1fc33bb12 100644 --- a/merk/src/element/costs.rs +++ b/merk/src/element/costs.rs @@ -457,3 +457,55 @@ impl ElementCostExtensions for Element { element.value_defined_cost(grove_version) } } + +#[cfg(test)] +mod tests { + use grovedb_version::version::GroveVersion; + + use super::*; + + #[test] + fn private_document_store_cost_paths() { + let grove_version = GroveVersion::latest(); + let element = Element::new_private_document_store(5, 64, 4, Some(vec![1, 2])); + + // Specialized (layered) cost uses the entry-size-carrying constant. + assert_eq!( + element.get_specialized_cost(grove_version).expect("cost"), + crate::tree_type::PRIVATE_DOCUMENT_STORE_COST_SIZE + ); + + // Layered value defined cost = specialized cost + flags overhead. + let layered = element + .layered_value_defined_cost(grove_version) + .expect("layered cost"); + assert_eq!( + layered, + crate::tree_type::PRIVATE_DOCUMENT_STORE_COST_SIZE + 3 + ); + assert!(matches!( + element.value_defined_cost(grove_version), + Some(LayeredValueDefinedCost(c)) if c == layered + )); + // Not an item-like element: no specialized (sum-item) value cost. + assert!(element + .specialized_value_defined_cost(grove_version) + .is_none()); + + // The serialized-value cost path takes the PDS block in + // specialized_costs_for_key_value. + let serialized = element.serialize(grove_version).expect("serialize"); + let cost = Element::specialized_costs_for_key_value( + b"key", + &serialized, + NodeType::NormalNode, + grove_version, + ) + .expect("cost for key value"); + assert!(cost > 0); + assert_eq!( + Element::value_defined_cost_for_serialized_value(&serialized, grove_version), + Some(LayeredValueDefinedCost(layered)) + ); + } +} diff --git a/merk/src/element/insert.rs b/merk/src/element/insert.rs index 1d24cde65..6dca2c18b 100644 --- a/merk/src/element/insert.rs +++ b/merk/src/element/insert.rs @@ -939,6 +939,60 @@ mod tests { TreeType, }; + #[test] + fn private_document_store_merk_rejects_all_child_element_inserts() { + // A PrivateDocumentStore's Merk must stay empty forever: every + // element-insert entry point rejects when the destination Merk is + // PDS-typed — validate_insertable_into (used by insert / + // insert_if_not_exists), the inline guards in insert_reference and + // insert_subtree, and insert_count_indexed_subtree. + let grove_version = GroveVersion::latest(); + let mut merk = + TempMerk::new_with_tree_type(grove_version, TreeType::PrivateDocumentStore(4)); + + let item = Element::new_item(b"value".to_vec()); + assert!(item + .validate_insertable_into(TreeType::PrivateDocumentStore(4)) + .is_err()); + assert!(item + .insert(&mut merk, b"k", None, grove_version) + .unwrap() + .is_err()); + assert!(item + .insert_if_not_exists(&mut merk, b"k", None, grove_version) + .unwrap() + .is_err()); + + let reference = Element::new_reference( + grovedb_element::reference_path::ReferencePathType::AbsolutePathReference(vec![ + b"a".to_vec() + ]), + ); + assert!(reference + .insert_reference(&mut merk, b"k", [0u8; 32], None, grove_version) + .unwrap() + .is_err()); + + let subtree = Element::empty_tree(); + assert!(subtree + .insert_subtree(&mut merk, b"k", [0u8; 32], None, grove_version) + .unwrap() + .is_err()); + + let cidx = Element::empty_provable_count_indexed_tree(); + assert!(cidx + .insert_count_indexed_subtree( + &mut merk, + b"k", + [0u8; 32], + [0u8; 32], + None, + grove_version + ) + .unwrap() + .is_err()); + } + #[test] fn test_success_insert() { let grove_version = GroveVersion::latest(); diff --git a/merk/src/element/reconstruct.rs b/merk/src/element/reconstruct.rs index c943a3ab8..26587dcfe 100644 --- a/merk/src/element/reconstruct.rs +++ b/merk/src/element/reconstruct.rs @@ -203,6 +203,18 @@ mod tests { use super::ElementReconstructExtensions; use crate::tree::AggregateData; + #[test] + fn reconstruct_private_document_store_is_verbatim() { + // Like the other non-Merk data trees, a PrivateDocumentStore has no + // root key and no Merk aggregate: reconstruction ignores both inputs + // and returns the element verbatim. + let element = Element::new_private_document_store(9, 64, 4, Some(vec![1, 2])); + let reconstructed = element + .reconstruct_with_root_key(Some(b"ignored".to_vec()), AggregateData::NoAggregateData) + .expect("reconstruct ok"); + assert_eq!(reconstructed, element); + } + #[test] fn reconstruct_preserves_non_counted_wrapper() { // A NonCounted-wrapped tree must come back wrapped after a root-key diff --git a/merk/src/element/tree_type.rs b/merk/src/element/tree_type.rs index 7518e0914..c911cc701 100644 --- a/merk/src/element/tree_type.rs +++ b/merk/src/element/tree_type.rs @@ -674,6 +674,46 @@ mod tests { assert_eq!(e.tree_feature_type(), Some(BasicMerkNode)); } + #[test] + fn private_document_store_extension_arms_direct() { + // PrivateDocumentStore carries {entry_size, chunk_power}; only the + // chunk_power flows into the TreeType. Drive every dispatch arm. + let entry_size = 64u32; + let chunk_power = 4u8; + let e = Element::PrivateDocumentStore(3, entry_size, chunk_power, Some(vec![1])); + + let (rk, tt) = e.root_key_and_tree_type().expect("Some"); + assert!(rk.is_none()); + assert_eq!(tt, TreeType::PrivateDocumentStore(chunk_power)); + + let (rk, tt) = e.clone().root_key_and_tree_type_owned().expect("Some"); + assert!(rk.is_none()); + assert_eq!(tt, TreeType::PrivateDocumentStore(chunk_power)); + + assert_eq!( + e.tree_type(), + Some(TreeType::PrivateDocumentStore(chunk_power)) + ); + assert_eq!( + e.maybe_tree_type(), + MaybeTree::Tree(TreeType::PrivateDocumentStore(chunk_power)) + ); + + let (flags, tt) = e.tree_flags_and_type().expect("Some"); + assert!(flags.is_some()); + assert_eq!(tt, TreeType::PrivateDocumentStore(chunk_power)); + assert_eq!(e.tree_feature_type(), Some(BasicMerkNode)); + + // Children of a PDS parent (unreachable in practice — inserts are + // rejected) still resolve to BasicMerkNode for exhaustiveness. + assert_eq!( + Element::new_item(b"x".to_vec()) + .get_feature_type(TreeType::PrivateDocumentStore(chunk_power)) + .expect("feature type"), + BasicMerkNode + ); + } + #[test] fn bulk_append_tree_extension_arms_direct() { let chunk_power = 8u8; From fff836337e48225016cbd67807a412976381a98a Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Mon, 3 Aug 2026 13:57:42 +0700 Subject: [PATCH 04/19] fix: address CodeRabbit review on PrivateDocumentStore (PR #787) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Seven findings, all addressed: 1. Serialization discriminant matrix: add the PrivateDocumentStore row (discriminant 24) and bump the expected base-variant count to 22. 2. Config validation at every ingress: an invalid committed config (entry_size 0 or chunk_power outside 1..=16) is now unrepresentable — Element::serialize, Element::deserialize, and the serde codec all reject it via the new validate_private_document_store_config helper (which looks through NonCounted). Safe to enforce at the codec level because no validly-written bytes can violate it: the checked constructors and both insert paths already enforce the same bound. new_private_document_store stays an unchecked restoration constructor (mirroring new_commitment_tree / new_bulk_append_tree) and is now documented as such. 3. Average-case hash calls: AVG_HASH_CALLS bumped 1 -> 2 for PrivateDocumentStoreInsert — a PDS append unconditionally derives the composite pds_state root on top of the bulk state root. 4. Wrong-entry-size test now asserts the grove root hash is unchanged by rejected appends, not just the count. 5. Delete test now proves non-Merk storage reclamation: after deletion the store's data namespace is raw-iterated and asserted empty, and a store recreated at the same path starts from position 0. 6. The V0 terminal proof test verifies the proof (root-hash binding + result set) instead of only generating it. 7. Batch-builder bypass: rather than re-plumbing TreeType through every *_into_batch_operations signature, a single chokepoint in Merk::apply_unchecked — the funnel every public apply variant goes through — rejects any non-empty batch aimed at a PrivateDocumentStore-typed Merk. Queued builder ops can only take effect through an apply, so this closes every builder route at once; covered by a test that queues via the builders and asserts the apply is rejected. The chokepoint exposed a pre-existing quirk in delete: the !is_empty branch reopens the PARENT merk labeled with the DELETED CHILD's tree type (see the long-standing `todo` there). For a PDS child that label tripped the guard, so PDS deletions now label the reopened parent with the parent's actual tree type; every existing type keeps the historical label byte-for-byte to avoid any behavior change on released paths. Co-Authored-By: Claude Fable 5 --- grovedb-element/src/element/constructor.rs | 10 ++- grovedb-element/src/element/mod.rs | 33 ++++++++ grovedb-element/src/element/serialize.rs | 22 ++++++ grovedb-element/src/element_type.rs | 14 +++- .../element_display_and_serialization.rs | 36 +++++++++ .../estimated_costs/average_case_costs.rs | 15 ++-- grovedb/src/operations/delete/mod.rs | 16 +++- .../src/tests/private_document_store_tests.rs | 76 ++++++++++++++++++- merk/src/element/insert.rs | 50 ++++++++++++ merk/src/merk/apply.rs | 17 ++++- 10 files changed, 272 insertions(+), 17 deletions(-) diff --git a/grovedb-element/src/element/constructor.rs b/grovedb-element/src/element/constructor.rs index cbc41d626..9bc7984fd 100644 --- a/grovedb-element/src/element/constructor.rs +++ b/grovedb-element/src/element/constructor.rs @@ -541,7 +541,15 @@ impl Element { )) } - /// Set element to a private document store with all fields + /// Set element to a private document store with all fields. + /// + /// Restoration constructor: unchecked, mirroring `new_commitment_tree` / + /// `new_bulk_append_tree` — it rebuilds an element from already-validated + /// state (stored bytes, batch metadata). Invalid configurations are + /// rejected at every real ingress: the `empty_*` constructors, the direct + /// and batch insert paths, and both (de)serialization codecs + /// (`Element::serialize` / `Element::deserialize` / serde) via + /// [`Element::validate_private_document_store_config`]. pub fn new_private_document_store( total_count: u64, entry_size: u32, diff --git a/grovedb-element/src/element/mod.rs b/grovedb-element/src/element/mod.rs index 372976148..7fdd54083 100644 --- a/grovedb-element/src/element/mod.rs +++ b/grovedb-element/src/element/mod.rs @@ -772,6 +772,34 @@ impl Element { self.element_type().as_str() } + /// Validate the committed configuration of a `PrivateDocumentStore` + /// element, looking through `NonCounted`: `entry_size` must be non-zero + /// and `chunk_power` must be in `1..=16` (the underlying `BulkAppendTree` + /// dense-buffer height range). Returns `Ok(())` for every other variant. + /// + /// The configuration is committed into the store's state root, so an + /// unusable configuration must not be representable: the checked + /// constructors, the insert paths, and both (de)serialization codecs + /// (bincode and serde) all enforce this. `new_private_document_store` + /// itself stays unchecked — it is the restoration constructor used to + /// rebuild elements from already-validated on-disk state, mirroring + /// `new_commitment_tree` / `new_bulk_append_tree`. + pub fn validate_private_document_store_config(&self) -> Result<(), crate::error::ElementError> { + if let Element::PrivateDocumentStore(_, entry_size, chunk_power, _) = self.underlying() { + if *entry_size == 0 { + return Err(crate::error::ElementError::InvalidInput( + "private document store entry_size must be non-zero", + )); + } + if !(1..=16).contains(chunk_power) { + return Err(crate::error::ElementError::InvalidInput( + "private document store chunk_power must be between 1 and 16", + )); + } + } + Ok(()) + } + /// Verify the wrapper invariants for `self`: /// - `NonCounted`, `NotSummed`, and `NotCountedOrSummed` may not nest /// in any combination. @@ -992,6 +1020,11 @@ mod serde_impl { // built by recursive `From` calls, so the check // at each level catches a violation at any depth. Self::check_recursive_wrapper_invariants(&element).map_err(D::Error::custom)?; + // A PrivateDocumentStore's committed config must be valid at + // every ingress — including this external-tooling codec. + element + .validate_private_document_store_config() + .map_err(D::Error::custom)?; Ok(element) } } diff --git a/grovedb-element/src/element/serialize.rs b/grovedb-element/src/element/serialize.rs index 3ea841bcb..16a7ad338 100644 --- a/grovedb-element/src/element/serialize.rs +++ b/grovedb-element/src/element/serialize.rs @@ -79,6 +79,16 @@ impl Element { } } } + // A PrivateDocumentStore's committed config is bound into its state + // root; an unusable config must never reach disk. The checked + // constructors and insert paths already enforce this — the codec + // check closes the caller-built-element gap. + if let Err(e) = self.validate_private_document_store_config() { + return Err(ElementError::CorruptedData(format!( + "invalid private document store config: {}", + e + ))); + } let config = config::standard().with_big_endian().with_no_limit(); bincode::encode_to_vec(self, config) .map_err(|e| ElementError::CorruptedData(format!("unable to serialize element {}", e))) @@ -188,6 +198,18 @@ impl Element { } } } + // Reject a PrivateDocumentStore with an unusable committed config + // (entry_size 0 or chunk_power outside 1..=16). No such bytes can + // legitimately exist — serialization and every insert path enforce + // the same bound — so this cannot reject previously-valid data; + // it makes the invalid configuration unrepresentable, mirroring + // the wrapper-invariant checks above. + if let Err(e) = elem.validate_private_document_store_config() { + return Err(ElementError::CorruptedData(format!( + "deserialized private document store with invalid config: {}", + e + ))); + } Ok(elem) } } diff --git a/grovedb-element/src/element_type.rs b/grovedb-element/src/element_type.rs index 3b19b9d4d..dc22990b4 100644 --- a/grovedb-element/src/element_type.rs +++ b/grovedb-element/src/element_type.rs @@ -2001,16 +2001,22 @@ mod tests { ElementType::ProvableCountProvableSumIndexedTree, "ProvableCountProvableSumIndexedTree", ), + // discriminant 24 + ( + Element::PrivateDocumentStore(0, 64, 4, None), + ElementType::PrivateDocumentStore, + "PrivateDocumentStore", + ), ]; - // Verify we're testing all 21 base discriminants: 0..=14, 18, 19, - // 20, 21, 22, 23. (15 = NonCounted wrapper byte, 16 = NotSummed + // Verify we're testing all 22 base discriminants: 0..=14, 18, 19, + // 20, 21, 22, 23, 24. (15 = NonCounted wrapper byte, 16 = NotSummed // wrapper byte, 17 = NotCountedOrSummed wrapper byte — none has // a base ElementType variant.) assert_eq!( test_cases.len(), - 21, - "Expected 21 base Element variants in test, got {}", + 22, + "Expected 22 base Element variants in test, got {}", test_cases.len() ); diff --git a/grovedb-element/tests/element_display_and_serialization.rs b/grovedb-element/tests/element_display_and_serialization.rs index 40aed6c23..9a565d3be 100644 --- a/grovedb-element/tests/element_display_and_serialization.rs +++ b/grovedb-element/tests/element_display_and_serialization.rs @@ -445,3 +445,39 @@ fn element_display_without_flags_covers_none_branches() { ); } } + +#[test] +fn private_document_store_invalid_config_is_unrepresentable() { + let grove_version = GroveVersion::latest(); + // entry_size 0, chunk_power 0, chunk_power 17 must be rejected by both + // directions of the bincode codec (the config is committed into the + // store's state root, so an unusable config must never round-trip). + for bad in [ + Element::new_private_document_store(0, 0, 4, None), + Element::new_private_document_store(0, 64, 0, None), + Element::new_private_document_store(0, 64, 17, None), + ] { + assert!( + bad.serialize(grove_version).is_err(), + "serialize must reject {:?}", + bad + ); + } + // Craft the bytes directly (serialize refuses) and check deserialize + // rejects them: a valid element re-encoded with a zeroed entry_size. + let good = Element::new_private_document_store(0, 64, 4, None); + let mut bytes = good.serialize(grove_version).expect("serialize valid"); + // Layout: [24 (discriminant), total_count varint (0 = 1 byte), + // entry_size varint (64 = 1 byte), chunk_power, flags None] + assert_eq!(bytes[0], 24); + assert_eq!(bytes[2], 64); + bytes[2] = 0; // entry_size -> 0 + assert!( + Element::deserialize(&bytes, grove_version).is_err(), + "deserialize must reject a zero entry_size" + ); + // NonCounted-wrapped invalid config is rejected too (validation looks + // through the wrapper). + let wrapped = Element::NonCounted(Box::new(Element::new_private_document_store(0, 0, 4, None))); + assert!(wrapped.serialize(grove_version).is_err()); +} diff --git a/grovedb/src/batch/estimated_costs/average_case_costs.rs b/grovedb/src/batch/estimated_costs/average_case_costs.rs index d37517e85..901d99d76 100644 --- a/grovedb/src/batch/estimated_costs/average_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/average_case_costs.rs @@ -318,16 +318,17 @@ impl GroveOp { propagate, grove_version, ); - // Additional cost: buffer write + running hash, identical to - // the underlying BulkAppend (the composite pds_state blake3 - // is folded into the same single-hash average as the bulk - // state-root hash it replaces per append). Most appends only + // Additional cost: buffer write + hashing. Most appends only // write to the buffer (O(1)); compaction happens once per - // epoch_size appends and is amortized. + // epoch_size appends and is amortized. Unlike BulkAppend, a + // PDS append unconditionally derives the composite + // `pds_state` root on top of the bulk state root, so the + // average models both blake3 calls. use grovedb_costs::storage_cost::{removal::StorageRemovedBytes, StorageCost}; let entry_size = entry.len() as u32; - // 1 blake3 hash for the running buffer hash chain - const AVG_HASH_CALLS: u32 = 1; + // 1 blake3 for the bulk state root + 1 for the composite + // config-binding pds_state root + const AVG_HASH_CALLS: u32 = 2; item_cost.add_cost(OperationCost { seek_count: 1, // 1 buffer entry write storage_cost: StorageCost { diff --git a/grovedb/src/operations/delete/mod.rs b/grovedb/src/operations/delete/mod.rs index 6f49f9f24..aaf56779e 100644 --- a/grovedb/src/operations/delete/mod.rs +++ b/grovedb/src/operations/delete/mod.rs @@ -976,12 +976,26 @@ impl GroveDb { .get_transactional_storage_context(path.clone(), Some(batch), transaction) .unwrap_add_cost(&mut cost); + // The merk reopened here is the PARENT merk (at `path`), but + // the historical code labels it with the DELETED CHILD's + // tree type. For a PrivateDocumentStore child that label + // would trip the merk-level "no ops on a PDS Merk" + // chokepoint (the delete below applies to the parent), so + // use the parent's actual tree type for PDS deletions. + // Existing types keep the historical label byte-for-byte to + // avoid any behavior change on released paths. + let reopen_tree_type = + if matches!(tree_type, grovedb_merk::TreeType::PrivateDocumentStore(_)) { + subtree_to_delete_from.tree_type + } else { + tree_type + }; let mut merk_to_delete_tree_from = cost_return_on_error!( &mut cost, Merk::open_layered_with_root_key( storage, subtree_to_delete_from.root_key(), - tree_type, + reopen_tree_type, Some(&Element::value_defined_cost_for_serialized_value), grove_version, ) diff --git a/grovedb/src/tests/private_document_store_tests.rs b/grovedb/src/tests/private_document_store_tests.rs index 6441fcaab..47f9c8ca0 100644 --- a/grovedb/src/tests/private_document_store_tests.rs +++ b/grovedb/src/tests/private_document_store_tests.rs @@ -186,6 +186,7 @@ fn test_private_document_store_insert_get_count_roundtrip() { fn test_private_document_store_insert_rejects_wrong_entry_size() { let grove_version = GroveVersion::latest(); let db = make_db_with_store(); + let root_before = db.root_hash(None, grove_version).unwrap().unwrap(); for bad in [ vec![0u8; TEST_ENTRY_SIZE as usize - 1], @@ -203,6 +204,11 @@ fn test_private_document_store_insert_rejects_wrong_entry_size() { } // Nothing was appended and the root hash is unchanged by rejections. + assert_eq!( + root_before, + db.root_hash(None, grove_version).unwrap().unwrap(), + "rejected appends must not change the grove root hash" + ); assert_eq!( db.private_document_store_count(&[b"root"], b"docs", None, grove_version) .unwrap() @@ -537,6 +543,52 @@ fn test_private_document_store_delete() { Err(Error::PathKeyNotFound(_)) )); + // The store's non-Merk data namespace (buffer entries, chunk blobs, MMR + // nodes) must be reclaimed by the delete — not just the parent element. + { + use grovedb_storage::{RawIterator, Storage, StorageContext}; + let tx = db.start_transaction(); + let ctx = db + .db + .get_transactional_storage_context( + grovedb_path::SubtreePath::from([b"root".as_slice(), b"docs".as_slice()].as_ref()), + None, + &tx, + ) + .unwrap(); + let mut iter = ctx.raw_iter(); + iter.seek_to_first().unwrap(); + assert!( + !iter.valid().unwrap(), + "deleted store left orphaned rows in its data namespace" + ); + } + + // A store recreated at the same path starts from scratch: appends begin + // at position 0 and nothing from the deleted store is visible. + db.insert( + &[b"root"], + b"docs", + Element::empty_private_document_store(TEST_ENTRY_SIZE, TEST_CHUNK_POWER) + .expect("valid config"), + None, + None, + grove_version, + ) + .unwrap() + .expect("recreate store"); + let (_, position) = db + .private_document_store_insert(&[b"root"], b"docs", entry(99), None, grove_version) + .unwrap() + .expect("append to recreated store"); + assert_eq!(position, 0); + assert_eq!( + db.private_document_store_get_value(&[b"root"], b"docs", 0, None, grove_version) + .unwrap() + .expect("get"), + Some(entry(99)) + ); + let issues = db .verify_grovedb(None, true, false, grove_version) .expect("verify_grovedb"); @@ -1148,13 +1200,31 @@ fn test_private_document_store_v0_prover_rejects_subqueries() { result ); - // ...while a terminal query for the element itself still proves (the - // node passes through as a result, V0 shape unchanged). + // ...while a terminal query for the element itself still proves and + // verifies (the node passes through as a result, V0 shape unchanged). let terminal = PathQuery::new_unsized( vec![b"root".to_vec()], Query::new_single_key(b"docs".to_vec()), ); - db.prove_query(&terminal, None, &GROVE_V2) + let proof_bytes = db + .prove_query(&terminal, None, &GROVE_V2) .unwrap() .expect("V0 terminal proof over a store element generates"); + let (root_hash, result_set) = GroveDb::verify_query_with_options( + &proof_bytes, + &terminal, + grovedb_merk::proofs::query::VerifyOptions { + absence_proofs_for_non_existing_searched_keys: false, + verify_proof_succinctness: false, + include_empty_trees_in_result: true, + }, + &GROVE_V2, + ) + .expect("verify V0 terminal proof"); + assert_eq!( + root_hash, + db.root_hash(None, grove_version).unwrap().unwrap(), + "V0 proof must bind the current grove root" + ); + assert_eq!(result_set.len(), 1); } diff --git a/merk/src/element/insert.rs b/merk/src/element/insert.rs index 6dca2c18b..7871b273c 100644 --- a/merk/src/element/insert.rs +++ b/merk/src/element/insert.rs @@ -991,6 +991,56 @@ mod tests { ) .unwrap() .is_err()); + + // The `*_into_batch_operations` builders cannot see the destination + // Merk, so ops CAN be queued — but the apply chokepoint + // (`Merk::apply_unchecked`, which every apply variant funnels + // through) rejects any non-empty batch aimed at a PDS Merk, so the + // queued ops can never take effect. + let mut ops: Vec>> = Vec::new(); + item.insert_into_batch_operations( + b"k".to_vec(), + &mut ops, + crate::TreeFeatureType::BasicMerkNode, + grove_version, + ) + .unwrap() + .expect("builder itself queues"); + subtree + .insert_subtree_into_batch_operations( + b"k2".to_vec(), + [0u8; 32], + true, + &mut ops, + crate::TreeFeatureType::BasicMerkNode, + grove_version, + ) + .unwrap() + .expect("builder itself queues"); + reference + .insert_reference_into_batch_operations( + b"k3".to_vec(), + [0u8; 32], + &mut ops, + crate::TreeFeatureType::BasicMerkNode, + grove_version, + ) + .unwrap() + .expect("builder itself queues"); + let apply_result = merk + .apply_with_specialized_costs::<_, Vec>( + &ops, + &[], + None, + &|_, _| Ok(0), + Some(&Element::value_defined_cost_for_serialized_value), + grove_version, + ) + .unwrap(); + assert!( + apply_result.is_err(), + "applying queued ops to a PDS Merk must be rejected" + ); } #[test] diff --git a/merk/src/merk/apply.rs b/merk/src/merk/apply.rs index 4ae8b3c9a..153abe680 100644 --- a/merk/src/merk/apply.rs +++ b/merk/src/merk/apply.rs @@ -16,7 +16,7 @@ use crate::{ kv::{ValueDefinedCostType, KV}, AuxMerkBatch, Walker, }, - Error, Merk, MerkBatch, MerkOptions, + Error, Merk, MerkBatch, MerkOptions, TreeType, }; const MAX_KEY_LENGTH: usize = u8::MAX as usize; @@ -346,6 +346,21 @@ where ) -> Result<(bool, Option), Error>, R: FnMut(&Vec, u32, u32) -> Result<(StorageRemovedBytes, StorageRemovedBytes), Error>, { + // A PrivateDocumentStore's Merk must stay empty forever — its + // entries live in the non-Merk data namespace and immutability is + // enforced by the type. The element-insert entry points already + // reject PDS destinations; this chokepoint additionally covers ops + // assembled through the `*_into_batch_operations` builders (which + // cannot see the destination), since queued ops can only take + // effect through an apply on the destination Merk. Every public + // apply variant funnels through here. + if !batch.is_empty() && matches!(self.tree_type, TreeType::PrivateDocumentStore(_)) { + return Err(Error::InvalidInputError( + "private document stores cannot hold child elements; entries are appended \ + via the private_document_store_insert API", + )) + .wrap_with_cost(Default::default()); + } for (key, ..) in batch.iter() { if key.as_ref().len() > MAX_KEY_LENGTH { return Err(Error::InvalidInputError( From 3f1d03ef5af21601ae498627e0666ad4063fe645 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Wed, 19 Aug 2026 23:20:52 +0700 Subject: [PATCH 05/19] fix: address the PrivateDocumentStore review findings (PR #787) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two review passes (thepastaclaw, and QuantumExplorer's P1/P2 pass) plus a self-review raised 13 distinct issues. All are addressed here. CONSENSUS * Appending stripped the NonCounted wrapper. Both append paths unwrapped the stored element and wrote back a bare PrivateDocumentStore, so a NonCounted store under a CountTree became counted on its first append, changing the parent aggregate and the root hash. The direct path now captures and restores the wrapper; the batch path restores it in the ReplaceNonMerkTreeRoot apply arm, which already re-reads the stored element for its flags (no extra read, no cost change). Scoped to PDS: CommitmentTree/Mmr/BulkAppend/Dense share the defect but are live on V1..V3, so their fix needs its own version gate. * check_pds_enabled was fail-open. `slot < 1` accepted 2 and above, so a future slot meaning new semantics would silently run v1 code. Now an exact `slot == 1`, matching every other guard in the codebase. * Write-once was not enforced. A plain InsertOrReplace of a fresh store over a populated one was accepted with default options, resetting the element to empty while its chunk blobs and MMR nodes stayed behind. Both the batch and direct paths now reject it. COST ACCOUNTING (fees; all pre-activation, so free to correct now) * The composite pds_state blake3 was computed but never charged. * Opening a store derives the committed-config hash, also uncharged: from_state now returns a CostResult and bills it. * Reads billed nothing for fetching the document. Added BulkAppendTree::{get_buffer_value,get_chunk_value}_with_cost (additive; the plain accessors delegate and discard exactly as before, so released paths are byte-identical), and threaded CostResult through PrivateDocumentStore::get_value and the grovedb read op. * The worst-case estimate was not an upper bound: 1091 modeled hashes against 131,070 real ones at chunk_power 16, and a flat 64 KiB blob against 2^16 * entry_size. Now derived from the permitted maximum. It deliberately over-estimates smaller configs because the op carries no config — over-estimating is the safe direction for a fee admission bound. The average-case arm now models the dense walk instead of a flat constant. CORRECTNESS / PERFORMANCE * Batch appends were O(N^2): try_insert recomputes the dense root on every insert (~4.3 billion hashes to fill one epoch at chunk_power 16), which append_no_state_root inherits. Added DenseFixedSizedMerkleTree::try_insert_no_root, BulkAppendTree::append_deferred_roots and PrivateDocumentStore::append_many, all additive; the batch preprocess uses append_many. A test pins byte-for-byte equivalence with a loop of append. * Batch preprocessing iterated a HashMap while doing cost-bearing work, so on the failure path the accumulated cost and the surfaced error varied by iteration order. Now a BTreeMap. * get_value returned Ok(None) for a truncated chunk, conflating corruption with absence. It now enforces the epoch_size invariant. * verify_grovedb laundered entry-size violations into an opaque hash mismatch and reported transient storage errors as corruption. The walk is now a separate check reporting its real message. CLEANUP * Deleted a 227-line byte-identical copy of test_utils.rs; the harness is shared from grovedb-bulk-append-tree behind a `test-utils` feature. * Replaced three identical path helpers with util::subtree_path_with_key. Two self-review findings were withdrawn on closer inspection: the batch meta ops cannot be constructed externally (the variants are #[non_exhaustive]), and verify_grovedb walking every entry is consistent with it walking every Merk element everywhere else. Full workspace suite green: 45 binaries, 2739 grovedb tests, 0 failures. Co-Authored-By: Claude Fable 5 --- grovedb-bulk-append-tree/Cargo.toml | 3 + grovedb-bulk-append-tree/src/lib.rs | 4 +- grovedb-bulk-append-tree/src/test_utils.rs | 6 +- grovedb-bulk-append-tree/src/tree/append.rs | 55 +++ grovedb-bulk-append-tree/src/tree/fetch.rs | 66 +++- .../src/tree.rs | 29 ++ grovedb-private-document-store/Cargo.toml | 5 + grovedb-private-document-store/src/lib.rs | 2 - grovedb-private-document-store/src/store.rs | 366 +++++++++++++++--- .../src/test_utils.rs | 227 ----------- .../estimated_costs/average_case_costs.rs | 27 +- .../batch/estimated_costs/worst_case_costs.rs | 53 ++- grovedb/src/batch/mod.rs | 97 +++-- grovedb/src/lib.rs | 88 ++++- grovedb/src/operations/bulk_append_tree.rs | 19 +- grovedb/src/operations/commitment_tree.rs | 13 +- .../insert/add_element_on_transaction/v0.rs | 25 +- .../insert/add_element_on_transaction/v1.rs | 25 +- .../src/operations/private_document_store.rs | 129 +++--- .../proof/bind_terminal_non_merk_tree/v1.rs | 8 +- .../src/tests/private_document_store_tests.rs | 298 ++++++++++++++ grovedb/src/util.rs | 15 + 22 files changed, 1118 insertions(+), 442 deletions(-) delete mode 100644 grovedb-private-document-store/src/test_utils.rs diff --git a/grovedb-bulk-append-tree/Cargo.toml b/grovedb-bulk-append-tree/Cargo.toml index 196061260..a15f9c262 100644 --- a/grovedb-bulk-append-tree/Cargo.toml +++ b/grovedb-bulk-append-tree/Cargo.toml @@ -18,6 +18,9 @@ storage = [ "grovedb-dense-fixed-sized-merkle-tree/storage", ] mem_store = ["grovedb-merkle-mountain-range/mem_store"] +# Exposes `test_utils::MemStorageContext` to other crates' test builds so the +# in-memory StorageContext harness lives in exactly one place. +test-utils = ["storage"] [dependencies] grovedb-merkle-mountain-range = { version = "5.0.1", path = "../grovedb-merkle-mountain-range", default-features = false } diff --git a/grovedb-bulk-append-tree/src/lib.rs b/grovedb-bulk-append-tree/src/lib.rs index f46436b85..0b2673174 100644 --- a/grovedb-bulk-append-tree/src/lib.rs +++ b/grovedb-bulk-append-tree/src/lib.rs @@ -13,8 +13,8 @@ mod error; pub mod proof; mod tree; -#[cfg(all(test, feature = "storage"))] -pub(crate) mod test_utils; +#[cfg(all(feature = "storage", any(test, feature = "test-utils")))] +pub mod test_utils; // Re-export main types pub use chunk::{deserialize_chunk_blob, serialize_chunk_blob}; diff --git a/grovedb-bulk-append-tree/src/test_utils.rs b/grovedb-bulk-append-tree/src/test_utils.rs index fbf79a715..0949473b4 100644 --- a/grovedb-bulk-append-tree/src/test_utils.rs +++ b/grovedb-bulk-append-tree/src/test_utils.rs @@ -14,7 +14,7 @@ use grovedb_storage::{Batch, RawIterator, StorageContext}; /// (data storage) have real implementations; all other `StorageContext` /// methods panic if called. #[derive(Default)] -pub(crate) struct MemStorageContext { +pub struct MemStorageContext { pub data: RefCell, Vec>>, } @@ -141,7 +141,7 @@ impl<'db> StorageContext<'db> for MemStorageContext { // ── Batch and RawIterator stubs ─────────────────────────────────────── /// No-op batch (never used — MemStorageContext does immediate writes). -pub(crate) struct MemBatch; +pub struct MemBatch; impl Batch for MemBatch { fn put>( @@ -186,7 +186,7 @@ impl Batch for MemBatch { } /// Stub iterator (never used by the bulk append tree). -pub(crate) struct MemRawIterator; +pub struct MemRawIterator; impl RawIterator for MemRawIterator { fn seek_to_first(&mut self) -> CostContext<()> { diff --git a/grovedb-bulk-append-tree/src/tree/append.rs b/grovedb-bulk-append-tree/src/tree/append.rs index a3ca53b5c..36a873746 100644 --- a/grovedb-bulk-append-tree/src/tree/append.rs +++ b/grovedb-bulk-append-tree/src/tree/append.rs @@ -128,6 +128,61 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { }) } + /// Append a value deferring **both** the dense-tree root and the + /// state root. + /// + /// Storage effect is identical to [`append`](Self::append), but the + /// per-insert `compute_root_hash` walk over the dense buffer is skipped. + /// [`append_no_state_root`](Self::append_no_state_root) still pays that + /// walk on every call (via `try_insert`), which makes a run of N appends + /// O(N^2) in hash calls — 65,535 entries at `height = 16` costs ~4.3 + /// billion. This variant is O(N) plus one final root computation. + /// + /// The caller MUST recover the state root once at the end via + /// [`compute_current_state_root`](Self::compute_current_state_root); + /// until then the dense root is stale in-memory only (it is always + /// recomputed from stored values, never cached). + /// + /// Compaction still happens inline when the buffer fills, because the + /// chunk blob is built from the stored values, not from the root. + pub fn append_deferred_roots( + &mut self, + value: &[u8], + ) -> Result { + let mut hash_count: u32 = 0; + let global_position = self.total_count; + + let try_result = self + .dense_tree + .try_insert_no_root(value) + .unwrap() + .map_err(|e| { + BulkAppendError::StorageError(format!("dense tree insert failed: {}", e)) + })?; + + let compacted = match try_result { + // Inserted into the buffer; no root walk, so no hashes yet. + Some(_position) => false, + None => { + // Buffer full — compact existing entries plus this value. + // Must run before incrementing total_count so self.mmr_size() + // reflects the pre-compaction state. + let (compact_hashes, mmr_root) = self.compact_with_value(value)?; + hash_count += compact_hashes; + self.last_mmr_root = Some(mmr_root); + true + } + }; + + self.total_count += 1; + + Ok(AppendNoStateRootResult { + global_position, + hash_count, + compacted, + }) + } + /// Compute the current state root without modifying the tree. /// /// Uses the cached MMR root when available, so this is O(1) on the diff --git a/grovedb-bulk-append-tree/src/tree/fetch.rs b/grovedb-bulk-append-tree/src/tree/fetch.rs index 3ad8bed63..5257ee4f7 100644 --- a/grovedb-bulk-append-tree/src/tree/fetch.rs +++ b/grovedb-bulk-append-tree/src/tree/fetch.rs @@ -1,5 +1,6 @@ //! Read operations for BulkAppendTree. +use grovedb_costs::{CostResult, CostsExt, OperationCost}; use grovedb_dense_fixed_sized_merkle_tree::DenseTreeProof; use grovedb_merkle_mountain_range::{leaf_to_pos, MmrKeySize, MmrStore, MMR}; use grovedb_query::Query; @@ -38,12 +39,34 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { /// from completed chunks. The position is relative to the current buffer /// cycle (0-based). pub fn get_buffer_value(&self, position: u16) -> Result>, BulkAppendError> { + self.get_buffer_value_with_cost(position).unwrap() + } + + /// Cost-propagating variant of + /// [`get_buffer_value`](Self::get_buffer_value). + /// + /// Identical behavior; the difference is that the dense-tree read's + /// `OperationCost` (seek count and loaded bytes) reaches the caller + /// instead of being discarded. Callers that bill reads — anything + /// returning a `CostResult` to GroveDB — must use this variant, or the + /// read is served for free. + pub fn get_buffer_value_with_cost( + &self, + position: u16, + ) -> CostResult>, BulkAppendError> { + let mut cost = OperationCost::default(); if position >= self.buffer_count() { - return Ok(None); + return Ok(None).wrap_with_cost(cost); } - self.dense_tree.get(position).unwrap().map_err(|e| { - BulkAppendError::StorageError(format!("dense tree get at {} failed: {}", position, e)) - }) + let result = self.dense_tree.get(position).unwrap_add_cost(&mut cost); + result + .map_err(|e| { + BulkAppendError::StorageError(format!( + "dense tree get at {} failed: {}", + position, e + )) + }) + .wrap_with_cost(cost) } /// Query the buffer using a dense tree query. @@ -73,28 +96,45 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { /// Uses the MMR overlay to find nodes that were pushed during this session /// but not yet committed to storage. pub fn get_chunk_value(&self, chunk_index: u64) -> Result>, BulkAppendError> { + self.get_chunk_value_with_cost(chunk_index).unwrap() + } + + /// Cost-propagating variant of + /// [`get_chunk_value`](Self::get_chunk_value). See + /// [`get_buffer_value_with_cost`](Self::get_buffer_value_with_cost) for + /// why the cost-bearing form exists. + pub fn get_chunk_value_with_cost( + &self, + chunk_index: u64, + ) -> CostResult>, BulkAppendError> { + let mut cost = OperationCost::default(); if chunk_index >= self.chunk_count() { - return Ok(None); + return Ok(None).wrap_with_cost(cost); } let mmr_pos = leaf_to_pos(chunk_index); let mmr_store = MmrStore::with_key_size(&self.dense_tree.storage, MmrKeySize::U32); let mmr = MMR::new_with_overlay(self.mmr_size(), &mmr_store, self.mmr_overlay.clone()); - let node = mmr + let node = match mmr .batch .element_at_position(mmr_pos) - .unwrap() - .map_err(|e| { - BulkAppendError::MmrError(format!( + .unwrap_add_cost(&mut cost) + { + Ok(n) => n, + Err(e) => { + return Err(BulkAppendError::MmrError(format!( "failed to read MMR node for chunk {}: {}", chunk_index, e - )) - })?; + ))) + .wrap_with_cost(cost); + } + }; match node { - Some(n) => Ok(n.into_value()), + Some(n) => Ok(n.into_value()).wrap_with_cost(cost), None => Err(BulkAppendError::CorruptedData(format!( "missing MMR leaf for chunk {}", chunk_index - ))), + ))) + .wrap_with_cost(cost), } } diff --git a/grovedb-dense-fixed-sized-merkle-tree/src/tree.rs b/grovedb-dense-fixed-sized-merkle-tree/src/tree.rs index 905e74088..53e8a8a30 100644 --- a/grovedb-dense-fixed-sized-merkle-tree/src/tree.rs +++ b/grovedb-dense-fixed-sized-merkle-tree/src/tree.rs @@ -187,6 +187,35 @@ impl<'db, S: StorageContext<'db>> DenseFixedSizedMerkleTree { } } + /// Insert a value **without** recomputing the root hash. + /// + /// Same storage effect as [`try_insert`](Self::try_insert) — the value is + /// written at the next free position and `count` is incremented — but the + /// O(count) `compute_root_hash` walk is skipped. Returns the position, or + /// `None` when the tree is already full. + /// + /// Use this when inserting a run of values and only the FINAL root + /// matters: calling [`try_insert`](Self::try_insert) in a loop is + /// O(n^2) in hash calls, because every insert re-walks every filled + /// position. Recover the root once at the end with + /// [`root_hash`](Self::root_hash). + pub fn try_insert_no_root( + &mut self, + value: &[u8], + ) -> CostResult, DenseMerkleError> { + let mut cost = OperationCost::default(); + + if self.count >= self.capacity() { + return Ok(None).wrap_with_cost(cost); + } + + let position = self.count; + cost_return_on_error!(cost, self.put_value(position, value)); + self.count += 1; + + Ok(Some(position)).wrap_with_cost(cost) + } + /// Get a value by position. /// /// Returns `None` if position >= count. Returns an error if position < diff --git a/grovedb-private-document-store/Cargo.toml b/grovedb-private-document-store/Cargo.toml index f1eeb34f3..4def59e14 100644 --- a/grovedb-private-document-store/Cargo.toml +++ b/grovedb-private-document-store/Cargo.toml @@ -20,3 +20,8 @@ grovedb-costs = { version = "5.0.1", path = "../costs" } grovedb-storage = { version = "5.0.1", path = "../storage", optional = true } blake3 = { workspace = true } thiserror = { workspace = true } + +[dev-dependencies] +# Reuses the in-memory StorageContext harness rather than copying it; see the +# `test-utils` feature on grovedb-bulk-append-tree. +grovedb-bulk-append-tree = { version = "5.0.1", path = "../grovedb-bulk-append-tree", features = ["test-utils"] } diff --git a/grovedb-private-document-store/src/lib.rs b/grovedb-private-document-store/src/lib.rs index 88c055939..2b0870c71 100644 --- a/grovedb-private-document-store/src/lib.rs +++ b/grovedb-private-document-store/src/lib.rs @@ -30,8 +30,6 @@ mod error; #[cfg(feature = "storage")] mod store; -#[cfg(all(test, feature = "storage"))] -pub(crate) mod test_utils; pub use error::PrivateDocumentStoreError; pub use grovedb_bulk_append_tree::{ diff --git a/grovedb-private-document-store/src/store.rs b/grovedb-private-document-store/src/store.rs index 25efd8349..f58e18274 100644 --- a/grovedb-private-document-store/src/store.rs +++ b/grovedb-private-document-store/src/store.rs @@ -66,7 +66,7 @@ impl<'db, S: StorageContext<'db>> PrivateDocumentStore { entry_size: u32, chunk_power: u8, storage: S, - ) -> Result { + ) -> CostResult { Self::from_state(0, entry_size, chunk_power, storage) } @@ -80,19 +80,34 @@ impl<'db, S: StorageContext<'db>> PrivateDocumentStore { entry_size: u32, chunk_power: u8, storage: S, - ) -> Result { + ) -> CostResult { + let mut cost = OperationCost::default(); if entry_size == 0 { return Err(PrivateDocumentStoreError::InvalidConfig( "entry_size must be non-zero".to_string(), - )); + )) + .wrap_with_cost(cost); } - let bulk_tree = BulkAppendTree::from_state(total_count, chunk_power, storage) - .map_err(|e| PrivateDocumentStoreError::InvalidData(format!("bulk tree: {}", e)))?; + let bulk_tree = match BulkAppendTree::from_state(total_count, chunk_power, storage) { + Ok(t) => t, + Err(e) => { + return Err(PrivateDocumentStoreError::InvalidData(format!( + "bulk tree: {}", + e + ))) + .wrap_with_cost(cost); + } + }; + // Opening the store derives the committed-config hash, which is one + // blake3 call — charged here so every path that opens a store pays + // for it rather than getting it for free. + cost.hash_node_calls += 1; Ok(Self { entry_size, config_hash: private_document_store_config_hash(entry_size, chunk_power), bulk_tree, }) + .wrap_with_cost(cost) } /// Append an entry to the store. @@ -126,8 +141,13 @@ impl<'db, S: StorageContext<'db>> PrivateDocumentStore { }; cost.hash_node_calls += bulk_result.hash_count; + // `bulk_result.hash_count` covers the BulkAppendTree work INCLUDING + // its own state-root blake3. The composite `pds_state` root below is + // an ADDITIONAL blake3 on top of it — charge it, or every append + // under-reports one hash call. let state_root = compute_private_document_store_state_root(&self.config_hash, &bulk_result.state_root); + cost.hash_node_calls += 1; Ok(PrivateDocumentStoreAppendResult { state_root, @@ -147,9 +167,11 @@ impl<'db, S: StorageContext<'db>> PrivateDocumentStore { pub fn get_value( &self, global_position: u64, - ) -> Result>, PrivateDocumentStoreError> { + ) -> CostResult>, PrivateDocumentStoreError> { + let mut cost = OperationCost::default(); + if global_position >= self.bulk_tree.total_count { - return Ok(None); + return Ok(None).wrap_with_cost(cost); } let epoch_size = self.bulk_tree.epoch_size(); @@ -159,26 +181,61 @@ impl<'db, S: StorageContext<'db>> PrivateDocumentStore { let value = if global_position >= buffer_start { // Entry is in the current buffer. let buffer_pos = (global_position - buffer_start) as u16; - self.bulk_tree - .get_buffer_value(buffer_pos) - .map_err(|e| PrivateDocumentStoreError::InvalidData(format!("{}", e)))? + match self + .bulk_tree + .get_buffer_value_with_cost(buffer_pos) + .unwrap_add_cost(&mut cost) + { + Ok(v) => v, + Err(e) => { + return Err(PrivateDocumentStoreError::InvalidData(format!("{}", e))) + .wrap_with_cost(cost); + } + } } else { // Entry is in a completed chunk. let chunk_idx = global_position / epoch_size; let pos_in_chunk = (global_position % epoch_size) as usize; - let blob = self + let blob = match self .bulk_tree - .get_chunk_value(chunk_idx) - .map_err(|e| PrivateDocumentStoreError::InvalidData(format!("{}", e)))? - .ok_or_else(|| { - PrivateDocumentStoreError::CorruptedData(format!( + .get_chunk_value_with_cost(chunk_idx) + .unwrap_add_cost(&mut cost) + { + Ok(Some(b)) => b, + Ok(None) => { + return Err(PrivateDocumentStoreError::CorruptedData(format!( "missing chunk blob for index {}", chunk_idx - )) - })?; - let entries = grovedb_bulk_append_tree::deserialize_chunk_blob(&blob) - .map_err(|e| PrivateDocumentStoreError::CorruptedData(format!("{}", e)))?; - entries.get(pos_in_chunk).cloned() + ))) + .wrap_with_cost(cost); + } + Err(e) => { + return Err(PrivateDocumentStoreError::InvalidData(format!("{}", e))) + .wrap_with_cost(cost); + } + }; + let entries = match grovedb_bulk_append_tree::deserialize_chunk_blob(&blob) { + Ok(e) => e, + Err(e) => { + return Err(PrivateDocumentStoreError::CorruptedData(format!("{}", e))) + .wrap_with_cost(cost); + } + }; + // The bounds check above already proved this position exists, so + // a completed chunk that holds fewer than `epoch_size` entries is + // corruption — NOT absence. Enforce the same invariant + // `verify_entry_sizes` checks, otherwise a truncated blob would + // be reported to the caller as "no such document". + if entries.len() as u64 != epoch_size { + return Err(PrivateDocumentStoreError::CorruptedData(format!( + "chunk {} has {} entries, expected {}", + chunk_idx, + entries.len(), + epoch_size + ))) + .wrap_with_cost(cost); + } + Some(entries[pos_in_chunk].clone()) }; // Defensive: a stored entry that violates the committed entry size @@ -191,10 +248,103 @@ impl<'db, S: StorageContext<'db>> PrivateDocumentStore { global_position, v.len(), self.entry_size - ))); + ))) + .wrap_with_cost(cost); } - Ok(value) + Ok(value).wrap_with_cost(cost) + } + + /// Append many entries in one pass, computing roots once at the end. + /// + /// Byte-for-byte equivalent to calling [`append`](Self::append) once per + /// entry — same stored values, same chunk blobs, same final state root — + /// but O(N) in hash calls instead of O(N^2). + /// + /// [`append`](Self::append) recomputes the dense-buffer Merkle root on + /// every insert, so a run of N buffered appends re-walks every filled + /// position N times: at `chunk_power = 16` filling one epoch costs about + /// 4.3 billion blake3 calls for 65,535 entries. This method defers the + /// dense root, the bulk state root, and the composite `pds_state` root + /// until the whole run is written. + /// + /// Every entry is size-validated before it is written. On a size + /// violation the entries already appended remain — discard the + /// surrounding transaction for all-or-nothing semantics, matching + /// per-entry [`append`](Self::append) in a loop. + /// + /// Returns the same shape the FINAL per-entry [`append`](Self::append) + /// would have: post-run roots, the last entry's position, the summed + /// hash count, and whether any compaction occurred. For an empty input + /// nothing is written and the current state root is returned. + pub fn append_many<'e, I>( + &mut self, + entries: I, + ) -> CostResult + where + I: IntoIterator, + { + let mut cost = OperationCost::default(); + let mut hash_count: u32 = 0; + let mut any_compacted = false; + let starting_total = self.bulk_tree.total_count; + let mut last_global_position = starting_total.saturating_sub(1); + + for entry in entries { + if entry.len() != self.entry_size as usize { + return Err(PrivateDocumentStoreError::InvalidEntrySize { + expected: self.entry_size, + actual: entry.len(), + }) + .wrap_with_cost(cost); + } + let r = match self.bulk_tree.append_deferred_roots(entry) { + Ok(r) => r, + Err(e) => { + return Err(PrivateDocumentStoreError::InvalidData(format!( + "bulk append: {}", + e + ))) + .wrap_with_cost(cost); + } + }; + hash_count = hash_count.saturating_add(r.hash_count); + any_compacted |= r.compacted; + last_global_position = r.global_position; + } + + // Pay the deferred roots exactly once: the dense-buffer walk plus the + // bulk state-root blake3 (inside compute_current_state_root), then the + // composite pds_state blake3. + let bulk_state_root = match self.bulk_tree.compute_current_state_root() { + Ok(r) => r, + Err(e) => { + return Err(PrivateDocumentStoreError::InvalidData(format!( + "state root: {}", + e + ))) + .wrap_with_cost(cost); + } + }; + let state_root = + compute_private_document_store_state_root(&self.config_hash, &bulk_state_root); + if self.bulk_tree.total_count > starting_total { + // One dense-root walk over the live buffer + bulk state root + the + // composite root. + hash_count = hash_count + .saturating_add(self.bulk_tree.buffer_count() as u32 * 2) + .saturating_add(2); + } + cost.hash_node_calls += hash_count; + + Ok(PrivateDocumentStoreAppendResult { + state_root, + bulk_state_root, + global_position: last_global_position, + hash_count, + compacted: any_compacted, + }) + .wrap_with_cost(cost) } /// Verify that all stored entries respect the committed `entry_size`. @@ -320,14 +470,102 @@ impl<'db, S: StorageContext<'db>> PrivateDocumentStore { } } +#[cfg(test)] +mod append_many_tests { + use super::*; + use grovedb_bulk_append_tree::test_utils::MemStorageContext; + + /// `append_many` must be byte-for-byte equivalent to `append` in a loop: + /// same state root, same positions, same stored bytes. It only differs in + /// how many hashes it takes to get there. + #[test] + fn append_many_matches_per_entry_append() { + // 10 entries at chunk_power 2 (epoch size 4) spans two compactions + // plus a partial buffer, so both storage tiers are exercised. + let entries: Vec> = (0..10u8).map(|i| vec![i; 8]).collect(); + + let mut one_by_one = PrivateDocumentStore::new(8, 2, MemStorageContext::new()) + .unwrap() + .expect("a"); + let mut last = None; + for e in &entries { + last = Some(one_by_one.append(e).unwrap().expect("append")); + } + let per_entry = last.expect("appended"); + + let mut batched = PrivateDocumentStore::new(8, 2, MemStorageContext::new()) + .unwrap() + .expect("b"); + let many = batched + .append_many(entries.iter().map(|e| e.as_slice())) + .unwrap() + .expect("append_many"); + + assert_eq!(many.state_root, per_entry.state_root); + assert_eq!(many.bulk_state_root, per_entry.bulk_state_root); + assert_eq!(many.global_position, per_entry.global_position); + assert_eq!(batched.total_count(), one_by_one.total_count()); + assert_eq!( + batched.compute_current_state_root().expect("root"), + one_by_one.compute_current_state_root().expect("root") + ); + for i in 0..10u64 { + assert_eq!( + batched.get_value(i).unwrap().expect("get"), + one_by_one.get_value(i).unwrap().expect("get"), + "position {}", + i + ); + } + batched.verify_entry_sizes().expect("sizes ok"); + + // The whole point: the batched path does asymptotically less hashing. + assert!( + many.hash_count < per_entry.hash_count * 10, + "append_many should not pay the per-entry dense walk" + ); + } + + #[test] + fn append_many_validates_entry_size_and_handles_empty() { + let mut store = PrivateDocumentStore::new(8, 2, MemStorageContext::new()) + .unwrap() + .expect("new"); + + // Empty input writes nothing and reports the current state root. + let empty: Vec> = Vec::new(); + let r = store + .append_many(empty.iter().map(|e| e.as_slice())) + .unwrap() + .expect("empty append_many"); + assert_eq!(store.total_count(), 0); + assert_eq!( + r.state_root, + store.compute_current_state_root().expect("root") + ); + + // A wrong-size entry is rejected. + let bad = [vec![0u8; 8], vec![0u8; 7]]; + assert!(matches!( + store.append_many(bad.iter().map(|e| e.as_slice())).unwrap(), + Err(PrivateDocumentStoreError::InvalidEntrySize { + expected: 8, + actual: 7 + }) + )); + } +} + #[cfg(test)] mod error_path_tests { use super::*; - use crate::test_utils::MemStorageContext; + use grovedb_bulk_append_tree::test_utils::MemStorageContext; #[test] fn test_debug_and_error_display() { - let store = PrivateDocumentStore::new(64, 4, MemStorageContext::new()).expect("new"); + let store = PrivateDocumentStore::new(64, 4, MemStorageContext::new()) + .unwrap() + .expect("new"); let dbg = format!("{:?}", store); assert!(dbg.contains("PrivateDocumentStore") && dbg.contains("entry_size: 64")); assert_eq!(store.entry_size(), 64); @@ -358,7 +596,9 @@ mod error_path_tests { // then wipe the backing storage and reopen with the same claimed // state: chunk reads and the integrity walk must surface errors // rather than fabricate data. - let mut store = PrivateDocumentStore::new(8, 2, MemStorageContext::new()).expect("new"); + let mut store = PrivateDocumentStore::new(8, 2, MemStorageContext::new()) + .unwrap() + .expect("new"); for i in 0..6u8 { store.append(&[i; 8]).unwrap().expect("append"); } @@ -366,14 +606,18 @@ mod error_path_tests { let storage = PrivateDocumentStore::into_storage_for_test(store); storage.data.borrow_mut().clear(); - let broken = PrivateDocumentStore::from_state(6, 8, 2, storage).expect("reopen"); + let broken = PrivateDocumentStore::from_state(6, 8, 2, storage) + .unwrap() + .expect("reopen"); // Position 0 lives in the (now missing) completed chunk. - assert!(broken.get_value(0).is_err()); + assert!(broken.get_value(0).unwrap().is_err()); // The integrity walk fails on the missing chunk too. assert!(broken.verify_entry_sizes().is_err()); // Buffer positions read as missing entries in the walk; direct // get_value returns the underlying error or None consistently. - assert!(broken.compute_current_state_root().is_err() || broken.get_value(5).is_err()); + assert!( + broken.compute_current_state_root().is_err() || broken.get_value(5).unwrap().is_err() + ); } } @@ -389,14 +633,15 @@ impl PrivateDocumentStore { #[cfg(test)] mod tests { use super::*; - use crate::{ - empty_private_document_store_state_root, test_utils::MemStorageContext, - EMPTY_BULK_APPEND_TREE_STATE_ROOT, - }; + use grovedb_bulk_append_tree::test_utils::MemStorageContext; + + use crate::{empty_private_document_store_state_root, EMPTY_BULK_APPEND_TREE_STATE_ROOT}; #[test] fn test_empty_store_state_root_matches_helper() { - let store = PrivateDocumentStore::new(64, 4, MemStorageContext::new()).expect("new store"); + let store = PrivateDocumentStore::new(64, 4, MemStorageContext::new()) + .unwrap() + .expect("new store"); assert_eq!( store.compute_current_state_root().expect("state root"), empty_private_document_store_state_root(64, 4), @@ -411,21 +656,26 @@ mod tests { #[test] fn test_zero_entry_size_rejected() { assert!(matches!( - PrivateDocumentStore::new(0, 4, MemStorageContext::new()), + PrivateDocumentStore::new(0, 4, MemStorageContext::new()).unwrap(), Err(PrivateDocumentStoreError::InvalidConfig(_)) )); } #[test] fn test_invalid_chunk_power_rejected() { - assert!(PrivateDocumentStore::new(64, 0, MemStorageContext::new()).is_err()); - assert!(PrivateDocumentStore::new(64, 17, MemStorageContext::new()).is_err()); + assert!(PrivateDocumentStore::new(64, 0, MemStorageContext::new()) + .unwrap() + .is_err()); + assert!(PrivateDocumentStore::new(64, 17, MemStorageContext::new()) + .unwrap() + .is_err()); } #[test] fn test_append_validates_entry_size() { - let mut store = - PrivateDocumentStore::new(8, 2, MemStorageContext::new()).expect("new store"); + let mut store = PrivateDocumentStore::new(8, 2, MemStorageContext::new()) + .unwrap() + .expect("new store"); assert!(matches!( store.append(&[0u8; 7]).unwrap(), Err(PrivateDocumentStoreError::InvalidEntrySize { @@ -451,8 +701,9 @@ mod tests { fn test_append_get_roundtrip_across_compaction() { // chunk_power 2 → capacity 3, epoch size 4: 10 appends span two // completed chunks plus a partial buffer. - let mut store = - PrivateDocumentStore::new(8, 2, MemStorageContext::new()).expect("new store"); + let mut store = PrivateDocumentStore::new(8, 2, MemStorageContext::new()) + .unwrap() + .expect("new store"); let mut roots = Vec::new(); for i in 0..10u8 { let entry = [i; 8]; @@ -468,10 +719,10 @@ mod tests { assert_eq!(store.chunk_count(), 2); for i in 0..10u8 { - let v = store.get_value(i as u64).expect("get"); + let v = store.get_value(i as u64).unwrap().expect("get"); assert_eq!(v, Some(vec![i; 8]), "position {}", i); } - assert_eq!(store.get_value(10).expect("get"), None); + assert_eq!(store.get_value(10).unwrap().expect("get"), None); // The append-path state root matches a fresh computation. assert_eq!( @@ -488,8 +739,12 @@ mod tests { // The composite root must bind the config: it can never equal the // raw bulk root, and two stores with identical data but different // configs must have different roots. - let mut a = PrivateDocumentStore::new(8, 2, MemStorageContext::new()).expect("a"); - let mut b = PrivateDocumentStore::new(8, 3, MemStorageContext::new()).expect("b"); + let mut a = PrivateDocumentStore::new(8, 2, MemStorageContext::new()) + .unwrap() + .expect("a"); + let mut b = PrivateDocumentStore::new(8, 3, MemStorageContext::new()) + .unwrap() + .expect("b"); let ra = a.append(&[7u8; 8]).unwrap().expect("append a"); let rb = b.append(&[7u8; 8]).unwrap().expect("append b"); assert_ne!(ra.state_root, ra.bulk_state_root); @@ -499,7 +754,9 @@ mod tests { #[test] fn test_reopen_from_state() { let storage = MemStorageContext::new(); - let mut store = PrivateDocumentStore::new(8, 2, storage).expect("new store"); + let mut store = PrivateDocumentStore::new(8, 2, storage) + .unwrap() + .expect("new store"); for i in 0..6u8 { store.append(&[i; 8]).unwrap().expect("append"); } @@ -507,13 +764,18 @@ mod tests { let root_before = store.compute_current_state_root().expect("root"); let storage = PrivateDocumentStore::into_storage_for_test(store); - let reopened = PrivateDocumentStore::from_state(6, 8, 2, storage).expect("reopen"); + let reopened = PrivateDocumentStore::from_state(6, 8, 2, storage) + .unwrap() + .expect("reopen"); assert_eq!( reopened.compute_current_state_root().expect("root"), root_before ); for i in 0..6u8 { - assert_eq!(reopened.get_value(i as u64).expect("get"), Some(vec![i; 8])); + assert_eq!( + reopened.get_value(i as u64).unwrap().expect("get"), + Some(vec![i; 8]) + ); } reopened.verify_entry_sizes().expect("sizes ok"); } @@ -523,19 +785,23 @@ mod tests { // Write entries of size 8, then reopen claiming entry_size 16 — the // integrity walk must flag the first entry. let storage = MemStorageContext::new(); - let mut store = PrivateDocumentStore::new(8, 2, storage).expect("new store"); + let mut store = PrivateDocumentStore::new(8, 2, storage) + .unwrap() + .expect("new store"); for i in 0..6u8 { store.append(&[i; 8]).unwrap().expect("append"); } store.commit_mmr().expect("commit mmr"); let storage = PrivateDocumentStore::into_storage_for_test(store); - let reopened = PrivateDocumentStore::from_state(6, 16, 2, storage).expect("reopen"); + let reopened = PrivateDocumentStore::from_state(6, 16, 2, storage) + .unwrap() + .expect("reopen"); assert!(matches!( reopened.verify_entry_sizes(), Err(PrivateDocumentStoreError::CorruptedData(_)) )); // get_value performs the same defensive check. - assert!(reopened.get_value(0).is_err()); + assert!(reopened.get_value(0).unwrap().is_err()); } } diff --git a/grovedb-private-document-store/src/test_utils.rs b/grovedb-private-document-store/src/test_utils.rs deleted file mode 100644 index 67978db5f..000000000 --- a/grovedb-private-document-store/src/test_utils.rs +++ /dev/null @@ -1,227 +0,0 @@ -//! Test utilities: in-memory StorageContext for PrivateDocumentStore tests. - -use std::{cell::RefCell, collections::HashMap}; - -use grovedb_costs::{ - storage_cost::key_value_cost::KeyValueStorageCost, ChildrenSizesWithIsSumTree, CostContext, - CostResult, CostsExt, OperationCost, -}; -use grovedb_storage::{Batch, RawIterator, StorageContext}; - -/// In-memory storage context for testing. -/// -/// Immediate reads and writes backed by a `HashMap`. Only `get` and `put` -/// (data storage) have real implementations; all other `StorageContext` -/// methods panic if called. -#[derive(Default)] -pub(crate) struct MemStorageContext { - pub data: RefCell, Vec>>, -} - -impl MemStorageContext { - pub fn new() -> Self { - Self::default() - } -} - -impl<'db> StorageContext<'db> for MemStorageContext { - type Batch = MemBatch; - type RawIterator = MemRawIterator; - - fn get>(&self, key: K) -> CostResult>, grovedb_storage::Error> { - Ok(self.data.borrow().get(key.as_ref()).cloned()).wrap_with_cost(OperationCost::default()) - } - - fn put>( - &self, - key: K, - value: &[u8], - _children_sizes: ChildrenSizesWithIsSumTree, - _cost_info: Option, - ) -> CostResult<(), grovedb_storage::Error> { - self.data - .borrow_mut() - .insert(key.as_ref().to_vec(), value.to_vec()); - Ok(()).wrap_with_cost(OperationCost::default()) - } - - fn put_aux>( - &self, - _key: K, - _value: &[u8], - _cost_info: Option, - ) -> CostResult<(), grovedb_storage::Error> { - unimplemented!("MemStorageContext::put_aux") - } - - fn put_root>( - &self, - _key: K, - _value: &[u8], - _cost_info: Option, - ) -> CostResult<(), grovedb_storage::Error> { - unimplemented!("MemStorageContext::put_root") - } - - fn put_meta>( - &self, - _key: K, - _value: &[u8], - _cost_info: Option, - ) -> CostResult<(), grovedb_storage::Error> { - unimplemented!("MemStorageContext::put_meta") - } - - fn delete>( - &self, - _key: K, - _cost_info: Option, - ) -> CostResult<(), grovedb_storage::Error> { - unimplemented!("MemStorageContext::delete") - } - - fn delete_aux>( - &self, - _key: K, - _cost_info: Option, - ) -> CostResult<(), grovedb_storage::Error> { - unimplemented!("MemStorageContext::delete_aux") - } - - fn delete_root>( - &self, - _key: K, - _cost_info: Option, - ) -> CostResult<(), grovedb_storage::Error> { - unimplemented!("MemStorageContext::delete_root") - } - - fn delete_meta>( - &self, - _key: K, - _cost_info: Option, - ) -> CostResult<(), grovedb_storage::Error> { - unimplemented!("MemStorageContext::delete_meta") - } - - fn get_aux>( - &self, - _key: K, - ) -> CostResult>, grovedb_storage::Error> { - unimplemented!("MemStorageContext::get_aux") - } - - fn get_root>( - &self, - _key: K, - ) -> CostResult>, grovedb_storage::Error> { - unimplemented!("MemStorageContext::get_root") - } - - fn get_meta>( - &self, - _key: K, - ) -> CostResult>, grovedb_storage::Error> { - unimplemented!("MemStorageContext::get_meta") - } - - fn new_batch(&self) -> Self::Batch { - MemBatch - } - - fn commit_batch(&self, _batch: Self::Batch) -> CostResult<(), grovedb_storage::Error> { - Ok(()).wrap_with_cost(OperationCost::default()) - } - - fn raw_iter(&self) -> Self::RawIterator { - unimplemented!("MemStorageContext::raw_iter") - } -} - -// ── Batch and RawIterator stubs ─────────────────────────────────────── - -/// No-op batch (never used — MemStorageContext does immediate writes). -pub(crate) struct MemBatch; - -impl Batch for MemBatch { - fn put>( - &mut self, - _key: K, - _value: &[u8], - _children_sizes: ChildrenSizesWithIsSumTree, - _cost_info: Option, - ) -> Result<(), grovedb_costs::error::Error> { - unimplemented!("MemBatch::put") - } - - fn put_aux>( - &mut self, - _key: K, - _value: &[u8], - _cost_info: Option, - ) -> Result<(), grovedb_costs::error::Error> { - unimplemented!("MemBatch::put_aux") - } - - fn put_root>( - &mut self, - _key: K, - _value: &[u8], - _cost_info: Option, - ) -> Result<(), grovedb_costs::error::Error> { - unimplemented!("MemBatch::put_root") - } - - fn delete>(&mut self, _key: K, _cost_info: Option) { - unimplemented!("MemBatch::delete") - } - - fn delete_aux>(&mut self, _key: K, _cost_info: Option) { - unimplemented!("MemBatch::delete_aux") - } - - fn delete_root>(&mut self, _key: K, _cost_info: Option) { - unimplemented!("MemBatch::delete_root") - } -} - -/// Stub iterator (never used by the bulk append tree). -pub(crate) struct MemRawIterator; - -impl RawIterator for MemRawIterator { - fn seek_to_first(&mut self) -> CostContext<()> { - unimplemented!() - } - - fn seek_to_last(&mut self) -> CostContext<()> { - unimplemented!() - } - - fn seek>(&mut self, _key: K) -> CostContext<()> { - unimplemented!() - } - - fn seek_for_prev>(&mut self, _key: K) -> CostContext<()> { - unimplemented!() - } - - fn next(&mut self) -> CostContext<()> { - unimplemented!() - } - - fn prev(&mut self) -> CostContext<()> { - unimplemented!() - } - - fn value(&self) -> CostContext> { - unimplemented!() - } - - fn key(&self) -> CostContext> { - unimplemented!() - } - - fn valid(&self) -> CostContext { - unimplemented!() - } -} diff --git a/grovedb/src/batch/estimated_costs/average_case_costs.rs b/grovedb/src/batch/estimated_costs/average_case_costs.rs index 901d99d76..a3ed04c15 100644 --- a/grovedb/src/batch/estimated_costs/average_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/average_case_costs.rs @@ -319,16 +319,27 @@ impl GroveOp { grove_version, ); // Additional cost: buffer write + hashing. Most appends only - // write to the buffer (O(1)); compaction happens once per - // epoch_size appends and is amortized. Unlike BulkAppend, a - // PDS append unconditionally derives the composite - // `pds_state` root on top of the bulk state root, so the - // average models both blake3 calls. + // write to the buffer; compaction happens once per epoch and + // is amortized away here. + // + // Unlike the flat constant this arm previously used, the + // dominant term is the dense-buffer root walk `append` + // performs on every insert (two hashes per filled position). + // That scales with the committed `chunk_power`, which the op + // does not carry, so this models a typical small store + // (`chunk_power = 8`, i.e. a 255-entry buffer averaging + // half-full). This is an average-case estimate, NOT a bound — + // `worst_case_cost` carries the real upper bound over the + // whole permitted configuration range. use grovedb_costs::storage_cost::{removal::StorageRemovedBytes, StorageCost}; let entry_size = entry.len() as u32; - // 1 blake3 for the bulk state root + 1 for the composite - // config-binding pds_state root - const AVG_HASH_CALLS: u32 = 2; + /// Assumed typical buffer occupancy (half of a + /// `chunk_power = 8` buffer), two hashes per filled position. + const AVG_DENSE_HASHES: u32 = 128 * 2; + /// Bulk state root + composite config-binding pds_state root + /// + the committed-config hash paid when opening the store. + const AVG_ROOT_AND_CONFIG_HASHES: u32 = 3; + const AVG_HASH_CALLS: u32 = AVG_DENSE_HASHES + AVG_ROOT_AND_CONFIG_HASHES; item_cost.add_cost(OperationCost { seek_count: 1, // 1 buffer entry write storage_cost: StorageCost { diff --git a/grovedb/src/batch/estimated_costs/worst_case_costs.rs b/grovedb/src/batch/estimated_costs/worst_case_costs.rs index 4d40925a7..aeed49ef4 100644 --- a/grovedb/src/batch/estimated_costs/worst_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/worst_case_costs.rs @@ -313,30 +313,55 @@ impl GroveOp { propagate, grove_version, ); - // Worst case mirrors the underlying BulkAppend: compaction - // trigger (buffer fills -> serialize chunk blob -> dense - // Merkle root -> MMR push), plus one blake3 for the - // config-binding composite pds_state root. The per-append - // write is entry-size-parametrized (entry.len() is the - // store's committed entry_size). + // A genuine UPPER BOUND over every configuration the type + // permits, not a typical-case figure. `chunk_power` is + // validated to 1..=16, so the dense buffer holds at most + // `2^16 - 1 = 65535` entries and an epoch is at most + // `2^16 = 65536` entries. + // + // The costly op is the compacting append: the dense-root walk + // hashes every filled position twice, the epoch is serialized + // into one chunk blob, and the blob is pushed to the MMR. + // + // This deliberately OVER-estimates smaller configurations — + // a `chunk_power = 4` store pays the `chunk_power = 16` + // bound — because `GroveOp::PrivateDocumentStoreInsert` + // carries only the entry, not the store's committed config, + // and the config is not knowable here. Over-estimating is the + // safe direction for a fee admission bound (an under-estimate + // makes a legitimate block fail replay). Threading the + // committed `{entry_size, chunk_power}` into the estimate is + // the way to tighten this; it needs the op or the layer + // estimate to carry the config. use grovedb_costs::storage_cost::{removal::StorageRemovedBytes, StorageCost}; let entry_size = entry.len() as u32; - // Max compaction overhead: 64KB safe bound for chunk blob - const MAX_COMPACTION_BLOB: u32 = 65536; - // Dense Merkle root: epoch_size hashes. Buffer hash: 1. - // MMR push: up to 64 merges. Composite pds_state root: 1. - const MAX_HASH_CALLS: u32 = 1024 + 1 + 65 + 1; + /// Largest epoch the type permits: `2^16` entries. + const MAX_EPOCH_ENTRIES: u32 = 1 << 16; + /// Largest dense buffer: `2^16 - 1` filled positions, each + /// costing a value hash and a node hash on the root walk. + const MAX_DENSE_HASHES: u32 = 2 * (MAX_EPOCH_ENTRIES - 1); + /// MMR push merges (bounded by the 64-bit position space). + const MAX_MMR_MERGES: u32 = 65; + /// Bulk state root + composite pds_state root + the + /// committed-config hash paid when the store is opened. + const ROOT_AND_CONFIG_HASHES: u32 = 3; + const MAX_HASH_CALLS: u32 = + MAX_DENSE_HASHES + MAX_MMR_MERGES + ROOT_AND_CONFIG_HASHES; // Writes: buffer entry + chunk blob + MMR nodes - const MAX_WRITES: u32 = 1 + 1 + 65; + const MAX_WRITES: u32 = 1 + 1 + MAX_MMR_MERGES; const MAX_READS: u32 = 64; // MMR sibling reads + // The compaction blob holds a whole epoch of entries, so its + // size scales with the committed entry size — a flat byte + // constant is not a bound. + let max_compaction_blob = MAX_EPOCH_ENTRIES.saturating_mul(entry_size); item_cost.add_cost(OperationCost { seek_count: MAX_WRITES + MAX_READS, storage_cost: StorageCost { - added_bytes: entry_size + MAX_COMPACTION_BLOB, + added_bytes: entry_size.saturating_add(max_compaction_blob), replaced_bytes: 0, removed_bytes: StorageRemovedBytes::NoStorageRemoval, }, - storage_loaded_bytes: (33 * MAX_READS) as u64, + storage_loaded_bytes: (33 * MAX_READS) as u64 + max_compaction_blob as u64, hash_node_calls: MAX_HASH_CALLS, sinsemilla_hash_calls: 0, }) diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index 3f56b3754..aa82c2d20 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -3024,30 +3024,44 @@ where )) .wrap_with_cost(cost); } - if is_insert_if_not_exists - || batch_apply_options.validate_insertion_does_not_override - { - let merk = self.merks.get_mut(path).expect("the Merk is cached"); - let existing = cost_return_on_error_into!( - &mut cost, - element.element_at_key_already_exists( - merk, - key_info.get_key_clone().as_slice(), - grove_version, - ) - ); - if existing { - if error_if_exists - || batch_apply_options.validate_insertion_does_not_override - { - return Err(Error::InvalidBatchOperation( - "attempting to insert PrivateDocumentStore element \ - that already exists", - )) - .wrap_with_cost(cost); - } + // A private document store is write-once, so + // creating one over an existing element is + // ALWAYS rejected — not just under + // `InsertIfNotExists` or + // `validate_insertion_does_not_override`. + // + // Without this, a plain `InsertOrReplace` of a + // fresh (total_count = 0) store over a populated + // one is accepted with default options: the + // element resets to empty and re-binds the empty + // state root while the old chunk blobs, MMR nodes + // and buffer entries stay behind in the subtree's + // data namespace. That both leaks storage and + // breaks the type's central promise, since a + // wholesale overwrite is a delete of every entry. + // Replacing a store means deleting it first, + // which clears the data namespace. + let merk = self.merks.get_mut(path).expect("the Merk is cached"); + let existing = cost_return_on_error_into!( + &mut cost, + element.element_at_key_already_exists( + merk, + key_info.get_key_clone().as_slice(), + grove_version, + ) + ); + if existing { + if is_insert_if_not_exists && !error_if_exists { + // `InsertIfNotExists` semantics: not an + // error, just nothing to do. continue; } + return Err(Error::InvalidBatchOperation( + "a PrivateDocumentStore already exists at this key; it is \ + append-only and cannot be overwritten \u{2014} delete it \ + first, which clears its data namespace", + )) + .wrap_with_cost(cost); } let merk_feature_type = cost_return_on_error_into!( &mut cost, @@ -3424,15 +3438,46 @@ where ); } GroveOp::ReplaceNonMerkTreeRoot { hash, meta } => { - // Read existing element to preserve flags + // Read existing element to preserve flags (and, for a + // PrivateDocumentStore, its NonCounted wrapper). let merk = self.merks.get(path).expect("the Merk is cached"); - let existing_flags = cost_return_on_error!( + let existing = cost_return_on_error!( &mut cost, GroveDb::get_element_from_subtree(merk, key_info.as_slice(), grove_version) - ) - .get_flags_owned(); + ); + let existing_non_counted = existing.is_non_counted(); + let existing_flags = existing.get_flags_owned(); let element = meta.to_element(existing_flags); + // `meta.to_element` always builds a BARE element, so a + // stored `NonCounted(tree)` would come back counted and + // change its parent count tree's aggregate — and with it + // the root hash. Restore the wrapper. + // + // Scoped to PrivateDocumentStore deliberately: the same + // latent defect exists for CommitmentTree / MmrTree / + // BulkAppendTree / DenseTree, but those are live on + // GROVE_V1..V3, so repairing them changes a released + // consensus outcome and belongs in its own version-gated + // change. PDS cannot exist before V4, so fixing it here + // alters nothing that has ever been committed. The + // element was already read above, so this costs nothing + // extra. + let element = if existing_non_counted + && matches!(meta, NonMerkTreeMeta::PrivateDocumentStore { .. }) + { + cost_return_on_error_no_add!( + cost, + element.into_non_counted().map_err(|_| { + Error::CorruptedCodeExecution( + "into_non_counted called on a wrapped element during \ + ReplaceNonMerkTreeRoot", + ) + }) + ) + } else { + element + }; let merk_feature_type = cost_return_on_error_into_no_add!( cost, element.get_feature_type(in_tree_type) diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index a59af481c..27447cf01 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -1992,6 +1992,30 @@ impl GroveDb { ); } + // PrivateDocumentStore integrity: the state root + // authenticates whatever bytes were written, so a buggy + // or gate-bypassing writer could persist entries that + // violate the committed `entry_size` under a perfectly + // consistent hash chain. Report that as its own issue, + // carrying the specific failure, rather than folding it + // into the hash comparison above. + if let Some(label) = self.private_document_store_entry_size_issue( + &element, + new_path_ref.clone(), + transaction, + ) { + let expected_placeholder: CryptoHash = blake3::hash( + b"private document store entries match committed entry_size", + ) + .into(); + let actual_placeholder: CryptoHash = blake3::hash(label.as_bytes()).into(); + issues.entry(new_path.to_vec()).or_insert(( + root_hash, + expected_placeholder, + actual_placeholder, + )); + } + // Software-consistency check: the aggregate fields // stored in the parent's tree element (e.g. // `sum_value` in `ProvableSumTree(_, sum_value, _)`) @@ -2390,6 +2414,44 @@ impl GroveDb { Ok(issues) } + /// Run the PrivateDocumentStore entry-size integrity walk, returning a + /// human-readable description of the first violation, or `None` when the + /// element is not a store, is empty, or every entry is well-formed. + /// + /// Kept separate from `compute_non_merk_child_hash` so a violation is + /// reported with its actual message instead of being signalled by + /// returning a deliberately-wrong hash. + fn private_document_store_entry_size_issue<'b, B: AsRef<[u8]>>( + &self, + element: &Element, + subtree_path: SubtreePath<'b, B>, + transaction: &Transaction, + ) -> Option { + let Element::PrivateDocumentStore(total_count, entry_size, chunk_power, _) = + element.underlying() + else { + return None; + }; + if *total_count == 0 { + return None; + } + let storage_ctx = self + .db + .get_transactional_storage_context(subtree_path, None, transaction) + .unwrap(); + match grovedb_private_document_store::PrivateDocumentStore::from_state( + *total_count, + *entry_size, + *chunk_power, + storage_ctx, + ) + .unwrap() + { + Ok(store) => store.verify_entry_sizes().err().map(|e| e.to_string()), + Err(e) => Some(format!("cannot open private document store: {e}")), + } + } + /// Compute the child hash for a non-Merk tree element by reconstructing /// its tree from storage and computing the state root. /// Falls back to `merk_root_hash` on any error or for standard Merk trees. @@ -2488,20 +2550,18 @@ impl GroveDb { *entry_size, *chunk_power, storage_ctx, - ) { - Ok(store) => { - // Integrity walk: every stored entry must respect the - // committed entry size (the state root authenticates - // whatever bytes were written, so a buggy or bypassing - // writer could persist wrong-size entries under a - // consistent root). On violation, fall back to - // `merk_root_hash` — the caller's chain check then - // reports the path as an issue. - if store.verify_entry_sizes().is_err() { - return merk_root_hash; - } - store.compute_current_state_root().unwrap_or(merk_root_hash) - } + ) + .unwrap() + { + // Report only the state root here. The entry-size + // integrity walk is a SEPARATE check reported by + // `private_document_store_entry_size_issue`, so a + // violation surfaces its real message ("entry at + // position N has size X, committed entry size is Y") + // instead of being laundered into an opaque hash + // mismatch — and a transient storage error during that + // walk no longer masquerades as corruption. + Ok(store) => store.compute_current_state_root().unwrap_or(merk_root_hash), Err(_) => merk_root_hash, } } diff --git a/grovedb/src/operations/bulk_append_tree.rs b/grovedb/src/operations/bulk_append_tree.rs index 50d13cccb..26df4dc58 100644 --- a/grovedb/src/operations/bulk_append_tree.rs +++ b/grovedb/src/operations/bulk_append_tree.rs @@ -68,7 +68,7 @@ impl GroveDb { // 2. Open transactional storage (write-through cache + MMR overlay // provide read-after-write visibility) - let subtree_path_vec = self.build_subtree_path_for_bulk(&path, key); + let subtree_path_vec = crate::util::subtree_path_with_key(&path, key); let subtree_path_refs: Vec<&[u8]> = subtree_path_vec.iter().map(|v| v.as_slice()).collect(); let subtree_path = SubtreePath::from(subtree_path_refs.as_slice()); @@ -203,7 +203,7 @@ impl GroveDb { return Ok(None).wrap_with_cost(cost); } - let subtree_path_vec = self.build_subtree_path_for_bulk(&path, key); + let subtree_path_vec = crate::util::subtree_path_with_key(&path, key); let subtree_path_refs: Vec<&[u8]> = subtree_path_vec.iter().map(|v| v.as_slice()).collect(); let subtree_path = SubtreePath::from(subtree_path_refs.as_slice()); @@ -284,7 +284,7 @@ impl GroveDb { } }; - let subtree_path_vec = self.build_subtree_path_for_bulk(&path, key); + let subtree_path_vec = crate::util::subtree_path_with_key(&path, key); let subtree_path_refs: Vec<&[u8]> = subtree_path_vec.iter().map(|v| v.as_slice()).collect(); let subtree_path = SubtreePath::from(subtree_path_refs.as_slice()); @@ -338,7 +338,7 @@ impl GroveDb { } }; - let subtree_path_vec = self.build_subtree_path_for_bulk(&path, key); + let subtree_path_vec = crate::util::subtree_path_with_key(&path, key); let subtree_path_refs: Vec<&[u8]> = subtree_path_vec.iter().map(|v| v.as_slice()).collect(); let subtree_path = SubtreePath::from(subtree_path_refs.as_slice()); @@ -426,17 +426,6 @@ impl GroveDb { } } - /// Build subtree path for a BulkAppendTree at path/key. - fn build_subtree_path_for_bulk>( - &self, - path: &SubtreePath, - key: &[u8], - ) -> Vec> { - let mut v = path.to_vec(); - v.push(key.to_vec()); - v - } - /// Preprocess `BulkAppend` ops in a batch. /// /// Groups ops by (path, key), executes all appends (including compactions) diff --git a/grovedb/src/operations/commitment_tree.rs b/grovedb/src/operations/commitment_tree.rs index b55a2f4e9..3ce75a3e3 100644 --- a/grovedb/src/operations/commitment_tree.rs +++ b/grovedb/src/operations/commitment_tree.rs @@ -136,7 +136,7 @@ impl GroveDb { // 2. Build subtree path and open transactional storage (write-through // cache + MMR overlay provide read-after-write visibility) - let ct_path_vec = self.build_ct_path(&path, key); + let ct_path_vec = crate::util::subtree_path_with_key(&path, key); let ct_path_refs: Vec<&[u8]> = ct_path_vec.iter().map(|v| v.as_slice()).collect(); let ct_path = SubtreePath::from(ct_path_refs.as_slice()); @@ -284,7 +284,7 @@ impl GroveDb { } }; - let ct_path_vec = self.build_ct_path(&path, key); + let ct_path_vec = crate::util::subtree_path_with_key(&path, key); let ct_path_refs: Vec<&[u8]> = ct_path_vec.iter().map(|v| v.as_slice()).collect(); let ct_path = SubtreePath::from(ct_path_refs.as_slice()); @@ -340,7 +340,7 @@ impl GroveDb { return Ok(None).wrap_with_cost(cost); } - let ct_path_vec = self.build_ct_path(&path, key); + let ct_path_vec = crate::util::subtree_path_with_key(&path, key); let ct_path_refs: Vec<&[u8]> = ct_path_vec.iter().map(|v| v.as_slice()).collect(); let ct_path = SubtreePath::from(ct_path_refs.as_slice()); @@ -415,13 +415,6 @@ impl GroveDb { } } - /// Build the subtree path for a commitment tree at path/key. - fn build_ct_path>(&self, path: &SubtreePath, key: &[u8]) -> Vec> { - let mut v = path.to_vec(); - v.push(key.to_vec()); - v - } - /// Preprocess `CommitmentTreeInsert` ops in a batch. /// /// For each group of insert ops targeting the same path: diff --git a/grovedb/src/operations/insert/add_element_on_transaction/v0.rs b/grovedb/src/operations/insert/add_element_on_transaction/v0.rs index 82aeee237..5b2f91915 100644 --- a/grovedb/src/operations/insert/add_element_on_transaction/v0.rs +++ b/grovedb/src/operations/insert/add_element_on_transaction/v0.rs @@ -27,8 +27,9 @@ use grovedb_costs::{ use grovedb_element::reference_path::path_from_reference_path_type; use grovedb_merk::{ element::{ - costs::ElementCostExtensions, get::ElementFetchFromStorageExtensions, - insert::ElementInsertToStorageExtensions, ElementExt, + costs::ElementCostExtensions, exists::ElementExistsInStorageExtensions, + get::ElementFetchFromStorageExtensions, insert::ElementInsertToStorageExtensions, + ElementExt, }, tree::NULL_HASH, tree_type::TreeType, @@ -221,6 +222,26 @@ impl GroveDb { )) .wrap_with_cost(cost); } + // Write-once: creating a store over an existing element is + // rejected on the direct path too, matching the batch arm. + // A silent overwrite would reset the element to empty while + // leaving the old chunk blobs and MMR nodes in the data + // namespace. + let already_exists = cost_return_on_error_into!( + &mut cost, + element.element_at_key_already_exists( + &mut subtree_to_insert_into, + key, + grove_version, + ) + ); + if already_exists { + return Err(Error::InvalidInput( + "a private document store already exists at this key; it is append-only \ + and cannot be overwritten", + )) + .wrap_with_cost(cost); + } cost_return_on_error_into!( &mut cost, element.insert_subtree( diff --git a/grovedb/src/operations/insert/add_element_on_transaction/v1.rs b/grovedb/src/operations/insert/add_element_on_transaction/v1.rs index e2e0a5799..e2edc0b43 100644 --- a/grovedb/src/operations/insert/add_element_on_transaction/v1.rs +++ b/grovedb/src/operations/insert/add_element_on_transaction/v1.rs @@ -19,8 +19,9 @@ use grovedb_costs::{ use grovedb_element::reference_path::path_from_reference_path_type; use grovedb_merk::{ element::{ - costs::ElementCostExtensions, get::ElementFetchFromStorageExtensions, - insert::ElementInsertToStorageExtensions, ElementExt, + costs::ElementCostExtensions, exists::ElementExistsInStorageExtensions, + get::ElementFetchFromStorageExtensions, insert::ElementInsertToStorageExtensions, + ElementExt, }, tree::NULL_HASH, tree_type::TreeType, @@ -217,6 +218,26 @@ impl GroveDb { )) .wrap_with_cost(cost); } + // Write-once: creating a store over an existing element is + // rejected on the direct path too, matching the batch arm. + // A silent overwrite would reset the element to empty while + // leaving the old chunk blobs and MMR nodes in the data + // namespace. + let already_exists = cost_return_on_error_into!( + &mut cost, + element.element_at_key_already_exists( + &mut subtree_to_insert_into, + key, + grove_version, + ) + ); + if already_exists { + return Err(Error::InvalidInput( + "a private document store already exists at this key; it is append-only \ + and cannot be overwritten", + )) + .wrap_with_cost(cost); + } cost_return_on_error_into!( &mut cost, element.insert_subtree( diff --git a/grovedb/src/operations/private_document_store.rs b/grovedb/src/operations/private_document_store.rs index 631afb416..b03c79c4b 100644 --- a/grovedb/src/operations/private_document_store.rs +++ b/grovedb/src/operations/private_document_store.rs @@ -11,7 +11,7 @@ //! type. Every entry point here fails closed on protocol versions that //! predate the element type (all slots are 0 before `GROVE_V4`). -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use grovedb_costs::{ cost_return_on_error, cost_return_on_error_into, cost_return_on_error_no_add, CostResult, @@ -37,15 +37,21 @@ fn map_pds_err(e: grovedb_private_document_store::PrivateDocumentStoreError) -> /// Fail-closed capability gate for the PrivateDocumentStore family. /// /// Slot `0` (every version before `GROVE_V4`) means the operation is -/// unavailable and returns a version-mismatch error; slot `1` is the active -/// v1 implementation. Unlike the `check_grovedb_v0!` family this rejects -/// *older* versions rather than newer ones — the element type must not be -/// creatable or operable under released protocol versions. +/// unavailable; slot `1` is the active v1 implementation. Unlike the +/// `check_grovedb_v0!` family this also rejects *older* versions — the +/// element type must not be creatable or operable under released protocol +/// versions. +/// +/// The comparison is an EXACT match, like every other version guard in the +/// codebase. Accepting `slot > 1` would be fail-open on a protocol +/// discriminator: a future GROVE_V5 that sets the slot to `2` to mean new +/// semantics would silently run v1 code on a node that only knows v1, which +/// is exactly the divergence this gate exists to prevent. pub(crate) fn check_pds_enabled( method: &str, slot: grovedb_version::version::FeatureVersion, ) -> Result<(), Error> { - if slot < 1 { + if slot != 1 { return Err(GroveVersionError::UnknownVersionMismatch { method: method.to_string(), known_versions: vec![1], @@ -109,7 +115,11 @@ impl GroveDb { self.get_raw_caching_optional(path.clone(), key, true, transaction, grove_version) ); - // Look through NonCounted: a wrapped PrivateDocumentStore is still one. + // Look through NonCounted: a wrapped PrivateDocumentStore is still + // one. The wrapper must be RESTORED on the way out (below) — it is + // part of the stored element and controls the parent count tree's + // aggregate. + let was_non_counted = element.is_non_counted(); let (total_count, entry_size, chunk_power, existing_flags) = match element.underlying() { Element::PrivateDocumentStore(tc, es, cp, flags) => (*tc, *es, *cp, flags.clone()), _ => { @@ -122,7 +132,7 @@ impl GroveDb { // 2. Open transactional storage (write-through cache + MMR overlay // provide read-after-write visibility). - let store_path_vec = self.build_pds_path(&path, key); + let store_path_vec = crate::util::subtree_path_with_key(&path, key); let store_path_refs: Vec<&[u8]> = store_path_vec.iter().map(|v| v.as_slice()).collect(); let store_path = SubtreePath::from(store_path_refs.as_slice()); @@ -133,10 +143,10 @@ impl GroveDb { .unwrap_add_cost(&mut cost); // 3. Open the store and append (validates the entry size). - let mut store = cost_return_on_error_no_add!( - cost, + let mut store = cost_return_on_error!( + &mut cost, PrivateDocumentStore::from_state(total_count, entry_size, chunk_power, storage_ctx) - .map_err(map_pds_err) + .map(|r| r.map_err(map_pds_err)) ); let append_result = cost_return_on_error!( @@ -187,6 +197,22 @@ impl GroveDb { chunk_power, existing_flags, ); + // Re-wrap when the stored element was NonCounted. Dropping the + // wrapper here would flip the store's contribution to a CountTree / + // CountSumTree parent from 0 to 1 on its first append, changing the + // parent aggregate and the consensus root hash. + let updated_element = if was_non_counted { + cost_return_on_error_no_add!( + cost, + Element::new_non_counted(updated_element).map_err(|e| { + Error::CorruptedData(format!( + "failed to re-wrap private document store in NonCounted: {e}" + )) + }) + ) + } else { + updated_element + }; cost_return_on_error_into!( &mut cost, @@ -281,7 +307,7 @@ impl GroveDb { return Ok(None).wrap_with_cost(cost); } - let store_path_vec = self.build_pds_path(&path, key); + let store_path_vec = crate::util::subtree_path_with_key(&path, key); let store_path_refs: Vec<&[u8]> = store_path_vec.iter().map(|v| v.as_slice()).collect(); let store_path = SubtreePath::from(store_path_refs.as_slice()); @@ -290,15 +316,19 @@ impl GroveDb { .get_transactional_storage_context(store_path, None, tx.as_ref()) .unwrap_add_cost(&mut cost); - let store = cost_return_on_error_no_add!( - cost, + let store = cost_return_on_error!( + &mut cost, PrivateDocumentStore::from_state(total_count, entry_size, chunk_power, storage_ctx) - .map_err(map_pds_err) + .map(|r| r.map_err(map_pds_err)) ); - let value = cost_return_on_error_no_add!( - cost, - store.get_value(global_position).map_err(map_pds_err) + // `get_value` is cost-bearing: the dense-tree / MMR reads that fetch + // the document must be billed, not served for free. + let value = cost_return_on_error!( + &mut cost, + store + .get_value(global_position) + .map(|r| r.map_err(map_pds_err)) ); Ok(value).wrap_with_cost(cost) @@ -346,13 +376,6 @@ impl GroveDb { } } - /// Build the subtree path for a private document store at path/key. - fn build_pds_path>(&self, path: &SubtreePath, key: &[u8]) -> Vec> { - let mut v = path.to_vec(); - v.push(key.to_vec()); - v - } - /// Preprocess `PrivateDocumentStoreInsert` ops in a batch. /// /// For each group of insert ops targeting the same store: @@ -396,7 +419,15 @@ impl GroveDb { type TreePath = Vec>; // Group insert ops by path (which includes tree key). - let mut pds_groups: HashMap>> = HashMap::new(); + // + // A BTreeMap, not a HashMap: the loop below performs cost-bearing + // storage reads and entry-size validation per group, so with a + // randomized iteration order two processes replaying the same batch + // could fail on different groups, having accumulated different + // operation costs, and surface different errors. Costs feed fees and + // errors are consensus-visible, so the order must be canonical. + // Insertion order WITHIN a group is preserved by the Vec. + let mut pds_groups: BTreeMap>> = BTreeMap::new(); for op in ops.iter() { if let GroveOp::PrivateDocumentStoreInsert { entry } = &op.op { let tree_path = op.path.to_path(); @@ -404,7 +435,7 @@ impl GroveDb { } } - let mut replacements: HashMap = HashMap::new(); + let mut replacements: BTreeMap = BTreeMap::new(); for (tree_path, entries) in pds_groups.iter() { // Extract parent path and tree key from the full path. @@ -438,7 +469,9 @@ impl GroveDb { ); // Look through NonCounted: a wrapped PrivateDocumentStore is - // still one. + // still one. The wrapper is restored when the replacement op is + // applied (see the `ReplaceNonMerkTreeRoot` arm in + // `batch/mod.rs`, which re-reads the stored element anyway). let (total_count, entry_size, chunk_power) = match element.underlying() { Element::PrivateDocumentStore(tc, es, cp, _) => (*tc, *es, *cp), _ => { @@ -461,30 +494,26 @@ impl GroveDb { .get_transactional_storage_context(st_path, Some(storage_batch), transaction) .unwrap_add_cost(&mut cost); - let mut store = cost_return_on_error_no_add!( - cost, + let mut store = cost_return_on_error!( + &mut cost, PrivateDocumentStore::from_state(total_count, entry_size, chunk_power, storage_ctx) - .map_err(map_pds_err) + .map(|r| r.map_err(map_pds_err)) ); - // Execute all inserts in order; each validates the entry size. - let mut last_state_root = None; - for entry in entries { - let r = cost_return_on_error!( - &mut cost, - store.append(entry).map(|r| r.map_err(map_pds_err)) - ); - last_state_root = Some(r.state_root); - } - - let new_state_root = match last_state_root { - Some(root) => root, - // Unreachable: groups only exist for at least one op. - None => cost_return_on_error_no_add!( - cost, - store.compute_current_state_root().map_err(map_pds_err) - ), - }; + // Execute all inserts in ONE pass. `append_many` is byte-for-byte + // equivalent to calling `append` per entry (same stored values, + // same final state root) but O(N) in hash calls instead of + // O(N^2): `append` recomputes the dense-buffer Merkle root on + // every insert, so a full epoch at chunk_power 16 would cost + // ~4.3 billion blake3 calls for 65,535 entries. Each entry is + // still size-validated before it is written. + let append_result = cost_return_on_error!( + &mut cost, + store + .append_many(entries.iter().map(|e| e.as_slice())) + .map(|r| r.map_err(map_pds_err)) + ); + let new_state_root = append_result.state_root; let current_total_count = store.total_count(); // Flush MMR overlay to storage (through the batch). @@ -513,7 +542,7 @@ impl GroveDb { // Build the new ops list: keep non-PDS ops, replace the first PDS // insert op per group with the replacement, skip the rest. - let mut first_seen: HashMap = HashMap::new(); + let mut first_seen: BTreeMap = BTreeMap::new(); let mut result = Vec::with_capacity(ops.len()); for op in ops.into_iter() { diff --git a/grovedb/src/operations/proof/bind_terminal_non_merk_tree/v1.rs b/grovedb/src/operations/proof/bind_terminal_non_merk_tree/v1.rs index 45b017f27..abeb1d815 100644 --- a/grovedb/src/operations/proof/bind_terminal_non_merk_tree/v1.rs +++ b/grovedb/src/operations/proof/bind_terminal_non_merk_tree/v1.rs @@ -278,18 +278,18 @@ impl GroveDb { .db .get_transactional_storage_context(storage_path, None, tx) .unwrap_add_cost(&mut cost); - let store = cost_return_on_error_no_add!( - cost, + let store = cost_return_on_error!( + &mut cost, grovedb_private_document_store::PrivateDocumentStore::from_state( *total_count, *entry_size, *chunk_power, storage_ctx, ) - .map_err(|e| Error::CorruptedData(format!( + .map(|r| r.map_err(|e| Error::CorruptedData(format!( "failed to open PrivateDocumentStore: {}", e - ))) + )))) ); let state_root = cost_return_on_error_no_add!( cost, diff --git a/grovedb/src/tests/private_document_store_tests.rs b/grovedb/src/tests/private_document_store_tests.rs index 2e241abfe..a26629d1c 100644 --- a/grovedb/src/tests/private_document_store_tests.rs +++ b/grovedb/src/tests/private_document_store_tests.rs @@ -1231,3 +1231,301 @@ fn test_private_document_store_v0_prover_rejects_subqueries() { ); assert_eq!(result_set.len(), 1); } + +// =========================================================================== +// Regression: wrapper preservation, write-once enforcement, guard exactness +// =========================================================================== + +/// Read a `CountTree`'s recorded aggregate count. +fn count_tree_value(db: &crate::tests::TempGroveDb, key: &[u8]) -> u64 { + let grove_version = GroveVersion::latest(); + match db + .get(EMPTY_PATH, key, None, grove_version) + .unwrap() + .expect("get count tree") + { + Element::CountTree(_, count, _) => count, + other => panic!("expected CountTree, got {}", other.type_str()), + } +} + +/// Build a `CountTree` at `counted` holding a `NonCounted`-wrapped store at +/// `counted/docs`, and return the parent's count before any append. +fn make_non_counted_store_under_count_tree() -> (crate::tests::TempGroveDb, u64) { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + db.insert( + EMPTY_PATH, + b"counted", + Element::empty_count_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert count tree"); + + let wrapped = Element::new_non_counted( + Element::empty_private_document_store(TEST_ENTRY_SIZE, TEST_CHUNK_POWER) + .expect("valid config"), + ) + .expect("wrap in NonCounted"); + db.insert(&[b"counted"], b"docs", wrapped, None, None, grove_version) + .unwrap() + .expect("insert NonCounted store"); + + let count = count_tree_value(&db, b"counted"); + assert_eq!(count, 0, "a NonCounted child must not be counted"); + (db, count) +} + +#[test] +fn test_direct_append_preserves_non_counted_wrapper() { + let grove_version = GroveVersion::latest(); + let (db, count_before) = make_non_counted_store_under_count_tree(); + + db.private_document_store_insert(&[b"counted"], b"docs", entry(1), None, grove_version) + .unwrap() + .expect("append"); + + // `get` resolves through wrappers by design, so read the RAW stored + // element to see whether the wrapper survived. + let stored = db + .get_raw( + [b"counted".as_slice()].as_ref().into(), + b"docs", + None, + grove_version, + ) + .unwrap() + .expect("get raw store"); + assert!( + stored.is_non_counted(), + "a direct append must not strip the NonCounted wrapper" + ); + assert!(stored.is_private_document_store()); + assert_eq!( + count_tree_value(&db, b"counted"), + count_before, + "appending to a NonCounted store must not change the parent's count" + ); + + let issues = db + .verify_grovedb(None, true, false, grove_version) + .expect("verify_grovedb"); + assert!(issues.is_empty(), "issues: {:?}", issues); +} + +#[test] +fn test_batch_append_preserves_non_counted_wrapper() { + let grove_version = GroveVersion::latest(); + let (db, count_before) = make_non_counted_store_under_count_tree(); + + // Several entries so the batch path goes through append_many and the + // ReplaceNonMerkTreeRoot rebuild. + let ops = (0..6u8) + .map(|i| { + QualifiedGroveDbOp::private_document_store_insert_op( + vec![b"counted".to_vec(), b"docs".to_vec()], + entry(i), + ) + }) + .collect(); + db.apply_batch(ops, None, None, grove_version) + .unwrap() + .expect("batch append"); + + let stored = db + .get_raw( + [b"counted".as_slice()].as_ref().into(), + b"docs", + None, + grove_version, + ) + .unwrap() + .expect("get raw store"); + assert!( + stored.is_non_counted(), + "a batch append must not strip the NonCounted wrapper" + ); + assert_eq!(stored.non_merk_entry_count(), Some(6)); + assert_eq!( + count_tree_value(&db, b"counted"), + count_before, + "batch appending to a NonCounted store must not change the parent's count" + ); + + let issues = db + .verify_grovedb(None, true, false, grove_version) + .expect("verify_grovedb"); + assert!(issues.is_empty(), "issues: {:?}", issues); +} + +#[test] +fn test_private_document_store_is_write_once() { + let grove_version = GroveVersion::latest(); + let db = make_db_with_store(); + for i in 0..5u8 { + db.private_document_store_insert(&[b"root"], b"docs", entry(i), None, grove_version) + .unwrap() + .expect("append"); + } + let root_before = db.root_hash(None, grove_version).unwrap().unwrap(); + + // Direct overwrite with a fresh empty store must be rejected, not + // silently accepted (which would orphan the existing chunk data). + let result = db + .insert( + &[b"root"], + b"docs", + Element::empty_private_document_store(TEST_ENTRY_SIZE, TEST_CHUNK_POWER) + .expect("valid config"), + None, + None, + grove_version, + ) + .unwrap(); + assert!( + matches!( + result, + Err(Error::InvalidInput(_)) | Err(Error::OverrideNotAllowed(_)) + ), + "direct overwrite must be rejected, got {:?}", + result + ); + + // Batch overwrite likewise. + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( + vec![b"root".to_vec()], + b"docs".to_vec(), + Element::empty_private_document_store(TEST_ENTRY_SIZE, TEST_CHUNK_POWER) + .expect("valid config"), + )]; + let result = db.apply_batch(ops, None, None, grove_version).unwrap(); + assert!( + matches!(result, Err(Error::InvalidBatchOperation(_))), + "batch overwrite must be rejected, got {:?}", + result + ); + + // The store is untouched by the rejected overwrites. + assert_eq!( + db.private_document_store_count(&[b"root"], b"docs", None, grove_version) + .unwrap() + .expect("count"), + 5 + ); + assert_eq!( + root_before, + db.root_hash(None, grove_version).unwrap().unwrap(), + "rejected overwrites must not change the root hash" + ); +} + +#[test] +fn test_version_guard_rejects_unknown_future_slot() { + // The guard is an exact match: a future slot value must NOT silently run + // v1 semantics, or an un-upgraded node diverges from an upgraded one. + let mut future = GROVE_V4.clone(); + future + .grovedb_versions + .operations + .private_document_store + .insert = 2; + let db = make_db_with_store(); + + let result = db + .private_document_store_insert(&[b"root"], b"docs", entry(0), None, &future) + .unwrap(); + assert!( + matches!(result, Err(Error::VersionError(_))), + "an unknown future slot must fail closed, got {:?}", + result + ); + + let ops = vec![QualifiedGroveDbOp::private_document_store_insert_op( + vec![b"root".to_vec(), b"docs".to_vec()], + entry(0), + )]; + let result = db.apply_batch(ops, None, None, &future).unwrap(); + assert!( + matches!(result, Err(Error::VersionError(_))), + "an unknown future slot must fail closed in batch too, got {:?}", + result + ); +} + +#[test] +fn test_private_document_store_reads_are_billed() { + // An in-range read must charge for actually fetching the document, not + // just for reading the parent element. + let grove_version = GroveVersion::latest(); + let db = make_db_with_store(); + // Enough entries to span a completed chunk and the live buffer. + for i in 0..6u8 { + db.private_document_store_insert(&[b"root"], b"docs", entry(i), None, grove_version) + .unwrap() + .expect("append"); + } + + let out_of_range = db + .private_document_store_get_value(&[b"root"], b"docs", 99, None, grove_version) + .cost; + // Position 0 lives in a completed chunk (MMR read), position 5 in the + // buffer (dense-tree read); both must cost more than the element lookup. + for pos in [0u64, 5] { + let cost = db + .private_document_store_get_value(&[b"root"], b"docs", pos, None, grove_version) + .cost; + assert!( + cost.storage_loaded_bytes > out_of_range.storage_loaded_bytes, + "reading position {} must load the document bytes (got {:?} vs out-of-range {:?})", + pos, + cost.storage_loaded_bytes, + out_of_range.storage_loaded_bytes + ); + } +} + +#[test] +fn test_private_document_store_delete_empty_and_via_batch() { + let grove_version = GroveVersion::latest(); + + // Deleting an EMPTY store takes the is_empty branch, which the populated + // delete test never reaches. + let db = make_db_with_store(); + db.delete(&[b"root"], b"docs", None, None, grove_version) + .unwrap() + .expect("delete empty store"); + assert!(matches!( + db.get(&[b"root"], b"docs", None, grove_version).unwrap(), + Err(Error::PathKeyNotFound(_)) + )); + + // Batch DeleteTree is a wholly different path (scan_delete_tree_ops / + // non_merk_delete_paths) from the direct delete. + let db = make_db_with_store(); + for i in 0..6u8 { + db.private_document_store_insert(&[b"root"], b"docs", entry(i), None, grove_version) + .unwrap() + .expect("append"); + } + let ops = vec![QualifiedGroveDbOp::delete_tree_op( + vec![b"root".to_vec()], + b"docs".to_vec(), + grovedb_merk::tree_type::TreeType::PrivateDocumentStore(TEST_CHUNK_POWER), + crate::batch::SubelementsDeletionBehavior::DeleteChildren, + )]; + db.apply_batch(ops, None, None, grove_version) + .unwrap() + .expect("batch delete populated store"); + assert!(matches!( + db.get(&[b"root"], b"docs", None, grove_version).unwrap(), + Err(Error::PathKeyNotFound(_)) + )); + + let issues = db + .verify_grovedb(None, true, false, grove_version) + .expect("verify_grovedb"); + assert!(issues.is_empty(), "issues: {:?}", issues); +} diff --git a/grovedb/src/util.rs b/grovedb/src/util.rs index c91205a2c..ad346fa9f 100644 --- a/grovedb/src/util.rs +++ b/grovedb/src/util.rs @@ -37,3 +37,18 @@ impl<'db> AsRef> for TxRef<'_, 'db> { } } } + +/// Build the storage path of a subtree living at `path`/`key`. +/// +/// Every non-Merk tree type (commitment tree, bulk-append tree, private +/// document store) needs its own subtree path to open the data namespace, +/// and each had grown a private copy of this three-line helper under a +/// different name. One shared function keeps them provably identical. +pub(crate) fn subtree_path_with_key>( + path: &grovedb_path::SubtreePath, + key: &[u8], +) -> Vec> { + let mut v = path.to_vec(); + v.push(key.to_vec()); + v +} From 4db7b4791fa5741d1a1679df8f57affd7b9ac9d4 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Wed, 19 Aug 2026 23:44:38 +0700 Subject: [PATCH 06/19] fix: thread the declared chunk power into PrivateDocumentStore estimates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Merges develop and adopts PR #813's mechanism for the new element type. #813 fixed the same class of defect for CommitmentTree — an estimated cost that could not see the tree's epoch scale — by threading the chunk power in from the tree's OWN declared layer and erroring loudly when the caller did not declare it. That is exactly the tension left open on this PR's worst-case bound: a config-blind estimate must either under-bound or grotesquely over-reserve. Rather than add a second parallel parameter, the existing one is generalized from `ct_chunk_power` to `append_tree_chunk_power`: both CommitmentTree and PrivateDocumentStore size their dense-recompute and compaction terms by 2^chunk_power, so one threaded config serves both. The layer lookup now matches `TreeType::PrivateDocumentStore(chunk_power)` alongside the commitment-tree case. The PDS average-case arm now derives its dense-walk and compaction terms from the declared epoch instead of assuming a typical store, and raises `PathNotFoundInCacheForEstimatedCosts` when the layer is undeclared, matching the CommitmentTreeInsert contract. This removes the up-to-64x over-charge the previous commit had to accept and flag for review. `entry.len()` is the committed entry size (the append path rejects any other length), so the byte terms need no separate declaration. Each entry is charged twice — once into the dense buffer, once into the chunk blob its epoch compacts into — which is what the amortized model now reflects. The worst-case arm is deliberately left as a true upper bound over the whole permitted range; that is the correct semantic there, and #813 left the worst-case path alone for the same reason. Tests: the estimate scales with the declared chunk power, an undeclared layer errors, and the entry-size parametrization assertion is corrected to the 2x amortized charge. Verification now matches CI (--all-features, against the merge with develop) rather than the branch alone with default features, which is why the previous commit passed locally and failed in CI. Co-Authored-By: Claude Fable 5 --- .../estimated_costs/average_case_costs.rs | 134 ++++++++++++------ 1 file changed, 92 insertions(+), 42 deletions(-) diff --git a/grovedb/src/batch/estimated_costs/average_case_costs.rs b/grovedb/src/batch/estimated_costs/average_case_costs.rs index 0f35d030a..678262c73 100644 --- a/grovedb/src/batch/estimated_costs/average_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/average_case_costs.rs @@ -47,11 +47,13 @@ impl GroveOp { &self, key: &KeyInfo, layer_element_estimates: &EstimatedLayerInformation, - // The declared chunk power of the commitment tree a - // `CommitmentTreeInsert` op targets (from the tree's own layer in - // the estimation paths), or `None` to charge the constructor- - // enforced cap. Ignored by every other op type. - ct_chunk_power: Option, + // The declared chunk power of the append-only tree this op targets + // — a `CommitmentTreeInsert`'s commitment tree or a + // `PrivateDocumentStoreInsert`'s store — read from that tree's own + // layer in the estimation paths. Both types size their dense-recompute + // and compaction terms by `2^chunk_power`, which the op itself does + // not carry. Ignored by every other op type. + append_tree_chunk_power: Option, propagate: bool, grove_version: &GroveVersion, ) -> CostResult<(), Error> { @@ -219,7 +221,7 @@ impl GroveOp { payload, key, layer_element_estimates, - ct_chunk_power, + append_tree_chunk_power, propagate, grove_version, ) @@ -288,50 +290,64 @@ impl GroveOp { }) } GroveOp::PrivateDocumentStoreInsert { entry } => { - // Cost of updating parent element in the Merk. The entry - // length is the store's committed entry_size, so the added - // storage bytes are entry-size-parametrized. + // The dense-recompute and compaction terms scale with + // `2^chunk_power`, which the op does not carry, so the + // store's own layer MUST be declared with + // `TreeType::PrivateDocumentStore(chunk_power)` — the same + // declare-your-layers contract `CommitmentTreeInsert` + // follows. A silent fallback would either under-bound or + // grotesquely over-reserve; both are worse than a loud error + // at integration time. + let Some(chunk_power) = append_tree_chunk_power else { + return Err(Error::PathNotFoundInCacheForEstimatedCosts( + "PrivateDocumentStoreInsert estimation requires the store's own layer \ + declared with TreeType::PrivateDocumentStore(chunk_power) in the \ + estimated layer information" + .to_string(), + )) + .wrap_with_cost(OperationCost::default()); + }; let item_cost = GroveDb::average_case_merk_replace_tree( key, layer_element_estimates, - TreeType::PrivateDocumentStore(0), + TreeType::PrivateDocumentStore(chunk_power), propagate, grove_version, ); - // Additional cost: buffer write + hashing. Most appends only - // write to the buffer; compaction happens once per epoch and - // is amortized away here. - // - // Unlike the flat constant this arm previously used, the - // dominant term is the dense-buffer root walk `append` - // performs on every insert (two hashes per filled position). - // That scales with the committed `chunk_power`, which the op - // does not carry, so this models a typical small store - // (`chunk_power = 8`, i.e. a 255-entry buffer averaging - // half-full). This is an average-case estimate, NOT a bound — - // `worst_case_cost` carries the real upper bound over the - // whole permitted configuration range. use grovedb_costs::storage_cost::{removal::StorageRemovedBytes, StorageCost}; + // `entry.len()` IS the store's committed entry size — the + // append path rejects any other length — so the byte terms + // need no separate declaration. let entry_size = entry.len() as u32; - /// Assumed typical buffer occupancy (half of a - /// `chunk_power = 8` buffer), two hashes per filled position. - const AVG_DENSE_HASHES: u32 = 128 * 2; - /// Bulk state root + composite config-binding pds_state root - /// + the committed-config hash paid when opening the store. - const AVG_ROOT_AND_CONFIG_HASHES: u32 = 3; - const AVG_HASH_CALLS: u32 = AVG_DENSE_HASHES + AVG_ROOT_AND_CONFIG_HASHES; + let epoch_entries: u32 = 1u32 << chunk_power.min(16) as u32; + // Amortized over one epoch: every entry is written once to + // the buffer, and once more into the chunk blob when the + // epoch compacts. + let amortized_compaction_bytes = entry_size; + // The dense-buffer root walk costs two hashes per filled + // position and runs on every append, so across an epoch it + // averages half the buffer. + let avg_dense_hashes = epoch_entries.saturating_sub(1); + // Bulk state root + composite pds_state root + the + // committed-config hash paid when the store is opened. + const ROOT_AND_CONFIG_HASHES: u32 = 3; + // MMR push work, amortized across the epoch it serves. + const AMORTIZED_MMR_HASHES: u32 = 1; item_cost.add_cost(OperationCost { seek_count: 1, // 1 buffer entry write storage_cost: StorageCost { - added_bytes: entry_size, + added_bytes: entry_size.saturating_add(amortized_compaction_bytes), replaced_bytes: 0, removed_bytes: StorageRemovedBytes::NoStorageRemoval, }, storage_loaded_bytes: 0, - hash_node_calls: AVG_HASH_CALLS, + hash_node_calls: avg_dense_hashes + .saturating_add(ROOT_AND_CONFIG_HASHES) + .saturating_add(AMORTIZED_MMR_HASHES), sinsemilla_hash_calls: 0, }) } + GroveOp::DenseTreeInsert { value } => { // Cost of updating parent element in the Merk let item_cost = GroveDb::average_case_merk_replace_tree( @@ -443,7 +459,7 @@ impl GroveOp { payload: &[u8], key: &KeyInfo, layer_element_estimates: &EstimatedLayerInformation, - ct_chunk_power: Option, + append_tree_chunk_power: Option, propagate: bool, grove_version: &GroveVersion, ) -> CostResult<(), Error> { @@ -464,7 +480,7 @@ impl GroveOp { payload, key, layer_element_estimates, - ct_chunk_power, + append_tree_chunk_power, propagate, grove_version, ), @@ -531,7 +547,7 @@ impl GroveOp { payload: &[u8], key: &KeyInfo, layer_element_estimates: &EstimatedLayerInformation, - ct_chunk_power: Option, + append_tree_chunk_power: Option, propagate: bool, grove_version: &GroveVersion, ) -> CostResult<(), Error> { @@ -542,7 +558,7 @@ impl GroveOp { // other estimated op follows. A silent fallback here would either // under-bound (too small) or grotesquely over-reserve (the physical // ceiling), both worse than a loud error at integration time. - let Some(chunk_power) = ct_chunk_power else { + let Some(chunk_power) = append_tree_chunk_power else { return Err(Error::PathNotFoundInCacheForEstimatedCosts( "CommitmentTreeInsert estimation requires the commitment tree's own layer \ declared with TreeType::CommitmentTree(chunk_power) in the estimated layer \ @@ -760,12 +776,16 @@ impl TreeCache for AverageCaseTreeCacheKnownPaths { // `TreeType::CommitmentTree(chunk_power)` — as Dash Platform // does — the estimate uses the tree's ACTUAL epoch scale // instead of the constructor-enforced cap. - let ct_chunk_power = if matches!(op, GroveOp::CommitmentTreeInsert { .. }) { + let append_tree_chunk_power = if matches!( + op, + GroveOp::CommitmentTreeInsert { .. } | GroveOp::PrivateDocumentStoreInsert { .. } + ) { crate::batch::batch_structure::keyless_op_tree_key(&key).and_then(|tree_key| { let mut tree_path = path.clone(); tree_path.push(KeyInfo::KnownKey(tree_key.to_vec())); match self.paths.get(&tree_path).map(|layer| layer.tree_type) { - Some(TreeType::CommitmentTree(chunk_power)) => Some(chunk_power), + Some(TreeType::CommitmentTree(chunk_power)) + | Some(TreeType::PrivateDocumentStore(chunk_power)) => Some(chunk_power), _ => None, } }) @@ -777,7 +797,7 @@ impl TreeCache for AverageCaseTreeCacheKnownPaths { op.average_case_cost( &key, layer_element_estimates, - ct_chunk_power, + append_tree_chunk_power, false, grove_version ) @@ -1968,7 +1988,7 @@ mod tests { estimated_layer_sizes: AllSubtrees(4, NoSumTrees, None), }; let cost = op - .average_case_cost(&key, &layer_info, false, grove_version) + .average_case_cost(&key, &layer_info, Some(4), false, grove_version) .cost_as_result() .expect("expected cost for private document store insert"); // PrivateDocumentStoreInsert mirrors BulkAppend: parent replace cost @@ -1996,12 +2016,42 @@ mod tests { entry: vec![42u8; 128], }; let cost_large = op_large - .average_case_cost(&key, &layer_info, false, grove_version) + .average_case_cost(&key, &layer_info, Some(4), false, grove_version) .cost_as_result() .expect("expected cost for larger entry"); + + // Undeclared layer must fail loudly rather than silently guessing a + // chunk power, matching the CommitmentTreeInsert contract. + assert!( + op.average_case_cost(&key, &layer_info, None, false, grove_version) + .cost_as_result() + .is_err(), + "estimation without a declared PrivateDocumentStore layer must error" + ); + + // The estimate tracks the declared chunk power: a larger epoch means + // a deeper dense-buffer walk. + let small = op + .average_case_cost(&key, &layer_info, Some(2), false, grove_version) + .cost_as_result() + .expect("cost at chunk_power 2"); + let big = op + .average_case_cost(&key, &layer_info, Some(10), false, grove_version) + .cost_as_result() + .expect("cost at chunk_power 10"); + assert!( + big.hash_node_calls > small.hash_node_calls, + "a larger declared epoch must cost more hashing ({} vs {})", + big.hash_node_calls, + small.hash_node_calls + ); + // Each entry is written TWICE across its lifetime: once into the + // dense buffer and once more into the chunk blob when the epoch + // compacts. The amortized per-append charge is therefore 2x the + // entry size, so doubling the entry grows added_bytes by 2 x 64. assert_eq!( cost_large.storage_cost.added_bytes - cost.storage_cost.added_bytes, - 64 + 128 ); } From 637b3ab03942f824122f613e4cb98b61c721e3ba Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 20 Aug 2026 00:21:20 +0700 Subject: [PATCH 07/19] fix: address the second review round on PrivateDocumentStore (PR #787) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ten distinct findings from CodeRabbit and QuantumExplorer, most of them consequences of the previous round's fixes. ATOMICITY / API * append_many appended valid entries before a later wrong-sized one failed, leaving the store mutated behind an error — no transaction to discard for a direct caller. Every entry is now validated before any is written, with a regression test asserting count, root and stored values are untouched. * append_many reported a sentinel position for empty input (0 on a fresh store, the previous last entry otherwise), indistinguishable from a real append. It now returns a dedicated result carrying `last_global_position: Option` and `appended`. COST ACCOUNTING * An empty append_many computed both roots but charged neither; the root hashes are now charged unconditionally and only the dense walk stays conditional. * Proof-path state-root derivation was free: compute_current_state_root discarded the dense walk's reads and hashes, and the empty branch did two uncharged blake3 calls. Added cost-bearing variants down through BulkAppendTree (additive; the plain forms delegate and discard exactly as before) and charged the empty branch explicitly. * The dense-root walk READS every filled position; the average-case arm charged one seek and zero loaded bytes, understating I/O by O(epoch). Both terms now scale with the epoch. * A preserved NonCounted wrapper adds one serialized byte that neither replacement estimator counted. Charged unconditionally in both arms — neither the op nor the declared layer records the wrapper, and over-charging one byte is harmless where omitting it is not. BOUNDS * The worst-case byte "bound" counted only the raw epoch payload, missing the 9-byte chunk header, the 37-byte MMR leaf envelope and 33 bytes per internal node — for entry_size = 1 the first compaction already exceeded it. Now included. * saturating_mul silently broke the bound for entry_size >= 65536. entry_size is capped at u16::MAX at all six creation/validation sites, which makes 2^16 * entry_size representable in the u32 added_bytes field so the bound holds for every accepted configuration. An entry larger than 64 KiB is outside this type's design envelope. ESTIMATION LOOKUP * The declared-layer lookup rebuilt the path segment as KeyInfo::KnownKey and used exact equality, but KeyInfo deliberately reports KnownKey and MaxKeySize as unequal — so a layer declared with MaxKeySize was missed and valid estimation failed with PathNotFoundInCacheForEstimatedCosts. It now matches by key bytes. This affects CommitmentTree too, since the mechanism is shared. VERIFICATION * The entry-size violation was recorded with entry().or_insert() at the same path as the child-hash mismatch, so it was silently dropped exactly when both checks failed. It now lands under a dedicated `__pds_entry_size__` sentinel child path, matching the indexed-tree integrity checks. Full --all-features workspace suite green; clippy clean under --all-features. Co-Authored-By: Claude Fable 5 --- grovedb-bulk-append-tree/src/tree/append.rs | 37 ++++ grovedb-element/src/element/constructor.rs | 9 +- grovedb-element/src/element/mod.rs | 4 +- grovedb-private-document-store/src/lib.rs | 4 +- grovedb-private-document-store/src/store.rs | 172 +++++++++++++++--- .../estimated_costs/average_case_costs.rs | 65 ++++++- .../batch/estimated_costs/worst_case_costs.rs | 33 +++- grovedb/src/batch/mod.rs | 7 +- grovedb/src/lib.rs | 18 +- .../insert/add_element_on_transaction/v0.rs | 8 +- .../insert/add_element_on_transaction/v1.rs | 8 +- .../proof/bind_terminal_non_merk_tree/v1.rs | 19 +- .../src/tests/private_document_store_tests.rs | 7 +- 13 files changed, 327 insertions(+), 64 deletions(-) diff --git a/grovedb-bulk-append-tree/src/tree/append.rs b/grovedb-bulk-append-tree/src/tree/append.rs index 36a873746..e28bf1b83 100644 --- a/grovedb-bulk-append-tree/src/tree/append.rs +++ b/grovedb-bulk-append-tree/src/tree/append.rs @@ -1,5 +1,6 @@ //! Append and compaction logic for BulkAppendTree. +use grovedb_costs::{CostResult, CostsExt, OperationCost}; use grovedb_merkle_mountain_range::{ hash_count_for_push, mmr_size_to_leaf_count, MmrKeySize, MmrNode, MmrStore, MMR, }; @@ -200,6 +201,42 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { Ok(compute_state_root(&mmr_root, &dense_root)) } + /// Cost-propagating variant of + /// [`compute_current_state_root`](Self::compute_current_state_root). + /// + /// Identical result; the difference is that the dense-tree root walk's + /// storage reads reach the caller instead of being discarded, and the + /// hash calls it performs (two per filled buffer position, plus the + /// state-root blake3) are charged. Callers that bill work — anything + /// returning a `CostResult` — should prefer this. + pub fn compute_current_state_root_with_cost(&self) -> CostResult<[u8; 32], BulkAppendError> { + let mut cost = OperationCost::default(); + let mmr_root = match self.last_mmr_root { + Some(r) => r, + None => match self.get_mmr_root() { + Ok(r) => r, + Err(e) => return Err(e).wrap_with_cost(cost), + }, + }; + let dense_root = match self.dense_tree.root_hash().unwrap_add_cost(&mut cost) { + Ok(r) => r, + Err(e) => { + return Err(BulkAppendError::StorageError(format!( + "dense tree root_hash failed: {}", + e + ))) + .wrap_with_cost(cost); + } + }; + // The root walk hashes every filled position twice, then one blake3 + // combines the MMR and dense roots into the state root. + cost.hash_node_calls = cost + .hash_node_calls + .saturating_add(self.dense_tree.count() as u32 * 2) + .saturating_add(1); + Ok(compute_state_root(&mmr_root, &dense_root)).wrap_with_cost(cost) + } + /// Compact all dense tree entries plus a new value into a chunk blob /// and append to the chunk MMR. Resets the dense tree. /// Returns `(hash_count, mmr_root)`. diff --git a/grovedb-element/src/element/constructor.rs b/grovedb-element/src/element/constructor.rs index 9bc7984fd..9386802a5 100644 --- a/grovedb-element/src/element/constructor.rs +++ b/grovedb-element/src/element/constructor.rs @@ -523,9 +523,14 @@ impl Element { chunk_power: u8, flags: Option, ) -> Result { - if entry_size == 0 { + if entry_size == 0 || entry_size > u16::MAX as u32 { + // Upper bound keeps `2^16 * entry_size` (the worst-case + // compaction blob) inside the u32 `added_bytes` field, so the + // worst-case storage estimate stays a real bound. An entry + // larger than 64 KiB is outside this type's design envelope + // anyway. return Err(ElementError::InvalidInput( - "private document store entry_size must be non-zero", + "private document store entry_size must be in 1..=65535", )); } if !(1..=16).contains(&chunk_power) { diff --git a/grovedb-element/src/element/mod.rs b/grovedb-element/src/element/mod.rs index f60a85a92..f2cf40bea 100644 --- a/grovedb-element/src/element/mod.rs +++ b/grovedb-element/src/element/mod.rs @@ -793,9 +793,9 @@ impl Element { /// `new_commitment_tree` / `new_bulk_append_tree`. pub fn validate_private_document_store_config(&self) -> Result<(), crate::error::ElementError> { if let Element::PrivateDocumentStore(_, entry_size, chunk_power, _) = self.underlying() { - if *entry_size == 0 { + if *entry_size == 0 || *entry_size > u16::MAX as u32 { return Err(crate::error::ElementError::InvalidInput( - "private document store entry_size must be non-zero", + "private document store entry_size must be in 1..=65535", )); } if !(1..=16).contains(chunk_power) { diff --git a/grovedb-private-document-store/src/lib.rs b/grovedb-private-document-store/src/lib.rs index 2b0870c71..0e484f5bb 100644 --- a/grovedb-private-document-store/src/lib.rs +++ b/grovedb-private-document-store/src/lib.rs @@ -36,7 +36,9 @@ pub use grovedb_bulk_append_tree::{ deserialize_chunk_blob, serialize_chunk_blob, BulkAppendError, BulkAppendTree, }; #[cfg(feature = "storage")] -pub use store::{PrivateDocumentStore, PrivateDocumentStoreAppendResult}; +pub use store::{ + PrivateDocumentStore, PrivateDocumentStoreAppendManyResult, PrivateDocumentStoreAppendResult, +}; /// Pre-computed state root of an empty [`BulkAppendTree`]: /// `blake3("bulk_state" || [0; 32] || [0; 32])`. diff --git a/grovedb-private-document-store/src/store.rs b/grovedb-private-document-store/src/store.rs index f58e18274..62444839e 100644 --- a/grovedb-private-document-store/src/store.rs +++ b/grovedb-private-document-store/src/store.rs @@ -34,6 +34,29 @@ pub struct PrivateDocumentStoreAppendResult { pub compacted: bool, } +/// Result of [`PrivateDocumentStore::append_many`]. +/// +/// Distinct from [`PrivateDocumentStoreAppendResult`] because a batch may be +/// empty, and there is then no appended position to report. Reporting a +/// sentinel (position 0 on a fresh store, or the previous last entry on a +/// populated one) would be indistinguishable from a real append. +#[derive(Debug, Clone)] +pub struct PrivateDocumentStoreAppendManyResult { + /// The new composite state root. + pub state_root: [u8; 32], + /// The underlying BulkAppendTree state root. + pub bulk_state_root: [u8; 32], + /// Position of the last entry appended BY THIS CALL, or `None` when the + /// input was empty and nothing was written. + pub last_global_position: Option, + /// How many entries this call appended. + pub appended: u64, + /// Number of blake3 hash calls performed. + pub hash_count: u32, + /// Whether compaction occurred during this call. + pub compacted: bool, +} + /// An append-only store of fixed-size opaque entries. /// /// Thin wrapper over [`BulkAppendTree`]: appends are validated against the @@ -82,9 +105,9 @@ impl<'db, S: StorageContext<'db>> PrivateDocumentStore { storage: S, ) -> CostResult { let mut cost = OperationCost::default(); - if entry_size == 0 { + if entry_size == 0 || entry_size > u16::MAX as u32 { return Err(PrivateDocumentStoreError::InvalidConfig( - "entry_size must be non-zero".to_string(), + "entry_size must be in 1..=65535".to_string(), )) .wrap_with_cost(cost); } @@ -280,17 +303,18 @@ impl<'db, S: StorageContext<'db>> PrivateDocumentStore { pub fn append_many<'e, I>( &mut self, entries: I, - ) -> CostResult + ) -> CostResult where I: IntoIterator, { let mut cost = OperationCost::default(); - let mut hash_count: u32 = 0; - let mut any_compacted = false; - let starting_total = self.bulk_tree.total_count; - let mut last_global_position = starting_total.saturating_sub(1); - for entry in entries { + // Validate EVERY entry before writing any of them. Validating inline + // would let a valid entry land and a later wrong-sized one fail, + // leaving the store mutated behind an error — which breaks atomicity + // for a direct caller that has no surrounding transaction to discard. + let entries: Vec<&[u8]> = entries.into_iter().collect(); + for entry in &entries { if entry.len() != self.entry_size as usize { return Err(PrivateDocumentStoreError::InvalidEntrySize { expected: self.entry_size, @@ -298,6 +322,14 @@ impl<'db, S: StorageContext<'db>> PrivateDocumentStore { }) .wrap_with_cost(cost); } + } + + let mut hash_count: u32 = 0; + let mut any_compacted = false; + let starting_total = self.bulk_tree.total_count; + let mut last_global_position = None; + + for entry in &entries { let r = match self.bulk_tree.append_deferred_roots(entry) { Ok(r) => r, Err(e) => { @@ -310,12 +342,10 @@ impl<'db, S: StorageContext<'db>> PrivateDocumentStore { }; hash_count = hash_count.saturating_add(r.hash_count); any_compacted |= r.compacted; - last_global_position = r.global_position; + last_global_position = Some(r.global_position); } - // Pay the deferred roots exactly once: the dense-buffer walk plus the - // bulk state-root blake3 (inside compute_current_state_root), then the - // composite pds_state blake3. + // Pay the deferred roots exactly once. let bulk_state_root = match self.bulk_tree.compute_current_state_root() { Ok(r) => r, Err(e) => { @@ -328,19 +358,22 @@ impl<'db, S: StorageContext<'db>> PrivateDocumentStore { }; let state_root = compute_private_document_store_state_root(&self.config_hash, &bulk_state_root); + + // The two root hashes are computed on EVERY call, including an empty + // one, so they are charged unconditionally. Only the dense-buffer + // walk is conditional, since it re-hashes the live buffer that the + // appends just changed. if self.bulk_tree.total_count > starting_total { - // One dense-root walk over the live buffer + bulk state root + the - // composite root. - hash_count = hash_count - .saturating_add(self.bulk_tree.buffer_count() as u32 * 2) - .saturating_add(2); + hash_count = hash_count.saturating_add(self.bulk_tree.buffer_count() as u32 * 2); } + hash_count = hash_count.saturating_add(2); cost.hash_node_calls += hash_count; - Ok(PrivateDocumentStoreAppendResult { + Ok(PrivateDocumentStoreAppendManyResult { state_root, bulk_state_root, - global_position: last_global_position, + last_global_position, + appended: self.bulk_tree.total_count - starting_total, hash_count, compacted: any_compacted, }) @@ -432,6 +465,36 @@ impl<'db, S: StorageContext<'db>> PrivateDocumentStore { )) } + /// Cost-propagating variant of + /// [`compute_current_state_root`](Self::compute_current_state_root): + /// charges the underlying dense-root walk (reads and hashes) plus the + /// composite `pds_state` blake3. + pub fn compute_current_state_root_with_cost( + &self, + ) -> CostResult<[u8; 32], PrivateDocumentStoreError> { + let mut cost = OperationCost::default(); + let bulk_root = match self + .bulk_tree + .compute_current_state_root_with_cost() + .unwrap_add_cost(&mut cost) + { + Ok(r) => r, + Err(e) => { + return Err(PrivateDocumentStoreError::InvalidData(format!( + "state root: {}", + e + ))) + .wrap_with_cost(cost); + } + }; + cost.hash_node_calls = cost.hash_node_calls.saturating_add(1); + Ok(compute_private_document_store_state_root( + &self.config_hash, + &bulk_root, + )) + .wrap_with_cost(cost) + } + /// Flush the MMR overlay to storage. /// /// Delegates to [`BulkAppendTree::commit_mmr`]. Call this at the end of @@ -503,7 +566,8 @@ mod append_many_tests { assert_eq!(many.state_root, per_entry.state_root); assert_eq!(many.bulk_state_root, per_entry.bulk_state_root); - assert_eq!(many.global_position, per_entry.global_position); + assert_eq!(many.last_global_position, Some(per_entry.global_position)); + assert_eq!(many.appended, 10); assert_eq!(batched.total_count(), one_by_one.total_count()); assert_eq!( batched.compute_current_state_root().expect("root"), @@ -532,17 +596,24 @@ mod append_many_tests { .unwrap() .expect("new"); - // Empty input writes nothing and reports the current state root. + // Empty input writes nothing, reports NO appended position (rather + // than a sentinel indistinguishable from a real append), and still + // charges the two root hashes it actually performs. let empty: Vec> = Vec::new(); - let r = store - .append_many(empty.iter().map(|e| e.as_slice())) - .unwrap() - .expect("empty append_many"); + let ctx = store.append_many(empty.iter().map(|e| e.as_slice())); + let empty_cost = ctx.cost.clone(); + let r = ctx.value.expect("empty append_many"); assert_eq!(store.total_count(), 0); + assert_eq!(r.last_global_position, None); + assert_eq!(r.appended, 0); assert_eq!( r.state_root, store.compute_current_state_root().expect("root") ); + assert_eq!( + empty_cost.hash_node_calls, 2, + "an empty batch still computes the bulk and composite roots" + ); // A wrong-size entry is rejected. let bad = [vec![0u8; 8], vec![0u8; 7]]; @@ -556,6 +627,57 @@ mod append_many_tests { } } +#[cfg(test)] +mod atomicity_tests { + use grovedb_bulk_append_tree::test_utils::MemStorageContext; + + use super::*; + + /// A wrong-sized entry anywhere in the batch must leave the store + /// completely untouched — not partially appended behind an error. A + /// direct caller has no surrounding transaction to discard. + #[test] + fn append_many_is_atomic_on_a_bad_entry() { + let mut store = PrivateDocumentStore::new(8, 2, MemStorageContext::new()) + .unwrap() + .expect("new"); + store.append(&[1u8; 8]).unwrap().expect("seed"); + + let count_before = store.total_count(); + let root_before = store.compute_current_state_root().expect("root"); + let value_before = store.get_value(0).unwrap().expect("get"); + + // First entry is valid, second is the wrong size. + let batch = [vec![2u8; 8], vec![3u8; 7]]; + assert!(matches!( + store + .append_many(batch.iter().map(|e| e.as_slice())) + .unwrap(), + Err(PrivateDocumentStoreError::InvalidEntrySize { + expected: 8, + actual: 7 + }) + )); + + assert_eq!(store.total_count(), count_before, "count must be unchanged"); + assert_eq!( + store.compute_current_state_root().expect("root"), + root_before, + "state root must be unchanged" + ); + assert_eq!( + store.get_value(0).unwrap().expect("get"), + value_before, + "stored values must be unchanged" + ); + assert_eq!( + store.get_value(1).unwrap().expect("get"), + None, + "the valid entry preceding the bad one must not have landed" + ); + } +} + #[cfg(test)] mod error_path_tests { use super::*; diff --git a/grovedb/src/batch/estimated_costs/average_case_costs.rs b/grovedb/src/batch/estimated_costs/average_case_costs.rs index 678262c73..086d1a0cd 100644 --- a/grovedb/src/batch/estimated_costs/average_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/average_case_costs.rs @@ -333,14 +333,32 @@ impl GroveOp { const ROOT_AND_CONFIG_HASHES: u32 = 3; // MMR push work, amortized across the epoch it serves. const AMORTIZED_MMR_HASHES: u32 = 1; + // The dense-root walk does not just hash: it READS every + // filled position. Averaged across an epoch that is about + // half a buffer per append, and compaction adds one more + // pass, so the I/O terms have to scale with the epoch too — + // charging one seek and zero loaded bytes understated this by + // O(epoch size). + let avg_dense_reads = epoch_entries / 2; + // A NonCounted-wrapped store serializes one byte wider, and + // the apply path now preserves that wrapper. Neither the op + // nor the declared layer records whether this store is + // wrapped, so charge the byte unconditionally: over-charging + // one byte is harmless, whereas omitting it understates every + // append to a non-counted store. + const NON_COUNTED_WRAPPER_BYTE: u32 = 1; item_cost.add_cost(OperationCost { - seek_count: 1, // 1 buffer entry write + // 1 buffer entry write + the root walk's reads. + seek_count: 1u32.saturating_add(avg_dense_reads), storage_cost: StorageCost { - added_bytes: entry_size.saturating_add(amortized_compaction_bytes), + added_bytes: entry_size + .saturating_add(amortized_compaction_bytes) + .saturating_add(NON_COUNTED_WRAPPER_BYTE), replaced_bytes: 0, removed_bytes: StorageRemovedBytes::NoStorageRemoval, }, - storage_loaded_bytes: 0, + storage_loaded_bytes: (avg_dense_reads as u64) + .saturating_mul(entry_size as u64), hash_node_calls: avg_dense_hashes .saturating_add(ROOT_AND_CONFIG_HASHES) .saturating_add(AMORTIZED_MMR_HASHES), @@ -781,13 +799,40 @@ impl TreeCache for AverageCaseTreeCacheKnownPaths { GroveOp::CommitmentTreeInsert { .. } | GroveOp::PrivateDocumentStoreInsert { .. } ) { crate::batch::batch_structure::keyless_op_tree_key(&key).and_then(|tree_key| { - let mut tree_path = path.clone(); - tree_path.push(KeyInfo::KnownKey(tree_key.to_vec())); - match self.paths.get(&tree_path).map(|layer| layer.tree_type) { - Some(TreeType::CommitmentTree(chunk_power)) - | Some(TreeType::PrivateDocumentStore(chunk_power)) => Some(chunk_power), - _ => None, - } + // Match the declared layer by KEY BYTES, not by + // `KeyInfo` equality. A caller may legitimately declare a + // layer with `KeyInfo::MaxKeySize { unique_id, .. }`, and + // `KeyInfo`'s `PartialEq` deliberately reports + // `KnownKey` and `MaxKeySize` as unequal — so an exact + // `paths.get` with a synthesized `KnownKey` misses such a + // declaration and the estimate then fails with + // `PathNotFoundInCacheForEstimatedCosts` even though the + // layer WAS declared. `KeyInfo::as_slice` yields the key + // for `KnownKey` and the `unique_id` for `MaxKeySize`, + // which is the identity callers declare with in both + // cases. + let parent_segments: Vec<&[u8]> = path.0.iter().map(|k| k.as_slice()).collect(); + self.paths + .iter() + .find(|(declared, _)| { + declared.0.len() == parent_segments.len() + 1 + && declared + .0 + .iter() + .map(|k| k.as_slice()) + .zip( + parent_segments + .iter() + .copied() + .chain(std::iter::once(tree_key)), + ) + .all(|(a, b)| a == b) + }) + .and_then(|(_, layer)| match layer.tree_type { + TreeType::CommitmentTree(chunk_power) + | TreeType::PrivateDocumentStore(chunk_power) => Some(chunk_power), + _ => None, + }) }) } else { None diff --git a/grovedb/src/batch/estimated_costs/worst_case_costs.rs b/grovedb/src/batch/estimated_costs/worst_case_costs.rs index 68aebb8f2..8df587a05 100644 --- a/grovedb/src/batch/estimated_costs/worst_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/worst_case_costs.rs @@ -325,14 +325,37 @@ impl GroveOp { // Writes: buffer entry + chunk blob + MMR nodes const MAX_WRITES: u32 = 1 + 1 + MAX_MMR_MERGES; const MAX_READS: u32 = 64; // MMR sibling reads - // The compaction blob holds a whole epoch of entries, so its - // size scales with the committed entry size — a flat byte - // constant is not a bound. - let max_compaction_blob = MAX_EPOCH_ENTRIES.saturating_mul(entry_size); + // A compacted epoch is not stored as a bare payload: the + // chunk blob carries a 9-byte header, sits inside a 37-byte + // MMR leaf envelope, and every internal MMR node the push + // creates costs a further 33 bytes. Counting only the raw + // payload made the "upper bound" fall short — for + // `entry_size = 1` the very first compaction already exceeded + // it. + const CHUNK_HEADER_BYTES: u32 = 9; + const MMR_LEAF_ENVELOPE_BYTES: u32 = 37; + const MMR_INTERNAL_NODE_BYTES: u32 = 33; + const MMR_SERIALIZATION_OVERHEAD: u32 = CHUNK_HEADER_BYTES + + MMR_LEAF_ENVELOPE_BYTES + + MMR_INTERNAL_NODE_BYTES * MAX_MMR_MERGES; + // The compaction blob holds a whole epoch of entries, so its + // size scales with the committed entry size — a flat byte + // constant is not a bound. `entry_size` is capped at + // `u16::MAX` at every creation site precisely so this product + // stays representable in the u32 `added_bytes` field. + let max_compaction_blob = MAX_EPOCH_ENTRIES + .saturating_mul(entry_size) + .saturating_add(MMR_SERIALIZATION_OVERHEAD); item_cost.add_cost(OperationCost { seek_count: MAX_WRITES + MAX_READS, storage_cost: StorageCost { - added_bytes: entry_size.saturating_add(max_compaction_blob), + // +1 for the NonCounted wrapper byte a preserved + // wrapper adds to the replaced parent element; the op + // does not record whether this store is wrapped, so + // the bound charges it unconditionally. + added_bytes: entry_size + .saturating_add(max_compaction_blob) + .saturating_add(1), replaced_bytes: 0, removed_bytes: StorageRemovedBytes::NoStorageRemoval, }, diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index fdf38036e..325a68e72 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -3017,9 +3017,12 @@ where // enforce — a caller-built element must not bypass // it, since the config is committed into the state // root. - if *entry_size == 0 || !(1..=16).contains(chunk_power) { + if *entry_size == 0 + || *entry_size > u16::MAX as u32 + || !(1..=16).contains(chunk_power) + { return Err(Error::InvalidBatchOperation( - "a PrivateDocumentStore requires entry_size >= 1 and \ + "a PrivateDocumentStore requires entry_size in 1..=65535 and \ chunk_power in 1..=16", )) .wrap_with_cost(cost); diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index f2c8f6f4d..98389473a 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -2001,11 +2001,19 @@ impl GroveDb { ) .into(); let actual_placeholder: CryptoHash = blake3::hash(label.as_bytes()).into(); - issues.entry(new_path.to_vec()).or_insert(( - root_hash, - expected_placeholder, - actual_placeholder, - )); + // Record under a dedicated sentinel child path, the + // way the indexed-tree integrity checks do. Keying it + // on `new_path` with `or_insert` would silently drop + // this violation whenever a child-hash mismatch was + // already recorded at the same path — i.e. exactly + // when both checks fail — which defeats the point of + // reporting it as its own diagnostic. + let mut issue_path = new_path.to_vec(); + issue_path.push(b"__pds_entry_size__".to_vec()); + issues.insert( + issue_path, + (root_hash, expected_placeholder, actual_placeholder), + ); } // Software-consistency check: the aggregate fields diff --git a/grovedb/src/operations/insert/add_element_on_transaction/v0.rs b/grovedb/src/operations/insert/add_element_on_transaction/v0.rs index 5b2f91915..e014f0b6f 100644 --- a/grovedb/src/operations/insert/add_element_on_transaction/v0.rs +++ b/grovedb/src/operations/insert/add_element_on_transaction/v0.rs @@ -216,9 +216,13 @@ impl GroveDb { )) .wrap_with_cost(cost); } - if *entry_size == 0 || !(1..=16).contains(chunk_power) { + if *entry_size == 0 + || *entry_size > u16::MAX as u32 + || !(1..=16).contains(chunk_power) + { return Err(Error::InvalidInput( - "a PrivateDocumentStore requires entry_size >= 1 and chunk_power in 1..=16", + "a PrivateDocumentStore requires entry_size in 1..=65535 and chunk_power \ + in 1..=16", )) .wrap_with_cost(cost); } diff --git a/grovedb/src/operations/insert/add_element_on_transaction/v1.rs b/grovedb/src/operations/insert/add_element_on_transaction/v1.rs index e2edc0b43..7b913d5c3 100644 --- a/grovedb/src/operations/insert/add_element_on_transaction/v1.rs +++ b/grovedb/src/operations/insert/add_element_on_transaction/v1.rs @@ -212,9 +212,13 @@ impl GroveDb { )) .wrap_with_cost(cost); } - if *entry_size == 0 || !(1..=16).contains(chunk_power) { + if *entry_size == 0 + || *entry_size > u16::MAX as u32 + || !(1..=16).contains(chunk_power) + { return Err(Error::InvalidInput( - "a PrivateDocumentStore requires entry_size >= 1 and chunk_power in 1..=16", + "a PrivateDocumentStore requires entry_size in 1..=65535 and chunk_power \ + in 1..=16", )) .wrap_with_cost(cost); } diff --git a/grovedb/src/operations/proof/bind_terminal_non_merk_tree/v1.rs b/grovedb/src/operations/proof/bind_terminal_non_merk_tree/v1.rs index abeb1d815..0de66d30d 100644 --- a/grovedb/src/operations/proof/bind_terminal_non_merk_tree/v1.rs +++ b/grovedb/src/operations/proof/bind_terminal_non_merk_tree/v1.rs @@ -266,6 +266,9 @@ impl GroveDb { // store is empty, so the empty case is the precomputed // config-parametrized root rather than NULL_HASH. if *total_count == 0 { + // Two blake3 calls: the committed-config hash and the + // composite pds_state root. + cost.hash_node_calls = cost.hash_node_calls.saturating_add(2); return Ok( grovedb_private_document_store::empty_private_document_store_state_root( *entry_size, @@ -291,13 +294,15 @@ impl GroveDb { e )))) ); - let state_root = cost_return_on_error_no_add!( - cost, - store.compute_current_state_root().map_err(|e| { - Error::CorruptedData(format!( - "private document store state root failed: {}", - e - )) + let state_root = cost_return_on_error!( + &mut cost, + store.compute_current_state_root_with_cost().map(|r| { + r.map_err(|e| { + Error::CorruptedData(format!( + "private document store state root failed: {}", + e + )) + }) }) ); Ok(state_root).wrap_with_cost(cost) diff --git a/grovedb/src/tests/private_document_store_tests.rs b/grovedb/src/tests/private_document_store_tests.rs index a26629d1c..1aff33d63 100644 --- a/grovedb/src/tests/private_document_store_tests.rs +++ b/grovedb/src/tests/private_document_store_tests.rs @@ -71,9 +71,14 @@ fn test_private_document_store_constructor_validation() { // chunk_power outside 1..=16 is rejected. assert!(Element::empty_private_document_store(TEST_ENTRY_SIZE, 0).is_err()); assert!(Element::empty_private_document_store(TEST_ENTRY_SIZE, 17).is_err()); + // entry_size is capped at u16::MAX so the worst-case compaction blob + // (2^16 * entry_size) stays representable in the u32 added_bytes field, + // keeping the worst-case storage estimate a real upper bound. + assert!(Element::empty_private_document_store(u16::MAX as u32 + 1, 4).is_err()); + assert!(Element::empty_private_document_store(u32::MAX, 4).is_err()); // Boundary values are accepted. assert!(Element::empty_private_document_store(1, 1).is_ok()); - assert!(Element::empty_private_document_store(u32::MAX, 16).is_ok()); + assert!(Element::empty_private_document_store(u16::MAX as u32, 16).is_ok()); } #[test] From a6533cdf3a53abedc0732417a90426321d08ad9f Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 20 Aug 2026 00:41:54 +0700 Subject: [PATCH 08/19] fix: bill the uncached MMR root read on the lazy path (PR #787) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `compute_current_state_root_with_cost` propagated the dense-tree read cost but fell back to `get_mmr_root()`, which returns a plain `Result` and discards the MMR read's `CostContext`. That fallback is taken exactly when `last_mmr_root` is `None` — the state `from_state` leaves behind — so a REOPENED non-empty tree, which is what proof binding and the integrity walk operate on, undercharged its storage I/O. A gap in the previous commit's own cost fix. Added `BulkAppendTree::get_mmr_root_with_cost` and routed the lazy path through it; the plain `get_mmr_root` now delegates and discards exactly as before, so released callers are unchanged. Testing note: the shared in-memory harness reports `OperationCost::default()` from `get`, so storage seeks and loaded bytes are invisible to crate-level tests — only hash accounting is observable there. The billing itself is asserted against real RocksDB storage by `test_private_document_store_reopened_reads_are_billed`, which proves a terminal store proof and checks it charges seeks and loaded bytes. Full --all-features workspace suite green; clippy clean under --all-features. Co-Authored-By: Claude Fable 5 --- grovedb-bulk-append-tree/src/tree/append.rs | 30 +++++++++++---- grovedb-private-document-store/src/store.rs | 38 +++++++++++++++++++ .../src/tests/private_document_store_tests.rs | 30 +++++++++++++++ 3 files changed, 91 insertions(+), 7 deletions(-) diff --git a/grovedb-bulk-append-tree/src/tree/append.rs b/grovedb-bulk-append-tree/src/tree/append.rs index e28bf1b83..00daaf4fc 100644 --- a/grovedb-bulk-append-tree/src/tree/append.rs +++ b/grovedb-bulk-append-tree/src/tree/append.rs @@ -213,7 +213,9 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { let mut cost = OperationCost::default(); let mmr_root = match self.last_mmr_root { Some(r) => r, - None => match self.get_mmr_root() { + // Lazy path: a reopened tree has no cached root, so this read is + // real I/O and must be billed. + None => match self.get_mmr_root_with_cost().unwrap_add_cost(&mut cost) { Ok(r) => r, Err(e) => return Err(e).wrap_with_cost(cost), }, @@ -317,17 +319,31 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { /// Get the MMR root hash, or `[0; 32]` if no chunks exist. pub(crate) fn get_mmr_root(&self) -> Result<[u8; 32], BulkAppendError> { + self.get_mmr_root_with_cost().unwrap() + } + + /// Cost-propagating variant of [`get_mmr_root`](Self::get_mmr_root). + /// + /// Matters on the lazy path: `from_state` leaves `last_mmr_root` as + /// `None`, so a REOPENED non-empty tree resolves its root through here — + /// exactly the case proof binding and the integrity walk hit. Discarding + /// the read cost there undercharges their storage I/O. + pub(crate) fn get_mmr_root_with_cost(&self) -> CostResult<[u8; 32], BulkAppendError> { + let mut cost = OperationCost::default(); let mmr_size = self.mmr_size(); if mmr_size == 0 { - return Ok([0u8; 32]); + return Ok([0u8; 32]).wrap_with_cost(cost); } let mmr_store = MmrStore::with_key_size(&self.dense_tree.storage, MmrKeySize::U32); let mmr = MMR::new_with_overlay(mmr_size, &mmr_store, self.mmr_overlay.clone()); - let root_node = mmr - .get_root() - .unwrap() - .map_err(|e| BulkAppendError::MmrError(format!("MMR get_root failed: {}", e)))?; - Ok(root_node.hash()) + match mmr.get_root().unwrap_add_cost(&mut cost) { + Ok(root_node) => Ok(root_node.hash()).wrap_with_cost(cost), + Err(e) => Err(BulkAppendError::MmrError(format!( + "MMR get_root failed: {}", + e + ))) + .wrap_with_cost(cost), + } } /// Flush the MMR overlay to storage. diff --git a/grovedb-private-document-store/src/store.rs b/grovedb-private-document-store/src/store.rs index 62444839e..06bbf9028 100644 --- a/grovedb-private-document-store/src/store.rs +++ b/grovedb-private-document-store/src/store.rs @@ -633,6 +633,44 @@ mod atomicity_tests { use super::*; + /// A reopened store resolves its MMR root through the lazy (uncached) + /// path, which is what proof binding and the integrity walk hit. That + /// read must be billed, not silently free. + #[test] + fn reopened_store_charges_the_uncached_mmr_root_read() { + let storage = MemStorageContext::new(); + let mut store = PrivateDocumentStore::new(8, 2, storage) + .unwrap() + .expect("new"); + // Past a compaction so the MMR actually holds a chunk. + for i in 0..6u8 { + store.append(&[i; 8]).unwrap().expect("append"); + } + store.commit_mmr().expect("commit mmr"); + let storage = PrivateDocumentStore::into_storage_for_test(store); + + // `from_state` leaves the MMR root cache empty, so this computation + // takes the lazy path. + let reopened = PrivateDocumentStore::from_state(6, 8, 2, storage) + .unwrap() + .expect("reopen"); + let ctx = reopened.compute_current_state_root_with_cost(); + let cost = ctx.cost.clone(); + ctx.value.expect("state root"); + // NOTE: `MemStorageContext::get` reports `OperationCost::default()`, + // so storage seeks and loaded bytes are invisible to this harness — + // only the hash accounting is observable here. That the underlying + // storage reads are genuinely billed is covered against real + // RocksDB storage by + // `test_private_document_store_reopened_reads_are_billed` in the + // grovedb crate. + assert!( + cost.hash_node_calls > 0, + "the dense walk and root hashes must be billed, got {:?}", + cost + ); + } + /// A wrong-sized entry anywhere in the batch must leave the store /// completely untouched — not partially appended behind an error. A /// direct caller has no surrounding transaction to discard. diff --git a/grovedb/src/tests/private_document_store_tests.rs b/grovedb/src/tests/private_document_store_tests.rs index 1aff33d63..ccc576a4c 100644 --- a/grovedb/src/tests/private_document_store_tests.rs +++ b/grovedb/src/tests/private_document_store_tests.rs @@ -1534,3 +1534,33 @@ fn test_private_document_store_delete_empty_and_via_batch() { .expect("verify_grovedb"); assert!(issues.is_empty(), "issues: {:?}", issues); } + +#[test] +fn test_private_document_store_reopened_reads_are_billed() { + // A store reopened from persisted state resolves its MMR root through + // the lazy (uncached) path — the path proof binding and the integrity + // walk take. Against real storage that read must be billed. + let grove_version = GroveVersion::latest(); + let db = make_db_with_store(); + for i in 0..6u8 { + db.private_document_store_insert(&[b"root"], b"docs", entry(i), None, grove_version) + .unwrap() + .expect("append"); + } + + // A terminal proof over the store derives its state root from a freshly + // opened (cache-cold) store. + let path_query = PathQuery::new_unsized( + vec![b"root".to_vec()], + Query::new_single_key(b"docs".to_vec()), + ); + let ctx = db.prove_query(&path_query, None, grove_version); + let cost = ctx.cost.clone(); + ctx.value.expect("prove terminal store element"); + assert!( + cost.storage_loaded_bytes > 0 && cost.seek_count > 0, + "deriving a reopened store's state root during proof binding must bill \ + its storage reads, got {:?}", + cost + ); +} From 1d2304cfa70370e4258e38deb9e8e151b64db8ff Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 20 Aug 2026 01:23:18 +0700 Subject: [PATCH 09/19] fix: address the review-body findings on PrivateDocumentStore (PR #787) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These came from CodeRabbit's "outside diff range" findings, which are posted in the REVIEW BODY rather than as inline comments — so an unanswered-inline-comment sweep does not see them. Noting that here because it was the gap that let them sit. CORRECTNESS * `append_many` still computed its bulk root through the plain `compute_current_state_root`, so it re-derived hash counts from a hand-rolled model and dropped the dense walk's real storage reads. It now uses `compute_current_state_root_with_cost` and charges only the composite root on top. * The estimator accepted EITHER append-tree layer type for EITHER op, so a private document store's epoch could be estimated from a commitment tree's declaration (or vice versa) — a confident but wrong figure. The declared layer must now match the op, and a chunk power outside 1..=16 is treated as undeclared, falling through to the loud error rather than being estimated from. DOCUMENTATION THAT HAD GONE STALE * `append_many`'s doc still promised that "on a size violation the entries already appended remain" — untrue since prevalidation landed. It now states the real contract: a size violation writes nothing at all, while a mid-run storage fault is NOT rolled back and needs the caller's transaction. * The `entry_size` constraint was documented as "non-zero" in three places after the cap made it `1..=65535`. TESTS * serde and bincode both reject `entry_size = 0`, `entry_size > 65535`, and `chunk_power` of 0/17, including behind a `NonCounted` wrapper. * The empty-store delete now runs `verify_grovedb`. * Renamed a test that claimed to cover `visualize` but asserted `Display` and `type_str`. DEDUPLICATION * The four PrivateDocumentStore creation rules were written out in both direct-insert versions; the `entry_size` cap had to be applied to each copy separately, which is the drift this invites. Extracted `validate_private_document_store_creation` so a rule change lands once. * Dropped the redundant `first_seen` map in batch preprocessing — `replacements.remove` already yields each store exactly once. Full --all-features workspace suite green; clippy clean under --all-features. Co-Authored-By: Claude Fable 5 --- grovedb-element/src/element/constructor.rs | 4 +- grovedb-element/src/element/mod.rs | 50 +++++++++++++++-- .../element_display_and_serialization.rs | 26 +++++++++ grovedb-private-document-store/src/store.rs | 47 ++++++++++------ .../estimated_costs/average_case_costs.rs | 28 ++++++++-- .../insert/add_element_on_transaction/v0.rs | 28 ++-------- .../insert/add_element_on_transaction/v1.rs | 28 ++-------- .../src/operations/private_document_store.rs | 55 +++++++++++++++---- .../src/tests/private_document_store_tests.rs | 6 +- 9 files changed, 186 insertions(+), 86 deletions(-) diff --git a/grovedb-element/src/element/constructor.rs b/grovedb-element/src/element/constructor.rs index 9386802a5..b426397f7 100644 --- a/grovedb-element/src/element/constructor.rs +++ b/grovedb-element/src/element/constructor.rs @@ -502,8 +502,8 @@ impl Element { /// Set element to an empty private document store. /// - /// Returns `InvalidInput` unless `entry_size >= 1` and `chunk_power` is - /// in `1..=16` (the underlying `BulkAppendTree` dense-buffer height + /// Returns `InvalidInput` unless `entry_size` is in `1..=65535` and + /// `chunk_power` is in `1..=16` (the underlying `BulkAppendTree` dense-buffer height /// range). Unlike `empty_commitment_tree` / `empty_bulk_append_tree`, /// the constraints are enforced eagerly here: the configuration is /// committed into the state root, so an unusable config must never be diff --git a/grovedb-element/src/element/mod.rs b/grovedb-element/src/element/mod.rs index f2cf40bea..a8b9c1013 100644 --- a/grovedb-element/src/element/mod.rs +++ b/grovedb-element/src/element/mod.rs @@ -331,8 +331,8 @@ pub enum Element { /// /// Fields: `(total_count, entry_size, chunk_power, flags)` /// - `total_count`: Number of entries appended so far. - /// - `entry_size`: Committed byte length of every entry; appends of any - /// other length are rejected. + /// - `entry_size`: Committed byte length of every entry, in `1..=65535`; + /// appends of any other length are rejected. /// - `chunk_power`: Log2 of the chunk size (actual size = `1 << /// chunk_power`). /// - `flags`: Optional per-element metadata. @@ -780,9 +780,13 @@ impl Element { } /// Validate the committed configuration of a `PrivateDocumentStore` - /// element, looking through `NonCounted`: `entry_size` must be non-zero - /// and `chunk_power` must be in `1..=16` (the underlying `BulkAppendTree` - /// dense-buffer height range). Returns `Ok(())` for every other variant. + /// element, looking through `NonCounted`: `entry_size` must be in + /// `1..=65535` + /// (the upper bound keeps `2^16 * entry_size` — the worst-case + /// compaction blob — representable in the u32 storage-cost field, so the + /// worst-case estimate stays a real bound), and `chunk_power` must be in + /// `1..=16` (the underlying `BulkAppendTree` dense-buffer height range). + /// Returns `Ok(())` for every other variant. /// /// The configuration is committed into the store's state root, so an /// unusable configuration must not be representable: the checked @@ -1136,6 +1140,42 @@ mod serde_impl { assert_eq!(back, pcpsit); } + /// A `PrivateDocumentStore` carrying an unusable committed config + /// must be rejected by the serde codec as well as bincode: the + /// config is bound into the store's state root, so an invalid one + /// must not be representable through any ingress. + #[test] + fn serde_rejects_invalid_private_document_store_config() { + for payload in [ + // entry_size = 0 + r#"{"PrivateDocumentStore":[0,0,4,null]}"#, + // entry_size above the 65535 cap + r#"{"PrivateDocumentStore":[0,65536,4,null]}"#, + // chunk_power = 0 and 17 (outside 1..=16) + r#"{"PrivateDocumentStore":[0,64,0,null]}"#, + r#"{"PrivateDocumentStore":[0,64,17,null]}"#, + // the same violations behind a NonCounted wrapper + r#"{"NonCounted":{"PrivateDocumentStore":[0,64,0,null]}}"#, + r#"{"NonCounted":{"PrivateDocumentStore":[0,0,4,null]}}"#, + ] { + let result: Result = serde_json::from_str(payload); + assert!( + result.is_err(), + "serde must reject {}; got {:?}", + payload, + result + ); + } + + // A valid config still round-trips. + let valid = Element::PrivateDocumentStore(3, 64, 4, None); + let json = serde_json::to_string(&valid).expect("serialize"); + assert_eq!( + serde_json::from_str::(&json).expect("deserialize"), + valid + ); + } + /// `NotCountedOrSummed(NotCountedOrSummed(_))` and cross-nestings /// involving the new wrapper must be rejected. #[test] diff --git a/grovedb-element/tests/element_display_and_serialization.rs b/grovedb-element/tests/element_display_and_serialization.rs index 9a565d3be..c65396315 100644 --- a/grovedb-element/tests/element_display_and_serialization.rs +++ b/grovedb-element/tests/element_display_and_serialization.rs @@ -481,3 +481,29 @@ fn private_document_store_invalid_config_is_unrepresentable() { let wrapped = Element::NonCounted(Box::new(Element::new_private_document_store(0, 0, 4, None))); assert!(wrapped.serialize(grove_version).is_err()); } + +#[test] +fn private_document_store_bincode_rejects_invalid_chunk_power() { + let grove_version = GroveVersion::latest(); + // Craft bytes from a valid element, then corrupt chunk_power. serialize + // refuses to produce these, so deserialize is the ingress under test. + let good = Element::new_private_document_store(0, 64, 4, None); + let bytes = good.serialize(grove_version).expect("serialize valid"); + // Layout: [24 (discriminant), total_count, entry_size, chunk_power, flags] + assert_eq!(bytes[0], 24); + assert_eq!(bytes[3], 4, "chunk_power is the fourth byte for this shape"); + + for bad_power in [0u8, 17, 255] { + let mut corrupted = bytes.clone(); + corrupted[3] = bad_power; + assert!( + Element::deserialize(&corrupted, grove_version).is_err(), + "deserialize must reject chunk_power {}", + bad_power + ); + } + + // And the constructors reject the same values up front. + assert!(Element::empty_private_document_store(64, 0).is_err()); + assert!(Element::empty_private_document_store(64, 17).is_err()); +} diff --git a/grovedb-private-document-store/src/store.rs b/grovedb-private-document-store/src/store.rs index 06bbf9028..dbf6ec07a 100644 --- a/grovedb-private-document-store/src/store.rs +++ b/grovedb-private-document-store/src/store.rs @@ -291,13 +291,22 @@ impl<'db, S: StorageContext<'db>> PrivateDocumentStore { /// dense root, the bulk state root, and the composite `pds_state` root /// until the whole run is written. /// - /// Every entry is size-validated before it is written. On a size - /// violation the entries already appended remain — discard the - /// surrounding transaction for all-or-nothing semantics, matching - /// per-entry [`append`](Self::append) in a loop. + /// # Failure semantics /// - /// Returns the same shape the FINAL per-entry [`append`](Self::append) - /// would have: post-run roots, the last entry's position, the summed + /// EVERY entry is size-validated before ANY is written, so a wrong-sized + /// entry anywhere in the input leaves the store completely untouched — + /// no partial write, nothing to roll back. + /// + /// A failure that can only be detected mid-run — a storage fault during + /// a dense-tree write or an MMR compaction — is NOT rolled back: entries + /// already written stay written. This method has no rollback path of its + /// own, so a caller that needs all-or-nothing under storage faults must + /// discard the surrounding transaction, which is what the GroveDB batch + /// path does. Callers holding no transaction should treat a mid-run + /// storage error as leaving the store in an indeterminate state. + /// + /// Returns post-run roots, the last position appended BY THIS CALL (or + /// `None` for empty input), how many entries landed, the summed /// hash count, and whether any compaction occurred. For an empty input /// nothing is written and the current state root is returned. pub fn append_many<'e, I>( @@ -345,10 +354,15 @@ impl<'db, S: StorageContext<'db>> PrivateDocumentStore { last_global_position = Some(r.global_position); } - // Pay the deferred roots exactly once. - let bulk_state_root = match self.bulk_tree.compute_current_state_root() { + // Pay the deferred roots exactly once, through the cost-aware path + // so the dense-buffer walk's real storage reads and hashes are + // billed rather than re-derived from a hand-rolled model. + let root_ctx = self.bulk_tree.compute_current_state_root_with_cost(); + let root_cost = root_ctx.cost; + let bulk_state_root = match root_ctx.value { Ok(r) => r, Err(e) => { + cost += root_cost; return Err(PrivateDocumentStoreError::InvalidData(format!( "state root: {}", e @@ -356,18 +370,15 @@ impl<'db, S: StorageContext<'db>> PrivateDocumentStore { .wrap_with_cost(cost); } }; + hash_count = hash_count.saturating_add(root_cost.hash_node_calls); + cost += root_cost; + let state_root = compute_private_document_store_state_root(&self.config_hash, &bulk_state_root); - - // The two root hashes are computed on EVERY call, including an empty - // one, so they are charged unconditionally. Only the dense-buffer - // walk is conditional, since it re-hashes the live buffer that the - // appends just changed. - if self.bulk_tree.total_count > starting_total { - hash_count = hash_count.saturating_add(self.bulk_tree.buffer_count() as u32 * 2); - } - hash_count = hash_count.saturating_add(2); - cost.hash_node_calls += hash_count; + // The composite root is computed on EVERY call, including an empty + // one, so it is charged unconditionally. + hash_count = hash_count.saturating_add(1); + cost.hash_node_calls = cost.hash_node_calls.saturating_add(1); Ok(PrivateDocumentStoreAppendManyResult { state_root, diff --git a/grovedb/src/batch/estimated_costs/average_case_costs.rs b/grovedb/src/batch/estimated_costs/average_case_costs.rs index 086d1a0cd..1ef3fa5fa 100644 --- a/grovedb/src/batch/estimated_costs/average_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/average_case_costs.rs @@ -828,10 +828,30 @@ impl TreeCache for AverageCaseTreeCacheKnownPaths { ) .all(|(a, b)| a == b) }) - .and_then(|(_, layer)| match layer.tree_type { - TreeType::CommitmentTree(chunk_power) - | TreeType::PrivateDocumentStore(chunk_power) => Some(chunk_power), - _ => None, + .and_then(|(_, layer)| { + // The declared layer must be the RIGHT KIND of + // append tree for this op. Accepting either kind + // would let a store's epoch be estimated from a + // commitment tree's declaration (or vice versa), + // silently producing a confident but wrong + // figure; a mismatch should fall through to the + // loud "declare your layer" error instead. + let chunk_power = match (&op, layer.tree_type) { + ( + GroveOp::CommitmentTreeInsert { .. }, + TreeType::CommitmentTree(cp), + ) + | ( + GroveOp::PrivateDocumentStoreInsert { .. }, + TreeType::PrivateDocumentStore(cp), + ) => cp, + _ => return None, + }; + // A declared chunk power outside the range the + // constructors accept cannot describe a real + // tree, so treat it as undeclared rather than + // estimating from it. + (1..=16).contains(&chunk_power).then_some(chunk_power) }) }) } else { diff --git a/grovedb/src/operations/insert/add_element_on_transaction/v0.rs b/grovedb/src/operations/insert/add_element_on_transaction/v0.rs index e014f0b6f..334597ca9 100644 --- a/grovedb/src/operations/insert/add_element_on_transaction/v0.rs +++ b/grovedb/src/operations/insert/add_element_on_transaction/v0.rs @@ -201,31 +201,13 @@ impl GroveDb { Element::PrivateDocumentStore(total_count, entry_size, chunk_power, _) => { cost_return_on_error_no_add!( cost, - crate::operations::private_document_store::check_pds_enabled( - "insert Element::PrivateDocumentStore", - grove_version - .grovedb_versions - .operations - .private_document_store - .element_creation, + crate::operations::private_document_store::validate_private_document_store_creation( + *total_count, + *entry_size, + *chunk_power, + grove_version, ) ); - if *total_count != 0 { - return Err(Error::InvalidCodeExecution( - "a private document store should be empty at the moment of insertion", - )) - .wrap_with_cost(cost); - } - if *entry_size == 0 - || *entry_size > u16::MAX as u32 - || !(1..=16).contains(chunk_power) - { - return Err(Error::InvalidInput( - "a PrivateDocumentStore requires entry_size in 1..=65535 and chunk_power \ - in 1..=16", - )) - .wrap_with_cost(cost); - } // Write-once: creating a store over an existing element is // rejected on the direct path too, matching the batch arm. // A silent overwrite would reset the element to empty while diff --git a/grovedb/src/operations/insert/add_element_on_transaction/v1.rs b/grovedb/src/operations/insert/add_element_on_transaction/v1.rs index 7b913d5c3..fac51ba34 100644 --- a/grovedb/src/operations/insert/add_element_on_transaction/v1.rs +++ b/grovedb/src/operations/insert/add_element_on_transaction/v1.rs @@ -197,31 +197,13 @@ impl GroveDb { Element::PrivateDocumentStore(total_count, entry_size, chunk_power, _) => { cost_return_on_error_no_add!( cost, - crate::operations::private_document_store::check_pds_enabled( - "insert Element::PrivateDocumentStore", - grove_version - .grovedb_versions - .operations - .private_document_store - .element_creation, + crate::operations::private_document_store::validate_private_document_store_creation( + *total_count, + *entry_size, + *chunk_power, + grove_version, ) ); - if *total_count != 0 { - return Err(Error::InvalidCodeExecution( - "a private document store should be empty at the moment of insertion", - )) - .wrap_with_cost(cost); - } - if *entry_size == 0 - || *entry_size > u16::MAX as u32 - || !(1..=16).contains(chunk_power) - { - return Err(Error::InvalidInput( - "a PrivateDocumentStore requires entry_size in 1..=65535 and chunk_power \ - in 1..=16", - )) - .wrap_with_cost(cost); - } // Write-once: creating a store over an existing element is // rejected on the direct path too, matching the batch arm. // A silent overwrite would reset the element to empty while diff --git a/grovedb/src/operations/private_document_store.rs b/grovedb/src/operations/private_document_store.rs index b03c79c4b..9e23e78ac 100644 --- a/grovedb/src/operations/private_document_store.rs +++ b/grovedb/src/operations/private_document_store.rs @@ -62,6 +62,44 @@ pub(crate) fn check_pds_enabled( Ok(()) } +/// The rules a `PrivateDocumentStore` element must satisfy to be created, +/// shared by the direct insert paths (`add_element_on_transaction` v0 and +/// v1) and mirrored by the batch arm. +/// +/// Kept in one place because every one of these rules has to hold on all +/// three paths: the version gate, the empty-at-creation requirement, and the +/// committed-config bounds. They were previously written out per path, and +/// the `entry_size` cap had to be applied to each copy separately — exactly +/// the drift this prevents. +pub(crate) fn validate_private_document_store_creation( + total_count: u64, + entry_size: u32, + chunk_power: u8, + grove_version: &GroveVersion, +) -> Result<(), Error> { + check_pds_enabled( + "create Element::PrivateDocumentStore", + grove_version + .grovedb_versions + .operations + .private_document_store + .element_creation, + )?; + if total_count != 0 { + return Err(Error::InvalidCodeExecution( + "a private document store should be empty at the moment of insertion", + )); + } + // `entry_size` is capped at u16::MAX so the worst-case compaction blob + // (2^16 * entry_size) stays representable in the u32 storage-cost field. + if entry_size == 0 || entry_size > u16::MAX as u32 || !(1..=16).contains(&chunk_power) { + return Err(Error::InvalidInput( + "a PrivateDocumentStore requires entry_size in 1..=65535 and chunk_power in 1..=16", + )); + } + Ok(()) +} + impl GroveDb { /// Append an entry to a PrivateDocumentStore subtree. /// @@ -540,21 +578,18 @@ impl GroveDb { replacements.insert(tree_path.clone(), replacement); } - // Build the new ops list: keep non-PDS ops, replace the first PDS - // insert op per group with the replacement, skip the rest. - let mut first_seen: BTreeMap = BTreeMap::new(); + // Build the new ops list: keep non-PDS ops, and emit each store's + // single replacement in place of that store's FIRST insert op. + // `replacements.remove` yields a given store exactly once, so it is + // its own "first seen" marker — subsequent inserts for the same + // store find nothing and are dropped. let mut result = Vec::with_capacity(ops.len()); for op in ops.into_iter() { if matches!(op.op, GroveOp::PrivateDocumentStoreInsert { .. }) { - let tree_path = op.path.to_path(); - if !first_seen.contains_key(&tree_path) { - first_seen.insert(tree_path.clone(), true); - if let Some(replacement) = replacements.remove(&tree_path) { - result.push(replacement); - } + if let Some(replacement) = replacements.remove(&op.path.to_path()) { + result.push(replacement); } - // Skip subsequent PDS ops for the same store. } else { result.push(op); } diff --git a/grovedb/src/tests/private_document_store_tests.rs b/grovedb/src/tests/private_document_store_tests.rs index ccc576a4c..aa12ffb4e 100644 --- a/grovedb/src/tests/private_document_store_tests.rs +++ b/grovedb/src/tests/private_document_store_tests.rs @@ -1149,7 +1149,7 @@ fn test_private_document_store_apply_without_batching() { } #[test] -fn test_private_document_store_element_display_and_visualize() { +fn test_private_document_store_element_display_and_type_str() { let element = Element::new_private_document_store(3, TEST_ENTRY_SIZE, TEST_CHUNK_POWER, None); let display = format!("{}", element); assert!( @@ -1506,6 +1506,10 @@ fn test_private_document_store_delete_empty_and_via_batch() { db.get(&[b"root"], b"docs", None, grove_version).unwrap(), Err(Error::PathKeyNotFound(_)) )); + let issues = db + .verify_grovedb(None, true, false, grove_version) + .expect("verify_grovedb after empty-store delete"); + assert!(issues.is_empty(), "issues: {:?}", issues); // Batch DeleteTree is a wholly different path (scan_delete_tree_ops / // non_merk_delete_paths) from the direct delete. From 2a901395404f10897996685dd3c6272f0f52c2a4 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 20 Aug 2026 01:42:47 +0700 Subject: [PATCH 10/19] fix(costs): correct five hash and seek accounting errors on the store paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every figure a caller is billed for an append now matches the work actually performed. Each of these was found by review; none was caught by a test, because the cost assertions in place were loose (`hash_node_calls > 0`), so this also pins the accounting exactly. - `compute_current_state_root_with_cost` double-charged the dense walk. `DenseFixedSizedMerkleTree::hash_node` already bills a value hash and a node hash per filled position, and those reach us through `unwrap_add_cost`; adding `count * 2` on top charged the same work twice. - `PrivateDocumentStore::append` walked the dense buffer twice per entry. `BulkAppendTree::append` computes a dense root inside the insert that nothing reads, then recomputes it for the state root, and returns a plain `Result` so the second walk's reads and hashes are discarded. `append` now runs its single entry through the deferred batch path, which walks once and bills what it walks; the two paths also stop being able to drift apart. - `MMR::get_root` billed the peak reads but not the folds. `bag_peaks` calls `MmrNode::merge` once per extra peak, so a multi-peak root performed uncharged blake3 work. Fixed in the MMR crate rather than at the call site: the live CommitmentTree reaches MMR roots only through paths that discard cost, so no released cost surface moves. - The worst-case seek bound omitted dense reads, counting only the MMR's 64 sibling reads. The dense buffer lives in storage and is read position by position by both the root walk and compaction, so at `chunk_power = 16` a single append can perform ~131k reads. The arm claims to be a genuine upper bound; at three orders of magnitude low it was not one. - Store creation never billed its two hashes. Deriving the empty root performs the config hash and the composite `pds_state` hash, neither visible to `insert_subtree`, which receives a finished array. Charged at all three creation sites (insert v0, insert v1, batch). Tests, each verified to fail without its fix: - exact per-append hash counts, derived from what is hashed rather than asserted loosely - MMR peak-bagging billed for 1, 2, 3 and 7 leaves (0, 0, 1, 2 merges) - creation billing as a difference against a `BulkAppendTree` insert, which starts from `NULL_HASH` and so performs neither hash — the gap is exactly the two under test and survives unrelated Merk cost changes Full --all-features workspace suite green (4806 tests); clippy clean. Co-Authored-By: Claude Fable 5 --- grovedb-bulk-append-tree/src/tree/append.rs | 19 ++- grovedb-merkle-mountain-range/src/mmr.rs | 7 + .../src/tests/test_coverage.rs | 57 ++++++++ grovedb-private-document-store/src/store.rs | 122 ++++++++++++++---- .../batch/estimated_costs/worst_case_costs.rs | 37 ++++-- grovedb/src/batch/mod.rs | 7 + .../insert/add_element_on_transaction/v0.rs | 6 + .../insert/add_element_on_transaction/v1.rs | 6 + .../src/tests/private_document_store_tests.rs | 73 +++++++++++ 9 files changed, 291 insertions(+), 43 deletions(-) diff --git a/grovedb-bulk-append-tree/src/tree/append.rs b/grovedb-bulk-append-tree/src/tree/append.rs index 00daaf4fc..0b3682aca 100644 --- a/grovedb-bulk-append-tree/src/tree/append.rs +++ b/grovedb-bulk-append-tree/src/tree/append.rs @@ -205,10 +205,10 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { /// [`compute_current_state_root`](Self::compute_current_state_root). /// /// Identical result; the difference is that the dense-tree root walk's - /// storage reads reach the caller instead of being discarded, and the - /// hash calls it performs (two per filled buffer position, plus the - /// state-root blake3) are charged. Callers that bill work — anything - /// returning a `CostResult` — should prefer this. + /// storage reads and hash calls reach the caller instead of being + /// discarded, and the final state-root blake3 is charged on top of them. + /// Callers that bill work — anything returning a `CostResult` — should + /// prefer this. pub fn compute_current_state_root_with_cost(&self) -> CostResult<[u8; 32], BulkAppendError> { let mut cost = OperationCost::default(); let mmr_root = match self.last_mmr_root { @@ -230,12 +230,11 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { .wrap_with_cost(cost); } }; - // The root walk hashes every filled position twice, then one blake3 - // combines the MMR and dense roots into the state root. - cost.hash_node_calls = cost - .hash_node_calls - .saturating_add(self.dense_tree.count() as u32 * 2) - .saturating_add(1); + // `root_hash` already charged the walk itself: `hash_node` bills a + // value hash and a node hash for every filled position it visits, and + // those reached us through `unwrap_add_cost` above. Only the final + // blake3 combining the MMR and dense roots is still unbilled. + cost.hash_node_calls = cost.hash_node_calls.saturating_add(1); Ok(compute_state_root(&mmr_root, &dense_root)).wrap_with_cost(cost) } diff --git a/grovedb-merkle-mountain-range/src/mmr.rs b/grovedb-merkle-mountain-range/src/mmr.rs index 880c8f9aa..1328bd860 100644 --- a/grovedb-merkle-mountain-range/src/mmr.rs +++ b/grovedb-merkle-mountain-range/src/mmr.rs @@ -145,6 +145,13 @@ impl MMR { Ok(p) => p, Err(e) => return Err(e).wrap_with_cost(cost), }; + // `bag_peaks` folds the peaks right-to-left with one `MmrNode::merge` + // — a blake3 — per fold, so it performs `peaks - 1` hashes. Reading + // the peaks was billed above but the merges were not, which made a + // multi-peak root look free beyond its I/O. + cost.hash_node_calls = cost + .hash_node_calls + .saturating_add(peaks.len().saturating_sub(1) as u32); match bag_peaks(peaks) { Ok(Some(root)) => Ok(root).wrap_with_cost(cost), Ok(None) => Err(Error::InconsistentStore).wrap_with_cost(cost), diff --git a/grovedb-merkle-mountain-range/src/tests/test_coverage.rs b/grovedb-merkle-mountain-range/src/tests/test_coverage.rs index 2d008694c..bcb4892b1 100644 --- a/grovedb-merkle-mountain-range/src/tests/test_coverage.rs +++ b/grovedb-merkle-mountain-range/src/tests/test_coverage.rs @@ -295,3 +295,60 @@ fn verify_and_get_root_surfaces_calculate_root_error() { msg ); } + +// ============================================================================= +// mmr.rs: get_root bills the peak-bagging merges +// ============================================================================= + +/// `get_root` reads the peaks with cost, but folding them into a single root +/// calls `MmrNode::merge` — a blake3 — once per extra peak. Those merges went +/// uncharged, so a multi-peak root looked free beyond its I/O. +#[test] +fn get_root_charges_one_hash_per_peak_merge() { + // 1 leaf: mmr_size 1 takes the single-element path, no bagging at all. + let store = MemStore::default(); + let mut mmr = MMR::new(0, &store); + mmr.push(leaf(0)).unwrap().expect("push"); + let ctx = mmr.get_root(); + ctx.value.expect("root"); + assert_eq!( + ctx.cost.hash_node_calls, 0, + "a single-element MMR bags nothing" + ); + + // 2 leaves: one perfect peak, so still nothing to fold. + let store = MemStore::default(); + let mut mmr = MMR::new(0, &store); + for i in 0..2 { + mmr.push(leaf(i)).unwrap().expect("push"); + } + let ctx = mmr.get_root(); + ctx.value.expect("root"); + assert_eq!(ctx.cost.hash_node_calls, 0, "one peak needs no merge"); + + // 3 leaves: two peaks, so exactly one merge. + let store = MemStore::default(); + let mut mmr = MMR::new(0, &store); + for i in 0..3 { + mmr.push(leaf(i)).unwrap().expect("push"); + } + let ctx = mmr.get_root(); + ctx.value.expect("root"); + assert_eq!( + ctx.cost.hash_node_calls, 1, + "two peaks fold with one blake3 merge" + ); + + // 7 leaves: three peaks (4 + 2 + 1), so two merges. + let store = MemStore::default(); + let mut mmr = MMR::new(0, &store); + for i in 0..7 { + mmr.push(leaf(i)).unwrap().expect("push"); + } + let ctx = mmr.get_root(); + ctx.value.expect("root"); + assert_eq!( + ctx.cost.hash_node_calls, 2, + "three peaks fold with two blake3 merges" + ); +} diff --git a/grovedb-private-document-store/src/store.rs b/grovedb-private-document-store/src/store.rs index dbf6ec07a..8e448009c 100644 --- a/grovedb-private-document-store/src/store.rs +++ b/grovedb-private-document-store/src/store.rs @@ -152,32 +152,43 @@ impl<'db, S: StorageContext<'db>> PrivateDocumentStore { .wrap_with_cost(cost); } - let bulk_result = match self.bulk_tree.append(entry) { + // Run the single entry through the batch path rather than + // `BulkAppendTree::append`. + // + // `BulkAppendTree::append` walks the dense buffer TWICE per entry: + // once inside `append_no_state_root`, whose dense root nothing here + // reads, and again in `compute_current_state_root` to derive the root + // we actually return. It also returns a plain `Result`, so the second + // walk's storage reads and hash calls are discarded outright — a + // one-entry append billed 4 hash calls while performing 6. The + // deferred path walks once and bills what it walks, and sharing one + // implementation keeps the single and batch paths from drifting. + let many = match self + .append_many(core::iter::once(entry)) + .unwrap_add_cost(&mut cost) + { Ok(r) => r, - Err(e) => { - return Err(PrivateDocumentStoreError::InvalidData(format!( - "bulk append: {}", - e - ))) + Err(e) => return Err(e).wrap_with_cost(cost), + }; + + let global_position = match many.last_global_position { + Some(p) => p, + None => { + // Unreachable: exactly one entry was supplied and its size was + // validated above, so `append_many` recorded a position. + return Err(PrivateDocumentStoreError::InvalidData( + "single append recorded no position".to_string(), + )) .wrap_with_cost(cost); } }; - cost.hash_node_calls += bulk_result.hash_count; - - // `bulk_result.hash_count` covers the BulkAppendTree work INCLUDING - // its own state-root blake3. The composite `pds_state` root below is - // an ADDITIONAL blake3 on top of it — charge it, or every append - // under-reports one hash call. - let state_root = - compute_private_document_store_state_root(&self.config_hash, &bulk_result.state_root); - cost.hash_node_calls += 1; Ok(PrivateDocumentStoreAppendResult { - state_root, - bulk_state_root: bulk_result.state_root, - global_position: bulk_result.global_position, - hash_count: bulk_result.hash_count, - compacted: bulk_result.compacted, + state_root: many.state_root, + bulk_state_root: many.bulk_state_root, + global_position, + hash_count: many.hash_count, + compacted: many.compacted, }) .wrap_with_cost(cost) } @@ -675,13 +686,78 @@ mod atomicity_tests { // RocksDB storage by // `test_private_document_store_reopened_reads_are_billed` in the // grovedb crate. - assert!( - cost.hash_node_calls > 0, - "the dense walk and root hashes must be billed, got {:?}", + // Pinned exactly rather than `> 0`: a loose bound is what let a + // double-charge of the dense walk sit here unnoticed. With + // `chunk_power = 2` the buffer holds 3 and an epoch is 4, so 6 total + // entries leave one completed chunk (mmr_size 1, which takes the + // single-element path and bags no peaks) and 2 live buffer + // positions. `hash_node` bills a value hash and a node hash per + // filled position, so the dense walk is 4; the bulk state root and + // the composite `pds_state` root are one each. + assert_eq!( + cost.hash_node_calls, 6, + "expected 2*2 dense-walk hashes + 1 bulk state root + 1 composite \ + pds_state root, got {:?}", cost ); } + /// The hash accounting for an append, pinned entry by entry. + /// + /// Every figure here is derived from what the code actually hashes, so a + /// change to either the model or the charging breaks this test rather + /// than silently shifting fees. + #[test] + fn append_bills_exactly_the_hashes_it_performs() { + // `chunk_power = 4` gives a 15-slot buffer, so none of these appends + // compacts and the MMR stays empty (its root is the zero hash, taken + // without hashing). + let mut store = PrivateDocumentStore::new(8, 4, MemStorageContext::new()) + .unwrap() + .expect("new"); + + // First append: the dense walk visits 1 filled position (2 hashes), + // then the bulk state root (1) and the composite pds_state root (1). + let ctx = store.append(&[1u8; 8]); + ctx.value.expect("append"); + assert_eq!( + ctx.cost.hash_node_calls, 4, + "2 dense + 1 bulk root + 1 composite, got {:?}", + ctx.cost + ); + + // Second append: 2 filled positions now, so the walk costs 4. + let ctx = store.append(&[2u8; 8]); + ctx.value.expect("append"); + assert_eq!( + ctx.cost.hash_node_calls, 6, + "4 dense + 1 bulk root + 1 composite, got {:?}", + ctx.cost + ); + + // Third: 6 dense + 2 roots. + let ctx = store.append(&[3u8; 8]); + ctx.value.expect("append"); + assert_eq!( + ctx.cost.hash_node_calls, 8, + "6 dense + 1 bulk root + 1 composite, got {:?}", + ctx.cost + ); + } + + /// Opening a store derives the committed-config hash, which is real work + /// and must not be free. + #[test] + fn opening_a_store_bills_the_config_hash() { + let ctx = PrivateDocumentStore::new(8, 4, MemStorageContext::new()); + ctx.value.expect("new"); + assert_eq!( + ctx.cost.hash_node_calls, 1, + "the committed-config blake3, got {:?}", + ctx.cost + ); + } + /// A wrong-sized entry anywhere in the batch must leave the store /// completely untouched — not partially appended behind an error. A /// direct caller has no surrounding transaction to discard. diff --git a/grovedb/src/batch/estimated_costs/worst_case_costs.rs b/grovedb/src/batch/estimated_costs/worst_case_costs.rs index 8df587a05..eb8b8d247 100644 --- a/grovedb/src/batch/estimated_costs/worst_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/worst_case_costs.rs @@ -324,14 +324,24 @@ impl GroveOp { MAX_DENSE_HASHES + MAX_MMR_MERGES + ROOT_AND_CONFIG_HASHES; // Writes: buffer entry + chunk blob + MMR nodes const MAX_WRITES: u32 = 1 + 1 + MAX_MMR_MERGES; - const MAX_READS: u32 = 64; // MMR sibling reads - // A compacted epoch is not stored as a bare payload: the - // chunk blob carries a 9-byte header, sits inside a 37-byte - // MMR leaf envelope, and every internal MMR node the push - // creates costs a further 33 bytes. Counting only the raw - // payload made the "upper bound" fall short — for - // `entry_size = 1` the very first compaction already exceeded - // it. + const MAX_MMR_READS: u32 = 64; // MMR sibling reads + /// Dense-buffer reads, which dominate the seek count and are + /// easy to miss: the buffer lives in storage, so BOTH the + /// dense-root walk and compaction read it position by + /// position. A reopened store at `chunk_power = 16` walks up + /// to `2^16 - 1` filled positions to derive the root, and a + /// compacting append reads the whole epoch again to build the + /// chunk blob. Counting only the MMR's 64 sibling reads left + /// `seek_count` three orders of magnitude below the true + /// worst case, which is not an upper bound at all. + const MAX_DENSE_READS: u32 = 2 * (MAX_EPOCH_ENTRIES - 1); + // A compacted epoch is not stored as a bare payload: the + // chunk blob carries a 9-byte header, sits inside a 37-byte + // MMR leaf envelope, and every internal MMR node the push + // creates costs a further 33 bytes. Counting only the raw + // payload made the "upper bound" fall short — for + // `entry_size = 1` the very first compaction already exceeded + // it. const CHUNK_HEADER_BYTES: u32 = 9; const MMR_LEAF_ENVELOPE_BYTES: u32 = 37; const MMR_INTERNAL_NODE_BYTES: u32 = 33; @@ -347,7 +357,9 @@ impl GroveOp { .saturating_mul(entry_size) .saturating_add(MMR_SERIALIZATION_OVERHEAD); item_cost.add_cost(OperationCost { - seek_count: MAX_WRITES + MAX_READS, + seek_count: MAX_WRITES + .saturating_add(MAX_MMR_READS) + .saturating_add(MAX_DENSE_READS), storage_cost: StorageCost { // +1 for the NonCounted wrapper byte a preserved // wrapper adds to the replaced parent element; the op @@ -359,7 +371,12 @@ impl GroveOp { replaced_bytes: 0, removed_bytes: StorageRemovedBytes::NoStorageRemoval, }, - storage_loaded_bytes: (33 * MAX_READS) as u64 + max_compaction_blob as u64, + // Each dense read returns one entry, so the bytes those + // reads load scale with the committed entry size. This + // product exceeds u32, hence the u64 arithmetic. + storage_loaded_bytes: (33 * MAX_MMR_READS) as u64 + + max_compaction_blob as u64 + + (MAX_DENSE_READS as u64).saturating_mul(entry_size as u64), hash_node_calls: MAX_HASH_CALLS, sinsemilla_hash_calls: 0, }) diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index 325a68e72..a92d41e02 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -3072,6 +3072,13 @@ where .get_feature_type(in_tree_type) .wrap_with_cost(OperationCost::default()) ); + // Deriving the empty root performs two blake3 + // calls — the committed-config hash and the + // composite `pds_state` hash — neither of which + // the helper can bill, since it returns a bare + // array. Charge them here so creating a store + // through a batch matches the direct path. + cost.hash_node_calls = cost.hash_node_calls.saturating_add(2); cost_return_on_error_into!( &mut cost, element.insert_subtree_into_batch_operations( diff --git a/grovedb/src/operations/insert/add_element_on_transaction/v0.rs b/grovedb/src/operations/insert/add_element_on_transaction/v0.rs index 334597ca9..e4b5b0916 100644 --- a/grovedb/src/operations/insert/add_element_on_transaction/v0.rs +++ b/grovedb/src/operations/insert/add_element_on_transaction/v0.rs @@ -228,6 +228,12 @@ impl GroveDb { )) .wrap_with_cost(cost); } + // Deriving the empty root performs two blake3 calls — the + // committed-config hash and the composite `pds_state` hash — + // neither of which the helper can bill, since it returns a + // bare array. Charge them here so creating a store is not + // two hashes cheaper than it really is. + cost.hash_node_calls = cost.hash_node_calls.saturating_add(2); cost_return_on_error_into!( &mut cost, element.insert_subtree( diff --git a/grovedb/src/operations/insert/add_element_on_transaction/v1.rs b/grovedb/src/operations/insert/add_element_on_transaction/v1.rs index fac51ba34..e66ba3dc2 100644 --- a/grovedb/src/operations/insert/add_element_on_transaction/v1.rs +++ b/grovedb/src/operations/insert/add_element_on_transaction/v1.rs @@ -224,6 +224,12 @@ impl GroveDb { )) .wrap_with_cost(cost); } + // Deriving the empty root performs two blake3 calls — the + // committed-config hash and the composite `pds_state` hash — + // neither of which the helper can bill, since it returns a + // bare array. Charge them here so creating a store is not + // two hashes cheaper than it really is. + cost.hash_node_calls = cost.hash_node_calls.saturating_add(2); cost_return_on_error_into!( &mut cost, element.insert_subtree( diff --git a/grovedb/src/tests/private_document_store_tests.rs b/grovedb/src/tests/private_document_store_tests.rs index aa12ffb4e..9e93b68ed 100644 --- a/grovedb/src/tests/private_document_store_tests.rs +++ b/grovedb/src/tests/private_document_store_tests.rs @@ -1568,3 +1568,76 @@ fn test_private_document_store_reopened_reads_are_billed() { cost ); } + +/// Creating a store derives its empty state root, which is two blake3 calls: +/// the committed-config hash and the composite `pds_state` hash. Neither is +/// visible to `insert_subtree`, which receives an already-computed array, so +/// both have to be charged at the creation site or the operation is billed two +/// hashes short. +/// +/// Asserted as a difference against a `BulkAppendTree` creation rather than as +/// an absolute figure: that type creates from `NULL_HASH` and so performs +/// neither hash, while every other part of the insert — the parent Merk write, +/// the element value hash — is the same work. The gap is therefore exactly the +/// two hashes under test, and the test does not break when unrelated Merk +/// costs shift. +#[test] +fn test_private_document_store_creation_bills_its_two_hashes() { + let grove_version = GroveVersion::latest(); + + let pds_cost = { + let db = make_empty_grovedb(); + db.insert( + EMPTY_PATH, + b"root", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert root tree"); + db.insert( + &[b"root"], + b"store", + Element::empty_private_document_store(TEST_ENTRY_SIZE, TEST_CHUNK_POWER) + .expect("valid config"), + None, + None, + grove_version, + ) + .cost + }; + + let bulk_cost = { + let db = make_empty_grovedb(); + db.insert( + EMPTY_PATH, + b"root", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert root tree"); + db.insert( + &[b"root"], + b"store", + Element::empty_bulk_append_tree(TEST_CHUNK_POWER).expect("valid config"), + None, + None, + grove_version, + ) + .cost + }; + + assert_eq!( + pds_cost.hash_node_calls, + bulk_cost.hash_node_calls + 2, + "store creation must bill the config hash and the composite root \ + hash (pds {:?} vs bulk {:?})", + pds_cost, + bulk_cost + ); +} From f05222d4f11f9c876f0713339273ae11346e7496 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 20 Aug 2026 02:26:44 +0700 Subject: [PATCH 11/19] test(pds): cover the corruption and storage-fault paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit codecov/patch failed at 81.86% against a 90% target. Rather than chase the number, this covers the paths that were genuinely untested — the ones that decide whether a damaged or misconfigured store is detected or silently misread. Locally, store.rs goes 87.61% -> 95.26% (72 -> 33 uncovered lines). Reopening under a wrong config. Building a store and reopening the same bytes under a different declared `{entry_size, chunk_power}` is exactly what the config-binding state root exists to stop, and it was untested. A wrong chunk_power makes a stored chunk's length disagree with the declared epoch; a wrong entry_size makes every entry the wrong width. Both must read as corruption, never as a missing document, on both `get_value` and `verify_entry_sizes`. Claiming more than storage holds. A store whose `total_count` names chunks or buffer slots that were never written must refuse rather than report the store as intact. Storage faults. `MemStorageContext` gains `fail_reads`/`fail_writes`, so the arms that only run when the backing store errors mid-operation are reachable at all. Reads must surface a fault instead of answering "absent" — conflating the two on an append-only store lets an I/O error look like an empty position — and a failed write must fail the append rather than return a state root for bytes that were never stored. The read test reopens first: the dense tree's write-through cache serves live buffer reads from memory, so a fault injected on a warm handle proves nothing. Also drops the duplicated size check in `append`: `append_many` already validates every entry before writing any and returns the same `InvalidEntrySize` with the same empty cost, so the second copy was only another place to drift — and it made its own error arm unreachable. The `last_global_position` arm becomes an `expect`, since a `None` there is a broken postcondition in this file rather than anything a caller can produce. Two arms are left uncovered deliberately, now documented as defensive: the store's own "missing buffer entry" branch (the dense tree detects the shortfall against its own count and errors first) and the batch preprocessor's empty-path and wrong-element-type guards (unreachable given how ops are grouped). Full --all-features workspace suite green (4811 tests); clippy clean. Co-Authored-By: Claude Fable 5 --- grovedb-bulk-append-tree/src/test_utils.rs | 40 +++- grovedb-private-document-store/src/store.rs | 253 ++++++++++++++++++-- 2 files changed, 273 insertions(+), 20 deletions(-) diff --git a/grovedb-bulk-append-tree/src/test_utils.rs b/grovedb-bulk-append-tree/src/test_utils.rs index 0949473b4..d68915dbe 100644 --- a/grovedb-bulk-append-tree/src/test_utils.rs +++ b/grovedb-bulk-append-tree/src/test_utils.rs @@ -1,6 +1,9 @@ //! Test utilities: in-memory StorageContext for BulkAppendTree tests. -use std::{cell::RefCell, collections::HashMap}; +use std::{ + cell::{Cell, RefCell}, + collections::HashMap, +}; use grovedb_costs::{ storage_cost::key_value_cost::KeyValueStorageCost, ChildrenSizesWithIsSumTree, CostContext, @@ -13,15 +16,38 @@ use grovedb_storage::{Batch, RawIterator, StorageContext}; /// Immediate reads and writes backed by a `HashMap`. Only `get` and `put` /// (data storage) have real implementations; all other `StorageContext` /// methods panic if called. +/// +/// `fail_get` / `fail_put` inject storage faults. Tree code is full of arms +/// that can only be reached when the backing store errors mid-operation — +/// exactly the paths that decide whether a fault surfaces or is silently +/// swallowed — and without injection those arms are untestable. #[derive(Default)] pub struct MemStorageContext { pub data: RefCell, Vec>>, + pub fail_get: Cell, + pub fail_put: Cell, } impl MemStorageContext { pub fn new() -> Self { Self::default() } + + /// Make every subsequent `get` return a storage error. + pub fn fail_reads(&self) { + self.fail_get.set(true); + } + + /// Make every subsequent `put` return a storage error. + pub fn fail_writes(&self) { + self.fail_put.set(true); + } + + /// Resume normal operation. + pub fn heal(&self) { + self.fail_get.set(false); + self.fail_put.set(false); + } } impl<'db> StorageContext<'db> for MemStorageContext { @@ -29,6 +55,12 @@ impl<'db> StorageContext<'db> for MemStorageContext { type RawIterator = MemRawIterator; fn get>(&self, key: K) -> CostResult>, grovedb_storage::Error> { + if self.fail_get.get() { + return Err(grovedb_storage::Error::StorageError( + "injected read failure".to_string(), + )) + .wrap_with_cost(OperationCost::default()); + } Ok(self.data.borrow().get(key.as_ref()).cloned()).wrap_with_cost(OperationCost::default()) } @@ -39,6 +71,12 @@ impl<'db> StorageContext<'db> for MemStorageContext { _children_sizes: ChildrenSizesWithIsSumTree, _cost_info: Option, ) -> CostResult<(), grovedb_storage::Error> { + if self.fail_put.get() { + return Err(grovedb_storage::Error::StorageError( + "injected write failure".to_string(), + )) + .wrap_with_cost(OperationCost::default()); + } self.data .borrow_mut() .insert(key.as_ref().to_vec(), value.to_vec()); diff --git a/grovedb-private-document-store/src/store.rs b/grovedb-private-document-store/src/store.rs index 8e448009c..9f96b52e2 100644 --- a/grovedb-private-document-store/src/store.rs +++ b/grovedb-private-document-store/src/store.rs @@ -144,14 +144,11 @@ impl<'db, S: StorageContext<'db>> PrivateDocumentStore { ) -> CostResult { let mut cost = OperationCost::default(); - if entry.len() != self.entry_size as usize { - return Err(PrivateDocumentStoreError::InvalidEntrySize { - expected: self.entry_size, - actual: entry.len(), - }) - .wrap_with_cost(cost); - } - + // Size validation is NOT repeated here: `append_many` validates every + // entry before writing any, and returns the same + // `InvalidEntrySize { expected, actual }` with the same (empty) cost. + // A second copy here would be one more place to drift. + // // Run the single entry through the batch path rather than // `BulkAppendTree::append`. // @@ -171,17 +168,14 @@ impl<'db, S: StorageContext<'db>> PrivateDocumentStore { Err(e) => return Err(e).wrap_with_cost(cost), }; - let global_position = match many.last_global_position { - Some(p) => p, - None => { - // Unreachable: exactly one entry was supplied and its size was - // validated above, so `append_many` recorded a position. - return Err(PrivateDocumentStoreError::InvalidData( - "single append recorded no position".to_string(), - )) - .wrap_with_cost(cost); - } - }; + // Exactly one entry was supplied and `append_many` returned `Ok`, so + // it appended it and recorded the position. `expect` rather than an + // error arm: a `None` here would mean `append_many` reported success + // without appending, which is a broken postcondition in this file, not + // a runtime condition a caller can produce or handle. + let global_position = many + .last_global_position + .expect("append_many returned Ok for one entry without a position"); Ok(PrivateDocumentStoreAppendResult { state_root: many.state_root, @@ -866,6 +860,227 @@ mod error_path_tests { broken.compute_current_state_root().is_err() || broken.get_value(5).unwrap().is_err() ); } + + /// Build a store, then reopen the same bytes under a DIFFERENT declared + /// config. This is the attack the config-binding state root exists to stop, + /// and the read paths must refuse rather than reinterpret the bytes. + #[test] + fn test_reopen_under_wrong_config_is_reported_as_corruption() { + // 10 entries at chunk_power 3 (epoch 8): one completed chunk of 8, + // two live in the buffer. + let mut store = PrivateDocumentStore::new(8, 3, MemStorageContext::new()) + .unwrap() + .expect("new"); + for i in 0..10u8 { + store.append(&[i; 8]).unwrap().expect("append"); + } + store.commit_mmr().expect("commit mmr"); + let storage = PrivateDocumentStore::into_storage_for_test(store); + + // Wrong chunk_power: epoch is now 4, so the stored 8-entry chunk no + // longer matches the declared epoch. A truncated/oversized chunk must + // read as corruption, NOT as a missing document. + let wrong_power = PrivateDocumentStore::from_state(10, 8, 2, storage) + .unwrap() + .expect("reopen"); + assert!( + matches!( + wrong_power.get_value(0).unwrap(), + Err(PrivateDocumentStoreError::CorruptedData(ref m)) + if m.contains("expected 4") + ), + "got {:?}", + wrong_power.get_value(0).unwrap() + ); + assert!(matches!( + wrong_power.verify_entry_sizes(), + Err(PrivateDocumentStoreError::CorruptedData(_)) + )); + + // Wrong entry_size: the chunk deserializes and has the right count, + // but every entry is the wrong width. + let storage = PrivateDocumentStore::into_storage_for_test(wrong_power); + let wrong_size = PrivateDocumentStore::from_state(10, 16, 3, storage) + .unwrap() + .expect("reopen"); + assert!( + matches!( + wrong_size.verify_entry_sizes(), + Err(PrivateDocumentStoreError::CorruptedData(ref m)) + if m.contains("committed entry size is 16") + ), + "got {:?}", + wrong_size.verify_entry_sizes() + ); + // The same violation is caught on the direct read path. + assert!(matches!( + wrong_size.get_value(0).unwrap(), + Err(PrivateDocumentStoreError::CorruptedData(_)) + )); + } + + /// A store that claims more entries than its storage holds must report + /// the absence as corruption on every path — a claimed-but-absent chunk + /// and a claimed-but-absent buffer slot are different branches. + #[test] + fn test_claimed_entries_beyond_storage_are_corruption() { + let mut store = PrivateDocumentStore::new(8, 2, MemStorageContext::new()) + .unwrap() + .expect("new"); + // 6 entries at chunk_power 2 (epoch 4): one chunk, two buffered. + for i in 0..6u8 { + store.append(&[i; 8]).unwrap().expect("append"); + } + store.commit_mmr().expect("commit mmr"); + let storage = PrivateDocumentStore::into_storage_for_test(store); + + // Claim 20 entries: chunks 1..=3 and their buffer slots do not exist. + let overclaimed = PrivateDocumentStore::from_state(20, 8, 2, storage) + .unwrap() + .expect("reopen"); + // Position 8 sits in chunk 2, which was never written. + assert!( + overclaimed.get_value(8).unwrap().is_err(), + "a claimed-but-absent chunk must not read as success" + ); + assert!(overclaimed.verify_entry_sizes().is_err()); + + // A store claiming buffer entries it never stored: 2 written, 3 + // claimed, and no completed chunk involved. + let mut store = PrivateDocumentStore::new(8, 3, MemStorageContext::new()) + .unwrap() + .expect("new"); + for i in 0..2u8 { + store.append(&[i; 8]).unwrap().expect("append"); + } + let storage = PrivateDocumentStore::into_storage_for_test(store); + let overclaimed = PrivateDocumentStore::from_state(3, 8, 3, storage) + .unwrap() + .expect("reopen"); + // Surfaces as `InvalidData`, not `CorruptedData`: the dense tree + // detects the shortfall against its own count and errors before + // returning `None`, so the store's own "missing buffer entry" arm is + // defensive rather than reachable from here. What matters is that the + // walk refuses rather than reporting a short store as intact. + let err = overclaimed + .verify_entry_sizes() + .expect_err("claimed buffer entry does not exist"); + assert!( + format!("{}", err).contains("position 2"), + "error should name the missing position, got {:?}", + err + ); + } + + /// Deriving the state root over storage that cannot satisfy the claimed + /// state must surface the error, not a plausible-looking root. + #[test] + fn test_state_root_over_missing_storage_errors() { + let mut store = PrivateDocumentStore::new(8, 2, MemStorageContext::new()) + .unwrap() + .expect("new"); + for i in 0..6u8 { + store.append(&[i; 8]).unwrap().expect("append"); + } + store.commit_mmr().expect("commit mmr"); + let storage = PrivateDocumentStore::into_storage_for_test(store); + storage.data.borrow_mut().clear(); + + let broken = PrivateDocumentStore::from_state(6, 8, 2, storage) + .unwrap() + .expect("reopen"); + let ctx = broken.compute_current_state_root_with_cost(); + assert!( + ctx.value.is_err(), + "a state root over wiped storage must be an error, got {:?}", + ctx.value + ); + // And appending onto that broken state fails rather than writing. + let mut broken = broken; + assert!(broken.append(&[9u8; 8]).unwrap().is_err()); + } + + /// Every read path must surface a storage fault instead of reporting the + /// document as absent. "Not found" and "could not be read" are different + /// answers, and conflating them on an append-only store would let an I/O + /// fault look like a legitimately empty position. + #[test] + fn test_read_faults_surface_rather_than_reading_as_absent() { + let mut store = PrivateDocumentStore::new(8, 2, MemStorageContext::new()) + .unwrap() + .expect("new"); + // Past a compaction so both a completed chunk and the buffer exist. + for i in 0..6u8 { + store.append(&[i; 8]).unwrap().expect("append"); + } + store.commit_mmr().expect("commit mmr"); + + // Reopen before injecting the fault. The dense tree keeps a + // write-through cache, so on the original handle a live buffer read is + // served from memory and never reaches storage — a fault there would + // prove nothing. A reopened store has a cold cache, which is also the + // state every real read after a restart is in. + let storage = PrivateDocumentStore::into_storage_for_test(store); + storage.fail_reads(); + let mut store = PrivateDocumentStore::from_state(6, 8, 2, storage) + .unwrap() + .expect("reopen"); + + // Position 5 is in the live buffer, position 0 in a completed chunk: + // these take different branches and both must error. + for pos in [5u64, 0] { + let r = store.get_value(pos).unwrap(); + assert!( + r.is_err(), + "position {} must report the read fault, got {:?}", + pos, + r + ); + } + + // The integrity walk and the state-root derivation likewise. + assert!(store.verify_entry_sizes().is_err()); + assert!(store.compute_current_state_root_with_cost().value.is_err()); + + // An out-of-range position is answered before any storage is touched, + // so it still reports absence rather than the fault. + assert_eq!(store.get_value(99).unwrap().expect("in-range check"), None); + + store.bulk_tree.dense_tree.storage.heal(); + assert!(store.get_value(0).unwrap().expect("healed read").is_some()); + } + + /// A write fault during an append must fail the append, not report a + /// success whose state root does not match what was stored. + #[test] + fn test_write_faults_fail_the_append() { + let mut store = PrivateDocumentStore::new(8, 2, MemStorageContext::new()) + .unwrap() + .expect("new"); + store.append(&[0u8; 8]).unwrap().expect("append"); + + store.bulk_tree.dense_tree.storage.fail_writes(); + let r = store.append(&[1u8; 8]).unwrap(); + assert!( + r.is_err(), + "a failed write must fail the append, got {:?}", + r + ); + + // The batch path too, and its size prevalidation still runs first: a + // wrong-sized entry is rejected on its own terms, not as a storage + // fault. + let bad = [vec![0u8; 7]]; + assert!(matches!( + store.append_many(bad.iter().map(|e| e.as_slice())).unwrap(), + Err(PrivateDocumentStoreError::InvalidEntrySize { .. }) + )); + let good = [vec![2u8; 8], vec![3u8; 8]]; + assert!(store + .append_many(good.iter().map(|e| e.as_slice())) + .unwrap() + .is_err()); + } } #[cfg(test)] From 540504b01cb7c5de21abc71d9e5fb97b1710be9e Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 20 Aug 2026 02:47:44 +0700 Subject: [PATCH 12/19] fix(costs): bill compaction work and MMR merge hashes Two gaps left by the previous cost fixes, both found by review. Compaction was free. `append_many` merged only the root computation's cost; `append_deferred_roots` returned a plain `Result` and unwrapped away the cost contexts from the buffer write, from reading every buffered entry back during `compact_with_value`, and from the MMR push and root. A compacting append at `chunk_power = 2` therefore reported 0 seeks and 0 loaded bytes despite reading all three buffered entries. `append_deferred_roots` and a new `compact_with_value_with_cost` are now cost-bearing, and `append_many` merges what they report. The released `append_no_state_root` path keeps its exact cost shape: `compact_with_value` remains, now as a wrapper that discards the cost inside the bulk crate rather than at its call site. MMR merges were uncharged in two more places. `bag_peaks` is shared, so charging it in `get_root` alone left `gen_proof` folding right-hand peaks for free; `push` likewise merged once per collapsed peak while billing only the sibling reads it fed. Both now charge one hash per merge, matching `get_root`. No live cost surface moves: every non-test caller of `gen_proof` and `push` discards the cost context, and the live CommitmentTree reaches this code only through `compact_with_value`, whose cost is discarded by design. Tests, each verified to fail without its fix: - a compacting append bills its read-back (>= 3 x 8 bytes) and exactly 3 hashes (chunk-blob leaf, bulk root, composite root), and is strictly more expensive in loaded bytes than the buffered append that follows it - `gen_proof` bills 1 merge for three peaks and 0 for one - `push` bills 0, 1, 0, 2 merges across the first four leaves Full --all-features workspace suite green (4814 tests); clippy clean. Co-Authored-By: Claude Fable 5 --- grovedb-bulk-append-tree/src/tree/append.rs | 94 ++++++++++++++----- grovedb-merkle-mountain-range/src/mmr.rs | 11 +++ .../src/tests/test_coverage.rs | 70 ++++++++++++++ grovedb-private-document-store/src/store.rs | 62 +++++++++++- 4 files changed, 212 insertions(+), 25 deletions(-) diff --git a/grovedb-bulk-append-tree/src/tree/append.rs b/grovedb-bulk-append-tree/src/tree/append.rs index 0b3682aca..86739f202 100644 --- a/grovedb-bulk-append-tree/src/tree/append.rs +++ b/grovedb-bulk-append-tree/src/tree/append.rs @@ -149,17 +149,25 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { pub fn append_deferred_roots( &mut self, value: &[u8], - ) -> Result { + ) -> CostResult { + let mut cost = OperationCost::default(); let mut hash_count: u32 = 0; let global_position = self.total_count; - let try_result = self + let try_result = match self .dense_tree .try_insert_no_root(value) - .unwrap() - .map_err(|e| { - BulkAppendError::StorageError(format!("dense tree insert failed: {}", e)) - })?; + .unwrap_add_cost(&mut cost) + { + Ok(r) => r, + Err(e) => { + return Err(BulkAppendError::StorageError(format!( + "dense tree insert failed: {}", + e + ))) + .wrap_with_cost(cost); + } + }; let compacted = match try_result { // Inserted into the buffer; no root walk, so no hashes yet. @@ -168,7 +176,13 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { // Buffer full — compact existing entries plus this value. // Must run before incrementing total_count so self.mmr_size() // reflects the pre-compaction state. - let (compact_hashes, mmr_root) = self.compact_with_value(value)?; + let (compact_hashes, mmr_root) = match self + .compact_with_value_with_cost(value) + .unwrap_add_cost(&mut cost) + { + Ok(r) => r, + Err(e) => return Err(e).wrap_with_cost(cost), + }; hash_count += compact_hashes; self.last_mmr_root = Some(mmr_root); true @@ -182,6 +196,7 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { hash_count, compacted, }) + .wrap_with_cost(cost) } /// Compute the current state root without modifying the tree. @@ -241,26 +256,50 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { /// Compact all dense tree entries plus a new value into a chunk blob /// and append to the chunk MMR. Resets the dense tree. /// Returns `(hash_count, mmr_root)`. + /// + /// Cost-discarding wrapper over + /// [`compact_with_value_with_cost`](Self::compact_with_value_with_cost). + /// Kept so the released `append_no_state_root` path bills exactly what it + /// always has — its costs are dropped here, not at the call site. fn compact_with_value(&mut self, new_value: &[u8]) -> Result<(u32, [u8; 32]), BulkAppendError> { + self.compact_with_value_with_cost(new_value).unwrap() + } + + /// Compact the buffer plus `new_value` into a chunk, propagating cost. + /// + /// Compaction is the expensive branch of an append: it reads every + /// buffered entry back out of storage, hashes the serialized blob, and + /// pushes it through the MMR. All of that was previously discarded, so a + /// compacting append looked no more expensive than a buffered one. + fn compact_with_value_with_cost( + &mut self, + new_value: &[u8], + ) -> CostResult<(u32, [u8; 32]), BulkAppendError> { + let mut cost = OperationCost::default(); let mut hash_count: u32 = 0; let count = self.dense_tree.count(); // Read all existing entries from dense tree let mut entries: Vec> = Vec::with_capacity(count as usize + 1); for i in 0..count { - let value = self - .dense_tree - .get(i) - .unwrap() - .map_err(|e| { - BulkAppendError::StorageError(format!("dense tree get at {} failed: {}", i, e)) - })? - .ok_or_else(|| { - BulkAppendError::CorruptedData(format!( + let read = self.dense_tree.get(i).unwrap_add_cost(&mut cost); + let value = match read { + Ok(Some(v)) => v, + Ok(None) => { + return Err(BulkAppendError::CorruptedData(format!( "dense tree missing value at position {} (count={})", i, count - )) - })?; + ))) + .wrap_with_cost(cost); + } + Err(e) => { + return Err(BulkAppendError::StorageError(format!( + "dense tree get at {} failed: {}", + i, e + ))) + .wrap_with_cost(cost); + } + }; entries.push(value); } @@ -268,7 +307,12 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { entries.push(new_value.to_vec()); // Serialize chunk blob as a standard MMR leaf — hash = blake3(0x00 || blob) - let blob = serialize_chunk_blob(&entries)?; + let blob = match serialize_chunk_blob(&entries) { + Ok(b) => b, + Err(e) => return Err(e).wrap_with_cost(cost), + }; + // `MmrNode::leaf` hashes the blob eagerly. + cost.hash_node_calls = cost.hash_node_calls.saturating_add(1); let leaf = MmrNode::leaf(blob); // Append chunk root to MMR @@ -285,14 +329,15 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { let mut mmr = MMR::new_with_overlay(mmr_size, &mmr_store, std::mem::take(&mut self.mmr_overlay)); - let push_result = mmr.push(leaf).unwrap(); + let push_result = mmr.push(leaf).unwrap_add_cost(&mut cost); if let Err(e) = push_result { // Restore overlay before returning error self.mmr_overlay = mmr.batch.take_overlay(); - return Err(BulkAppendError::MmrError(format!("MMR push failed: {}", e))); + return Err(BulkAppendError::MmrError(format!("MMR push failed: {}", e))) + .wrap_with_cost(cost); } - let root_result = mmr.get_root().unwrap(); + let root_result = mmr.get_root().unwrap_add_cost(&mut cost); let root = match root_result { Ok(node) => node.hash(), Err(e) => { @@ -300,7 +345,8 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { return Err(BulkAppendError::MmrError(format!( "MMR get_root failed: {}", e - ))); + ))) + .wrap_with_cost(cost); } }; @@ -313,7 +359,7 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { // Reset dense tree (old values stay in store, overwritten on next cycle) self.dense_tree.reset(); - Ok((hash_count, mmr_root)) + Ok((hash_count, mmr_root)).wrap_with_cost(cost) } /// Get the MMR root hash, or `[0; 32]` if no chunks exist. diff --git a/grovedb-merkle-mountain-range/src/mmr.rs b/grovedb-merkle-mountain-range/src/mmr.rs index 1328bd860..7d156cfac 100644 --- a/grovedb-merkle-mountain-range/src/mmr.rs +++ b/grovedb-merkle-mountain-range/src/mmr.rs @@ -107,6 +107,10 @@ impl MMR { }; let right_elem = elems.last().expect("checked"); let parent_elem = MmrNode::merge(&left_elem, right_elem); + // `merge` is a blake3. The sibling read above was billed but the + // hash it feeds was not, so a push that collapsed several peaks + // reported only its I/O. + cost.hash_node_calls = cost.hash_node_calls.saturating_add(1); elems.push(parent_elem); } // store hashes @@ -269,6 +273,13 @@ impl MMR { if bagging_track > 1 { let rhs_peaks = proof.split_off(proof.len() - bagging_track); + // Same shared `bag_peaks` the root computation uses, and the same + // `bagging_track - 1` blake3 merges — charged here too, so proof + // generation does not get the folds for free just because it + // reaches them by a different route. + cost.hash_node_calls = cost + .hash_node_calls + .saturating_add(bagging_track.saturating_sub(1) as u32); match bag_peaks(rhs_peaks) { Ok(Some(bagged)) => proof.push(bagged), Ok(None) => { diff --git a/grovedb-merkle-mountain-range/src/tests/test_coverage.rs b/grovedb-merkle-mountain-range/src/tests/test_coverage.rs index bcb4892b1..7f9c302b6 100644 --- a/grovedb-merkle-mountain-range/src/tests/test_coverage.rs +++ b/grovedb-merkle-mountain-range/src/tests/test_coverage.rs @@ -352,3 +352,73 @@ fn get_root_charges_one_hash_per_peak_merge() { "three peaks fold with two blake3 merges" ); } + +/// `gen_proof` folds right-hand peaks through the same `bag_peaks` helper the +/// root computation uses, so it performs the same blake3 merges and must bill +/// them. Charging only in `get_root` left proof generation free. +#[test] +fn gen_proof_charges_the_peak_bagging_merges() { + // 7 leaves gives three peaks (4 + 2 + 1). A proof for the first leaf + // leaves the two right-hand peaks to be bagged: one merge. + let store = MemStore::default(); + let mut mmr = MMR::new(0, &store); + let mut positions = Vec::new(); + for i in 0..7 { + positions.push(mmr.push(leaf(i)).unwrap().expect("push")); + } + + let ctx = mmr.gen_proof(vec![positions[0]]); + ctx.value.expect("proof"); + assert_eq!( + ctx.cost.hash_node_calls, 1, + "bagging two right-hand peaks is one blake3 merge, got {:?}", + ctx.cost + ); + + // A single perfect peak has nothing to bag. + let store = MemStore::default(); + let mut mmr = MMR::new(0, &store); + let mut positions = Vec::new(); + for i in 0..4 { + positions.push(mmr.push(leaf(i)).unwrap().expect("push")); + } + let ctx = mmr.gen_proof(vec![positions[0]]); + ctx.value.expect("proof"); + assert_eq!( + ctx.cost.hash_node_calls, 0, + "one peak means no bagging, got {:?}", + ctx.cost + ); +} + +/// `push` merges once per peak it collapses, and those merges are blake3 +/// calls. The sibling reads were billed but the hashes they fed were not. +#[test] +fn push_charges_one_hash_per_peak_collapse() { + let store = MemStore::default(); + let mut mmr = MMR::new(0, &store); + + // Leaf 0: no collapse. + let ctx = mmr.push(leaf(0)); + ctx.value.expect("push"); + assert_eq!(ctx.cost.hash_node_calls, 0, "first leaf merges nothing"); + + // Leaf 1 collapses one pair. + let ctx = mmr.push(leaf(1)); + ctx.value.expect("push"); + assert_eq!(ctx.cost.hash_node_calls, 1, "one merge, got {:?}", ctx.cost); + + // Leaf 2: no collapse (new peak). + let ctx = mmr.push(leaf(2)); + ctx.value.expect("push"); + assert_eq!(ctx.cost.hash_node_calls, 0, "got {:?}", ctx.cost); + + // Leaf 3 collapses twice: the pair, then the two 2-leaf peaks. + let ctx = mmr.push(leaf(3)); + ctx.value.expect("push"); + assert_eq!( + ctx.cost.hash_node_calls, 2, + "two merges, got {:?}", + ctx.cost + ); +} diff --git a/grovedb-private-document-store/src/store.rs b/grovedb-private-document-store/src/store.rs index 9f96b52e2..c64eeae81 100644 --- a/grovedb-private-document-store/src/store.rs +++ b/grovedb-private-document-store/src/store.rs @@ -344,7 +344,16 @@ impl<'db, S: StorageContext<'db>> PrivateDocumentStore { let mut last_global_position = None; for entry in &entries { - let r = match self.bulk_tree.append_deferred_roots(entry) { + // The append's own cost — the buffer write, and on a compacting + // append the read-back of every buffered entry plus the MMR push + // and root — is merged into `cost` here. Copying only + // `r.hash_count` into the result field would leave that I/O and + // the MMR's bagging hashes free. + let r = match self + .bulk_tree + .append_deferred_roots(entry) + .unwrap_add_cost(&mut cost) + { Ok(r) => r, Err(e) => { return Err(PrivateDocumentStoreError::InvalidData(format!( @@ -739,6 +748,57 @@ mod atomicity_tests { ); } + /// Compaction is the expensive branch of an append — it reads every + /// buffered entry back out of storage, hashes the chunk blob, and pushes + /// it through the MMR — and all of that used to be discarded, so a + /// compacting append billed no more I/O than a buffered one. + #[test] + fn compacting_append_bills_its_reads_and_hashes() { + // chunk_power 2: the buffer holds 3, so the 4th append compacts. + let mut store = PrivateDocumentStore::new(8, 2, MemStorageContext::new()) + .unwrap() + .expect("new"); + for i in 0..3u8 { + store.append(&[i; 8]).unwrap().expect("append"); + } + + // The 4th append does not fit the buffer, so it compacts. + let compacting = store.append(&[3u8; 8]); + compacting.value.expect("compacting append"); + let compacting_cost = compacting.cost; + + assert!( + compacting_cost.seek_count > 0 && compacting_cost.storage_loaded_bytes > 0, + "compaction reads every buffered entry; those reads must be billed, got {:?}", + compacting_cost + ); + // 3 buffered entries read back, at the committed 8 bytes each. + assert!( + compacting_cost.storage_loaded_bytes >= 24, + "expected at least the 3 x 8 bytes compaction reads back, got {:?}", + compacting_cost + ); + // 1 chunk-blob leaf hash + 1 bulk state root + 1 composite root. The + // MMR push collapses no peaks at size 0 and the root takes the + // single-element path, so neither adds a hash here. + assert_eq!( + compacting_cost.hash_node_calls, 3, + "1 leaf + 1 bulk root + 1 composite, got {:?}", + compacting_cost + ); + + // A plain buffered append afterwards reads nothing back. + let plain = store.append(&[4u8; 8]); + plain.value.expect("buffered append"); + assert!( + plain.cost.storage_loaded_bytes < compacting_cost.storage_loaded_bytes, + "a buffered append must be cheaper in loaded bytes than a \ + compacting one (buffered {:?} vs compacting {:?})", + plain.cost, + compacting_cost + ); + } + /// Opening a store derives the committed-config hash, which is real work /// and must not be free. #[test] From b7aa74f19ad47f5cf6f2e4fabf5c34cbf25494d7 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 20 Aug 2026 03:08:14 +0700 Subject: [PATCH 13/19] fix(costs): drop the duplicated MMR merge charge, and report bagging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two regressions from the previous commit's MMR change, both found by review. Merges were charged twice on the MmrTree paths. Once `MMR::push` began billing one hash per collapsed peak, the direct and batch `mmr_tree.rs` call sites were still adding `hash_count_for_push` — which is the eager leaf hash PLUS those same merges — and then propagating push's cost on top. Both sites now charge only the leaf hash they actually perform, leaving the merges to `push`. The reported hash count omitted peak bagging. `hash_count_for_push` covers the leaf hash and push's merges but not the folds `get_root` performs during a compaction, so once the MMR had more than one peak the counter fell below the cost. `append_deferred_roots` now derives its counter from the accumulated `OperationCost`, which is exactly this append's own hashing, so the two cannot disagree again. That derivation is deliberately scoped to the deferred path. `compact_with_value` and `append_no_state_root` keep returning the model counter, because the live CommitmentTree adds `bulk_result.hash_count` straight into its own `hash_node_calls` — changing it there would move a released cost and would need a version gate. Tests, each verified to fail without its fix: - `mmr_tree_append` over four leaves, asserted as deltas against the first append so the Merk baseline cancels: 0, +1, +1, +2, matching leaf hash + push merges + root bagging per append - `append_many` over 12 entries at chunk_power 2 (three compactions, the last leaving two peaks): the reported `hash_count` equals the billed `hash_node_calls`, on both the batch and single-append paths Full --all-features workspace suite green (4816 tests); clippy clean. Co-Authored-By: Claude Fable 5 --- grovedb-bulk-append-tree/src/tree/append.rs | 20 ++++++- grovedb-private-document-store/src/store.rs | 37 ++++++++++++ grovedb/src/operations/mmr_tree.rs | 17 +++--- grovedb/src/tests/mmr_tree_tests.rs | 63 +++++++++++++++++++++ 4 files changed, 127 insertions(+), 10 deletions(-) diff --git a/grovedb-bulk-append-tree/src/tree/append.rs b/grovedb-bulk-append-tree/src/tree/append.rs index 86739f202..d5b5e2219 100644 --- a/grovedb-bulk-append-tree/src/tree/append.rs +++ b/grovedb-bulk-append-tree/src/tree/append.rs @@ -151,7 +151,6 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { value: &[u8], ) -> CostResult { let mut cost = OperationCost::default(); - let mut hash_count: u32 = 0; let global_position = self.total_count; let try_result = match self @@ -176,14 +175,15 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { // Buffer full — compact existing entries plus this value. // Must run before incrementing total_count so self.mmr_size() // reflects the pre-compaction state. - let (compact_hashes, mmr_root) = match self + // The model counter this returns is deliberately unused — see + // the `hash_count` derivation below. + let (_model_hash_count, mmr_root) = match self .compact_with_value_with_cost(value) .unwrap_add_cost(&mut cost) { Ok(r) => r, Err(e) => return Err(e).wrap_with_cost(cost), }; - hash_count += compact_hashes; self.last_mmr_root = Some(mmr_root); true } @@ -191,6 +191,20 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { self.total_count += 1; + // Derive the reported counter from what was actually billed rather + // than from `hash_count_for_push`. That helper covers the eager leaf + // hash and the merges `push` performs, but NOT the peak-bagging + // merges `get_root` performs during a compaction, so the model + // counter falls below the true figure as soon as the MMR has more + // than one peak. Everything accumulated in `cost` here is this + // append's own hashing, so the two cannot disagree. + // + // Scoped to this deferred path on purpose: `compact_with_value` and + // `append_no_state_root` keep returning the model counter, because + // the live CommitmentTree adds that value straight into its own + // `hash_node_calls` and changing it would move a released cost. + let hash_count = cost.hash_node_calls; + Ok(AppendNoStateRootResult { global_position, hash_count, diff --git a/grovedb-private-document-store/src/store.rs b/grovedb-private-document-store/src/store.rs index c64eeae81..c20389d72 100644 --- a/grovedb-private-document-store/src/store.rs +++ b/grovedb-private-document-store/src/store.rs @@ -799,6 +799,43 @@ mod atomicity_tests { ); } + /// The reported `hash_count` and the billed `hash_node_calls` describe the + /// same work, so they must agree. They diverged once the MMR started + /// bagging peaks: the counter was derived from `hash_count_for_push`, + /// which covers the leaf hash and `push`'s merges but not `get_root`'s + /// peak folds, while the cost picked the folds up. + #[test] + fn append_many_hash_count_matches_the_billed_hashes() { + // chunk_power 2 gives an epoch of 4, so 12 entries compact three + // times. The third compaction leaves the MMR with two peaks, which is + // the case that used to be under-counted. + let mut store = PrivateDocumentStore::new(8, 2, MemStorageContext::new()) + .unwrap() + .expect("new"); + let entries: Vec> = (0..12u8).map(|i| vec![i; 8]).collect(); + let ctx = store.append_many(entries.iter().map(|e| e.as_slice())); + let cost = ctx.cost.clone(); + let r = ctx.value.expect("append_many"); + + assert_eq!( + r.hash_count, cost.hash_node_calls, + "the reported hash count and the billed hashes describe the same \ + work: reported {}, billed {:?}", + r.hash_count, cost + ); + assert!(r.compacted, "12 entries at epoch 4 must compact"); + + // Same invariant on the single-append path, which forwards the counter. + let ctx = store.append(&[99u8; 8]); + let cost = ctx.cost.clone(); + let r = ctx.value.expect("append"); + assert_eq!( + r.hash_count, cost.hash_node_calls, + "reported {}, billed {:?}", + r.hash_count, cost + ); + } + /// Opening a store derives the committed-config hash, which is real work /// and must not be free. #[test] diff --git a/grovedb/src/operations/mmr_tree.rs b/grovedb/src/operations/mmr_tree.rs index 586c55b27..2a5f4fe0c 100644 --- a/grovedb/src/operations/mmr_tree.rs +++ b/grovedb/src/operations/mmr_tree.rs @@ -11,9 +11,7 @@ use std::collections::HashMap; use grovedb_costs::{cost_return_on_error, CostResult, CostsExt, OperationCost}; use grovedb_merk::element::insert::ElementInsertToStorageExtensions; -use grovedb_merkle_mountain_range::{ - hash_count_for_push, mmr_size_to_leaf_count, MmrNode, MmrStore, MMR, -}; +use grovedb_merkle_mountain_range::{mmr_size_to_leaf_count, MmrNode, MmrStore, MMR}; use grovedb_path::SubtreePath; use grovedb_storage::{rocksdb_storage::PrefixedRocksDbTransactionContext, Storage, StorageBatch}; use grovedb_version::version::GroveVersion; @@ -84,8 +82,12 @@ impl GroveDb { let store = MmrStore::new(&storage_ctx); let leaf_count = mmr_size_to_leaf_count(mmr_size); - // Track Blake3 hash cost for this push - cost.hash_node_calls += hash_count_for_push(leaf_count); + // Only the eager leaf hash is charged here. `MMR::push` now bills one + // hash per peak it collapses, and that cost is propagated by the + // `cost_return_on_error!` below, so adding `hash_count_for_push` — + // which is the leaf hash PLUS those same merges — would charge every + // merge twice. + cost.hash_node_calls += 1; let leaf = MmrNode::leaf(value); let mut mmr = MMR::new(mmr_size, &store); @@ -446,8 +448,9 @@ impl GroveDb { // Push all values into a single MMR instance let mut mmr = MMR::new(mmr_size, &store); for value in values { - let leaf_count = mmr_size_to_leaf_count(mmr.mmr_size); - cost.hash_node_calls += hash_count_for_push(leaf_count); + // Eager leaf hash only; `push` bills its own merges. See the + // note on the direct path above. + cost.hash_node_calls += 1; let leaf = MmrNode::leaf(value.clone()); cost_return_on_error!( diff --git a/grovedb/src/tests/mmr_tree_tests.rs b/grovedb/src/tests/mmr_tree_tests.rs index 5c5d75d77..89be16ebb 100644 --- a/grovedb/src/tests/mmr_tree_tests.rs +++ b/grovedb/src/tests/mmr_tree_tests.rs @@ -1992,3 +1992,66 @@ fn test_mmr_successful_batch_after_failed_batch() { .expect("leaf count"); assert_eq!(count, 1, "only the second batch's leaf should be present"); } + +/// `MMR::push` bills one hash per peak it collapses, and the ops layer hashes +/// the leaf eagerly before calling it. Charging `hash_count_for_push` here as +/// well — that helper is the leaf hash PLUS the same merges — double-counted +/// every merge. +/// +/// Asserted as deltas against the first append: each append does identical +/// Merk work (it replaces the same element with a same-sized value), so the +/// difference between appends isolates the MMR hashing and the test does not +/// break when unrelated Merk costs shift. +#[test] +fn test_mmr_tree_append_does_not_double_charge_merges() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + db.insert( + EMPTY_PATH, + b"log", + Element::empty_mmr_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert mmr tree"); + + let mut hashes = Vec::new(); + for i in 0..4u8 { + let ctx = db.mmr_tree_append(EMPTY_PATH, b"log", vec![i; 8], None, grove_version); + ctx.value.expect("append"); + hashes.push(ctx.cost.hash_node_calls); + } + + // MMR hashing per append is the eager leaf hash, plus the merges `push` + // performs, plus the peak bagging the trailing `get_root` performs: + // + // leaf 0: 1 leaf + 0 push + 0 bag = 1 (mmr_size 1, single-element root) + // leaf 1: 1 leaf + 1 push + 0 bag = 2 (mmr_size 3, one peak) + // leaf 2: 1 leaf + 0 push + 1 bag = 2 (mmr_size 4, two peaks to fold) + // leaf 3: 1 leaf + 2 push + 0 bag = 3 (mmr_size 7, one peak) + // + // so relative to the first append the deltas are 0, +1, +1, +2. Before the + // fix each `push` merge was charged twice, which showed up as 3 hashes on + // the second leaf for the 2 it performs. + let base = hashes[0]; + assert_eq!( + hashes[1].checked_sub(base), + Some(1), + "second leaf: one push merge, no bagging, got {:?}", + hashes + ); + assert_eq!( + hashes[2].checked_sub(base), + Some(1), + "third leaf: no push merge, but two peaks to bag, got {:?}", + hashes + ); + assert_eq!( + hashes[3].checked_sub(base), + Some(2), + "fourth leaf: two push merges, one peak so no bagging, got {:?}", + hashes + ); +} From f5fb2a7bec00555c192f13a289f4ccb89b02935f Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 20 Aug 2026 05:09:13 +0700 Subject: [PATCH 14/19] feat(version): gate the MMR hash-charge corrections behind V0/V1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The MMR cost fixes earlier in this PR changed `hash_node_calls` for `push`, `get_root` and `gen_proof` unconditionally. That was wrong regardless of who consumes those costs today: costs become fees, so a node replaying a historical block has to charge what the block was admitted under. A corrected charge has to arrive as a new version, not replace the old one in place. Version plumbing: - new `MmrVersions { cost: { push, get_root, gen_proof } }` in grovedb-version, wired into `GroveVersion` alongside `merk_versions` - GROVE_V1..V3 pin all three to 0 (the shipped accounting: bill the storage reads a merge consumes, but not the merge); GROVE_V4 selects 1 MMR crate: - new `cost` module with the usual `mod.rs` / `v0.rs` / `v1.rs` split. It versions the CHARGE rather than duplicating three algorithms that differ by one `+=`; the values returned are bit-identical either way, so copying the bodies would only create somewhere for them to diverge. - `push`/`get_root`/`gen_proof` keep their signatures and delegate to `*_with_version(GroveVersion::first())`, so every caller that predates the gate keeps its released cost by construction rather than by review. One body each, no duplication. - `gen_proof` dispatches the charge unconditionally rather than inside its `bagging_track > 1` branch: the charge is zero when there is nothing to fold, but an unknown version must still be rejected rather than slipping through whenever a proof happens not to fold peaks. Consumers: - the PDS append path threads the version end to end (`append`, `append_many`, `compute_current_state_root_with_cost`, and the bulk-append functions this PR introduced), so V4 gets the corrected charges - `compact_with_value` and `get_mmr_root` pin to `GroveVersion::first()`. Both discard the cost, so the choice is unobservable — pinning states that the released `append_no_state_root` path, and therefore CommitmentTree, must not pick up a newer charge just because one exists. - `mmr_tree.rs` goes back to charging `hash_count_for_push` (leaf + collapses) paired with the unversioned `push`, which is exactly the shipped total; its `get_root` takes the versioned entry point so only the bagging correction is gated in. This resolves the earlier double-charge by construction: the merges are counted once, at the call site, on every version. CommitmentTree's public API is untouched — no crate that is live gained a version parameter. Tests: - `push`, `get_root` and `gen_proof` asserted under both versions, with the root/proof compared across versions to pin that only cost moves - the bare entry points asserted to stay on v0 - unknown charge versions rejected for all three - the version table itself pinned: V1..V3 at 0, V4 at 1, and `GroveVersion::first()` on the shipped accounting Full --all-features workspace suite green (4819 tests); clippy clean. Co-Authored-By: Claude Fable 5 --- grovedb-bulk-append-tree/Cargo.toml | 1 + grovedb-bulk-append-tree/src/tree/append.rs | 45 +++- grovedb-merkle-mountain-range/Cargo.toml | 1 + grovedb-merkle-mountain-range/src/cost/mod.rs | 74 ++++++ grovedb-merkle-mountain-range/src/cost/v0.rs | 14 ++ grovedb-merkle-mountain-range/src/cost/v1.rs | 14 ++ grovedb-merkle-mountain-range/src/error.rs | 5 + grovedb-merkle-mountain-range/src/lib.rs | 1 + grovedb-merkle-mountain-range/src/mmr.rs | 93 ++++++-- .../src/tests/test_coverage.rs | 221 ++++++++++++------ grovedb-private-document-store/Cargo.toml | 1 + grovedb-private-document-store/src/store.rs | 151 ++++++++---- grovedb-version/src/tests.rs | 42 ++++ grovedb-version/src/version/mmr_versions.rs | 38 +++ grovedb-version/src/version/mod.rs | 5 +- grovedb-version/src/version/v1.rs | 12 + grovedb-version/src/version/v2.rs | 12 + grovedb-version/src/version/v3.rs | 12 + grovedb-version/src/version/v4.rs | 13 ++ grovedb/src/operations/mmr_tree.rs | 27 ++- .../src/operations/private_document_store.rs | 6 +- .../proof/bind_terminal_non_merk_tree/mod.rs | 2 +- .../proof/bind_terminal_non_merk_tree/v1.rs | 21 +- 23 files changed, 644 insertions(+), 167 deletions(-) create mode 100644 grovedb-merkle-mountain-range/src/cost/mod.rs create mode 100644 grovedb-merkle-mountain-range/src/cost/v0.rs create mode 100644 grovedb-merkle-mountain-range/src/cost/v1.rs create mode 100644 grovedb-version/src/version/mmr_versions.rs diff --git a/grovedb-bulk-append-tree/Cargo.toml b/grovedb-bulk-append-tree/Cargo.toml index a15f9c262..4583b0f91 100644 --- a/grovedb-bulk-append-tree/Cargo.toml +++ b/grovedb-bulk-append-tree/Cargo.toml @@ -27,6 +27,7 @@ grovedb-merkle-mountain-range = { version = "5.0.1", path = "../grovedb-merkle-m grovedb-dense-fixed-sized-merkle-tree = { version = "5.0.1", path = "../grovedb-dense-fixed-sized-merkle-tree", default-features = false } grovedb-query = { version = "5.0.1", path = "../grovedb-query" } grovedb-costs = { version = "5.0.1", path = "../costs" } +grovedb-version = { version = "5.0.1", path = "../grovedb-version" } grovedb-storage = { version = "5.0.1", path = "../storage", optional = true } blake3 = { workspace = true } bincode = { workspace = true, features = ["derive"] } diff --git a/grovedb-bulk-append-tree/src/tree/append.rs b/grovedb-bulk-append-tree/src/tree/append.rs index d5b5e2219..fc542a048 100644 --- a/grovedb-bulk-append-tree/src/tree/append.rs +++ b/grovedb-bulk-append-tree/src/tree/append.rs @@ -5,6 +5,7 @@ use grovedb_merkle_mountain_range::{ hash_count_for_push, mmr_size_to_leaf_count, MmrKeySize, MmrNode, MmrStore, MMR, }; use grovedb_storage::StorageContext; +use grovedb_version::version::GroveVersion; use super::{ capacity_for_height, hash::compute_state_root, AppendNoStateRootResult, AppendResult, @@ -149,6 +150,7 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { pub fn append_deferred_roots( &mut self, value: &[u8], + grove_version: &GroveVersion, ) -> CostResult { let mut cost = OperationCost::default(); let global_position = self.total_count; @@ -178,7 +180,7 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { // The model counter this returns is deliberately unused — see // the `hash_count` derivation below. let (_model_hash_count, mmr_root) = match self - .compact_with_value_with_cost(value) + .compact_with_value_with_cost(value, grove_version) .unwrap_add_cost(&mut cost) { Ok(r) => r, @@ -238,13 +240,19 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { /// discarded, and the final state-root blake3 is charged on top of them. /// Callers that bill work — anything returning a `CostResult` — should /// prefer this. - pub fn compute_current_state_root_with_cost(&self) -> CostResult<[u8; 32], BulkAppendError> { + pub fn compute_current_state_root_with_cost( + &self, + grove_version: &GroveVersion, + ) -> CostResult<[u8; 32], BulkAppendError> { let mut cost = OperationCost::default(); let mmr_root = match self.last_mmr_root { Some(r) => r, // Lazy path: a reopened tree has no cached root, so this read is // real I/O and must be billed. - None => match self.get_mmr_root_with_cost().unwrap_add_cost(&mut cost) { + None => match self + .get_mmr_root_with_cost(grove_version) + .unwrap_add_cost(&mut cost) + { Ok(r) => r, Err(e) => return Err(e).wrap_with_cost(cost), }, @@ -276,7 +284,13 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { /// Kept so the released `append_no_state_root` path bills exactly what it /// always has — its costs are dropped here, not at the call site. fn compact_with_value(&mut self, new_value: &[u8]) -> Result<(u32, [u8; 32]), BulkAppendError> { - self.compact_with_value_with_cost(new_value).unwrap() + // Pinned to the first grove version, i.e. the shipped MMR hash + // accounting. This wrapper discards the cost anyway, so the choice is + // unobservable here — but pinning states the intent: the released + // `append_no_state_root` path (and therefore CommitmentTree) must not + // pick up a newer charge just because one exists. + self.compact_with_value_with_cost(new_value, GroveVersion::first()) + .unwrap() } /// Compact the buffer plus `new_value` into a chunk, propagating cost. @@ -288,6 +302,7 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { fn compact_with_value_with_cost( &mut self, new_value: &[u8], + grove_version: &GroveVersion, ) -> CostResult<(u32, [u8; 32]), BulkAppendError> { let mut cost = OperationCost::default(); let mut hash_count: u32 = 0; @@ -343,7 +358,9 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { let mut mmr = MMR::new_with_overlay(mmr_size, &mmr_store, std::mem::take(&mut self.mmr_overlay)); - let push_result = mmr.push(leaf).unwrap_add_cost(&mut cost); + let push_result = mmr + .push_with_version(leaf, grove_version) + .unwrap_add_cost(&mut cost); if let Err(e) = push_result { // Restore overlay before returning error self.mmr_overlay = mmr.batch.take_overlay(); @@ -351,7 +368,9 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { .wrap_with_cost(cost); } - let root_result = mmr.get_root().unwrap_add_cost(&mut cost); + let root_result = mmr + .get_root_with_version(grove_version) + .unwrap_add_cost(&mut cost); let root = match root_result { Ok(node) => node.hash(), Err(e) => { @@ -378,7 +397,9 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { /// Get the MMR root hash, or `[0; 32]` if no chunks exist. pub(crate) fn get_mmr_root(&self) -> Result<[u8; 32], BulkAppendError> { - self.get_mmr_root_with_cost().unwrap() + // Cost discarded here, so the version is unobservable; pinned to the + // shipped accounting to match the released callers. + self.get_mmr_root_with_cost(GroveVersion::first()).unwrap() } /// Cost-propagating variant of [`get_mmr_root`](Self::get_mmr_root). @@ -387,7 +408,10 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { /// `None`, so a REOPENED non-empty tree resolves its root through here — /// exactly the case proof binding and the integrity walk hit. Discarding /// the read cost there undercharges their storage I/O. - pub(crate) fn get_mmr_root_with_cost(&self) -> CostResult<[u8; 32], BulkAppendError> { + pub(crate) fn get_mmr_root_with_cost( + &self, + grove_version: &GroveVersion, + ) -> CostResult<[u8; 32], BulkAppendError> { let mut cost = OperationCost::default(); let mmr_size = self.mmr_size(); if mmr_size == 0 { @@ -395,7 +419,10 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { } let mmr_store = MmrStore::with_key_size(&self.dense_tree.storage, MmrKeySize::U32); let mmr = MMR::new_with_overlay(mmr_size, &mmr_store, self.mmr_overlay.clone()); - match mmr.get_root().unwrap_add_cost(&mut cost) { + match mmr + .get_root_with_version(grove_version) + .unwrap_add_cost(&mut cost) + { Ok(root_node) => Ok(root_node.hash()).wrap_with_cost(cost), Err(e) => Err(BulkAppendError::MmrError(format!( "MMR get_root failed: {}", diff --git a/grovedb-merkle-mountain-range/Cargo.toml b/grovedb-merkle-mountain-range/Cargo.toml index 5b445d7d1..36ac59a0c 100644 --- a/grovedb-merkle-mountain-range/Cargo.toml +++ b/grovedb-merkle-mountain-range/Cargo.toml @@ -22,6 +22,7 @@ blake3 = { workspace = true } bincode = { workspace = true, features = ["derive"] } grovedb-costs = { version = "5.0.1", path = "../costs" } grovedb-storage = { version = "5.0.1", path = "../storage", optional = true } +grovedb-version = { version = "5.0.1", path = "../grovedb-version" } [dev-dependencies] faster-hex = "0.10.0" diff --git a/grovedb-merkle-mountain-range/src/cost/mod.rs b/grovedb-merkle-mountain-range/src/cost/mod.rs new file mode 100644 index 000000000..cf818943d --- /dev/null +++ b/grovedb-merkle-mountain-range/src/cost/mod.rs @@ -0,0 +1,74 @@ +//! Versioned hash charges for the MMR operations that merge internally. +//! +//! `push`, `get_root` and `gen_proof` all compute blake3 merges — collapsing +//! peaks on a push, folding peaks into a root or a proof. The shipped +//! accounting billed the storage reads those merges consume but not the +//! merges themselves. +//! +//! Correcting that changes `hash_node_calls`, and costs become fees, so the +//! correction cannot simply replace the old behaviour: a node replaying a +//! historical block has to charge what that block was admitted under. Each +//! charge is therefore version-dispatched, v0 being the shipped (uncharged) +//! accounting and v1 the corrected one. The values the operations return are +//! bit-identical either way. + +mod v0; +mod v1; + +use grovedb_version::{error::GroveVersionError, version::GroveVersion}; + +use crate::Error; + +/// Hashes to charge for the peak collapses a `push` performs. +pub(crate) fn push_merge_hashes(merges: u32, grove_version: &GroveVersion) -> Result { + match grove_version.mmr_versions.cost.push { + 0 => Ok(v0::merge_hashes(merges)), + 1 => Ok(v1::merge_hashes(merges)), + version => Err(Error::VersionError( + GroveVersionError::UnknownVersionMismatch { + method: "MMR::push hash charge".to_string(), + known_versions: vec![0, 1], + received: version, + } + .to_string(), + )), + } +} + +/// Hashes to charge for the peak bagging a `get_root` performs. +pub(crate) fn get_root_bagging_hashes( + peaks: usize, + grove_version: &GroveVersion, +) -> Result { + match grove_version.mmr_versions.cost.get_root { + 0 => Ok(v0::bagging_hashes(peaks)), + 1 => Ok(v1::bagging_hashes(peaks)), + version => Err(Error::VersionError( + GroveVersionError::UnknownVersionMismatch { + method: "MMR::get_root hash charge".to_string(), + known_versions: vec![0, 1], + received: version, + } + .to_string(), + )), + } +} + +/// Hashes to charge for the peak bagging a `gen_proof` performs. +pub(crate) fn gen_proof_bagging_hashes( + peaks: usize, + grove_version: &GroveVersion, +) -> Result { + match grove_version.mmr_versions.cost.gen_proof { + 0 => Ok(v0::bagging_hashes(peaks)), + 1 => Ok(v1::bagging_hashes(peaks)), + version => Err(Error::VersionError( + GroveVersionError::UnknownVersionMismatch { + method: "MMR::gen_proof hash charge".to_string(), + known_versions: vec![0, 1], + received: version, + } + .to_string(), + )), + } +} diff --git a/grovedb-merkle-mountain-range/src/cost/v0.rs b/grovedb-merkle-mountain-range/src/cost/v0.rs new file mode 100644 index 000000000..ae0fe4e09 --- /dev/null +++ b/grovedb-merkle-mountain-range/src/cost/v0.rs @@ -0,0 +1,14 @@ +//! Shipped MMR hash accounting: merges are not charged. +//! +//! Used by GROVE_V1..V3. These are released versions, so this is locked — +//! see [`super`] for why a cost correction cannot replace it in place. + +/// Charge for the peak collapses of a push: none. +pub(super) fn merge_hashes(_merges: u32) -> u32 { + 0 +} + +/// Charge for peak bagging: none. +pub(super) fn bagging_hashes(_peaks: usize) -> u32 { + 0 +} diff --git a/grovedb-merkle-mountain-range/src/cost/v1.rs b/grovedb-merkle-mountain-range/src/cost/v1.rs new file mode 100644 index 000000000..93ae42aab --- /dev/null +++ b/grovedb-merkle-mountain-range/src/cost/v1.rs @@ -0,0 +1,14 @@ +//! Corrected MMR hash accounting: one charge per blake3 merge computed. +//! +//! Used from GROVE_V4. + +/// A push calls `MmrNode::merge` once per peak it collapses. +pub(super) fn merge_hashes(merges: u32) -> u32 { + merges +} + +/// Bagging folds the peaks right-to-left, so `n` peaks cost `n - 1` merges. +/// One peak (or none) folds nothing. +pub(super) fn bagging_hashes(peaks: usize) -> u32 { + peaks.saturating_sub(1) as u32 +} diff --git a/grovedb-merkle-mountain-range/src/error.rs b/grovedb-merkle-mountain-range/src/error.rs index ec972ce87..9b828a932 100644 --- a/grovedb-merkle-mountain-range/src/error.rs +++ b/grovedb-merkle-mountain-range/src/error.rs @@ -26,6 +26,10 @@ pub enum Error { InvalidInput(String), /// Invalid proof during verification. InvalidProof(String), + /// The grove version selected an unknown version for a versioned MMR + /// method. Reaching this means a version set names a variant this build + /// does not implement. + VersionError(String), } impl core::fmt::Display for Error { @@ -41,6 +45,7 @@ impl core::fmt::Display for Error { InvalidData(msg) => write!(f, "Invalid MMR data: {}", msg), InvalidInput(msg) => write!(f, "Invalid input: {}", msg), InvalidProof(msg) => write!(f, "Invalid proof: {}", msg), + VersionError(e) => write!(f, "Version error: {}", e), } } } diff --git a/grovedb-merkle-mountain-range/src/lib.rs b/grovedb-merkle-mountain-range/src/lib.rs index 1bb8b58e9..5ab75c38f 100644 --- a/grovedb-merkle-mountain-range/src/lib.rs +++ b/grovedb-merkle-mountain-range/src/lib.rs @@ -20,6 +20,7 @@ #![deny(missing_docs)] +mod cost; mod error; /// MMR helper functions for position arithmetic, storage keys, and cost /// calculations. diff --git a/grovedb-merkle-mountain-range/src/mmr.rs b/grovedb-merkle-mountain-range/src/mmr.rs index 7d156cfac..de27eb755 100644 --- a/grovedb-merkle-mountain-range/src/mmr.rs +++ b/grovedb-merkle-mountain-range/src/mmr.rs @@ -3,8 +3,10 @@ use std::{borrow::Cow, collections::VecDeque}; use grovedb_costs::{CostResult, CostsExt, OperationCost}; +use grovedb_version::version::GroveVersion; use crate::{ + cost::{gen_proof_bagging_hashes, get_root_bagging_hashes, push_merge_hashes}, helper::{get_peak_map, get_peaks, parent_offset, pos_height_in_tree, sibling_offset}, mmr_store::{MMRBatch, MMRStoreReadOps, MMRStoreWriteOps}, proof::{take_while_vec, MerkleProof}, @@ -87,8 +89,26 @@ impl MMR { /// /// This may also create internal (merged) nodes. The new nodes are /// buffered until [`MMR::commit`] is called. + /// + /// Charges hashes under the shipped (v0) accounting. Callers that hold a + /// [`GroveVersion`] should use [`push_with_version`](Self::push_with_version) + /// instead; this entry point exists so paths that predate the gate keep + /// their released cost by construction. pub fn push(&mut self, elem: MmrNode) -> CostResult { + self.push_with_version(elem, GroveVersion::first()) + } + + /// Version-dispatched [`push`](Self::push). + /// + /// The MMR it builds is identical under every version; only the hash + /// charge differs — see [`crate::cost`]. + pub fn push_with_version( + &mut self, + elem: MmrNode, + grove_version: &GroveVersion, + ) -> CostResult { let mut cost = OperationCost::default(); + let mut merges: u32 = 0; let mut elems = vec![elem]; let elem_pos = self.mmr_size; let peak_map = get_peak_map(self.mmr_size); @@ -107,23 +127,39 @@ impl MMR { }; let right_elem = elems.last().expect("checked"); let parent_elem = MmrNode::merge(&left_elem, right_elem); - // `merge` is a blake3. The sibling read above was billed but the - // hash it feeds was not, so a push that collapsed several peaks - // reported only its I/O. - cost.hash_node_calls = cost.hash_node_calls.saturating_add(1); + merges = merges.saturating_add(1); elems.push(parent_elem); } // store hashes self.batch.append(elem_pos, elems); // update mmr_size self.mmr_size = pos + 1; + // Each `MmrNode::merge` above is a blake3. Whether it is billed is + // version-gated: the shipped accounting charged the sibling reads the + // merges consume but not the merges themselves. + match push_merge_hashes(merges, grove_version) { + Ok(h) => cost.hash_node_calls = cost.hash_node_calls.saturating_add(h), + Err(e) => return Err(e).wrap_with_cost(cost), + } Ok(elem_pos).wrap_with_cost(cost) } /// Compute the root hash by bagging all peaks right-to-left. /// /// Returns [`Error::GetRootOnEmpty`] for an empty MMR. + /// + /// Charges hashes under the shipped (v0) accounting; see + /// [`push`](Self::push) for why this entry point is kept. pub fn get_root(&self) -> CostResult { + self.get_root_with_version(GroveVersion::first()) + } + + /// Version-dispatched [`get_root`](Self::get_root). The root is identical + /// under every version; only the hash charge differs. + pub fn get_root_with_version( + &self, + grove_version: &GroveVersion, + ) -> CostResult { let mut cost = OperationCost::default(); if self.mmr_size == 0 { return Err(Error::GetRootOnEmpty).wrap_with_cost(cost); @@ -150,12 +186,13 @@ impl MMR { Err(e) => return Err(e).wrap_with_cost(cost), }; // `bag_peaks` folds the peaks right-to-left with one `MmrNode::merge` - // — a blake3 — per fold, so it performs `peaks - 1` hashes. Reading - // the peaks was billed above but the merges were not, which made a - // multi-peak root look free beyond its I/O. - cost.hash_node_calls = cost - .hash_node_calls - .saturating_add(peaks.len().saturating_sub(1) as u32); + // — a blake3 — per fold, so it performs `peaks - 1` hashes. Whether + // those are billed is version-gated; the shipped accounting billed + // the peak reads only. + match get_root_bagging_hashes(peaks.len(), grove_version) { + Ok(h) => cost.hash_node_calls = cost.hash_node_calls.saturating_add(h), + Err(e) => return Err(e).wrap_with_cost(cost), + } match bag_peaks(peaks) { Ok(Some(root)) => Ok(root).wrap_with_cost(cost), Ok(None) => Err(Error::InconsistentStore).wrap_with_cost(cost), @@ -235,7 +272,20 @@ impl MMR { /// Positions are sorted and deduplicated internally. Returns /// [`Error::GenProofForInvalidLeaves`] if any position is out of range /// or the list is empty. - pub fn gen_proof(&self, mut pos_list: Vec) -> CostResult { + /// + /// Charges hashes under the shipped (v0) accounting; see + /// [`push`](Self::push) for why this entry point is kept. + pub fn gen_proof(&self, pos_list: Vec) -> CostResult { + self.gen_proof_with_version(pos_list, GroveVersion::first()) + } + + /// Version-dispatched [`gen_proof`](Self::gen_proof). The proof is + /// identical under every version; only the hash charge differs. + pub fn gen_proof_with_version( + &self, + mut pos_list: Vec, + grove_version: &GroveVersion, + ) -> CostResult { let mut cost = OperationCost::default(); if pos_list.is_empty() { return Err(Error::GenProofForInvalidLeaves).wrap_with_cost(cost); @@ -271,15 +321,22 @@ impl MMR { return Err(Error::GenProofForInvalidLeaves).wrap_with_cost(cost); } + // Same shared `bag_peaks` the root computation uses, and the same + // `bagging_track - 1` blake3 merges. Version-gated identically, so + // proof generation neither gets the folds for free nor starts + // charging them on a released version. + // + // Dispatched unconditionally, not inside the `bagging_track > 1` + // branch below: with nothing to bag the charge is zero under every + // version, but an unknown version must still be rejected rather than + // slipping through whenever a proof happens not to fold peaks. + match gen_proof_bagging_hashes(bagging_track, grove_version) { + Ok(h) => cost.hash_node_calls = cost.hash_node_calls.saturating_add(h), + Err(e) => return Err(e).wrap_with_cost(cost), + } + if bagging_track > 1 { let rhs_peaks = proof.split_off(proof.len() - bagging_track); - // Same shared `bag_peaks` the root computation uses, and the same - // `bagging_track - 1` blake3 merges — charged here too, so proof - // generation does not get the folds for free just because it - // reaches them by a different route. - cost.hash_node_calls = cost - .hash_node_calls - .saturating_add(bagging_track.saturating_sub(1) as u32); match bag_peaks(rhs_peaks) { Ok(Some(bagged)) => proof.push(bagged), Ok(None) => { diff --git a/grovedb-merkle-mountain-range/src/tests/test_coverage.rs b/grovedb-merkle-mountain-range/src/tests/test_coverage.rs index 7f9c302b6..05e2a0e46 100644 --- a/grovedb-merkle-mountain-range/src/tests/test_coverage.rs +++ b/grovedb-merkle-mountain-range/src/tests/test_coverage.rs @@ -297,69 +297,123 @@ fn verify_and_get_root_surfaces_calculate_root_error() { } // ============================================================================= -// mmr.rs: get_root bills the peak-bagging merges +// mmr.rs: versioned hash charges for the internal blake3 merges // ============================================================================= -/// `get_root` reads the peaks with cost, but folding them into a single root -/// calls `MmrNode::merge` — a blake3 — once per extra peak. Those merges went -/// uncharged, so a multi-peak root looked free beyond its I/O. +/// `get_root` bags peaks with one blake3 merge per extra peak. v0 (shipped) +/// charges none of them; v1 charges `peaks - 1`. The root itself is identical +/// under both, so only the cost may differ. #[test] -fn get_root_charges_one_hash_per_peak_merge() { - // 1 leaf: mmr_size 1 takes the single-element path, no bagging at all. +fn get_root_peak_bagging_charge_is_versioned() { + use grovedb_version::version::{v1::GROVE_V1, v4::GROVE_V4}; + + // (leaves, expected v1 merges): 1 and 2 leaves leave a single peak (or + // take the single-element path), 3 leaves give two peaks, 7 give three. + for (leaves, expected) in [(1u32, 0u32), (2, 0), (3, 1), (7, 2)] { + let store = MemStore::default(); + let mut mmr = MMR::new(0, &store); + for i in 0..leaves { + mmr.push(leaf(i)).unwrap().expect("push"); + } + + let v0 = mmr.get_root_with_version(&GROVE_V1); + let v0_root = v0.value.expect("root"); + assert_eq!( + v0.cost.hash_node_calls, 0, + "v0 charges no bagging merges ({} leaves)", + leaves + ); + + let v1 = mmr.get_root_with_version(&GROVE_V4); + let v1_root = v1.value.expect("root"); + assert_eq!( + v1.cost.hash_node_calls, expected, + "v1 charges one hash per fold ({} leaves), got {:?}", + leaves, v1.cost + ); + + assert_eq!( + v0_root, v1_root, + "the root must not depend on the cost version ({} leaves)", + leaves + ); + } + + // The un-suffixed entry point must keep the shipped accounting. let store = MemStore::default(); let mut mmr = MMR::new(0, &store); - mmr.push(leaf(0)).unwrap().expect("push"); + for i in 0..7 { + mmr.push(leaf(i)).unwrap().expect("push"); + } let ctx = mmr.get_root(); ctx.value.expect("root"); assert_eq!( ctx.cost.hash_node_calls, 0, - "a single-element MMR bags nothing" + "bare get_root must stay on v0, got {:?}", + ctx.cost ); +} - // 2 leaves: one perfect peak, so still nothing to fold. - let store = MemStore::default(); - let mut mmr = MMR::new(0, &store); - for i in 0..2 { - mmr.push(leaf(i)).unwrap().expect("push"); - } - let ctx = mmr.get_root(); - ctx.value.expect("root"); - assert_eq!(ctx.cost.hash_node_calls, 0, "one peak needs no merge"); +/// `push` merges once per peak it collapses. v0 billed the sibling reads +/// those merges consume but not the merges; v1 charges them. +#[test] +fn push_peak_collapse_charge_is_versioned() { + use grovedb_version::version::{v1::GROVE_V1, v4::GROVE_V4}; + + // Collapses for the first four leaves: 0, 1, 0, 2. + let expected = [0u32, 1, 0, 2]; + + let store_v0 = MemStore::default(); + let mut mmr_v0 = MMR::new(0, &store_v0); + let store_v1 = MemStore::default(); + let mut mmr_v1 = MMR::new(0, &store_v1); + + for (i, exp) in expected.iter().enumerate() { + let c0 = mmr_v0.push_with_version(leaf(i as u32), &GROVE_V1); + c0.value.expect("push"); + assert_eq!( + c0.cost.hash_node_calls, 0, + "v0 charges no merges (leaf {})", + i + ); - // 3 leaves: two peaks, so exactly one merge. - let store = MemStore::default(); - let mut mmr = MMR::new(0, &store); - for i in 0..3 { - mmr.push(leaf(i)).unwrap().expect("push"); + let c1 = mmr_v1.push_with_version(leaf(i as u32), &GROVE_V4); + c1.value.expect("push"); + assert_eq!( + c1.cost.hash_node_calls, *exp, + "v1 charges one hash per collapse (leaf {}), got {:?}", + i, c1.cost + ); } - let ctx = mmr.get_root(); - ctx.value.expect("root"); + + // Same MMR either way. assert_eq!( - ctx.cost.hash_node_calls, 1, - "two peaks fold with one blake3 merge" + mmr_v0.get_root().unwrap().expect("root"), + mmr_v1.get_root().unwrap().expect("root"), + "the MMR must not depend on the cost version" ); - // 7 leaves: three peaks (4 + 2 + 1), so two merges. + // The un-suffixed entry point must keep the shipped accounting. let store = MemStore::default(); let mut mmr = MMR::new(0, &store); - for i in 0..7 { - mmr.push(leaf(i)).unwrap().expect("push"); - } - let ctx = mmr.get_root(); - ctx.value.expect("root"); + mmr.push(leaf(0)).unwrap().expect("push"); + let ctx = mmr.push(leaf(1)); + ctx.value.expect("push"); assert_eq!( - ctx.cost.hash_node_calls, 2, - "three peaks fold with two blake3 merges" + ctx.cost.hash_node_calls, 0, + "bare push must stay on v0, got {:?}", + ctx.cost ); } -/// `gen_proof` folds right-hand peaks through the same `bag_peaks` helper the -/// root computation uses, so it performs the same blake3 merges and must bill -/// them. Charging only in `get_root` left proof generation free. +/// `gen_proof` folds right-hand peaks through the same `bag_peaks` helper, +/// so it carries the same versioned charge. #[test] -fn gen_proof_charges_the_peak_bagging_merges() { - // 7 leaves gives three peaks (4 + 2 + 1). A proof for the first leaf - // leaves the two right-hand peaks to be bagged: one merge. +fn gen_proof_peak_bagging_charge_is_versioned() { + use grovedb_version::version::{v1::GROVE_V1, v4::GROVE_V4}; + + // 7 leaves gives three peaks (4 + 2 + 1); a proof for the first leaf + // leaves the two right-hand peaks to bag: one merge. let store = MemStore::default(); let mut mmr = MMR::new(0, &store); let mut positions = Vec::new(); @@ -367,58 +421,75 @@ fn gen_proof_charges_the_peak_bagging_merges() { positions.push(mmr.push(leaf(i)).unwrap().expect("push")); } - let ctx = mmr.gen_proof(vec![positions[0]]); - ctx.value.expect("proof"); + let v0 = mmr.gen_proof_with_version(vec![positions[0]], &GROVE_V1); + let v0_proof = v0.value.expect("proof"); + assert_eq!(v0.cost.hash_node_calls, 0, "v0 charges no bagging merges"); + + let v1 = mmr.gen_proof_with_version(vec![positions[0]], &GROVE_V4); + let v1_proof = v1.value.expect("proof"); assert_eq!( - ctx.cost.hash_node_calls, 1, - "bagging two right-hand peaks is one blake3 merge, got {:?}", - ctx.cost + v1.cost.hash_node_calls, 1, + "v1 charges the one fold, got {:?}", + v1.cost ); - // A single perfect peak has nothing to bag. + assert_eq!( + v0_proof.proof_items(), + v1_proof.proof_items(), + "the proof must not depend on the cost version" + ); + + // A single perfect peak has nothing to bag under either version. let store = MemStore::default(); let mut mmr = MMR::new(0, &store); let mut positions = Vec::new(); for i in 0..4 { positions.push(mmr.push(leaf(i)).unwrap().expect("push")); } - let ctx = mmr.gen_proof(vec![positions[0]]); + let ctx = mmr.gen_proof_with_version(vec![positions[0]], &GROVE_V4); ctx.value.expect("proof"); - assert_eq!( - ctx.cost.hash_node_calls, 0, - "one peak means no bagging, got {:?}", - ctx.cost - ); + assert_eq!(ctx.cost.hash_node_calls, 0, "one peak means no bagging"); } -/// `push` merges once per peak it collapses, and those merges are blake3 -/// calls. The sibling reads were billed but the hashes they fed were not. +/// An unknown version must be rejected rather than silently falling back to +/// one of the implemented charges. #[test] -fn push_charges_one_hash_per_peak_collapse() { +fn mmr_cost_dispatch_rejects_unknown_version() { + use grovedb_version::version::{v4::GROVE_V4, GroveVersion}; + let store = MemStore::default(); let mut mmr = MMR::new(0, &store); + for i in 0..3 { + mmr.push(leaf(i)).unwrap().expect("push"); + } - // Leaf 0: no collapse. - let ctx = mmr.push(leaf(0)); - ctx.value.expect("push"); - assert_eq!(ctx.cost.hash_node_calls, 0, "first leaf merges nothing"); - - // Leaf 1 collapses one pair. - let ctx = mmr.push(leaf(1)); - ctx.value.expect("push"); - assert_eq!(ctx.cost.hash_node_calls, 1, "one merge, got {:?}", ctx.cost); + let mut bad: GroveVersion = GROVE_V4.clone(); + bad.mmr_versions.cost.get_root = 99; + assert!( + matches!( + mmr.get_root_with_version(&bad).unwrap(), + Err(Error::VersionError(_)) + ), + "an unknown get_root charge version must error" + ); - // Leaf 2: no collapse (new peak). - let ctx = mmr.push(leaf(2)); - ctx.value.expect("push"); - assert_eq!(ctx.cost.hash_node_calls, 0, "got {:?}", ctx.cost); + let mut bad: GroveVersion = GROVE_V4.clone(); + bad.mmr_versions.cost.push = 99; + assert!( + matches!( + mmr.push_with_version(leaf(9), &bad).unwrap(), + Err(Error::VersionError(_)) + ), + "an unknown push charge version must error" + ); - // Leaf 3 collapses twice: the pair, then the two 2-leaf peaks. - let ctx = mmr.push(leaf(3)); - ctx.value.expect("push"); - assert_eq!( - ctx.cost.hash_node_calls, 2, - "two merges, got {:?}", - ctx.cost + let mut bad: GroveVersion = GROVE_V4.clone(); + bad.mmr_versions.cost.gen_proof = 99; + assert!( + matches!( + mmr.gen_proof_with_version(vec![0], &bad).unwrap(), + Err(Error::VersionError(_)) + ), + "an unknown gen_proof charge version must error" ); } diff --git a/grovedb-private-document-store/Cargo.toml b/grovedb-private-document-store/Cargo.toml index 4def59e14..25df42d92 100644 --- a/grovedb-private-document-store/Cargo.toml +++ b/grovedb-private-document-store/Cargo.toml @@ -17,6 +17,7 @@ storage = ["grovedb-storage", "grovedb-bulk-append-tree/storage"] [dependencies] grovedb-bulk-append-tree = { version = "5.0.1", path = "../grovedb-bulk-append-tree", default-features = false } grovedb-costs = { version = "5.0.1", path = "../costs" } +grovedb-version = { version = "5.0.1", path = "../grovedb-version" } grovedb-storage = { version = "5.0.1", path = "../storage", optional = true } blake3 = { workspace = true } thiserror = { workspace = true } diff --git a/grovedb-private-document-store/src/store.rs b/grovedb-private-document-store/src/store.rs index c20389d72..d28bcb0bb 100644 --- a/grovedb-private-document-store/src/store.rs +++ b/grovedb-private-document-store/src/store.rs @@ -11,6 +11,7 @@ use grovedb_bulk_append_tree::BulkAppendTree; use grovedb_costs::{CostResult, CostsExt, OperationCost}; use grovedb_storage::StorageContext; +use grovedb_version::version::GroveVersion; use crate::{ compute_private_document_store_state_root, private_document_store_config_hash, @@ -141,6 +142,7 @@ impl<'db, S: StorageContext<'db>> PrivateDocumentStore { pub fn append( &mut self, entry: &[u8], + grove_version: &GroveVersion, ) -> CostResult { let mut cost = OperationCost::default(); @@ -161,7 +163,7 @@ impl<'db, S: StorageContext<'db>> PrivateDocumentStore { // deferred path walks once and bills what it walks, and sharing one // implementation keeps the single and batch paths from drifting. let many = match self - .append_many(core::iter::once(entry)) + .append_many(core::iter::once(entry), grove_version) .unwrap_add_cost(&mut cost) { Ok(r) => r, @@ -317,6 +319,7 @@ impl<'db, S: StorageContext<'db>> PrivateDocumentStore { pub fn append_many<'e, I>( &mut self, entries: I, + grove_version: &GroveVersion, ) -> CostResult where I: IntoIterator, @@ -351,7 +354,7 @@ impl<'db, S: StorageContext<'db>> PrivateDocumentStore { // the MMR's bagging hashes free. let r = match self .bulk_tree - .append_deferred_roots(entry) + .append_deferred_roots(entry, grove_version) .unwrap_add_cost(&mut cost) { Ok(r) => r, @@ -371,7 +374,9 @@ impl<'db, S: StorageContext<'db>> PrivateDocumentStore { // Pay the deferred roots exactly once, through the cost-aware path // so the dense-buffer walk's real storage reads and hashes are // billed rather than re-derived from a hand-rolled model. - let root_ctx = self.bulk_tree.compute_current_state_root_with_cost(); + let root_ctx = self + .bulk_tree + .compute_current_state_root_with_cost(grove_version); let root_cost = root_ctx.cost; let bulk_state_root = match root_ctx.value { Ok(r) => r, @@ -496,11 +501,12 @@ impl<'db, S: StorageContext<'db>> PrivateDocumentStore { /// composite `pds_state` blake3. pub fn compute_current_state_root_with_cost( &self, + grove_version: &GroveVersion, ) -> CostResult<[u8; 32], PrivateDocumentStoreError> { let mut cost = OperationCost::default(); let bulk_root = match self .bulk_tree - .compute_current_state_root_with_cost() + .compute_current_state_root_with_cost(grove_version) .unwrap_add_cost(&mut cost) { Ok(r) => r, @@ -577,7 +583,12 @@ mod append_many_tests { .expect("a"); let mut last = None; for e in &entries { - last = Some(one_by_one.append(e).unwrap().expect("append")); + last = Some( + one_by_one + .append(e, GroveVersion::latest()) + .unwrap() + .expect("append"), + ); } let per_entry = last.expect("appended"); @@ -585,7 +596,7 @@ mod append_many_tests { .unwrap() .expect("b"); let many = batched - .append_many(entries.iter().map(|e| e.as_slice())) + .append_many(entries.iter().map(|e| e.as_slice()), GroveVersion::latest()) .unwrap() .expect("append_many"); @@ -625,7 +636,7 @@ mod append_many_tests { // than a sentinel indistinguishable from a real append), and still // charges the two root hashes it actually performs. let empty: Vec> = Vec::new(); - let ctx = store.append_many(empty.iter().map(|e| e.as_slice())); + let ctx = store.append_many(empty.iter().map(|e| e.as_slice()), GroveVersion::latest()); let empty_cost = ctx.cost.clone(); let r = ctx.value.expect("empty append_many"); assert_eq!(store.total_count(), 0); @@ -643,7 +654,9 @@ mod append_many_tests { // A wrong-size entry is rejected. let bad = [vec![0u8; 8], vec![0u8; 7]]; assert!(matches!( - store.append_many(bad.iter().map(|e| e.as_slice())).unwrap(), + store + .append_many(bad.iter().map(|e| e.as_slice()), GroveVersion::latest()) + .unwrap(), Err(PrivateDocumentStoreError::InvalidEntrySize { expected: 8, actual: 7 @@ -669,7 +682,10 @@ mod atomicity_tests { .expect("new"); // Past a compaction so the MMR actually holds a chunk. for i in 0..6u8 { - store.append(&[i; 8]).unwrap().expect("append"); + store + .append(&[i; 8], GroveVersion::latest()) + .unwrap() + .expect("append"); } store.commit_mmr().expect("commit mmr"); let storage = PrivateDocumentStore::into_storage_for_test(store); @@ -679,7 +695,7 @@ mod atomicity_tests { let reopened = PrivateDocumentStore::from_state(6, 8, 2, storage) .unwrap() .expect("reopen"); - let ctx = reopened.compute_current_state_root_with_cost(); + let ctx = reopened.compute_current_state_root_with_cost(GroveVersion::latest()); let cost = ctx.cost.clone(); ctx.value.expect("state root"); // NOTE: `MemStorageContext::get` reports `OperationCost::default()`, @@ -721,7 +737,7 @@ mod atomicity_tests { // First append: the dense walk visits 1 filled position (2 hashes), // then the bulk state root (1) and the composite pds_state root (1). - let ctx = store.append(&[1u8; 8]); + let ctx = store.append(&[1u8; 8], GroveVersion::latest()); ctx.value.expect("append"); assert_eq!( ctx.cost.hash_node_calls, 4, @@ -730,7 +746,7 @@ mod atomicity_tests { ); // Second append: 2 filled positions now, so the walk costs 4. - let ctx = store.append(&[2u8; 8]); + let ctx = store.append(&[2u8; 8], GroveVersion::latest()); ctx.value.expect("append"); assert_eq!( ctx.cost.hash_node_calls, 6, @@ -739,7 +755,7 @@ mod atomicity_tests { ); // Third: 6 dense + 2 roots. - let ctx = store.append(&[3u8; 8]); + let ctx = store.append(&[3u8; 8], GroveVersion::latest()); ctx.value.expect("append"); assert_eq!( ctx.cost.hash_node_calls, 8, @@ -759,11 +775,14 @@ mod atomicity_tests { .unwrap() .expect("new"); for i in 0..3u8 { - store.append(&[i; 8]).unwrap().expect("append"); + store + .append(&[i; 8], GroveVersion::latest()) + .unwrap() + .expect("append"); } // The 4th append does not fit the buffer, so it compacts. - let compacting = store.append(&[3u8; 8]); + let compacting = store.append(&[3u8; 8], GroveVersion::latest()); compacting.value.expect("compacting append"); let compacting_cost = compacting.cost; @@ -788,7 +807,7 @@ mod atomicity_tests { ); // A plain buffered append afterwards reads nothing back. - let plain = store.append(&[4u8; 8]); + let plain = store.append(&[4u8; 8], GroveVersion::latest()); plain.value.expect("buffered append"); assert!( plain.cost.storage_loaded_bytes < compacting_cost.storage_loaded_bytes, @@ -813,7 +832,7 @@ mod atomicity_tests { .unwrap() .expect("new"); let entries: Vec> = (0..12u8).map(|i| vec![i; 8]).collect(); - let ctx = store.append_many(entries.iter().map(|e| e.as_slice())); + let ctx = store.append_many(entries.iter().map(|e| e.as_slice()), GroveVersion::latest()); let cost = ctx.cost.clone(); let r = ctx.value.expect("append_many"); @@ -826,7 +845,7 @@ mod atomicity_tests { assert!(r.compacted, "12 entries at epoch 4 must compact"); // Same invariant on the single-append path, which forwards the counter. - let ctx = store.append(&[99u8; 8]); + let ctx = store.append(&[99u8; 8], GroveVersion::latest()); let cost = ctx.cost.clone(); let r = ctx.value.expect("append"); assert_eq!( @@ -857,7 +876,10 @@ mod atomicity_tests { let mut store = PrivateDocumentStore::new(8, 2, MemStorageContext::new()) .unwrap() .expect("new"); - store.append(&[1u8; 8]).unwrap().expect("seed"); + store + .append(&[1u8; 8], GroveVersion::latest()) + .unwrap() + .expect("seed"); let count_before = store.total_count(); let root_before = store.compute_current_state_root().expect("root"); @@ -867,7 +889,7 @@ mod atomicity_tests { let batch = [vec![2u8; 8], vec![3u8; 7]]; assert!(matches!( store - .append_many(batch.iter().map(|e| e.as_slice())) + .append_many(batch.iter().map(|e| e.as_slice()), GroveVersion::latest()) .unwrap(), Err(PrivateDocumentStoreError::InvalidEntrySize { expected: 8, @@ -938,7 +960,10 @@ mod error_path_tests { .unwrap() .expect("new"); for i in 0..6u8 { - store.append(&[i; 8]).unwrap().expect("append"); + store + .append(&[i; 8], GroveVersion::latest()) + .unwrap() + .expect("append"); } store.commit_mmr().expect("commit mmr"); let storage = PrivateDocumentStore::into_storage_for_test(store); @@ -969,7 +994,10 @@ mod error_path_tests { .unwrap() .expect("new"); for i in 0..10u8 { - store.append(&[i; 8]).unwrap().expect("append"); + store + .append(&[i; 8], GroveVersion::latest()) + .unwrap() + .expect("append"); } store.commit_mmr().expect("commit mmr"); let storage = PrivateDocumentStore::into_storage_for_test(store); @@ -1026,7 +1054,10 @@ mod error_path_tests { .expect("new"); // 6 entries at chunk_power 2 (epoch 4): one chunk, two buffered. for i in 0..6u8 { - store.append(&[i; 8]).unwrap().expect("append"); + store + .append(&[i; 8], GroveVersion::latest()) + .unwrap() + .expect("append"); } store.commit_mmr().expect("commit mmr"); let storage = PrivateDocumentStore::into_storage_for_test(store); @@ -1048,7 +1079,10 @@ mod error_path_tests { .unwrap() .expect("new"); for i in 0..2u8 { - store.append(&[i; 8]).unwrap().expect("append"); + store + .append(&[i; 8], GroveVersion::latest()) + .unwrap() + .expect("append"); } let storage = PrivateDocumentStore::into_storage_for_test(store); let overclaimed = PrivateDocumentStore::from_state(3, 8, 3, storage) @@ -1077,7 +1111,10 @@ mod error_path_tests { .unwrap() .expect("new"); for i in 0..6u8 { - store.append(&[i; 8]).unwrap().expect("append"); + store + .append(&[i; 8], GroveVersion::latest()) + .unwrap() + .expect("append"); } store.commit_mmr().expect("commit mmr"); let storage = PrivateDocumentStore::into_storage_for_test(store); @@ -1086,7 +1123,7 @@ mod error_path_tests { let broken = PrivateDocumentStore::from_state(6, 8, 2, storage) .unwrap() .expect("reopen"); - let ctx = broken.compute_current_state_root_with_cost(); + let ctx = broken.compute_current_state_root_with_cost(GroveVersion::latest()); assert!( ctx.value.is_err(), "a state root over wiped storage must be an error, got {:?}", @@ -1094,7 +1131,10 @@ mod error_path_tests { ); // And appending onto that broken state fails rather than writing. let mut broken = broken; - assert!(broken.append(&[9u8; 8]).unwrap().is_err()); + assert!(broken + .append(&[9u8; 8], GroveVersion::latest()) + .unwrap() + .is_err()); } /// Every read path must surface a storage fault instead of reporting the @@ -1108,7 +1148,10 @@ mod error_path_tests { .expect("new"); // Past a compaction so both a completed chunk and the buffer exist. for i in 0..6u8 { - store.append(&[i; 8]).unwrap().expect("append"); + store + .append(&[i; 8], GroveVersion::latest()) + .unwrap() + .expect("append"); } store.commit_mmr().expect("commit mmr"); @@ -1137,7 +1180,10 @@ mod error_path_tests { // The integrity walk and the state-root derivation likewise. assert!(store.verify_entry_sizes().is_err()); - assert!(store.compute_current_state_root_with_cost().value.is_err()); + assert!(store + .compute_current_state_root_with_cost(GroveVersion::latest()) + .value + .is_err()); // An out-of-range position is answered before any storage is touched, // so it still reports absence rather than the fault. @@ -1154,10 +1200,13 @@ mod error_path_tests { let mut store = PrivateDocumentStore::new(8, 2, MemStorageContext::new()) .unwrap() .expect("new"); - store.append(&[0u8; 8]).unwrap().expect("append"); + store + .append(&[0u8; 8], GroveVersion::latest()) + .unwrap() + .expect("append"); store.bulk_tree.dense_tree.storage.fail_writes(); - let r = store.append(&[1u8; 8]).unwrap(); + let r = store.append(&[1u8; 8], GroveVersion::latest()).unwrap(); assert!( r.is_err(), "a failed write must fail the append, got {:?}", @@ -1169,12 +1218,14 @@ mod error_path_tests { // fault. let bad = [vec![0u8; 7]]; assert!(matches!( - store.append_many(bad.iter().map(|e| e.as_slice())).unwrap(), + store + .append_many(bad.iter().map(|e| e.as_slice()), GroveVersion::latest()) + .unwrap(), Err(PrivateDocumentStoreError::InvalidEntrySize { .. }) )); let good = [vec![2u8; 8], vec![3u8; 8]]; assert!(store - .append_many(good.iter().map(|e| e.as_slice())) + .append_many(good.iter().map(|e| e.as_slice()), GroveVersion::latest()) .unwrap() .is_err()); } @@ -1236,14 +1287,14 @@ mod tests { .unwrap() .expect("new store"); assert!(matches!( - store.append(&[0u8; 7]).unwrap(), + store.append(&[0u8; 7], GroveVersion::latest()).unwrap(), Err(PrivateDocumentStoreError::InvalidEntrySize { expected: 8, actual: 7 }) )); assert!(matches!( - store.append(&[0u8; 9]).unwrap(), + store.append(&[0u8; 9], GroveVersion::latest()).unwrap(), Err(PrivateDocumentStoreError::InvalidEntrySize { expected: 8, actual: 9 @@ -1251,7 +1302,10 @@ mod tests { )); // A rejected append must not mutate the store. assert_eq!(store.total_count(), 0); - let ok = store.append(&[1u8; 8]).unwrap().expect("valid append"); + let ok = store + .append(&[1u8; 8], GroveVersion::latest()) + .unwrap() + .expect("valid append"); assert_eq!(ok.global_position, 0); assert_eq!(store.total_count(), 1); } @@ -1266,7 +1320,10 @@ mod tests { let mut roots = Vec::new(); for i in 0..10u8 { let entry = [i; 8]; - let r = store.append(&entry).unwrap().expect("append"); + let r = store + .append(&entry, GroveVersion::latest()) + .unwrap() + .expect("append"); assert_eq!(r.global_position, i as u64); roots.push(r.state_root); } @@ -1304,8 +1361,14 @@ mod tests { let mut b = PrivateDocumentStore::new(8, 3, MemStorageContext::new()) .unwrap() .expect("b"); - let ra = a.append(&[7u8; 8]).unwrap().expect("append a"); - let rb = b.append(&[7u8; 8]).unwrap().expect("append b"); + let ra = a + .append(&[7u8; 8], GroveVersion::latest()) + .unwrap() + .expect("append a"); + let rb = b + .append(&[7u8; 8], GroveVersion::latest()) + .unwrap() + .expect("append b"); assert_ne!(ra.state_root, ra.bulk_state_root); assert_ne!(ra.state_root, rb.state_root); } @@ -1317,7 +1380,10 @@ mod tests { .unwrap() .expect("new store"); for i in 0..6u8 { - store.append(&[i; 8]).unwrap().expect("append"); + store + .append(&[i; 8], GroveVersion::latest()) + .unwrap() + .expect("append"); } store.commit_mmr().expect("commit mmr"); let root_before = store.compute_current_state_root().expect("root"); @@ -1348,7 +1414,10 @@ mod tests { .unwrap() .expect("new store"); for i in 0..6u8 { - store.append(&[i; 8]).unwrap().expect("append"); + store + .append(&[i; 8], GroveVersion::latest()) + .unwrap() + .expect("append"); } store.commit_mmr().expect("commit mmr"); let storage = PrivateDocumentStore::into_storage_for_test(store); diff --git a/grovedb-version/src/tests.rs b/grovedb-version/src/tests.rs index dcd718d8c..de764532a 100644 --- a/grovedb-version/src/tests.rs +++ b/grovedb-version/src/tests.rs @@ -549,3 +549,45 @@ fn feature_version_bounds_default() { assert!(bounds.check_version(0)); assert!(!bounds.check_version(1)); } + +// ── MMR cost version table ─────────────────────────────────────────── + +/// The MMR hash charges are consensus-visible through fees, so which version +/// each protocol version selects is pinned here rather than left to whoever +/// edits a version constant next. V1..V3 are released and must stay on the +/// shipped accounting; the corrected charges arrive with V4. +#[test] +fn mmr_cost_versions_are_pinned_per_protocol_version() { + for (name, version) in [ + ("GROVE_V1", &GROVE_V1), + ("GROVE_V2", &GROVE_V2), + ("GROVE_V3", &GROVE_V3), + ] { + let cost = &version.mmr_versions.cost; + assert_eq!(cost.push, 0, "{name} must keep the shipped push charge"); + assert_eq!( + cost.get_root, 0, + "{name} must keep the shipped get_root charge" + ); + assert_eq!( + cost.gen_proof, 0, + "{name} must keep the shipped gen_proof charge" + ); + } + + let cost = &GROVE_V4.mmr_versions.cost; + assert_eq!(cost.push, 1, "GROVE_V4 charges push merges"); + assert_eq!(cost.get_root, 1, "GROVE_V4 charges get_root bagging"); + assert_eq!(cost.gen_proof, 1, "GROVE_V4 charges gen_proof bagging"); +} + +/// `GroveVersion::first()` is what the unversioned MMR entry points delegate +/// to, so it has to be the shipped accounting — otherwise every caller that +/// predates the gate would silently pick up the new charges. +#[test] +fn grove_version_first_selects_shipped_mmr_costs() { + let cost = &GroveVersion::first().mmr_versions.cost; + assert_eq!(cost.push, 0); + assert_eq!(cost.get_root, 0); + assert_eq!(cost.gen_proof, 0); +} diff --git a/grovedb-version/src/version/mmr_versions.rs b/grovedb-version/src/version/mmr_versions.rs new file mode 100644 index 000000000..168a8c3e2 --- /dev/null +++ b/grovedb-version/src/version/mmr_versions.rs @@ -0,0 +1,38 @@ +//! Version gates for the Merkle Mountain Range crate. +//! +//! Only cost accounting is versioned here. Every gate below leaves the +//! returned hashes, roots and proofs bit-identical — what changes is how many +//! `hash_node_calls` the operation reports. That still has to be gated, +//! because costs become fees: a node replaying a historical block must charge +//! what the block was admitted under, so a corrected charge cannot simply +//! replace the old one. + +use versioned_feature_core::FeatureVersion; + +#[derive(Clone, Debug, Default)] +pub struct MmrVersions { + pub cost: MmrCostVersions, +} + +/// Hash-charge versions for the three MMR operations that perform blake3 +/// merges internally. +/// +/// In every case version 0 is the shipped behaviour, which billed the storage +/// reads an operation performed but not the merges those reads fed, and +/// version 1 charges one hash per merge actually computed. +#[derive(Clone, Debug, Default)] +pub struct MmrCostVersions { + /// `MMR::push`. A push collapses one peak per set trailing bit of the + /// leaf count, calling `MmrNode::merge` — a blake3 — each time. Version 0 + /// billed the sibling reads those merges consume but not the merges. + /// Version 1 charges one hash per collapse. + pub push: FeatureVersion, + /// `MMR::get_root`. Bagging folds the peaks right-to-left with one + /// `MmrNode::merge` per additional peak. Version 0 billed the peak reads + /// only; version 1 charges `peaks - 1` merges. + pub get_root: FeatureVersion, + /// `MMR::gen_proof`. Proof generation folds the right-hand peaks through + /// the same `bag_peaks` helper `get_root` uses. Version 0 charged none of + /// those merges; version 1 charges `bagging_track - 1`. + pub gen_proof: FeatureVersion, +} diff --git a/grovedb-version/src/version/mod.rs b/grovedb-version/src/version/mod.rs index 7965d6489..34a066ba6 100644 --- a/grovedb-version/src/version/mod.rs +++ b/grovedb-version/src/version/mod.rs @@ -1,5 +1,6 @@ pub mod grovedb_versions; pub mod merk_versions; +pub mod mmr_versions; pub mod v1; pub mod v2; pub mod v3; @@ -10,7 +11,8 @@ pub use versioned_feature_core::*; use crate::version::v3::GROVE_V3; use crate::version::v4::GROVE_V4; use crate::version::{ - grovedb_versions::GroveDBVersions, merk_versions::MerkVersions, v1::GROVE_V1, v2::GROVE_V2, + grovedb_versions::GroveDBVersions, merk_versions::MerkVersions, mmr_versions::MmrVersions, + v1::GROVE_V1, v2::GROVE_V2, }; #[derive(Clone, Debug, Default)] @@ -18,6 +20,7 @@ pub struct GroveVersion { pub protocol_version: u32, pub grovedb_versions: GroveDBVersions, pub merk_versions: MerkVersions, + pub mmr_versions: MmrVersions, } impl GroveVersion { diff --git a/grovedb-version/src/version/v1.rs b/grovedb-version/src/version/v1.rs index 03e5b5aa9..6f2f6d812 100644 --- a/grovedb-version/src/version/v1.rs +++ b/grovedb-version/src/version/v1.rs @@ -13,6 +13,7 @@ use crate::version::{ merk_versions::{ MerkAverageCaseCostsVersions, MerkBatchVersions, MerkProofVersions, MerkVersions, }, + mmr_versions::{MmrCostVersions, MmrVersions}, GroveVersion, }; @@ -254,4 +255,15 @@ pub const GROVE_V1: GroveVersion = GroveVersion { prove_count_offset_on_range: 0, }, }, + // MMR hash charges: the shipped accounting, which billed the + // storage reads each operation performed but not the blake3 merges + // those reads fed. Locked here — these versions are released and a + // replayed block must be charged what it was admitted under. + mmr_versions: MmrVersions { + cost: MmrCostVersions { + push: 0, + get_root: 0, + gen_proof: 0, + }, + }, }; diff --git a/grovedb-version/src/version/v2.rs b/grovedb-version/src/version/v2.rs index 92b6ff676..1dff47307 100644 --- a/grovedb-version/src/version/v2.rs +++ b/grovedb-version/src/version/v2.rs @@ -13,6 +13,7 @@ use crate::version::{ merk_versions::{ MerkAverageCaseCostsVersions, MerkBatchVersions, MerkProofVersions, MerkVersions, }, + mmr_versions::{MmrCostVersions, MmrVersions}, GroveVersion, }; @@ -253,4 +254,15 @@ pub const GROVE_V2: GroveVersion = GroveVersion { prove_count_offset_on_range: 0, }, }, + // MMR hash charges: the shipped accounting, which billed the + // storage reads each operation performed but not the blake3 merges + // those reads fed. Locked here — these versions are released and a + // replayed block must be charged what it was admitted under. + mmr_versions: MmrVersions { + cost: MmrCostVersions { + push: 0, + get_root: 0, + gen_proof: 0, + }, + }, }; diff --git a/grovedb-version/src/version/v3.rs b/grovedb-version/src/version/v3.rs index 769b2c169..7818e4a99 100644 --- a/grovedb-version/src/version/v3.rs +++ b/grovedb-version/src/version/v3.rs @@ -13,6 +13,7 @@ use crate::version::{ merk_versions::{ MerkAverageCaseCostsVersions, MerkBatchVersions, MerkProofVersions, MerkVersions, }, + mmr_versions::{MmrCostVersions, MmrVersions}, GroveVersion, }; @@ -268,4 +269,15 @@ pub const GROVE_V3: GroveVersion = GroveVersion { prove_count_offset_on_range: 0, }, }, + // MMR hash charges: the shipped accounting, which billed the + // storage reads each operation performed but not the blake3 merges + // those reads fed. Locked here — these versions are released and a + // replayed block must be charged what it was admitted under. + mmr_versions: MmrVersions { + cost: MmrCostVersions { + push: 0, + get_root: 0, + gen_proof: 0, + }, + }, }; diff --git a/grovedb-version/src/version/v4.rs b/grovedb-version/src/version/v4.rs index ba185b6fe..0ca8c09a7 100644 --- a/grovedb-version/src/version/v4.rs +++ b/grovedb-version/src/version/v4.rs @@ -114,6 +114,7 @@ use crate::version::{ merk_versions::{ MerkAverageCaseCostsVersions, MerkBatchVersions, MerkProofVersions, MerkVersions, }, + mmr_versions::{MmrCostVersions, MmrVersions}, GroveVersion, }; @@ -368,4 +369,16 @@ pub const GROVE_V4: GroveVersion = GroveVersion { prove_count_offset_on_range: 0, }, }, + // MMR hash charges: one hash per blake3 merge actually computed — + // `push` per collapsed peak, `get_root` and `gen_proof` per peak + // folded during bagging. V1..V3 bill the reads but not these + // merges. Roots and proofs are bit-identical across both versions; + // only `hash_node_calls` differs. + mmr_versions: MmrVersions { + cost: MmrCostVersions { + push: 1, + get_root: 1, + gen_proof: 1, + }, + }, }; diff --git a/grovedb/src/operations/mmr_tree.rs b/grovedb/src/operations/mmr_tree.rs index 2a5f4fe0c..d0488e9e8 100644 --- a/grovedb/src/operations/mmr_tree.rs +++ b/grovedb/src/operations/mmr_tree.rs @@ -11,7 +11,9 @@ use std::collections::HashMap; use grovedb_costs::{cost_return_on_error, CostResult, CostsExt, OperationCost}; use grovedb_merk::element::insert::ElementInsertToStorageExtensions; -use grovedb_merkle_mountain_range::{mmr_size_to_leaf_count, MmrNode, MmrStore, MMR}; +use grovedb_merkle_mountain_range::{ + hash_count_for_push, mmr_size_to_leaf_count, MmrNode, MmrStore, MMR, +}; use grovedb_path::SubtreePath; use grovedb_storage::{rocksdb_storage::PrefixedRocksDbTransactionContext, Storage, StorageBatch}; use grovedb_version::version::GroveVersion; @@ -82,12 +84,12 @@ impl GroveDb { let store = MmrStore::new(&storage_ctx); let leaf_count = mmr_size_to_leaf_count(mmr_size); - // Only the eager leaf hash is charged here. `MMR::push` now bills one - // hash per peak it collapses, and that cost is propagated by the - // `cost_return_on_error!` below, so adding `hash_count_for_push` — - // which is the leaf hash PLUS those same merges — would charge every - // merge twice. - cost.hash_node_calls += 1; + // `hash_count_for_push` is the eager leaf hash PLUS one per peak the + // push collapses. It pairs with the UNVERSIONED `push` below, which + // charges no merges of its own — so the merges are counted exactly + // once, here. Switching that call to `push_with_version` without + // dropping this helper would charge every merge twice. + cost.hash_node_calls += hash_count_for_push(leaf_count); let leaf = MmrNode::leaf(value); let mut mmr = MMR::new(mmr_size, &store); @@ -100,7 +102,7 @@ impl GroveDb { // Get root BEFORE commit — data is still in the MMRBatch overlay let new_root = cost_return_on_error!( &mut cost, - mmr.get_root() + mmr.get_root_with_version(grove_version) .map_err(|e| Error::CorruptedData(format!("MMR get_root failed: {}", e))) ); let new_mmr_root = new_root.hash(); @@ -448,9 +450,10 @@ impl GroveDb { // Push all values into a single MMR instance let mut mmr = MMR::new(mmr_size, &store); for value in values { - // Eager leaf hash only; `push` bills its own merges. See the - // note on the direct path above. - cost.hash_node_calls += 1; + // Leaf hash + peak collapses, paired with the unversioned + // `push` below. See the note on the direct path above. + let leaf_count = mmr_size_to_leaf_count(mmr.mmr_size); + cost.hash_node_calls += hash_count_for_push(leaf_count); let leaf = MmrNode::leaf(value.clone()); cost_return_on_error!( @@ -463,7 +466,7 @@ impl GroveDb { // Get root BEFORE commit — data is still in the MMRBatch overlay let new_root = cost_return_on_error!( &mut cost, - mmr.get_root() + mmr.get_root_with_version(grove_version) .map_err(|e| Error::CorruptedData(format!("MMR get_root failed: {}", e))) ); let new_mmr_root = new_root.hash(); diff --git a/grovedb/src/operations/private_document_store.rs b/grovedb/src/operations/private_document_store.rs index 9e23e78ac..d644c94aa 100644 --- a/grovedb/src/operations/private_document_store.rs +++ b/grovedb/src/operations/private_document_store.rs @@ -189,7 +189,9 @@ impl GroveDb { let append_result = cost_return_on_error!( &mut cost, - store.append(&entry).map(|r| r.map_err(map_pds_err)) + store + .append(&entry, grove_version) + .map(|r| r.map_err(map_pds_err)) ); let new_state_root = append_result.state_root; @@ -548,7 +550,7 @@ impl GroveDb { let append_result = cost_return_on_error!( &mut cost, store - .append_many(entries.iter().map(|e| e.as_slice())) + .append_many(entries.iter().map(|e| e.as_slice()), grove_version) .map(|r| r.map_err(map_pds_err)) ); let new_state_root = append_result.state_root; diff --git a/grovedb/src/operations/proof/bind_terminal_non_merk_tree/mod.rs b/grovedb/src/operations/proof/bind_terminal_non_merk_tree/mod.rs index 33e584c02..9ad19ec23 100644 --- a/grovedb/src/operations/proof/bind_terminal_non_merk_tree/mod.rs +++ b/grovedb/src/operations/proof/bind_terminal_non_merk_tree/mod.rs @@ -70,7 +70,7 @@ impl GroveDb { .terminal_non_merk_tree_child_hash { 0 => self.bind_terminal_non_merk_tree_v0(node, element, parent_path, tx), - 1 => self.bind_terminal_non_merk_tree_v1(node, element, parent_path, tx), + 1 => self.bind_terminal_non_merk_tree_v1(node, element, parent_path, tx, grove_version), version => Err(Error::VersionError( grovedb_version::error::GroveVersionError::UnknownVersionMismatch { method: "bind_terminal_non_merk_tree".to_string(), diff --git a/grovedb/src/operations/proof/bind_terminal_non_merk_tree/v1.rs b/grovedb/src/operations/proof/bind_terminal_non_merk_tree/v1.rs index 0de66d30d..03aaff208 100644 --- a/grovedb/src/operations/proof/bind_terminal_non_merk_tree/v1.rs +++ b/grovedb/src/operations/proof/bind_terminal_non_merk_tree/v1.rs @@ -27,6 +27,7 @@ use grovedb_merk::{ CryptoHash, TreeFeatureType, }; use grovedb_storage::{Storage, StorageContext}; +use grovedb_version::version::GroveVersion; use crate::{Element, Error, GroveDb, Transaction}; @@ -38,6 +39,7 @@ impl GroveDb { element: &Element, parent_path: &[&[u8]], tx: &Transaction, + grove_version: &GroveVersion, ) -> CostResult<(), Error> { let mut cost = OperationCost::default(); @@ -64,7 +66,7 @@ impl GroveDb { let child_hash = cost_return_on_error!( &mut cost, - self.non_merk_tree_child_hash(element, &child_path, tx) + self.non_merk_tree_child_hash(element, &child_path, tx, grove_version) ); // Reuse the value_hash the node already carries — it is the one the @@ -136,6 +138,7 @@ impl GroveDb { element: &Element, subtree_path: &[&[u8]], tx: &Transaction, + grove_version: &GroveVersion, ) -> CostResult { let mut cost = OperationCost::default(); @@ -296,14 +299,16 @@ impl GroveDb { ); let state_root = cost_return_on_error!( &mut cost, - store.compute_current_state_root_with_cost().map(|r| { - r.map_err(|e| { - Error::CorruptedData(format!( - "private document store state root failed: {}", - e - )) + store + .compute_current_state_root_with_cost(grove_version) + .map(|r| { + r.map_err(|e| { + Error::CorruptedData(format!( + "private document store state root failed: {}", + e + )) + }) }) - }) ); Ok(state_root).wrap_with_cost(cost) } From 0e404d6f747bc8c9969a0a73fb790ac4a2cae487 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 20 Aug 2026 05:43:17 +0700 Subject: [PATCH 15/19] fix(costs): gate the CommitmentTree compaction under-charge into V4 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A compacting append reported `hash_count_for_push` — the chunk-blob leaf hash plus one per peak the MMR push collapses — and omitted the peak bagging the compaction's own `get_root` performs. `CommitmentTree` adds that figure straight into its `hash_node_calls`, so the shielded pool has been under-charged one hash per multi-peak compaction since mainnet activation. Measured before changing anything, per compaction at chunk_power 2: chunks_after=0 model=1 actual=1 chunks_after=1 model=2 actual=2 chunks_after=3 model=1 actual=2 <- under-charged chunks_after=4 model=3 actual=3 chunks_after=7 model=1 actual=2 <- under-charged The gap appears exactly when the MMR has multiple peaks to fold. This is a live fee, so the correction lands as a version rather than in place: `bulk_append_tree_versions.cost.compaction_hash_count`, 0 for GROVE_V1..V3 and 1 for GROVE_V4. The v1 term is derived from the MMR shape via a new `hash_count_for_root_bagging(mmr_size)` rather than read back out of the accumulated `OperationCost`. Reading the cost would have made this gate depend on the MMR crate's own `get_root` charge being enabled for the same version — true today only because both flip at V4. Deriving it keeps the two gates independent. As with the MMR gate, the existing entry points keep their signatures and delegate to `GroveVersion::first()`, so callers that predate the gate keep the released figure by construction: `BulkAppendTree::{append, append_no_state_root}` and `CommitmentTree::{append, append_raw, append_many_raw}` are unchanged for external callers, each gaining a `*_with_version` sibling. GroveDB's own commitment-tree and bulk-append operations call the versioned ones, so V4 charges the corrected figure. Tests: - a 20-append run at chunk_power 2 under V3 vs V4: identical state roots and the same number of compactions, v1 never cheaper, and strictly dearer on at least one multi-peak compaction; GROVE_V1 and GROVE_V3 agree exactly - a compacting append rejects an unknown charge version - the version table pinned: V1..V3 at 0, V4 at 1, `first()` on the shipped figure Full --all-features workspace suite green (4822 tests); clippy clean. Co-Authored-By: Claude Fable 5 --- grovedb-bulk-append-tree/src/cost/mod.rs | 43 +++++ grovedb-bulk-append-tree/src/cost/v0.rs | 14 ++ grovedb-bulk-append-tree/src/cost/v1.rs | 13 ++ grovedb-bulk-append-tree/src/error.rs | 4 + grovedb-bulk-append-tree/src/lib.rs | 1 + grovedb-bulk-append-tree/src/tree/append.rs | 163 ++++++++++++++++-- grovedb-commitment-tree/Cargo.toml | 1 + .../src/commitment_tree/mod.rs | 55 +++++- grovedb-merkle-mountain-range/src/helper.rs | 18 ++ grovedb-merkle-mountain-range/src/lib.rs | 7 +- grovedb-version/src/tests.rs | 35 ++++ .../src/version/bulk_append_tree_versions.rs | 30 ++++ grovedb-version/src/version/mod.rs | 6 +- grovedb-version/src/version/v1.rs | 9 + grovedb-version/src/version/v2.rs | 9 + grovedb-version/src/version/v3.rs | 9 + grovedb-version/src/version/v4.rs | 9 + grovedb/src/operations/bulk_append_tree.rs | 13 +- grovedb/src/operations/commitment_tree.rs | 4 +- 19 files changed, 416 insertions(+), 27 deletions(-) create mode 100644 grovedb-bulk-append-tree/src/cost/mod.rs create mode 100644 grovedb-bulk-append-tree/src/cost/v0.rs create mode 100644 grovedb-bulk-append-tree/src/cost/v1.rs create mode 100644 grovedb-version/src/version/bulk_append_tree_versions.rs diff --git a/grovedb-bulk-append-tree/src/cost/mod.rs b/grovedb-bulk-append-tree/src/cost/mod.rs new file mode 100644 index 000000000..7b3a412db --- /dev/null +++ b/grovedb-bulk-append-tree/src/cost/mod.rs @@ -0,0 +1,43 @@ +//! Versioned cost accounting for the bulk-append tree. +//! +//! Only the reported hash count is versioned; chunk bytes, roots and stored +//! state are identical under every version. This gate reaches a live fee — +//! `CommitmentTree` adds the compaction hash count straight into its own +//! `hash_node_calls` — so the corrected figure arrives as a new version rather +//! than replacing the old one. + +mod v0; +mod v1; + +use grovedb_version::{error::GroveVersionError, version::GroveVersion}; + +use crate::BulkAppendError; + +/// Hashes to report for a compacting append. +/// +/// `leaf_count` is the MMR leaf count BEFORE the push (what +/// `hash_count_for_push` expects); `mmr_size_after_push` is the size the MMR +/// reached, which determines how many peaks the compaction's `get_root` had +/// to fold. +pub(crate) fn compaction_hash_count( + leaf_count: u64, + mmr_size_after_push: u64, + grove_version: &GroveVersion, +) -> Result { + match grove_version + .bulk_append_tree_versions + .cost + .compaction_hash_count + { + 0 => Ok(v0::compaction_hash_count(leaf_count)), + 1 => Ok(v1::compaction_hash_count(leaf_count, mmr_size_after_push)), + version => Err(BulkAppendError::VersionError( + GroveVersionError::UnknownVersionMismatch { + method: "BulkAppendTree compaction hash count".to_string(), + known_versions: vec![0, 1], + received: version, + } + .to_string(), + )), + } +} diff --git a/grovedb-bulk-append-tree/src/cost/v0.rs b/grovedb-bulk-append-tree/src/cost/v0.rs new file mode 100644 index 000000000..b66a16456 --- /dev/null +++ b/grovedb-bulk-append-tree/src/cost/v0.rs @@ -0,0 +1,14 @@ +//! Shipped compaction hash count. +//! +//! The chunk-blob leaf hash plus one per peak the MMR push collapses. It omits +//! the peak bagging the compaction's own `get_root` performs, so a compaction +//! landing on a multi-peak MMR under-reports by `peaks - 1`. +//! +//! Locked: GROVE_V1..V3 are released and CommitmentTree has been billing this +//! figure on mainnet. + +use grovedb_merkle_mountain_range::hash_count_for_push; + +pub(super) fn compaction_hash_count(leaf_count: u64) -> u32 { + hash_count_for_push(leaf_count) +} diff --git a/grovedb-bulk-append-tree/src/cost/v1.rs b/grovedb-bulk-append-tree/src/cost/v1.rs new file mode 100644 index 000000000..d7862b44d --- /dev/null +++ b/grovedb-bulk-append-tree/src/cost/v1.rs @@ -0,0 +1,13 @@ +//! Corrected compaction hash count. +//! +//! Adds the peak-bagging merges [`v0`](super::v0) omitted. Used from GROVE_V4. + +use grovedb_merkle_mountain_range::{hash_count_for_push, hash_count_for_root_bagging}; + +pub(super) fn compaction_hash_count(leaf_count: u64, mmr_size_after_push: u64) -> u32 { + // Derived from the MMR shape rather than read back out of the accumulated + // `OperationCost`, so this stays correct regardless of whether + // `MMR::get_root`'s own charge is enabled for the caller's version — the + // two gates are independent. + hash_count_for_push(leaf_count).saturating_add(hash_count_for_root_bagging(mmr_size_after_push)) +} diff --git a/grovedb-bulk-append-tree/src/error.rs b/grovedb-bulk-append-tree/src/error.rs index 28689bd58..db05c573a 100644 --- a/grovedb-bulk-append-tree/src/error.rs +++ b/grovedb-bulk-append-tree/src/error.rs @@ -15,4 +15,8 @@ pub enum BulkAppendError { InvalidProof(String), #[error("invalid input: {0}")] InvalidInput(String), + /// The grove version selected an unknown version for a versioned + /// bulk-append method. + #[error("version error: {0}")] + VersionError(String), } diff --git a/grovedb-bulk-append-tree/src/lib.rs b/grovedb-bulk-append-tree/src/lib.rs index 0b2673174..0100c00a2 100644 --- a/grovedb-bulk-append-tree/src/lib.rs +++ b/grovedb-bulk-append-tree/src/lib.rs @@ -9,6 +9,7 @@ //! CDN-cacheable. pub mod chunk; +mod cost; mod error; pub mod proof; mod tree; diff --git a/grovedb-bulk-append-tree/src/tree/append.rs b/grovedb-bulk-append-tree/src/tree/append.rs index fc542a048..d04906a99 100644 --- a/grovedb-bulk-append-tree/src/tree/append.rs +++ b/grovedb-bulk-append-tree/src/tree/append.rs @@ -1,9 +1,7 @@ //! Append and compaction logic for BulkAppendTree. use grovedb_costs::{CostResult, CostsExt, OperationCost}; -use grovedb_merkle_mountain_range::{ - hash_count_for_push, mmr_size_to_leaf_count, MmrKeySize, MmrNode, MmrStore, MMR, -}; +use grovedb_merkle_mountain_range::{mmr_size_to_leaf_count, MmrKeySize, MmrNode, MmrStore, MMR}; use grovedb_storage::StorageContext; use grovedb_version::version::GroveVersion; @@ -11,7 +9,7 @@ use super::{ capacity_for_height, hash::compute_state_root, AppendNoStateRootResult, AppendResult, BulkAppendTree, }; -use crate::{chunk::serialize_chunk_blob, BulkAppendError}; +use crate::{chunk::serialize_chunk_blob, cost::compaction_hash_count, BulkAppendError}; impl<'db, S: StorageContext<'db>> BulkAppendTree { /// Create a new empty tree. @@ -63,8 +61,19 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { /// state root computation. For batched inserts prefer /// [`append_many`](Self::append_many) or [`append_no_state_root`](Self::append_no_state_root) /// — they skip the per-leaf state-root blake3 call. + /// Reports the shipped (v0) hash count; see + /// [`append_no_state_root`](Self::append_no_state_root). pub fn append(&mut self, value: &[u8]) -> Result { - let r = self.append_no_state_root(value)?; + self.append_with_version(value, GroveVersion::first()) + } + + /// Version-dispatched [`append`](Self::append). + pub fn append_with_version( + &mut self, + value: &[u8], + grove_version: &GroveVersion, + ) -> Result { + let r = self.append_no_state_root_with_version(value, grove_version)?; let state_root = self.compute_current_state_root()?; Ok(AppendResult { state_root, @@ -85,9 +94,26 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { /// Storage mutation is identical to [`append`](Self::append). /// /// [`CommitmentTree::append_many_raw`]: ../../grovedb_commitment_tree/struct.CommitmentTree.html#method.append_many_raw + /// Reports the shipped (v0) hash count. Callers holding a + /// [`GroveVersion`] should prefer + /// [`append_no_state_root_with_version`](Self::append_no_state_root_with_version): + /// this entry point exists so paths that predate the gate keep their + /// released figure by construction. pub fn append_no_state_root( &mut self, value: &[u8], + ) -> Result { + self.append_no_state_root_with_version(value, GroveVersion::first()) + } + + /// Version-dispatched [`append_no_state_root`](Self::append_no_state_root). + /// + /// Stored bytes, chunks and roots are identical under every version; only + /// the reported `hash_count` differs, and only for an append that compacts. + pub fn append_no_state_root_with_version( + &mut self, + value: &[u8], + grove_version: &GroveVersion, ) -> Result { let mut hash_count: u32 = 0; let global_position = self.total_count; @@ -113,7 +139,7 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { // Dense tree is full — compact existing entries + new value. // Must run before incrementing total_count so that // self.mmr_size() reflects the pre-compaction state. - let (compact_hashes, mmr_root) = self.compact_with_value(value)?; + let (compact_hashes, mmr_root) = self.compact_with_value(value, grove_version)?; hash_count += compact_hashes; // MMR mutated by the compaction — refresh the cached root. self.last_mmr_root = Some(mmr_root); @@ -283,13 +309,15 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { /// [`compact_with_value_with_cost`](Self::compact_with_value_with_cost). /// Kept so the released `append_no_state_root` path bills exactly what it /// always has — its costs are dropped here, not at the call site. - fn compact_with_value(&mut self, new_value: &[u8]) -> Result<(u32, [u8; 32]), BulkAppendError> { - // Pinned to the first grove version, i.e. the shipped MMR hash - // accounting. This wrapper discards the cost anyway, so the choice is - // unobservable here — but pinning states the intent: the released - // `append_no_state_root` path (and therefore CommitmentTree) must not - // pick up a newer charge just because one exists. - self.compact_with_value_with_cost(new_value, GroveVersion::first()) + fn compact_with_value( + &mut self, + new_value: &[u8], + grove_version: &GroveVersion, + ) -> Result<(u32, [u8; 32]), BulkAppendError> { + // The accumulated `OperationCost` is discarded here — this path + // reports its work through the returned `hash_count` instead, which + // IS version-dependent and which CommitmentTree bills. + self.compact_with_value_with_cost(new_value, grove_version) .unwrap() } @@ -347,7 +375,7 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { // Append chunk root to MMR let mmr_size = self.mmr_size(); let leaf_count = mmr_size_to_leaf_count(mmr_size); - hash_count += hash_count_for_push(leaf_count); + let mut mmr_size_after_push = mmr_size; // Create MmrStore on the fly from the dense tree's storage. // Use the overlay from previous compactions so cross-compaction @@ -384,6 +412,7 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { }; // Take overlay back instead of committing + mmr_size_after_push = mmr.mmr_size; self.mmr_overlay = mmr.batch.take_overlay(); root @@ -392,6 +421,15 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { // Reset dense tree (old values stay in store, overwritten on next cycle) self.dense_tree.reset(); + // The reported count is version-gated: v0 is the shipped figure (leaf + // hash + push collapses), v1 adds the peak bagging the `get_root` + // above performed. `mmr.mmr_size` is read after the push, which is + // the shape that root had to fold. + hash_count = match compaction_hash_count(leaf_count, mmr_size_after_push, grove_version) { + Ok(h) => hash_count.saturating_add(h), + Err(e) => return Err(e).wrap_with_cost(cost), + }; + Ok((hash_count, mmr_root)).wrap_with_cost(cost) } @@ -463,3 +501,100 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { Ok(()) } } + +#[cfg(test)] +mod compaction_hash_count_gate_tests { + use grovedb_version::version::{v1::GROVE_V1, v3::GROVE_V3, v4::GROVE_V4}; + + use super::*; + use crate::test_utils::MemStorageContext; + + /// The reported hash count for a compacting append is version-gated: v0 + /// (V1..V3) omits the peak bagging the compaction's own `get_root` + /// performs, v1 (V4) includes it. Everything else about the append — + /// stored bytes, chunk contents, roots — must be identical. + #[test] + fn compaction_hash_count_gains_the_bagging_term_at_v4() { + // chunk_power 2 -> epoch 4. Enough appends to compact repeatedly so + // the MMR passes through single- and multi-peak shapes. + let build = |version: &_| { + let mut t = BulkAppendTree::new(2, MemStorageContext::new()).expect("new"); + let mut counts = Vec::new(); + let mut roots = Vec::new(); + for i in 0..20u8 { + let r = t + .append_no_state_root_with_version(&[i; 8], version) + .expect("append"); + if r.compacted { + counts.push(r.hash_count); + } + } + roots.push(t.compute_current_state_root().expect("root")); + (counts, roots) + }; + + let (v0_counts, v0_roots) = build(&GROVE_V3); + let (v1_counts, v1_roots) = build(&GROVE_V4); + + assert_eq!( + v0_roots, v1_roots, + "the tree itself must not depend on the cost version" + ); + assert_eq!( + v0_counts.len(), + v1_counts.len(), + "same number of compactions" + ); + + // v1 is never cheaper, and is strictly dearer on at least one + // compaction — the ones that landed on a multi-peak MMR. + let mut saw_increase = false; + for (i, (a, b)) in v0_counts.iter().zip(v1_counts.iter()).enumerate() { + assert!( + b >= a, + "v1 must never report fewer hashes (compaction {}): v0={} v1={}", + i, + a, + b + ); + if b > a { + saw_increase = true; + } + } + assert!( + saw_increase, + "expected at least one multi-peak compaction to gain the bagging \ + term: v0={:?} v1={:?}", + v0_counts, v1_counts + ); + + // V1 and V3 are both v0, so they must agree exactly. + let (v1_ver_counts, _) = build(&GROVE_V1); + assert_eq!( + v0_counts, v1_ver_counts, + "GROVE_V1 and GROVE_V3 both select the shipped figure" + ); + } + + /// An unknown charge version must be rejected, not silently treated as one + /// of the implemented ones. + #[test] + fn compaction_hash_count_rejects_unknown_version() { + let mut bad = GROVE_V4.clone(); + bad.bulk_append_tree_versions.cost.compaction_hash_count = 99; + + let mut t = BulkAppendTree::new(2, MemStorageContext::new()).expect("new"); + // Fill the buffer so the next append compacts and reaches the gate. + for i in 0..3u8 { + t.append_no_state_root_with_version(&[i; 8], &bad) + .expect("buffered appends do not reach the gate"); + } + assert!( + matches!( + t.append_no_state_root_with_version(&[9u8; 8], &bad), + Err(BulkAppendError::VersionError(_)) + ), + "a compacting append must reject an unknown charge version" + ); + } +} diff --git a/grovedb-commitment-tree/Cargo.toml b/grovedb-commitment-tree/Cargo.toml index 84689b7c0..caa1e7ed8 100644 --- a/grovedb-commitment-tree/Cargo.toml +++ b/grovedb-commitment-tree/Cargo.toml @@ -22,6 +22,7 @@ incrementalmerkletree = "0.8" shardtree = { version = "0.6", optional = true } rusqlite = { version = "0.38", features = ["bundled"], optional = true } grovedb-costs = { version = "5.0.1", path = "../costs" } +grovedb-version = { version = "5.0.1", path = "../grovedb-version" } grovedb-storage = { version = "5.0.1", path = "../storage", optional = true } grovedb-bulk-append-tree = { version = "5.0.1", path = "../grovedb-bulk-append-tree", default-features = false } blake3 = { workspace = true } diff --git a/grovedb-commitment-tree/src/commitment_tree/mod.rs b/grovedb-commitment-tree/src/commitment_tree/mod.rs index 8649211d0..d4282391f 100644 --- a/grovedb-commitment-tree/src/commitment_tree/mod.rs +++ b/grovedb-commitment-tree/src/commitment_tree/mod.rs @@ -14,6 +14,7 @@ use std::marker::PhantomData; use grovedb_bulk_append_tree::BulkAppendTree; use grovedb_costs::{CostResult, CostsExt, OperationCost}; use grovedb_storage::StorageContext; +use grovedb_version::version::GroveVersion; use orchard::{ memo::{DashMemo, MemoSize}, note::TransmittedNoteCiphertext, @@ -256,9 +257,25 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { rho: [u8; 32], cv_net: [u8; 32], ciphertext: &TransmittedNoteCiphertext, + ) -> CostResult { + self.append_with_version(cmx, rho, cv_net, ciphertext, GroveVersion::first()) + } + + /// Version-dispatched [`append`](Self::append). + /// + /// The note, chunk and roots are identical under every version; what the + /// version selects is the hash count a compacting append reports, which + /// this method bills. + pub fn append_with_version( + &mut self, + cmx: [u8; 32], + rho: [u8; 32], + cv_net: [u8; 32], + ciphertext: &TransmittedNoteCiphertext, + grove_version: &GroveVersion, ) -> CostResult { let payload = serialize_ciphertext(ciphertext); - self.append_raw(cmx, rho, cv_net, &payload) + self.append_raw_with_version(cmx, rho, cv_net, &payload, grove_version) } /// Append a note commitment and raw payload bytes to the commitment tree. @@ -287,6 +304,19 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { rho: [u8; 32], cv_net: [u8; 32], payload: &[u8], + ) -> CostResult { + self.append_raw_with_version(cmx, rho, cv_net, payload, GroveVersion::first()) + } + + /// Version-dispatched [`append_raw`](Self::append_raw). See + /// [`append_with_version`](Self::append_with_version). + pub fn append_raw_with_version( + &mut self, + cmx: [u8; 32], + rho: [u8; 32], + cv_net: [u8; 32], + payload: &[u8], + grove_version: &GroveVersion, ) -> CostResult { let mut cost = OperationCost::default(); @@ -314,7 +344,10 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { item_value.extend_from_slice(&cv_net); item_value.extend_from_slice(payload); - let bulk_result = match self.bulk_tree.append(&item_value) { + let bulk_result = match self + .bulk_tree + .append_with_version(&item_value, grove_version) + { Ok(r) => r, // codecov:ignore — requires BulkAppendTree::append to fail, which only happens on // storage faults (put/get errors) during dense tree insert or MMR compaction; @@ -419,6 +452,19 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { &mut self, entries: I, ) -> CostResult + where + I: IntoIterator, + { + self.append_many_raw_with_version(entries, GroveVersion::first()) + } + + /// Version-dispatched [`append_many_raw`](Self::append_many_raw). See + /// [`append_with_version`](Self::append_with_version). + pub fn append_many_raw_with_version( + &mut self, + entries: I, + grove_version: &GroveVersion, + ) -> CostResult where I: IntoIterator, { @@ -463,7 +509,10 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { item_value.extend_from_slice(&cv_net); item_value.extend_from_slice(&payload); - let r = match self.bulk_tree.append_no_state_root(&item_value) { + let r = match self + .bulk_tree + .append_no_state_root_with_version(&item_value, grove_version) + { Ok(r) => r, // codecov:ignore — only reachable on a storage fault during the // dense-tree insert or MMR compaction (see `append_raw`'s diff --git a/grovedb-merkle-mountain-range/src/helper.rs b/grovedb-merkle-mountain-range/src/helper.rs index 8d948855c..61db0c163 100644 --- a/grovedb-merkle-mountain-range/src/helper.rs +++ b/grovedb-merkle-mountain-range/src/helper.rs @@ -107,6 +107,24 @@ pub fn get_peak_map(mmr_size: u64) -> u64 { /// 2 /// / \ /// 0 1 3 +/// Blake3 merges [`MMR::get_root`](crate::MMR::get_root) performs for an MMR +/// of this size. +/// +/// Bagging folds the peaks right-to-left with one merge per additional peak, +/// so `n` peaks cost `n - 1`. Sizes 0 and 1 take the empty and single-element +/// paths, which fold nothing. +/// +/// Exposed so callers that report a hash count for an operation containing a +/// root computation can account for the bagging WITHOUT depending on whether +/// `get_root`'s own charge is enabled for their grove version — the two are +/// separately gated. +pub fn hash_count_for_root_bagging(mmr_size: u64) -> u32 { + if mmr_size <= 1 { + return 0; + } + get_peaks(mmr_size).len().saturating_sub(1) as u32 +} + pub fn get_peaks(mmr_size: u64) -> Vec { if mmr_size == 0 { return vec![]; diff --git a/grovedb-merkle-mountain-range/src/lib.rs b/grovedb-merkle-mountain-range/src/lib.rs index 5ab75c38f..5f93df0a1 100644 --- a/grovedb-merkle-mountain-range/src/lib.rs +++ b/grovedb-merkle-mountain-range/src/lib.rs @@ -40,9 +40,10 @@ mod tests; pub use error::{Error, Result}; pub use grovedb_costs::{CostResult, CostsExt, OperationCost}; pub use helper::{ - hash_count_for_push, leaf_index_to_mmr_size, leaf_index_to_mmr_size as leaf_to_mmr_size, - leaf_index_to_pos, leaf_index_to_pos as leaf_to_pos, mmr_node_key, mmr_node_key_sized, - mmr_size_to_leaf_count, MmrKey, MmrKeySize, MAX_U32_MMR_POSITION, + hash_count_for_push, hash_count_for_root_bagging, leaf_index_to_mmr_size, + leaf_index_to_mmr_size as leaf_to_mmr_size, leaf_index_to_pos, + leaf_index_to_pos as leaf_to_pos, mmr_node_key, mmr_node_key_sized, mmr_size_to_leaf_count, + MmrKey, MmrKeySize, MAX_U32_MMR_POSITION, }; #[cfg(any(test, feature = "mem_store"))] pub use mem_store::MemStore; diff --git a/grovedb-version/src/tests.rs b/grovedb-version/src/tests.rs index de764532a..2467d7911 100644 --- a/grovedb-version/src/tests.rs +++ b/grovedb-version/src/tests.rs @@ -591,3 +591,38 @@ fn grove_version_first_selects_shipped_mmr_costs() { assert_eq!(cost.get_root, 0); assert_eq!(cost.gen_proof, 0); } + +// ── Bulk-append cost version table ─────────────────────────────────── + +/// The compaction hash count reaches a live fee through CommitmentTree, so +/// which version each protocol version selects is pinned here. +#[test] +fn bulk_append_cost_versions_are_pinned_per_protocol_version() { + for (name, version) in [ + ("GROVE_V1", &GROVE_V1), + ("GROVE_V2", &GROVE_V2), + ("GROVE_V3", &GROVE_V3), + ] { + assert_eq!( + version.bulk_append_tree_versions.cost.compaction_hash_count, 0, + "{name} must keep the shipped compaction hash count" + ); + } + assert_eq!( + GROVE_V4 + .bulk_append_tree_versions + .cost + .compaction_hash_count, + 1, + "GROVE_V4 adds the peak-bagging term" + ); + assert_eq!( + GroveVersion::first() + .bulk_append_tree_versions + .cost + .compaction_hash_count, + 0, + "the unversioned entry points delegate here and must stay on the \ + shipped figure" + ); +} diff --git a/grovedb-version/src/version/bulk_append_tree_versions.rs b/grovedb-version/src/version/bulk_append_tree_versions.rs new file mode 100644 index 000000000..a9b3bfbe3 --- /dev/null +++ b/grovedb-version/src/version/bulk_append_tree_versions.rs @@ -0,0 +1,30 @@ +//! Version gates for the bulk-append tree. +//! +//! As in [`super::mmr_versions`], only cost accounting is versioned: the +//! chunks, roots and stored bytes are identical under every version. These +//! gates matter more than the MMR ones, though, because the value they change +//! reaches a live fee — `CommitmentTree` adds the reported hash count straight +//! into its own `hash_node_calls`, and the shielded pool has been running on +//! it since mainnet activation. + +use versioned_feature_core::FeatureVersion; + +#[derive(Clone, Debug, Default)] +pub struct BulkAppendTreeVersions { + pub cost: BulkAppendTreeCostVersions, +} + +#[derive(Clone, Debug, Default)] +pub struct BulkAppendTreeCostVersions { + /// The `hash_count` a compacting append reports, which + /// `append_no_state_root` forwards and `CommitmentTree` bills. + /// + /// Version 0 reports `hash_count_for_push` — the chunk-blob leaf hash plus + /// one per peak the MMR push collapses. That omits the peak bagging the + /// compaction's own `get_root` performs, so a compaction landing on a + /// multi-peak MMR under-reports by `peaks - 1`. + /// + /// Version 1 adds that bagging term. Shipped chunk bytes and roots are + /// unaffected; what moves is the fee a compacting append is charged. + pub compaction_hash_count: FeatureVersion, +} diff --git a/grovedb-version/src/version/mod.rs b/grovedb-version/src/version/mod.rs index 34a066ba6..148dce001 100644 --- a/grovedb-version/src/version/mod.rs +++ b/grovedb-version/src/version/mod.rs @@ -1,3 +1,4 @@ +pub mod bulk_append_tree_versions; pub mod grovedb_versions; pub mod merk_versions; pub mod mmr_versions; @@ -11,8 +12,8 @@ pub use versioned_feature_core::*; use crate::version::v3::GROVE_V3; use crate::version::v4::GROVE_V4; use crate::version::{ - grovedb_versions::GroveDBVersions, merk_versions::MerkVersions, mmr_versions::MmrVersions, - v1::GROVE_V1, v2::GROVE_V2, + bulk_append_tree_versions::BulkAppendTreeVersions, grovedb_versions::GroveDBVersions, + merk_versions::MerkVersions, mmr_versions::MmrVersions, v1::GROVE_V1, v2::GROVE_V2, }; #[derive(Clone, Debug, Default)] @@ -21,6 +22,7 @@ pub struct GroveVersion { pub grovedb_versions: GroveDBVersions, pub merk_versions: MerkVersions, pub mmr_versions: MmrVersions, + pub bulk_append_tree_versions: BulkAppendTreeVersions, } impl GroveVersion { diff --git a/grovedb-version/src/version/v1.rs b/grovedb-version/src/version/v1.rs index 6f2f6d812..43b129b2f 100644 --- a/grovedb-version/src/version/v1.rs +++ b/grovedb-version/src/version/v1.rs @@ -1,5 +1,6 @@ use crate::version::grovedb_versions::GroveDBAggregateSumPathQueryMethodVersions; use crate::version::{ + bulk_append_tree_versions::{BulkAppendTreeCostVersions, BulkAppendTreeVersions}, grovedb_versions::{ GroveDBApplyBatchVersions, GroveDBElementMethodVersions, GroveDBOperationsAverageCaseVersions, GroveDBOperationsDeleteUpTreeVersions, @@ -266,4 +267,12 @@ pub const GROVE_V1: GroveVersion = GroveVersion { gen_proof: 0, }, }, + // Compaction hash count: the shipped figure, which omits the peak + // bagging a compaction's own `get_root` performs. Locked — the + // shielded pool has been charged this since mainnet activation. + bulk_append_tree_versions: BulkAppendTreeVersions { + cost: BulkAppendTreeCostVersions { + compaction_hash_count: 0, + }, + }, }; diff --git a/grovedb-version/src/version/v2.rs b/grovedb-version/src/version/v2.rs index 1dff47307..d817794ed 100644 --- a/grovedb-version/src/version/v2.rs +++ b/grovedb-version/src/version/v2.rs @@ -1,5 +1,6 @@ use crate::version::grovedb_versions::GroveDBAggregateSumPathQueryMethodVersions; use crate::version::{ + bulk_append_tree_versions::{BulkAppendTreeCostVersions, BulkAppendTreeVersions}, grovedb_versions::{ GroveDBApplyBatchVersions, GroveDBElementMethodVersions, GroveDBOperationsAverageCaseVersions, GroveDBOperationsDeleteUpTreeVersions, @@ -265,4 +266,12 @@ pub const GROVE_V2: GroveVersion = GroveVersion { gen_proof: 0, }, }, + // Compaction hash count: the shipped figure, which omits the peak + // bagging a compaction's own `get_root` performs. Locked — the + // shielded pool has been charged this since mainnet activation. + bulk_append_tree_versions: BulkAppendTreeVersions { + cost: BulkAppendTreeCostVersions { + compaction_hash_count: 0, + }, + }, }; diff --git a/grovedb-version/src/version/v3.rs b/grovedb-version/src/version/v3.rs index 7818e4a99..6a141dce3 100644 --- a/grovedb-version/src/version/v3.rs +++ b/grovedb-version/src/version/v3.rs @@ -1,5 +1,6 @@ use crate::version::grovedb_versions::GroveDBAggregateSumPathQueryMethodVersions; use crate::version::{ + bulk_append_tree_versions::{BulkAppendTreeCostVersions, BulkAppendTreeVersions}, grovedb_versions::{ GroveDBApplyBatchVersions, GroveDBElementMethodVersions, GroveDBOperationsAverageCaseVersions, GroveDBOperationsDeleteUpTreeVersions, @@ -280,4 +281,12 @@ pub const GROVE_V3: GroveVersion = GroveVersion { gen_proof: 0, }, }, + // Compaction hash count: the shipped figure, which omits the peak + // bagging a compaction's own `get_root` performs. Locked — the + // shielded pool has been charged this since mainnet activation. + bulk_append_tree_versions: BulkAppendTreeVersions { + cost: BulkAppendTreeCostVersions { + compaction_hash_count: 0, + }, + }, }; diff --git a/grovedb-version/src/version/v4.rs b/grovedb-version/src/version/v4.rs index 0ca8c09a7..e08c49f27 100644 --- a/grovedb-version/src/version/v4.rs +++ b/grovedb-version/src/version/v4.rs @@ -101,6 +101,7 @@ use crate::version::grovedb_versions::GroveDBAggregateSumPathQueryMethodVersions; use crate::version::{ + bulk_append_tree_versions::{BulkAppendTreeCostVersions, BulkAppendTreeVersions}, grovedb_versions::{ GroveDBApplyBatchVersions, GroveDBElementMethodVersions, GroveDBOperationsAverageCaseVersions, GroveDBOperationsDeleteUpTreeVersions, @@ -381,4 +382,12 @@ pub const GROVE_V4: GroveVersion = GroveVersion { gen_proof: 1, }, }, + // Compaction hash count: adds the peak-bagging merges the shipped + // figure omitted, so a compacting append is charged the hashes it + // actually performs. Chunk bytes and roots are unchanged. + bulk_append_tree_versions: BulkAppendTreeVersions { + cost: BulkAppendTreeCostVersions { + compaction_hash_count: 1, + }, + }, }; diff --git a/grovedb/src/operations/bulk_append_tree.rs b/grovedb/src/operations/bulk_append_tree.rs index 26df4dc58..8d5b0bf16 100644 --- a/grovedb/src/operations/bulk_append_tree.rs +++ b/grovedb/src/operations/bulk_append_tree.rs @@ -84,7 +84,11 @@ impl GroveDb { BulkAppendTree::from_state(total_count, chunk_power, storage_ctx).map_err(map_bulk_err) ); - let result = cost_return_on_error_no_add!(cost, tree.append(&value).map_err(map_bulk_err)); + let result = cost_return_on_error_no_add!( + cost, + tree.append_with_version(&value, grove_version) + .map_err(map_bulk_err) + ); cost.hash_node_calls += result.hash_count; @@ -524,8 +528,11 @@ impl GroveDb { // Process each value for value in values { - let result = - cost_return_on_error_no_add!(cost, tree.append(value).map_err(map_bulk_err)); + let result = cost_return_on_error_no_add!( + cost, + tree.append_with_version(value, grove_version) + .map_err(map_bulk_err) + ); cost.hash_node_calls += result.hash_count; } diff --git a/grovedb/src/operations/commitment_tree.rs b/grovedb/src/operations/commitment_tree.rs index 3ce75a3e3..eadfad0d2 100644 --- a/grovedb/src/operations/commitment_tree.rs +++ b/grovedb/src/operations/commitment_tree.rs @@ -156,7 +156,7 @@ impl GroveDb { let append_result = cost_return_on_error!( &mut cost, - ct.append_raw(cmx, rho, cv_net, &payload) + ct.append_raw_with_version(cmx, rho, cv_net, &payload, grove_version) .map(|r| r.map_err(map_ct_err)) ); @@ -534,7 +534,7 @@ impl GroveDb { for (cmx, rho, cv_net, payload) in inserts { cost_return_on_error!( &mut cost, - ct.append_raw(*cmx, *rho, *cv_net, payload) + ct.append_raw_with_version(*cmx, *rho, *cv_net, payload, grove_version) .map(|r| r.map_err(map_ct_err)) ); } From 6fa32775d17d5744b9514ec98b128ff3ee01cde7 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 20 Aug 2026 05:50:40 +0700 Subject: [PATCH 16/19] fix(bulk-append): drop the dead initializer flagged by -D warnings `mmr_size_after_push` was initialized to the pre-push size and then unconditionally overwritten inside the MMR block, so the initializer was never read. Declared without one instead; the single assignment after the push is what the bagging term must be computed from. Caught by CI, not locally: the lint job runs `cargo clippy --workspace --all-features -- -D warnings`, which promotes this to an error, while my check counted only lines already starting with "error". Verified against every gate the lint and formatting jobs actually run. Co-Authored-By: Claude Fable 5 --- grovedb-bulk-append-tree/src/tree/append.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/grovedb-bulk-append-tree/src/tree/append.rs b/grovedb-bulk-append-tree/src/tree/append.rs index d04906a99..19fa891f8 100644 --- a/grovedb-bulk-append-tree/src/tree/append.rs +++ b/grovedb-bulk-append-tree/src/tree/append.rs @@ -375,7 +375,9 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { // Append chunk root to MMR let mmr_size = self.mmr_size(); let leaf_count = mmr_size_to_leaf_count(mmr_size); - let mut mmr_size_after_push = mmr_size; + // Assigned once inside the MMR block below, after the push, so the + // bagging term is computed from the shape `get_root` actually folded. + let mmr_size_after_push; // Create MmrStore on the fly from the dense tree's storage. // Use the overlay from previous compactions so cross-compaction From 2c2e95ca62d95671c77136e73068d1e09550b5fd Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 20 Aug 2026 14:07:00 +0700 Subject: [PATCH 17/19] refactor: take grove_version directly instead of paired _with_version methods MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Collapses the delegating pairs on CommitmentTree (`append`, `append_raw`, `append_many_raw`) and BulkAppendTree (`append`, `append_no_state_root`) into single methods that take `&GroveVersion`. The pairs existed to keep external call sites compiling unchanged, with the bare form pinned to `GroveVersion::first()`. That is a worse default than it looks: a caller gets the shipped cost accounting by omission rather than by decision, and the version a fee is computed under is exactly the thing a caller should have to state. One method that takes the version makes the choice explicit at every site. BulkAppendTree is collapsed alongside CommitmentTree because CommitmentTree calls straight into it — leaving a pinned bare form one layer down would have reintroduced the same implicit default underneath the explicit API. Call sites updated: the grovedb commitment-tree and bulk-append operations already had a version in scope; tests and the seeding bench now pass one explicitly. The MMR crate keeps its `push`/`get_root`/`gen_proof` pairs for now — those bare forms are reached from a dozen internal and bench call sites that have no version to hand, so collapsing them is a larger change than this one. Full CI gate set green: clippy -D warnings, check --all-targets, the verify feature build, fmt --check, and the --all-features suite (4822 tests). Co-Authored-By: Claude Fable 5 --- grovedb-bulk-append-tree/src/proof/tests.rs | 5 +- grovedb-bulk-append-tree/src/tree/append.rs | 39 ++------- grovedb-bulk-append-tree/src/tree/tests.rs | 82 +++++++++++-------- grovedb-commitment-tree/benches/seeding.rs | 3 +- .../src/commitment_tree/mod.rs | 51 ++---------- .../src/commitment_tree/tests.rs | 80 ++++++++++++++---- grovedb/src/operations/bulk_append_tree.rs | 6 +- grovedb/src/operations/commitment_tree.rs | 4 +- 8 files changed, 140 insertions(+), 130 deletions(-) diff --git a/grovedb-bulk-append-tree/src/proof/tests.rs b/grovedb-bulk-append-tree/src/proof/tests.rs index 83b6f30d3..d51452025 100644 --- a/grovedb-bulk-append-tree/src/proof/tests.rs +++ b/grovedb-bulk-append-tree/src/proof/tests.rs @@ -2,6 +2,7 @@ mod proof_tests { use grovedb_merkle_mountain_range::MmrTreeProof; use grovedb_query::{Query, QueryItem}; + use grovedb_version::version::GroveVersion; use crate::{proof::*, test_utils::MemStorageContext, BulkAppendTree}; @@ -17,7 +18,9 @@ mod proof_tests { let mut last_state_root = [0u8; 32]; for value in values { - let result = tree.append(value).expect("append value"); + let result = tree + .append(value, GroveVersion::latest()) + .expect("append value"); last_state_root = result.state_root; } diff --git a/grovedb-bulk-append-tree/src/tree/append.rs b/grovedb-bulk-append-tree/src/tree/append.rs index 19fa891f8..1a057cd97 100644 --- a/grovedb-bulk-append-tree/src/tree/append.rs +++ b/grovedb-bulk-append-tree/src/tree/append.rs @@ -61,19 +61,12 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { /// state root computation. For batched inserts prefer /// [`append_many`](Self::append_many) or [`append_no_state_root`](Self::append_no_state_root) /// — they skip the per-leaf state-root blake3 call. - /// Reports the shipped (v0) hash count; see - /// [`append_no_state_root`](Self::append_no_state_root). - pub fn append(&mut self, value: &[u8]) -> Result { - self.append_with_version(value, GroveVersion::first()) - } - - /// Version-dispatched [`append`](Self::append). - pub fn append_with_version( + pub fn append( &mut self, value: &[u8], grove_version: &GroveVersion, ) -> Result { - let r = self.append_no_state_root_with_version(value, grove_version)?; + let r = self.append_no_state_root(value, grove_version)?; let state_root = self.compute_current_state_root()?; Ok(AppendResult { state_root, @@ -94,23 +87,11 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { /// Storage mutation is identical to [`append`](Self::append). /// /// [`CommitmentTree::append_many_raw`]: ../../grovedb_commitment_tree/struct.CommitmentTree.html#method.append_many_raw - /// Reports the shipped (v0) hash count. Callers holding a - /// [`GroveVersion`] should prefer - /// [`append_no_state_root_with_version`](Self::append_no_state_root_with_version): - /// this entry point exists so paths that predate the gate keep their - /// released figure by construction. - pub fn append_no_state_root( - &mut self, - value: &[u8], - ) -> Result { - self.append_no_state_root_with_version(value, GroveVersion::first()) - } - - /// Version-dispatched [`append_no_state_root`](Self::append_no_state_root). /// - /// Stored bytes, chunks and roots are identical under every version; only - /// the reported `hash_count` differs, and only for an append that compacts. - pub fn append_no_state_root_with_version( + /// Stored bytes, chunks and roots are identical under every grove + /// version; only the reported `hash_count` differs, and only for an append + /// that compacts. + pub fn append_no_state_root( &mut self, value: &[u8], grove_version: &GroveVersion, @@ -524,9 +505,7 @@ mod compaction_hash_count_gate_tests { let mut counts = Vec::new(); let mut roots = Vec::new(); for i in 0..20u8 { - let r = t - .append_no_state_root_with_version(&[i; 8], version) - .expect("append"); + let r = t.append_no_state_root(&[i; 8], version).expect("append"); if r.compacted { counts.push(r.hash_count); } @@ -588,12 +567,12 @@ mod compaction_hash_count_gate_tests { let mut t = BulkAppendTree::new(2, MemStorageContext::new()).expect("new"); // Fill the buffer so the next append compacts and reaches the gate. for i in 0..3u8 { - t.append_no_state_root_with_version(&[i; 8], &bad) + t.append_no_state_root(&[i; 8], &bad) .expect("buffered appends do not reach the gate"); } assert!( matches!( - t.append_no_state_root_with_version(&[9u8; 8], &bad), + t.append_no_state_root(&[9u8; 8], &bad), Err(BulkAppendError::VersionError(_)) ), "a compacting append must reject an unknown charge version" diff --git a/grovedb-bulk-append-tree/src/tree/tests.rs b/grovedb-bulk-append-tree/src/tree/tests.rs index 3c3f2e490..f2bb87c72 100644 --- a/grovedb-bulk-append-tree/src/tree/tests.rs +++ b/grovedb-bulk-append-tree/src/tree/tests.rs @@ -2,6 +2,7 @@ use super::BulkAppendTree; use crate::{chunk::deserialize_chunk_blob, test_utils::MemStorageContext}; +use grovedb_version::version::GroveVersion; #[test] fn new_tree() { @@ -49,7 +50,7 @@ fn cached_mmr_root_matches_recomputation_across_compactions() { // height=2 → epoch_size=4, so 20 appends span 5 compaction cycles. let mut tree = BulkAppendTree::new(2u8, MemStorageContext::new()).expect("create tree"); for i in 0..20u8 { - tree.append(&[i]).expect("append"); + tree.append(&[i], GroveVersion::latest()).expect("append"); let fresh = tree.get_mmr_root().expect("recompute mmr root"); assert_eq!( tree.last_mmr_root, @@ -64,7 +65,9 @@ fn cached_mmr_root_matches_recomputation_across_compactions() { fn single_append() { let mut tree = BulkAppendTree::new(2u8, MemStorageContext::new()).expect("create tree"); - let result = tree.append(b"hello").expect("append hello"); + let result = tree + .append(b"hello", GroveVersion::latest()) + .expect("append hello"); assert_eq!(result.global_position, 0); assert!(!result.compacted); assert_eq!(tree.total_count, 1); @@ -82,7 +85,9 @@ fn multiple_appends_no_compaction() { // Height=2, capacity=3. Append 2 values (no compaction). for i in 0..2 { - let result = tree.append(&[i]).expect("append entry"); + let result = tree + .append(&[i], GroveVersion::latest()) + .expect("append entry"); assert_eq!(result.global_position, i as u64); assert!(!result.compacted); } @@ -98,10 +103,14 @@ fn compaction_trigger() { // Height=2, capacity=3, epoch_size=4. First 3 appends fill the buffer, // 4th triggers compaction (try_insert returns None when buffer is full). for i in 0..3u8 { - let r = tree.append(&[i]).expect("append pre-compaction entry"); + let r = tree + .append(&[i], GroveVersion::latest()) + .expect("append pre-compaction entry"); assert!(!r.compacted); } - let result = tree.append(&[3]).expect("append compacting entry"); + let result = tree + .append(&[3], GroveVersion::latest()) + .expect("append compacting entry"); assert!(result.compacted); assert_eq!(result.global_position, 3); assert_eq!(tree.total_count, 4); @@ -119,7 +128,8 @@ fn multi_chunk() { // append 2 → buffer (count=1), append 3 → compaction (chunk has [2,3]) // 4 appends = 2 chunks + 0 buffer for i in 0..4u8 { - tree.append(&[i]).expect("append entry"); + tree.append(&[i], GroveVersion::latest()) + .expect("append entry"); } assert_eq!(tree.total_count, 4); assert_eq!(tree.chunk_count(), 2); @@ -135,10 +145,10 @@ fn get_chunk_value_from_mmr() { // append b → try_insert fails (full), compact [a, b] → chunk 0 // append c → buffer (count=1) // append d → try_insert fails, compact [c, d] → chunk 1 - tree.append(b"a").expect("append a"); - tree.append(b"b").expect("append b"); - tree.append(b"c").expect("append c"); - tree.append(b"d").expect("append d"); + tree.append(b"a", GroveVersion::latest()).expect("append a"); + tree.append(b"b", GroveVersion::latest()).expect("append b"); + tree.append(b"c", GroveVersion::latest()).expect("append c"); + tree.append(b"d", GroveVersion::latest()).expect("append d"); assert_eq!(tree.chunk_count(), 2); assert_eq!(tree.buffer_count(), 0); @@ -167,8 +177,8 @@ fn get_buffer_value_from_dense_tree() { // capacity=3 let mut tree = BulkAppendTree::new(2u8, MemStorageContext::new()).expect("create tree"); - tree.append(b"a").expect("append a"); - tree.append(b"b").expect("append b"); + tree.append(b"a", GroveVersion::latest()).expect("append a"); + tree.append(b"b", GroveVersion::latest()).expect("append b"); // Both from the buffer (dense tree) assert_eq!( @@ -189,8 +199,8 @@ fn get_chunk_blob() { let mut tree = BulkAppendTree::new(1u8, MemStorageContext::new()).expect("create tree"); // Need 2 appends to trigger compaction (epoch_size=2) - tree.append(b"x").expect("append x"); - tree.append(b"y").expect("append y"); // compacts [x, y] + tree.append(b"x", GroveVersion::latest()).expect("append x"); + tree.append(b"y", GroveVersion::latest()).expect("append y"); // compacts [x, y] let blob = tree.get_chunk_value(0).expect("get chunk 0"); assert!(blob.is_some()); @@ -207,8 +217,8 @@ fn query_buffer_entries() { // capacity=3 let mut tree = BulkAppendTree::new(2u8, MemStorageContext::new()).expect("create tree"); - tree.append(b"a").expect("append a"); - tree.append(b"b").expect("append b"); + tree.append(b"a", GroveVersion::latest()).expect("append a"); + tree.append(b"b", GroveVersion::latest()).expect("append b"); // Query all buffer entries with RangeFull let query = grovedb_query::Query::new_range_full(); @@ -225,10 +235,10 @@ fn query_chunks_from_mmr() { let mut tree = BulkAppendTree::new(1u8, MemStorageContext::new()).expect("create tree"); // 4 appends → 2 chunks: chunk 0 = [a,b], chunk 1 = [c,d] - tree.append(b"a").expect("append a"); - tree.append(b"b").expect("append b"); - tree.append(b"c").expect("append c"); - tree.append(b"d").expect("append d"); + tree.append(b"a", GroveVersion::latest()).expect("append a"); + tree.append(b"b", GroveVersion::latest()).expect("append b"); + tree.append(b"c", GroveVersion::latest()).expect("append c"); + tree.append(b"d", GroveVersion::latest()).expect("append d"); // Query both chunks let result = tree.query_chunks(&[0, 1]).expect("query chunks"); @@ -268,8 +278,12 @@ fn state_root_determinism() { let mut tree2 = BulkAppendTree::new(2u8, MemStorageContext::new()).expect("create tree2"); for i in 0..5u8 { - tree1.append(&[i]).expect("append to tree1"); - tree2.append(&[i]).expect("append to tree2"); + tree1 + .append(&[i], GroveVersion::latest()) + .expect("append to tree1"); + tree2 + .append(&[i], GroveVersion::latest()) + .expect("append to tree2"); } let root1 = tree1.compute_current_state_root().expect("state root 1"); @@ -292,15 +306,17 @@ fn hash_count_accuracy() { let mut tree = BulkAppendTree::new(2u8, MemStorageContext::new()).expect("create tree"); // Non-compacting append includes dense tree hashing + state root - let r = tree.append(b"a").expect("append a"); + let r = tree.append(b"a", GroveVersion::latest()).expect("append a"); assert!(r.hash_count > 0); - tree.append(b"b").expect("append b"); - tree.append(b"c").expect("append c"); + tree.append(b"b", GroveVersion::latest()).expect("append b"); + tree.append(b"c", GroveVersion::latest()).expect("append c"); // 4th append triggers compaction: should have more hash calls (dense + mmr + // state root) - let r = tree.append(b"d").expect("append d (compaction)"); + let r = tree + .append(b"d", GroveVersion::latest()) + .expect("append d (compaction)"); assert!(r.compacted); assert!(r.hash_count > 1); } @@ -309,8 +325,10 @@ fn hash_count_accuracy() { fn from_state_roundtrip() { let mut tree = BulkAppendTree::new(2u8, MemStorageContext::new()).expect("create tree"); - tree.append(b"hello").expect("append hello"); - tree.append(b"world").expect("append world"); + tree.append(b"hello", GroveVersion::latest()) + .expect("append hello"); + tree.append(b"world", GroveVersion::latest()) + .expect("append world"); let total_count = tree.total_count; let mmr_size = tree.mmr_size(); @@ -334,7 +352,7 @@ fn compaction_and_continue() { // Fill one epoch and continue for i in 0..5u8 { - tree.append(&[i]).expect("append"); + tree.append(&[i], GroveVersion::latest()).expect("append"); } assert_eq!(tree.total_count, 5); assert_eq!(tree.chunk_count(), 1); // 5/4 = 1 full chunk @@ -362,7 +380,7 @@ fn multiple_compaction_cycles() { // 8 values = 2 full chunks (8/4 = 2) for i in 0..8u8 { - tree.append(&[i]).expect("append"); + tree.append(&[i], GroveVersion::latest()).expect("append"); } assert_eq!(tree.total_count, 8); assert_eq!(tree.chunk_count(), 2); @@ -382,8 +400,8 @@ fn multiple_compaction_cycles() { #[test] fn query_chunks_empty_indices_returns_empty_proof() { let mut tree = BulkAppendTree::new(1u8, MemStorageContext::new()).expect("create tree"); - tree.append(b"a").expect("append a"); - tree.append(b"b").expect("append b"); // one completed chunk exists + tree.append(b"a", GroveVersion::latest()).expect("append a"); + tree.append(b"b", GroveVersion::latest()).expect("append b"); // one completed chunk exists let result = tree.query_chunks(&[]).expect("query with empty indices"); assert!(result.chunks.is_empty()); diff --git a/grovedb-commitment-tree/benches/seeding.rs b/grovedb-commitment-tree/benches/seeding.rs index 7e22a42b0..3e6356961 100644 --- a/grovedb-commitment-tree/benches/seeding.rs +++ b/grovedb-commitment-tree/benches/seeding.rs @@ -26,6 +26,7 @@ fn main() { }; use grovedb_path::SubtreePath; use grovedb_storage::{rocksdb_storage::RocksDbStorage, Storage, StorageBatch}; + use grovedb_version::version::GroveVersion; use rand::{rngs::StdRng, Rng, SeedableRng}; let n: u64 = std::env::var("SEED_N") @@ -87,7 +88,7 @@ fn main() { // at the end, not per leaf. let t_seed = Instant::now(); let result = ct - .append_many_raw(notes) + .append_many_raw(notes, GroveVersion::latest()) .value .expect("batched commitment-tree seeding"); // Flush the MMR overlay into the storage batch now that we're done diff --git a/grovedb-commitment-tree/src/commitment_tree/mod.rs b/grovedb-commitment-tree/src/commitment_tree/mod.rs index d4282391f..9f674e583 100644 --- a/grovedb-commitment-tree/src/commitment_tree/mod.rs +++ b/grovedb-commitment-tree/src/commitment_tree/mod.rs @@ -251,31 +251,19 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { /// decrypts `out_ciphertext`, and it cannot be recomputed from the note. /// /// Call [`save`](Self::save) afterwards to persist the updated frontier. + /// The note, chunk and roots are identical under every grove version; + /// what the version selects is the hash count a compacting append + /// reports, which this method bills. pub fn append( &mut self, cmx: [u8; 32], rho: [u8; 32], cv_net: [u8; 32], ciphertext: &TransmittedNoteCiphertext, - ) -> CostResult { - self.append_with_version(cmx, rho, cv_net, ciphertext, GroveVersion::first()) - } - - /// Version-dispatched [`append`](Self::append). - /// - /// The note, chunk and roots are identical under every version; what the - /// version selects is the hash count a compacting append reports, which - /// this method bills. - pub fn append_with_version( - &mut self, - cmx: [u8; 32], - rho: [u8; 32], - cv_net: [u8; 32], - ciphertext: &TransmittedNoteCiphertext, grove_version: &GroveVersion, ) -> CostResult { let payload = serialize_ciphertext(ciphertext); - self.append_raw_with_version(cmx, rho, cv_net, &payload, grove_version) + self.append_raw(cmx, rho, cv_net, &payload, grove_version) } /// Append a note commitment and raw payload bytes to the commitment tree. @@ -304,18 +292,6 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { rho: [u8; 32], cv_net: [u8; 32], payload: &[u8], - ) -> CostResult { - self.append_raw_with_version(cmx, rho, cv_net, payload, GroveVersion::first()) - } - - /// Version-dispatched [`append_raw`](Self::append_raw). See - /// [`append_with_version`](Self::append_with_version). - pub fn append_raw_with_version( - &mut self, - cmx: [u8; 32], - rho: [u8; 32], - cv_net: [u8; 32], - payload: &[u8], grove_version: &GroveVersion, ) -> CostResult { let mut cost = OperationCost::default(); @@ -344,10 +320,7 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { item_value.extend_from_slice(&cv_net); item_value.extend_from_slice(payload); - let bulk_result = match self - .bulk_tree - .append_with_version(&item_value, grove_version) - { + let bulk_result = match self.bulk_tree.append(&item_value, grove_version) { Ok(r) => r, // codecov:ignore — requires BulkAppendTree::append to fail, which only happens on // storage faults (put/get errors) during dense tree insert or MMR compaction; @@ -451,18 +424,6 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { pub fn append_many_raw( &mut self, entries: I, - ) -> CostResult - where - I: IntoIterator, - { - self.append_many_raw_with_version(entries, GroveVersion::first()) - } - - /// Version-dispatched [`append_many_raw`](Self::append_many_raw). See - /// [`append_with_version`](Self::append_with_version). - pub fn append_many_raw_with_version( - &mut self, - entries: I, grove_version: &GroveVersion, ) -> CostResult where @@ -511,7 +472,7 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { let r = match self .bulk_tree - .append_no_state_root_with_version(&item_value, grove_version) + .append_no_state_root(&item_value, grove_version) { Ok(r) => r, // codecov:ignore — only reachable on a storage fault during the diff --git a/grovedb-commitment-tree/src/commitment_tree/tests.rs b/grovedb-commitment-tree/src/commitment_tree/tests.rs index 78b9c2489..5e8c65ebe 100644 --- a/grovedb-commitment-tree/src/commitment_tree/tests.rs +++ b/grovedb-commitment-tree/src/commitment_tree/tests.rs @@ -1,5 +1,6 @@ #[cfg(test)] mod storage_tests { + use grovedb_version::version::GroveVersion; use std::{collections::BTreeMap, marker::PhantomData}; use grovedb_bulk_append_tree::BulkAppendTree; @@ -515,6 +516,7 @@ mod storage_tests { test_rho(i as u8), test_cv_net(i as u8), &test_ciphertext(i as u8), + GroveVersion::latest(), ) .value .expect("append should succeed"); @@ -567,6 +569,7 @@ mod storage_tests { test_rho(0), test_cv_net(0), &test_ciphertext(0), + GroveVersion::latest(), ) .value .expect("append should succeed"); @@ -669,6 +672,7 @@ mod storage_tests { test_rho(i as u8), test_cv_net(i as u8), &test_ciphertext(i as u8), + GroveVersion::latest(), ) .value .expect("append should succeed"); @@ -708,6 +712,7 @@ mod storage_tests { test_rho(0), test_cv_net(0), &test_ciphertext(0), + GroveVersion::latest(), ) .value .expect("first append"); @@ -724,6 +729,7 @@ mod storage_tests { test_rho(1), test_cv_net(1), &test_ciphertext(1), + GroveVersion::latest(), ) .value .expect("second append"); @@ -753,7 +759,13 @@ mod storage_tests { .expect("open should succeed"); // Too small - let result = ct.append_raw(test_leaf(0), test_rho(0), test_cv_net(0), &[0u8; 10]); + let result = ct.append_raw( + test_leaf(0), + test_rho(0), + test_cv_net(0), + &[0u8; 10], + GroveVersion::latest(), + ); let err = result.value.expect_err("should reject wrong size"); let msg = format!("{}", err); assert!( @@ -763,7 +775,13 @@ mod storage_tests { ); // Too large - let result = ct.append_raw(test_leaf(0), test_rho(0), test_cv_net(0), &[0u8; 300]); + let result = ct.append_raw( + test_leaf(0), + test_rho(0), + test_cv_net(0), + &[0u8; 300], + GroveVersion::latest(), + ); assert!( result.value.is_err(), "should reject payload that is too large" @@ -776,6 +794,7 @@ mod storage_tests { test_rho(0), test_cv_net(0), &vec![0u8; expected_size], + GroveVersion::latest(), ); assert!(result.value.is_ok(), "correct size should succeed"); } @@ -854,6 +873,7 @@ mod storage_tests { test_rho(0), test_cv_net(0), &test_ciphertext(0), + GroveVersion::latest(), ) .value .expect("append should succeed"); @@ -896,6 +916,7 @@ mod storage_tests { test_rho(0), test_cv_net(0), &test_ciphertext(0), + GroveVersion::latest(), ) .value .expect("append 0"); @@ -905,6 +926,7 @@ mod storage_tests { test_rho(1), test_cv_net(1), &test_ciphertext(1), + GroveVersion::latest(), ) .value .expect("append 1"); @@ -948,6 +970,7 @@ mod storage_tests { test_rho(0), test_cv_net(0), &test_ciphertext(0), + GroveVersion::latest(), ) .value .expect("append 0"); @@ -979,6 +1002,7 @@ mod storage_tests { test_rho(0), test_cv_net(0), &test_ciphertext(0), + GroveVersion::latest(), ) .value .expect("append 0"); @@ -987,6 +1011,7 @@ mod storage_tests { test_rho(1), test_cv_net(1), &test_ciphertext(1), + GroveVersion::latest(), ) .value .expect("append 1"); @@ -999,6 +1024,7 @@ mod storage_tests { test_rho(2), test_cv_net(2), &test_ciphertext(2), + GroveVersion::latest(), ) .value .expect("append 2"); @@ -1007,6 +1033,7 @@ mod storage_tests { test_rho(3), test_cv_net(3), &test_ciphertext(3), + GroveVersion::latest(), ) .value .expect("append 3"); @@ -1033,6 +1060,7 @@ mod storage_tests { test_rho(0), test_cv_net(0), &test_ciphertext(0), + GroveVersion::latest(), ) .value .expect("append 0"); @@ -1067,6 +1095,7 @@ mod storage_tests { test_rho(0), test_cv_net(0), &test_ciphertext(0), + GroveVersion::latest(), ) .value .expect("append should succeed"); @@ -1105,6 +1134,7 @@ mod storage_tests { test_rho(0), test_cv_net(0), &test_ciphertext(0), + GroveVersion::latest(), ) .value .expect("append should succeed"); @@ -1133,7 +1163,13 @@ mod storage_tests { // All 0xFF is not a valid Pallas field element let payload = vec![0u8; ciphertext_payload_size::()]; - let result = ct.append_raw([0xFF; 32], test_rho(0), test_cv_net(0), &payload); + let result = ct.append_raw( + [0xFF; 32], + test_rho(0), + test_cv_net(0), + &payload, + GroveVersion::latest(), + ); assert!( result.value.is_err(), "should reject invalid cmx field element" @@ -1170,9 +1206,15 @@ mod storage_tests { CommitmentTree::<_, DashMemo>::new(TEST_CHUNK_POWER, MockDataStorageContext::new()) .expect("new should succeed"); for entry in entries { - ct.append_raw(entry.cmx, entry.rho, entry.cv_net, &entry.payload) - .value - .expect("per-leaf append_raw should succeed"); + ct.append_raw( + entry.cmx, + entry.rho, + entry.cv_net, + &entry.payload, + GroveVersion::latest(), + ) + .value + .expect("per-leaf append_raw should succeed"); } ct } @@ -1184,7 +1226,7 @@ mod storage_tests { let mut ct = CommitmentTree::<_, DashMemo>::new(TEST_CHUNK_POWER, MockDataStorageContext::new()) .expect("new should succeed"); - ct.append_many_raw(entries) + ct.append_many_raw(entries, GroveVersion::latest()) .value .expect("append_many_raw should succeed"); ct @@ -1438,7 +1480,7 @@ mod storage_tests { }, ]; let err = ct - .append_many_raw(entries) + .append_many_raw(entries, GroveVersion::latest()) .value .expect_err("invalid cmx must propagate out of the batch"); assert!( @@ -1472,7 +1514,7 @@ mod storage_tests { }, ]; let err = ct - .append_many_raw(entries) + .append_many_raw(entries, GroveVersion::latest()) .value .expect_err("bad payload size must propagate out of the batch"); assert!( @@ -1502,7 +1544,7 @@ mod storage_tests { payload: seed_payload(0), }]; let err = ct - .append_many_raw(entries) + .append_many_raw(entries, GroveVersion::latest()) .value .expect_err("bulk storage failure should surface"); assert!( @@ -1547,7 +1589,9 @@ mod storage_tests { cv_net: test_cv_net(i), payload: seed_payload(i), }); - ct.append_many_raw(notes).value.expect("warmup"); + ct.append_many_raw(notes, GroveVersion::latest()) + .value + .expect("warmup"); ct.commit_mmr().expect("commit_mmr should flush cleanly"); } @@ -1571,7 +1615,7 @@ mod storage_tests { // A single append keeps the item in the dense buffer (epoch_size = 2 for // TEST_CHUNK_POWER = 1), so we can read the raw item bytes straight back. - ct.append(cmx, rho, cv_net, &ciphertext) + ct.append(cmx, rho, cv_net, &ciphertext, GroveVersion::latest()) .value .expect("append should succeed"); @@ -1643,9 +1687,15 @@ mod storage_tests { .expect("new should succeed"); for (i, cmx) in fixed_cmx.iter().enumerate() { let idx = i as u8; - ct.append_raw(*cmx, test_rho(idx), test_cv_net(idx), &seed_payload(idx)) - .value - .expect("append_raw should succeed"); + ct.append_raw( + *cmx, + test_rho(idx), + test_cv_net(idx), + &seed_payload(idx), + GroveVersion::latest(), + ) + .value + .expect("append_raw should succeed"); } // Build a pure frontier from the SAME cmx sequence (no rho/cv_net/payload). diff --git a/grovedb/src/operations/bulk_append_tree.rs b/grovedb/src/operations/bulk_append_tree.rs index 8d5b0bf16..0947cbede 100644 --- a/grovedb/src/operations/bulk_append_tree.rs +++ b/grovedb/src/operations/bulk_append_tree.rs @@ -86,8 +86,7 @@ impl GroveDb { let result = cost_return_on_error_no_add!( cost, - tree.append_with_version(&value, grove_version) - .map_err(map_bulk_err) + tree.append(&value, grove_version).map_err(map_bulk_err) ); cost.hash_node_calls += result.hash_count; @@ -530,8 +529,7 @@ impl GroveDb { for value in values { let result = cost_return_on_error_no_add!( cost, - tree.append_with_version(value, grove_version) - .map_err(map_bulk_err) + tree.append(value, grove_version).map_err(map_bulk_err) ); cost.hash_node_calls += result.hash_count; } diff --git a/grovedb/src/operations/commitment_tree.rs b/grovedb/src/operations/commitment_tree.rs index eadfad0d2..79433e540 100644 --- a/grovedb/src/operations/commitment_tree.rs +++ b/grovedb/src/operations/commitment_tree.rs @@ -156,7 +156,7 @@ impl GroveDb { let append_result = cost_return_on_error!( &mut cost, - ct.append_raw_with_version(cmx, rho, cv_net, &payload, grove_version) + ct.append_raw(cmx, rho, cv_net, &payload, grove_version) .map(|r| r.map_err(map_ct_err)) ); @@ -534,7 +534,7 @@ impl GroveDb { for (cmx, rho, cv_net, payload) in inserts { cost_return_on_error!( &mut cost, - ct.append_raw_with_version(*cmx, *rho, *cv_net, payload, grove_version) + ct.append_raw(*cmx, *rho, *cv_net, payload, grove_version) .map(|r| r.map_err(map_ct_err)) ); } From 11768ae923e7cb714b7f924feb02bcc47bfbf4f9 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 20 Aug 2026 14:30:18 +0700 Subject: [PATCH 18/19] refactor(mmr): take grove_version directly on push, get_root and gen_proof MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Collapses the last delegating pairs. `MMR::{push, get_root, gen_proof}` now each take `&GroveVersion`; no `*_with_version` sibling remains anywhere in the workspace. The pinned bare forms were a bad default for the same reason as the CommitmentTree ones: a caller got the shipped hash accounting by omission rather than by decision. Every call site now states the version it is costing under — including the tests and benches, which pass one explicitly. Removing the bare forms broke an invariant that was being held implicitly, and a test caught it. `mmr_tree.rs` charges the eager leaf hash itself and then calls `push`; that split is version-dependent, because `push` bills its own merges from v1 on. With no bare form left, the call site was charging `hash_count_for_push` (leaf + collapses) while `push` also charged the collapses — the exact double-count this PR fixed earlier, reintroduced. The split is now explicit rather than implied by which entry point was picked: `push_call_site_hashes(leaf_count, grove_version)` returns what the caller still owes — `hash_count_for_push` under v0, just the leaf hash under v1 — so `call_site + push == 1 + merges` holds under both, and an MmrTree push costs the same either way. The test that caught the regression pins those totals. Also drops two now-meaningless assertions that the bare entry points stayed on v0; there are no bare entry points. `grovedb`'s verify walk gained the version it needed: `compute_non_merk_child_hash` takes `&GroveVersion`, threaded from `verify_merk_and_submerks_in_transaction`, which already had one. Full CI gate set green: clippy -D warnings, check --all-targets, the verify feature build, fmt --check, and the --all-features suite (4822 tests). Co-Authored-By: Claude Fable 5 --- grovedb-bulk-append-tree/src/tree/append.rs | 13 +- grovedb-bulk-append-tree/src/tree/fetch.rs | 12 +- .../benches/mmr_benchmark.rs | 24 ++- grovedb-merkle-mountain-range/src/cost/mod.rs | 28 ++++ grovedb-merkle-mountain-range/src/cost/v0.rs | 8 + grovedb-merkle-mountain-range/src/cost/v1.rs | 5 + grovedb-merkle-mountain-range/src/lib.rs | 1 + grovedb-merkle-mountain-range/src/mmr.rs | 47 +----- grovedb-merkle-mountain-range/src/proof.rs | 7 +- .../src/tests/test_coverage.rs | 110 ++++++------ .../src/tests/test_helper.rs | 5 +- .../src/tests/test_incremental.rs | 26 ++- .../src/tests/test_mmr.rs | 158 +++++++++++++----- .../src/tests/test_storage_adapter.rs | 22 ++- grovedb/src/lib.rs | 4 +- grovedb/src/operations/mmr_tree.rs | 43 +++-- .../proof/bind_terminal_non_merk_tree/v1.rs | 2 +- grovedb/src/tests/mmr_tree_tests.rs | 4 +- 18 files changed, 331 insertions(+), 188 deletions(-) diff --git a/grovedb-bulk-append-tree/src/tree/append.rs b/grovedb-bulk-append-tree/src/tree/append.rs index 1a057cd97..89819e2a1 100644 --- a/grovedb-bulk-append-tree/src/tree/append.rs +++ b/grovedb-bulk-append-tree/src/tree/append.rs @@ -369,9 +369,7 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { let mut mmr = MMR::new_with_overlay(mmr_size, &mmr_store, std::mem::take(&mut self.mmr_overlay)); - let push_result = mmr - .push_with_version(leaf, grove_version) - .unwrap_add_cost(&mut cost); + let push_result = mmr.push(leaf, grove_version).unwrap_add_cost(&mut cost); if let Err(e) = push_result { // Restore overlay before returning error self.mmr_overlay = mmr.batch.take_overlay(); @@ -379,9 +377,7 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { .wrap_with_cost(cost); } - let root_result = mmr - .get_root_with_version(grove_version) - .unwrap_add_cost(&mut cost); + let root_result = mmr.get_root(grove_version).unwrap_add_cost(&mut cost); let root = match root_result { Ok(node) => node.hash(), Err(e) => { @@ -440,10 +436,7 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { } let mmr_store = MmrStore::with_key_size(&self.dense_tree.storage, MmrKeySize::U32); let mmr = MMR::new_with_overlay(mmr_size, &mmr_store, self.mmr_overlay.clone()); - match mmr - .get_root_with_version(grove_version) - .unwrap_add_cost(&mut cost) - { + match mmr.get_root(grove_version).unwrap_add_cost(&mut cost) { Ok(root_node) => Ok(root_node.hash()).wrap_with_cost(cost), Err(e) => Err(BulkAppendError::MmrError(format!( "MMR get_root failed: {}", diff --git a/grovedb-bulk-append-tree/src/tree/fetch.rs b/grovedb-bulk-append-tree/src/tree/fetch.rs index 5257ee4f7..7a3baaea5 100644 --- a/grovedb-bulk-append-tree/src/tree/fetch.rs +++ b/grovedb-bulk-append-tree/src/tree/fetch.rs @@ -8,6 +8,7 @@ use grovedb_storage::StorageContext; use super::BulkAppendTree; use crate::{chunk::deserialize_chunk_blob, BulkAppendError}; +use grovedb_version::version::GroveVersion; /// Result of querying the dense tree buffer. #[derive(Debug, Clone)] @@ -178,14 +179,17 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { let mmr = MMR::new_with_overlay(mmr_size, &mmr_store, self.mmr_overlay.clone()); let positions: Vec = chunk_indices.iter().map(|&idx| leaf_to_pos(idx)).collect(); - let proof = mmr.gen_proof(positions).unwrap().map_err(|e| { - BulkAppendError::MmrError(format!("chunk MMR gen_proof failed: {}", e)) - })?; + let proof = mmr + .gen_proof(positions, GroveVersion::latest()) + .unwrap() + .map_err(|e| { + BulkAppendError::MmrError(format!("chunk MMR gen_proof failed: {}", e)) + })?; let proof_items: Vec<[u8; 32]> = proof.proof_items().iter().map(|node| node.hash()).collect(); - let root = mmr.get_root().unwrap().map_err(|e| { + let root = mmr.get_root(GroveVersion::latest()).unwrap().map_err(|e| { BulkAppendError::MmrError(format!("chunk MMR get_root failed: {}", e)) })?; diff --git a/grovedb-merkle-mountain-range/benches/mmr_benchmark.rs b/grovedb-merkle-mountain-range/benches/mmr_benchmark.rs index 75a75f947..14dd96713 100644 --- a/grovedb-merkle-mountain-range/benches/mmr_benchmark.rs +++ b/grovedb-merkle-mountain-range/benches/mmr_benchmark.rs @@ -3,6 +3,7 @@ extern crate criterion; use criterion::{BenchmarkId, Criterion}; use grovedb_merkle_mountain_range::{MMRStoreReadOps, MemStore, MmrNode, MMR}; +use grovedb_version::version::GroveVersion; use rand::seq::IndexedRandom; /// Create an MmrNode leaf from an integer (for benchmarking). @@ -14,7 +15,11 @@ fn prepare_mmr(count: u32) -> (u64, MemStore, Vec) { let store = MemStore::default(); let mut mmr = MMR::new(0, &store); let positions: Vec = (0u32..count) - .map(|i| mmr.push(leaf_from_u32(i)).unwrap().expect("push")) + .map(|i| { + mmr.push(leaf_from_u32(i), GroveVersion::latest()) + .unwrap() + .expect("push") + }) .collect(); let mmr_size = mmr.mmr_size; mmr.commit().unwrap().expect("write to store"); @@ -37,8 +42,11 @@ fn bench(c: &mut Criterion) { let mmr = MMR::new(mmr_size, &store); let mut rng = rand::rng(); b.iter(|| { - mmr.gen_proof(vec![*positions.choose(&mut rng).unwrap()]) - .unwrap() + mmr.gen_proof( + vec![*positions.choose(&mut rng).unwrap()], + GroveVersion::latest(), + ) + .unwrap() }); }); @@ -46,7 +54,10 @@ fn bench(c: &mut Criterion) { let (mmr_size, store, positions) = prepare_mmr(100_0000); let mmr = MMR::new(mmr_size, &store); let mut rng = rand::rng(); - let root = mmr.get_root().unwrap().expect("get root"); + let root = mmr + .get_root(GroveVersion::latest()) + .unwrap() + .expect("get root"); let proofs: Vec<_> = (0..10_000) .map(|_| { let pos = positions.choose(&mut rng).unwrap(); @@ -55,7 +66,10 @@ fn bench(c: &mut Criterion) { .unwrap() .expect("read") .expect("exists"); - let proof = mmr.gen_proof(vec![*pos]).unwrap().expect("gen proof"); + let proof = mmr + .gen_proof(vec![*pos], GroveVersion::latest()) + .unwrap() + .expect("gen proof"); (pos, elem, proof) }) .collect(); diff --git a/grovedb-merkle-mountain-range/src/cost/mod.rs b/grovedb-merkle-mountain-range/src/cost/mod.rs index cf818943d..483f85626 100644 --- a/grovedb-merkle-mountain-range/src/cost/mod.rs +++ b/grovedb-merkle-mountain-range/src/cost/mod.rs @@ -72,3 +72,31 @@ pub(crate) fn gen_proof_bagging_hashes( )), } } + +/// Hashes a CALLER must charge for a push, on top of what +/// [`MMR::push`](crate::MMR::push) charges itself. +/// +/// A caller that hashes the leaf before calling `push` — the ops layer does — +/// has to make up whatever `push` does not bill for the version in play: +/// +/// - v0: `push` charges no merges, so the caller owes the leaf hash AND the +/// collapses, i.e. `hash_count_for_push` +/// - v1: `push` charges the merges, so the caller owes only the leaf hash +/// +/// The invariant across both is `call_site + push == 1 + merges`, which is +/// why an MmrTree push costs the same under either version. Getting this +/// wrong in either direction double-charges or under-charges every merge. +pub fn push_call_site_hashes(leaf_count: u64, grove_version: &GroveVersion) -> Result { + match grove_version.mmr_versions.cost.push { + 0 => Ok(v0::call_site_hashes(leaf_count)), + 1 => Ok(v1::call_site_hashes(leaf_count)), + version => Err(Error::VersionError( + GroveVersionError::UnknownVersionMismatch { + method: "MMR push call-site hash charge".to_string(), + known_versions: vec![0, 1], + received: version, + } + .to_string(), + )), + } +} diff --git a/grovedb-merkle-mountain-range/src/cost/v0.rs b/grovedb-merkle-mountain-range/src/cost/v0.rs index ae0fe4e09..121f28ff8 100644 --- a/grovedb-merkle-mountain-range/src/cost/v0.rs +++ b/grovedb-merkle-mountain-range/src/cost/v0.rs @@ -3,6 +3,8 @@ //! Used by GROVE_V1..V3. These are released versions, so this is locked — //! see [`super`] for why a cost correction cannot replace it in place. +use crate::helper::hash_count_for_push; + /// Charge for the peak collapses of a push: none. pub(super) fn merge_hashes(_merges: u32) -> u32 { 0 @@ -12,3 +14,9 @@ pub(super) fn merge_hashes(_merges: u32) -> u32 { pub(super) fn bagging_hashes(_peaks: usize) -> u32 { 0 } + +/// The caller owes the leaf hash and every collapse, because `push` bills +/// none of them under this version. +pub(super) fn call_site_hashes(leaf_count: u64) -> u32 { + hash_count_for_push(leaf_count) +} diff --git a/grovedb-merkle-mountain-range/src/cost/v1.rs b/grovedb-merkle-mountain-range/src/cost/v1.rs index 93ae42aab..be7999180 100644 --- a/grovedb-merkle-mountain-range/src/cost/v1.rs +++ b/grovedb-merkle-mountain-range/src/cost/v1.rs @@ -12,3 +12,8 @@ pub(super) fn merge_hashes(merges: u32) -> u32 { pub(super) fn bagging_hashes(peaks: usize) -> u32 { peaks.saturating_sub(1) as u32 } + +/// The caller owes only the eager leaf hash; `push` bills its own merges. +pub(super) fn call_site_hashes(_leaf_count: u64) -> u32 { + 1 +} diff --git a/grovedb-merkle-mountain-range/src/lib.rs b/grovedb-merkle-mountain-range/src/lib.rs index 5f93df0a1..1ea9b28ba 100644 --- a/grovedb-merkle-mountain-range/src/lib.rs +++ b/grovedb-merkle-mountain-range/src/lib.rs @@ -21,6 +21,7 @@ #![deny(missing_docs)] mod cost; +pub use cost::push_call_site_hashes; mod error; /// MMR helper functions for position arithmetic, storage keys, and cost /// calculations. diff --git a/grovedb-merkle-mountain-range/src/mmr.rs b/grovedb-merkle-mountain-range/src/mmr.rs index de27eb755..8b47144c5 100644 --- a/grovedb-merkle-mountain-range/src/mmr.rs +++ b/grovedb-merkle-mountain-range/src/mmr.rs @@ -90,23 +90,9 @@ impl MMR { /// This may also create internal (merged) nodes. The new nodes are /// buffered until [`MMR::commit`] is called. /// - /// Charges hashes under the shipped (v0) accounting. Callers that hold a - /// [`GroveVersion`] should use [`push_with_version`](Self::push_with_version) - /// instead; this entry point exists so paths that predate the gate keep - /// their released cost by construction. - pub fn push(&mut self, elem: MmrNode) -> CostResult { - self.push_with_version(elem, GroveVersion::first()) - } - - /// Version-dispatched [`push`](Self::push). - /// - /// The MMR it builds is identical under every version; only the hash - /// charge differs — see [`crate::cost`]. - pub fn push_with_version( - &mut self, - elem: MmrNode, - grove_version: &GroveVersion, - ) -> CostResult { + /// The MMR it builds is identical under every grove version; only the + /// hash charge differs — see [`crate::cost`]. + pub fn push(&mut self, elem: MmrNode, grove_version: &GroveVersion) -> CostResult { let mut cost = OperationCost::default(); let mut merges: u32 = 0; let mut elems = vec![elem]; @@ -148,18 +134,9 @@ impl MMR { /// /// Returns [`Error::GetRootOnEmpty`] for an empty MMR. /// - /// Charges hashes under the shipped (v0) accounting; see - /// [`push`](Self::push) for why this entry point is kept. - pub fn get_root(&self) -> CostResult { - self.get_root_with_version(GroveVersion::first()) - } - - /// Version-dispatched [`get_root`](Self::get_root). The root is identical - /// under every version; only the hash charge differs. - pub fn get_root_with_version( - &self, - grove_version: &GroveVersion, - ) -> CostResult { + /// The root is identical under every grove version; only the hash charge + /// differs. + pub fn get_root(&self, grove_version: &GroveVersion) -> CostResult { let mut cost = OperationCost::default(); if self.mmr_size == 0 { return Err(Error::GetRootOnEmpty).wrap_with_cost(cost); @@ -273,15 +250,9 @@ impl MMR { /// [`Error::GenProofForInvalidLeaves`] if any position is out of range /// or the list is empty. /// - /// Charges hashes under the shipped (v0) accounting; see - /// [`push`](Self::push) for why this entry point is kept. - pub fn gen_proof(&self, pos_list: Vec) -> CostResult { - self.gen_proof_with_version(pos_list, GroveVersion::first()) - } - - /// Version-dispatched [`gen_proof`](Self::gen_proof). The proof is - /// identical under every version; only the hash charge differs. - pub fn gen_proof_with_version( + /// The proof is identical under every grove version; only the hash charge + /// differs. + pub fn gen_proof( &self, mut pos_list: Vec, grove_version: &GroveVersion, diff --git a/grovedb-merkle-mountain-range/src/proof.rs b/grovedb-merkle-mountain-range/src/proof.rs index 659d9ce5c..97a04f55e 100644 --- a/grovedb-merkle-mountain-range/src/proof.rs +++ b/grovedb-merkle-mountain-range/src/proof.rs @@ -12,6 +12,7 @@ use std::{ use bincode::{Decode, Encode}; use grovedb_costs::{CostResult, CostsExt, OperationCost}; +use grovedb_version::version::GroveVersion; use crate::{ helper::{ @@ -440,7 +441,7 @@ impl MmrTreeProof { // Generate the MerkleProof (drops zero costs from lazy store) let mmr = crate::MMR::new(mmr_size, &store); - let proof_result = mmr.gen_proof(positions).unwrap(); + let proof_result = mmr.gen_proof(positions, GroveVersion::latest()).unwrap(); // Check deferred storage errors first — if the store failed, the // error (e.g. InconsistentStore) is a symptom, not the root cause. @@ -715,7 +716,7 @@ mod tests { let store = MemStore::default(); let mut mmr = MMR::new(0, &store); for v in values { - mmr.push(MmrNode::leaf(v.to_vec())) + mmr.push(MmrNode::leaf(v.to_vec()), GroveVersion::latest()) .unwrap() .expect("push should succeed"); } @@ -727,7 +728,7 @@ mod tests { /// Get root hash from a MemStore + mmr_size. fn root_hash(store: &MemStore, mmr_size: u64) -> [u8; 32] { let mmr = MMR::new(mmr_size, store); - mmr.get_root() + mmr.get_root(GroveVersion::latest()) .unwrap() .expect("get_root should succeed") .hash() diff --git a/grovedb-merkle-mountain-range/src/tests/test_coverage.rs b/grovedb-merkle-mountain-range/src/tests/test_coverage.rs index 05e2a0e46..0604388fd 100644 --- a/grovedb-merkle-mountain-range/src/tests/test_coverage.rs +++ b/grovedb-merkle-mountain-range/src/tests/test_coverage.rs @@ -8,6 +8,7 @@ use crate::{ MMR, }; use grovedb_costs::{CostResult, CostsExt, OperationCost}; +use grovedb_version::version::GroveVersion; /// Create an MmrNode leaf from an integer. fn leaf(i: u32) -> MmrNode { @@ -59,8 +60,12 @@ fn batch_into_iterator() { let store = MemStore::default(); let mut mmr = MMR::new(0, &store); - mmr.push(leaf(10)).unwrap().expect("push"); - mmr.push(leaf(11)).unwrap().expect("push"); + mmr.push(leaf(10), GroveVersion::latest()) + .unwrap() + .expect("push"); + mmr.push(leaf(11), GroveVersion::latest()) + .unwrap() + .expect("push"); let entries: Vec<(u64, Vec)> = mmr.batch.into_iter().collect(); assert_eq!(entries.len(), 2); @@ -88,7 +93,9 @@ fn batch_commit_surfaces_store_error() { let store = FailingWriteStore; let mut mmr = MMR::new(0, &store); - mmr.push(leaf(0)).unwrap().expect("push to batch"); + mmr.push(leaf(0), GroveVersion::latest()) + .unwrap() + .expect("push to batch"); let result = mmr.commit().unwrap(); assert!(result.is_err(), "commit should surface store write error"); @@ -107,7 +114,9 @@ fn mmr_is_empty() { assert!(mmr.is_empty()); let mut mmr2 = MMR::new(0, &store); - mmr2.push(leaf(0)).unwrap().expect("push"); + mmr2.push(leaf(0), GroveVersion::latest()) + .unwrap() + .expect("push"); assert!(!mmr2.is_empty()); } @@ -145,7 +154,7 @@ impl MMRStoreWriteOps for &ErrorStore { fn get_root_single_element_missing_returns_inconsistent() { let store = EmptyStore; let mmr = MMR::new(1, &store); - let result = mmr.get_root().unwrap(); + let result = mmr.get_root(GroveVersion::latest()).unwrap(); assert_eq!(result, Err(Error::InconsistentStore)); } @@ -153,7 +162,7 @@ fn get_root_single_element_missing_returns_inconsistent() { fn get_root_single_element_store_error_propagates() { let store = ErrorStore; let mmr = MMR::new(1, &store); - let result = mmr.get_root().unwrap(); + let result = mmr.get_root(GroveVersion::latest()).unwrap(); assert!(result.is_err()); let msg = format!("{}", result.unwrap_err()); assert!(msg.contains("read error"), "error: {}", msg); @@ -163,7 +172,7 @@ fn get_root_single_element_store_error_propagates() { fn get_root_multi_peak_store_error_propagates() { let store = ErrorStore; let mmr = MMR::new(4, &store); - let result = mmr.get_root().unwrap(); + let result = mmr.get_root(GroveVersion::latest()).unwrap(); assert!(result.is_err()); } @@ -234,7 +243,7 @@ fn push_propagates_store_read_error_during_merge() { let mut mmr = MMR::new(1, &store); // Push triggers merge with element at position 0 → store read fails - let result = mmr.push(leaf(1)).unwrap(); + let result = mmr.push(leaf(1), GroveVersion::latest()).unwrap(); assert!(result.is_err()); let msg = format!("{}", result.unwrap_err()); assert!( @@ -254,7 +263,7 @@ fn push_returns_inconsistent_store_when_merge_element_missing() { // Push triggers merge with element at position 0, but EmptyStore // returns Ok(None) → InconsistentStore - let result = mmr.push(leaf(1)).unwrap(); + let result = mmr.push(leaf(1), GroveVersion::latest()).unwrap(); assert_eq!(result, Err(Error::InconsistentStore)); } @@ -265,7 +274,9 @@ fn push_returns_inconsistent_store_when_merge_element_missing() { fn batch_element_at_position_break_falls_through_to_store() { let store = MemStore::default(); let mut mmr = MMR::new(0, &store); - mmr.push(leaf(0)).unwrap().expect("push"); + mmr.push(leaf(0), GroveVersion::latest()) + .unwrap() + .expect("push"); // batch has entry (0, [leaf(0)]). Position 5 is past this range, // triggering the break and falling through to the store. let result = mmr @@ -313,10 +324,12 @@ fn get_root_peak_bagging_charge_is_versioned() { let store = MemStore::default(); let mut mmr = MMR::new(0, &store); for i in 0..leaves { - mmr.push(leaf(i)).unwrap().expect("push"); + mmr.push(leaf(i), GroveVersion::latest()) + .unwrap() + .expect("push"); } - let v0 = mmr.get_root_with_version(&GROVE_V1); + let v0 = mmr.get_root(&GROVE_V1); let v0_root = v0.value.expect("root"); assert_eq!( v0.cost.hash_node_calls, 0, @@ -324,7 +337,7 @@ fn get_root_peak_bagging_charge_is_versioned() { leaves ); - let v1 = mmr.get_root_with_version(&GROVE_V4); + let v1 = mmr.get_root(&GROVE_V4); let v1_root = v1.value.expect("root"); assert_eq!( v1.cost.hash_node_calls, expected, @@ -338,20 +351,6 @@ fn get_root_peak_bagging_charge_is_versioned() { leaves ); } - - // The un-suffixed entry point must keep the shipped accounting. - let store = MemStore::default(); - let mut mmr = MMR::new(0, &store); - for i in 0..7 { - mmr.push(leaf(i)).unwrap().expect("push"); - } - let ctx = mmr.get_root(); - ctx.value.expect("root"); - assert_eq!( - ctx.cost.hash_node_calls, 0, - "bare get_root must stay on v0, got {:?}", - ctx.cost - ); } /// `push` merges once per peak it collapses. v0 billed the sibling reads @@ -369,7 +368,7 @@ fn push_peak_collapse_charge_is_versioned() { let mut mmr_v1 = MMR::new(0, &store_v1); for (i, exp) in expected.iter().enumerate() { - let c0 = mmr_v0.push_with_version(leaf(i as u32), &GROVE_V1); + let c0 = mmr_v0.push(leaf(i as u32), &GROVE_V1); c0.value.expect("push"); assert_eq!( c0.cost.hash_node_calls, 0, @@ -377,7 +376,7 @@ fn push_peak_collapse_charge_is_versioned() { i ); - let c1 = mmr_v1.push_with_version(leaf(i as u32), &GROVE_V4); + let c1 = mmr_v1.push(leaf(i as u32), &GROVE_V4); c1.value.expect("push"); assert_eq!( c1.cost.hash_node_calls, *exp, @@ -388,22 +387,16 @@ fn push_peak_collapse_charge_is_versioned() { // Same MMR either way. assert_eq!( - mmr_v0.get_root().unwrap().expect("root"), - mmr_v1.get_root().unwrap().expect("root"), + mmr_v0 + .get_root(GroveVersion::latest()) + .unwrap() + .expect("root"), + mmr_v1 + .get_root(GroveVersion::latest()) + .unwrap() + .expect("root"), "the MMR must not depend on the cost version" ); - - // The un-suffixed entry point must keep the shipped accounting. - let store = MemStore::default(); - let mut mmr = MMR::new(0, &store); - mmr.push(leaf(0)).unwrap().expect("push"); - let ctx = mmr.push(leaf(1)); - ctx.value.expect("push"); - assert_eq!( - ctx.cost.hash_node_calls, 0, - "bare push must stay on v0, got {:?}", - ctx.cost - ); } /// `gen_proof` folds right-hand peaks through the same `bag_peaks` helper, @@ -418,14 +411,18 @@ fn gen_proof_peak_bagging_charge_is_versioned() { let mut mmr = MMR::new(0, &store); let mut positions = Vec::new(); for i in 0..7 { - positions.push(mmr.push(leaf(i)).unwrap().expect("push")); + positions.push( + mmr.push(leaf(i), GroveVersion::latest()) + .unwrap() + .expect("push"), + ); } - let v0 = mmr.gen_proof_with_version(vec![positions[0]], &GROVE_V1); + let v0 = mmr.gen_proof(vec![positions[0]], &GROVE_V1); let v0_proof = v0.value.expect("proof"); assert_eq!(v0.cost.hash_node_calls, 0, "v0 charges no bagging merges"); - let v1 = mmr.gen_proof_with_version(vec![positions[0]], &GROVE_V4); + let v1 = mmr.gen_proof(vec![positions[0]], &GROVE_V4); let v1_proof = v1.value.expect("proof"); assert_eq!( v1.cost.hash_node_calls, 1, @@ -444,9 +441,13 @@ fn gen_proof_peak_bagging_charge_is_versioned() { let mut mmr = MMR::new(0, &store); let mut positions = Vec::new(); for i in 0..4 { - positions.push(mmr.push(leaf(i)).unwrap().expect("push")); + positions.push( + mmr.push(leaf(i), GroveVersion::latest()) + .unwrap() + .expect("push"), + ); } - let ctx = mmr.gen_proof_with_version(vec![positions[0]], &GROVE_V4); + let ctx = mmr.gen_proof(vec![positions[0]], &GROVE_V4); ctx.value.expect("proof"); assert_eq!(ctx.cost.hash_node_calls, 0, "one peak means no bagging"); } @@ -460,16 +461,15 @@ fn mmr_cost_dispatch_rejects_unknown_version() { let store = MemStore::default(); let mut mmr = MMR::new(0, &store); for i in 0..3 { - mmr.push(leaf(i)).unwrap().expect("push"); + mmr.push(leaf(i), GroveVersion::latest()) + .unwrap() + .expect("push"); } let mut bad: GroveVersion = GROVE_V4.clone(); bad.mmr_versions.cost.get_root = 99; assert!( - matches!( - mmr.get_root_with_version(&bad).unwrap(), - Err(Error::VersionError(_)) - ), + matches!(mmr.get_root(&bad).unwrap(), Err(Error::VersionError(_))), "an unknown get_root charge version must error" ); @@ -477,7 +477,7 @@ fn mmr_cost_dispatch_rejects_unknown_version() { bad.mmr_versions.cost.push = 99; assert!( matches!( - mmr.push_with_version(leaf(9), &bad).unwrap(), + mmr.push(leaf(9), &bad).unwrap(), Err(Error::VersionError(_)) ), "an unknown push charge version must error" @@ -487,7 +487,7 @@ fn mmr_cost_dispatch_rejects_unknown_version() { bad.mmr_versions.cost.gen_proof = 99; assert!( matches!( - mmr.gen_proof_with_version(vec![0], &bad).unwrap(), + mmr.gen_proof(vec![0], &bad).unwrap(), Err(Error::VersionError(_)) ), "an unknown gen_proof charge version must error" diff --git a/grovedb-merkle-mountain-range/src/tests/test_helper.rs b/grovedb-merkle-mountain-range/src/tests/test_helper.rs index 0162410bf..7336b8018 100644 --- a/grovedb-merkle-mountain-range/src/tests/test_helper.rs +++ b/grovedb-merkle-mountain-range/src/tests/test_helper.rs @@ -1,3 +1,4 @@ +use grovedb_version::version::GroveVersion; use lazy_static::lazy_static; use proptest::prelude::*; @@ -19,7 +20,7 @@ lazy_static! { let store = MemStore::default(); let mut mmr = MMR::new(0, &store); (0u32..100_000) - .map(|i| mmr.push(leaf_from_u32(i)).unwrap().expect("push")) + .map(|i| mmr.push(leaf_from_u32(i), GroveVersion::latest()).unwrap().expect("push")) .collect() }; /// mmr size when 0..100_000 elem @@ -28,7 +29,7 @@ lazy_static! { let mut mmr = MMR::new(0, &store); (0u32..100_000) .map(|i| { - mmr.push(leaf_from_u32(i)).unwrap().expect("push"); + mmr.push(leaf_from_u32(i), GroveVersion::latest()).unwrap().expect("push"); mmr.mmr_size }) .collect() diff --git a/grovedb-merkle-mountain-range/src/tests/test_incremental.rs b/grovedb-merkle-mountain-range/src/tests/test_incremental.rs index 3db97b18d..52f3ebcd1 100644 --- a/grovedb-merkle-mountain-range/src/tests/test_incremental.rs +++ b/grovedb-merkle-mountain-range/src/tests/test_incremental.rs @@ -1,6 +1,7 @@ use proptest::proptest; use crate::{mem_store::MemStore, MmrNode, MMR}; +use grovedb_version::version::GroveVersion; /// Create an MmrNode leaf from an integer (for test convenience). fn leaf_from_u32(i: u32) -> MmrNode { @@ -22,7 +23,10 @@ fn test_incremental_with_params(start: u32, steps: usize, turns: usize) { let _positions: Vec = (0u32..start) .map(|_| { - let pos = mmr.push(leaf_from_u32(curr)).unwrap().expect("push"); + let pos = mmr + .push(leaf_from_u32(curr), GroveVersion::latest()) + .unwrap() + .expect("push"); curr += 1; pos }) @@ -30,12 +34,18 @@ fn test_incremental_with_params(start: u32, steps: usize, turns: usize) { mmr.commit().unwrap().expect("commit changes"); for turn in 0..turns { - let prev_root = mmr.get_root().unwrap().expect("get root"); + let prev_root = mmr + .get_root(GroveVersion::latest()) + .unwrap() + .expect("get root"); let (positions, leaves) = (0..steps).fold( (Vec::new(), Vec::new()), |(mut positions, mut leaves), _| { let leaf = leaf_from_u32(curr); - let pos = mmr.push(leaf.clone()).unwrap().expect("push"); + let pos = mmr + .push(leaf.clone(), GroveVersion::latest()) + .unwrap() + .expect("push"); curr += 1; positions.push(pos); leaves.push(leaf); @@ -43,8 +53,14 @@ fn test_incremental_with_params(start: u32, steps: usize, turns: usize) { }, ); mmr.commit().unwrap().expect("commit changes"); - let proof = mmr.gen_proof(positions).unwrap().expect("gen proof"); - let root = mmr.get_root().unwrap().expect("get root"); + let proof = mmr + .gen_proof(positions, GroveVersion::latest()) + .unwrap() + .expect("gen proof"); + let root = mmr + .get_root(GroveVersion::latest()) + .unwrap() + .expect("get root"); let result = proof .verify_incremental(root, prev_root, leaves) .expect("verify_incremental"); diff --git a/grovedb-merkle-mountain-range/src/tests/test_mmr.rs b/grovedb-merkle-mountain-range/src/tests/test_mmr.rs index a269e8a5b..3a3df26bc 100644 --- a/grovedb-merkle-mountain-range/src/tests/test_mmr.rs +++ b/grovedb-merkle-mountain-range/src/tests/test_mmr.rs @@ -1,4 +1,5 @@ use faster_hex::hex_string; +use grovedb_version::version::GroveVersion; use proptest::prelude::*; use rand::{seq::SliceRandom, RngExt}; @@ -16,15 +17,23 @@ fn test_mmr(count: u32, proof_elem: Vec) { let store = MemStore::default(); let mut mmr = MMR::new(0, &store); let positions: Vec = (0u32..count) - .map(|i| mmr.push(leaf_from_u32(i)).unwrap().expect("push")) + .map(|i| { + mmr.push(leaf_from_u32(i), GroveVersion::latest()) + .unwrap() + .expect("push") + }) .collect(); - let root = mmr.get_root().unwrap().expect("get root"); + let root = mmr + .get_root(GroveVersion::latest()) + .unwrap() + .expect("get root"); let proof = mmr .gen_proof( proof_elem .iter() .map(|elem| positions[*elem as usize]) .collect(), + GroveVersion::latest(), ) .unwrap() .expect("gen proof"); @@ -45,17 +54,27 @@ fn test_gen_new_root_from_proof(count: u32) { let store = MemStore::default(); let mut mmr = MMR::new(0, &store); let positions: Vec = (0u32..count) - .map(|i| mmr.push(leaf_from_u32(i)).unwrap().expect("push")) + .map(|i| { + mmr.push(leaf_from_u32(i), GroveVersion::latest()) + .unwrap() + .expect("push") + }) .collect(); let elem = count - 1; let pos = positions[elem as usize]; - let proof = mmr.gen_proof(vec![pos]).unwrap().expect("gen proof"); + let proof = mmr + .gen_proof(vec![pos], GroveVersion::latest()) + .unwrap() + .expect("gen proof"); let new_elem = count; let new_pos = mmr - .push(leaf_from_u32(new_elem)) + .push(leaf_from_u32(new_elem), GroveVersion::latest()) .unwrap() .expect("push new"); - let root = mmr.get_root().unwrap().expect("get root"); + let root = mmr + .get_root(GroveVersion::latest()) + .unwrap() + .expect("get root"); mmr.commit().unwrap().expect("commit changes"); let calculated_root = proof .calculate_root_with_new_leaf( @@ -73,9 +92,14 @@ fn test_mmr_root() { let store = MemStore::default(); let mut mmr = MMR::new(0, &store); (0u32..11).for_each(|i| { - mmr.push(leaf_from_u32(i)).unwrap().expect("push"); + mmr.push(leaf_from_u32(i), GroveVersion::latest()) + .unwrap() + .expect("push"); }); - let root = mmr.get_root().unwrap().expect("get root"); + let root = mmr + .get_root(GroveVersion::latest()) + .unwrap() + .expect("get root"); let hex_root = hex_string(&root.hash()); // This is the deterministic root for 11 leaves with MmrNode/blake3 assert_eq!(hex_root.len(), 64, "root hash should be 32 bytes hex"); @@ -85,7 +109,10 @@ fn test_mmr_root() { fn test_empty_mmr_root() { let store = MemStore::default(); let mmr = MMR::new(0, &store); - assert_eq!(Err(Error::GetRootOnEmpty), mmr.get_root().unwrap()); + assert_eq!( + Err(Error::GetRootOnEmpty), + mmr.get_root(GroveVersion::latest()).unwrap() + ); } #[test] @@ -165,10 +192,16 @@ fn test_invalid_proof_verification( let mut mmr = MMR::new(0, &store); let mut positions: Vec = Vec::new(); for i in 0u32..leaf_count { - let pos = mmr.push(leaf_from_u32(i)).unwrap().expect("push"); + let pos = mmr + .push(leaf_from_u32(i), GroveVersion::latest()) + .unwrap() + .expect("push"); positions.push(pos); } - let root = mmr.get_root().unwrap().expect("get root"); + let root = mmr + .get_root(GroveVersion::latest()) + .unwrap() + .expect("get root"); let entries_to_verify: Vec<(u64, MmrNode)> = positions_to_verify .iter() @@ -216,7 +249,10 @@ fn test_invalid_proof_verification( assert!(handrolled_proof_result.is_err() || !handrolled_proof_result.expect("verify")); } - match mmr.gen_proof(positions_to_verify.clone()).unwrap() { + match mmr + .gen_proof(positions_to_verify.clone(), GroveVersion::latest()) + .unwrap() + { Ok(proof) => { assert!(proof .verify(root.clone(), entries_to_verify) @@ -265,7 +301,10 @@ fn test_batch_cache_hit_returns_nonzero_cost() { // Push a leaf — it goes into MMRBatch.memory_batch let leaf = MmrNode::leaf(b"test value".to_vec()); let expected_size = leaf.serialized_size(); - let pos = mmr.push(leaf).unwrap().expect("push should succeed"); + let pos = mmr + .push(leaf, GroveVersion::latest()) + .unwrap() + .expect("push should succeed"); // Before commit, read from the batch (cache hit) let cost_result = mmr.batch.element_at_position(pos); @@ -290,7 +329,7 @@ fn test_push_cost_includes_read_costs() { let mut mmr = MMR::new(0, &store); // First push — no merging needed, no sibling reads - mmr.push(MmrNode::leaf(b"leaf0".to_vec())) + mmr.push(MmrNode::leaf(b"leaf0".to_vec()), GroveVersion::latest()) .unwrap() .expect("push should succeed"); @@ -299,10 +338,10 @@ fn test_push_cost_includes_read_costs() { let mut mmr = MMR::new(0, &store); // Push two leaves — second push triggers a merge with the first - let push0_result = mmr.push(MmrNode::leaf(b"leaf0".to_vec())); + let push0_result = mmr.push(MmrNode::leaf(b"leaf0".to_vec()), GroveVersion::latest()); let push0_cost = push0_result.cost; - let push1_result = mmr.push(MmrNode::leaf(b"leaf1".to_vec())); + let push1_result = mmr.push(MmrNode::leaf(b"leaf1".to_vec()), GroveVersion::latest()); let push1_cost = push1_result.cost; // Second push should have higher cost (reads the first leaf for merging) @@ -323,12 +362,12 @@ fn test_get_root_cost_reflects_peak_reads() { // Push 3 leaves → mmr_size=4, 2 peaks (pos 2 and pos 3) for i in 0..3u8 { - mmr.push(MmrNode::leaf(vec![i])) + mmr.push(MmrNode::leaf(vec![i]), GroveVersion::latest()) .unwrap() .expect("push should succeed"); } - let root_result = mmr.get_root(); + let root_result = mmr.get_root(GroveVersion::latest()); let root_cost = root_result.cost; // With 2 peaks, get_root reads 2 nodes → at least 2 seeks @@ -351,14 +390,20 @@ fn test_mmr_tree_proof_standard_leaf_verify_succeeds() { // Push standard leaves — leaf_hash(value) matches the stored hash for i in 0u32..5 { - mmr.push(MmrNode::leaf(i.to_le_bytes().to_vec())) - .unwrap() - .expect("push should succeed"); + mmr.push( + MmrNode::leaf(i.to_le_bytes().to_vec()), + GroveVersion::latest(), + ) + .unwrap() + .expect("push should succeed"); } mmr.commit().unwrap().expect("commit should succeed"); let mmr_size = mmr.mmr_size; - let root = mmr.get_root().unwrap().expect("get root should succeed"); + let root = mmr + .get_root(GroveVersion::latest()) + .unwrap() + .expect("get root should succeed"); let get_node = |pos: u64| -> crate::Result> { (&store) @@ -388,9 +433,14 @@ fn test_single_element_mmr_root() { let mut mmr = MMR::new(0, &store); let leaf = MmrNode::leaf(b"only leaf".to_vec()); let expected_hash = leaf.hash(); - mmr.push(leaf).unwrap().expect("push should succeed"); + mmr.push(leaf, GroveVersion::latest()) + .unwrap() + .expect("push should succeed"); - let root = mmr.get_root().unwrap().expect("get_root should succeed"); + let root = mmr + .get_root(GroveVersion::latest()) + .unwrap() + .expect("get_root should succeed"); assert_eq!( root.hash(), expected_hash, @@ -403,13 +453,13 @@ fn test_single_element_mmr_root() { fn test_gen_proof_empty_positions() { let store = MemStore::default(); let mut mmr = MMR::new(0, &store); - mmr.push(MmrNode::leaf(b"leaf".to_vec())) + mmr.push(MmrNode::leaf(b"leaf".to_vec()), GroveVersion::latest()) .unwrap() .expect("push should succeed"); assert!( matches!( - mmr.gen_proof(vec![]).unwrap(), + mmr.gen_proof(vec![], GroveVersion::latest()).unwrap(), Err(Error::GenProofForInvalidLeaves) ), "should reject empty positions" @@ -422,12 +472,14 @@ fn test_gen_proof_rejects_internal_positions() { let store = MemStore::default(); let mut mmr = MMR::new(0, &store); for i in 0u32..4 { - mmr.push(leaf_from_u32(i)).unwrap().expect("push"); + mmr.push(leaf_from_u32(i), GroveVersion::latest()) + .unwrap() + .expect("push"); } // Position 2 is an internal node (height 1, merge of pos 0 and 1) assert!( matches!( - mmr.gen_proof(vec![2]).unwrap(), + mmr.gen_proof(vec![2], GroveVersion::latest()).unwrap(), Err(Error::NodeProofsNotSupported) ), "should reject internal node positions" @@ -440,12 +492,14 @@ fn test_gen_proof_out_of_range_leaf_positions() { let store = MemStore::default(); let mut mmr = MMR::new(0, &store); for i in 0u32..4 { - mmr.push(leaf_from_u32(i)).unwrap().expect("push"); + mmr.push(leaf_from_u32(i), GroveVersion::latest()) + .unwrap() + .expect("push"); } // mmr_size = 7. Position 7 is a leaf position (height 0) but beyond range. assert!( matches!( - mmr.gen_proof(vec![7]).unwrap(), + mmr.gen_proof(vec![7], GroveVersion::latest()).unwrap(), Err(Error::GenProofForInvalidLeaves) ), "should reject leaf positions beyond MMR range" @@ -460,14 +514,21 @@ fn test_gen_proof_bags_trailing_peaks() { let mut mmr = MMR::new(0, &store); // 11 leaves → mmr_size=19, 3 peaks at positions [14, 17, 18] let positions: Vec = (0u32..11) - .map(|i| mmr.push(leaf_from_u32(i)).unwrap().expect("push")) + .map(|i| { + mmr.push(leaf_from_u32(i), GroveVersion::latest()) + .unwrap() + .expect("push") + }) .collect(); - let root = mmr.get_root().unwrap().expect("get_root"); + let root = mmr + .get_root(GroveVersion::latest()) + .unwrap() + .expect("get_root"); // Prove only leaf 0 (under peak 14). Peaks 17, 18 have no proved leaves // → bagging_track = 2 → triggers bag_peaks of the trailing peaks. let proof = mmr - .gen_proof(vec![positions[0]]) + .gen_proof(vec![positions[0]], GroveVersion::latest()) .unwrap() .expect("gen_proof should succeed"); let valid = proof @@ -496,9 +557,14 @@ fn test_verify_incremental_success() { // Build initial MMR with 4 leaves → single peak at position 6 for i in 0u32..4 { - mmr.push(leaf_from_u32(i)).unwrap().expect("push"); + mmr.push(leaf_from_u32(i), GroveVersion::latest()) + .unwrap() + .expect("push"); } - let prev_root = mmr.get_root().unwrap().expect("prev root"); + let prev_root = mmr + .get_root(GroveVersion::latest()) + .unwrap() + .expect("prev root"); let peak_node = mmr .batch .element_at_position(6) @@ -509,9 +575,14 @@ fn test_verify_incremental_success() { // Add 3 more incremental leaves let incremental_leaves: Vec = (4u32..7).map(leaf_from_u32).collect(); for leaf in &incremental_leaves { - mmr.push(leaf.clone()).unwrap().expect("push incremental"); + mmr.push(leaf.clone(), GroveVersion::latest()) + .unwrap() + .expect("push incremental"); } - let current_root = mmr.get_root().unwrap().expect("current root"); + let current_root = mmr + .get_root(GroveVersion::latest()) + .unwrap() + .expect("current root"); // Proof items = previous peak hashes (just [peak at 6]) let proof = MerkleProof::new(mmr.mmr_size, vec![peak_node]); @@ -529,7 +600,9 @@ fn test_verify_incremental_wrong_prev_root() { let mut mmr = MMR::new(0, &store); for i in 0u32..4 { - mmr.push(leaf_from_u32(i)).unwrap().expect("push"); + mmr.push(leaf_from_u32(i), GroveVersion::latest()) + .unwrap() + .expect("push"); } let peak_node = mmr .batch @@ -540,9 +613,14 @@ fn test_verify_incremental_wrong_prev_root() { let incremental_leaves: Vec = (4u32..7).map(leaf_from_u32).collect(); for leaf in &incremental_leaves { - mmr.push(leaf.clone()).unwrap().expect("push incremental"); + mmr.push(leaf.clone(), GroveVersion::latest()) + .unwrap() + .expect("push incremental"); } - let current_root = mmr.get_root().unwrap().expect("current root"); + let current_root = mmr + .get_root(GroveVersion::latest()) + .unwrap() + .expect("current root"); let proof = MerkleProof::new(mmr.mmr_size, vec![peak_node]); let wrong_prev = MmrNode::internal([0xFFu8; 32]); diff --git a/grovedb-merkle-mountain-range/src/tests/test_storage_adapter.rs b/grovedb-merkle-mountain-range/src/tests/test_storage_adapter.rs index af856bb17..fd0cda146 100644 --- a/grovedb-merkle-mountain-range/src/tests/test_storage_adapter.rs +++ b/grovedb-merkle-mountain-range/src/tests/test_storage_adapter.rs @@ -5,6 +5,7 @@ use grovedb_costs::{ CostResult, CostsExt, OperationCost, }; use grovedb_storage::StorageContext; +use grovedb_version::version::GroveVersion; use crate::{ helper::{mmr_node_key_sized, MmrKeySize}, @@ -485,18 +486,27 @@ fn mmr_store_full_mmr_roundtrip() { // Use MmrStore as the backend for a full MMR let mut mmr = MMR::new(0, &store); for i in 0u32..7 { - mmr.push(MmrNode::leaf(i.to_le_bytes().to_vec())) - .unwrap() - .expect("push should succeed"); - } - let root_before_commit = mmr.get_root().unwrap().expect("get_root should succeed"); + mmr.push( + MmrNode::leaf(i.to_le_bytes().to_vec()), + GroveVersion::latest(), + ) + .unwrap() + .expect("push should succeed"); + } + let root_before_commit = mmr + .get_root(GroveVersion::latest()) + .unwrap() + .expect("get_root should succeed"); // Commit to storage mmr.commit().unwrap().expect("commit should succeed"); // Re-open MMR from the same store and verify root let mmr2 = MMR::new(mmr.mmr_size, &store); - let root_after_reopen = mmr2.get_root().unwrap().expect("get_root should succeed"); + let root_after_reopen = mmr2 + .get_root(GroveVersion::latest()) + .unwrap() + .expect("get_root should succeed"); assert_eq!( root_before_commit.hash(), diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index 98389473a..5f06040dc 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -1972,6 +1972,7 @@ impl GroveDb { new_path_ref.clone(), transaction, merk_root_hash, + grove_version, ); let actual_value_hash = value_hash(&kv_value).unwrap(); @@ -2461,6 +2462,7 @@ impl GroveDb { subtree_path: SubtreePath<'b, B>, transaction: &Transaction, merk_root_hash: [u8; 32], + grove_version: &GroveVersion, ) -> [u8; 32] { match element { Element::CommitmentTree(total_count, chunk_power, _) => { @@ -2509,7 +2511,7 @@ impl GroveDb { .unwrap(); let store = grovedb_merkle_mountain_range::MmrStore::new(&storage_ctx); let mmr = grovedb_merkle_mountain_range::MMR::new(*mmr_size, &store); - match mmr.get_root().value { + match mmr.get_root(grove_version).value { Ok(root) => root.hash(), Err(_) => merk_root_hash, } diff --git a/grovedb/src/operations/mmr_tree.rs b/grovedb/src/operations/mmr_tree.rs index d0488e9e8..bd67cf78d 100644 --- a/grovedb/src/operations/mmr_tree.rs +++ b/grovedb/src/operations/mmr_tree.rs @@ -9,10 +9,12 @@ use std::collections::HashMap; -use grovedb_costs::{cost_return_on_error, CostResult, CostsExt, OperationCost}; +use grovedb_costs::{ + cost_return_on_error, cost_return_on_error_no_add, CostResult, CostsExt, OperationCost, +}; use grovedb_merk::element::insert::ElementInsertToStorageExtensions; use grovedb_merkle_mountain_range::{ - hash_count_for_push, mmr_size_to_leaf_count, MmrNode, MmrStore, MMR, + mmr_size_to_leaf_count, push_call_site_hashes, MmrNode, MmrStore, MMR, }; use grovedb_path::SubtreePath; use grovedb_storage::{rocksdb_storage::PrefixedRocksDbTransactionContext, Storage, StorageBatch}; @@ -84,25 +86,30 @@ impl GroveDb { let store = MmrStore::new(&storage_ctx); let leaf_count = mmr_size_to_leaf_count(mmr_size); - // `hash_count_for_push` is the eager leaf hash PLUS one per peak the - // push collapses. It pairs with the UNVERSIONED `push` below, which - // charges no merges of its own — so the merges are counted exactly - // once, here. Switching that call to `push_with_version` without - // dropping this helper would charge every merge twice. - cost.hash_node_calls += hash_count_for_push(leaf_count); + // The leaf is hashed here, before `push`. How much of the push's work + // this call site still owes depends on the version — `push` bills its + // own merges from v1 on — so the split comes from + // `push_call_site_hashes` rather than being hard-coded. Charging + // `hash_count_for_push` unconditionally would double-count every + // merge once `push` started billing them. + cost.hash_node_calls += cost_return_on_error_no_add!( + cost, + push_call_site_hashes(leaf_count, grove_version) + .map_err(|e| Error::CorruptedData(format!("MMR push cost: {}", e))) + ); let leaf = MmrNode::leaf(value); let mut mmr = MMR::new(mmr_size, &store); cost_return_on_error!( &mut cost, - mmr.push(leaf) + mmr.push(leaf, grove_version) .map_err(|e| Error::CorruptedData(format!("MMR push failed: {}", e))) ); // Get root BEFORE commit — data is still in the MMRBatch overlay let new_root = cost_return_on_error!( &mut cost, - mmr.get_root_with_version(grove_version) + mmr.get_root(grove_version) .map_err(|e| Error::CorruptedData(format!("MMR get_root failed: {}", e))) ); let new_mmr_root = new_root.hash(); @@ -233,7 +240,7 @@ impl GroveDb { let root = cost_return_on_error!( &mut cost, - mmr.get_root() + mmr.get_root(grove_version) .map_err(|e| Error::CorruptedData(format!("MMR get_root failed: {}", e))) ); @@ -450,15 +457,19 @@ impl GroveDb { // Push all values into a single MMR instance let mut mmr = MMR::new(mmr_size, &store); for value in values { - // Leaf hash + peak collapses, paired with the unversioned - // `push` below. See the note on the direct path above. + // Version-dependent split between this call site and `push`; + // see the note on the direct path above. let leaf_count = mmr_size_to_leaf_count(mmr.mmr_size); - cost.hash_node_calls += hash_count_for_push(leaf_count); + cost.hash_node_calls += cost_return_on_error_no_add!( + cost, + push_call_site_hashes(leaf_count, grove_version) + .map_err(|e| Error::CorruptedData(format!("MMR push cost: {}", e))) + ); let leaf = MmrNode::leaf(value.clone()); cost_return_on_error!( &mut cost, - mmr.push(leaf) + mmr.push(leaf, grove_version) .map_err(|e| Error::CorruptedData(format!("MMR push failed: {}", e))) ); } @@ -466,7 +477,7 @@ impl GroveDb { // Get root BEFORE commit — data is still in the MMRBatch overlay let new_root = cost_return_on_error!( &mut cost, - mmr.get_root_with_version(grove_version) + mmr.get_root(grove_version) .map_err(|e| Error::CorruptedData(format!("MMR get_root failed: {}", e))) ); let new_mmr_root = new_root.hash(); diff --git a/grovedb/src/operations/proof/bind_terminal_non_merk_tree/v1.rs b/grovedb/src/operations/proof/bind_terminal_non_merk_tree/v1.rs index 03aaff208..9507f614b 100644 --- a/grovedb/src/operations/proof/bind_terminal_non_merk_tree/v1.rs +++ b/grovedb/src/operations/proof/bind_terminal_non_merk_tree/v1.rs @@ -159,7 +159,7 @@ impl GroveDb { let mmr = grovedb_merkle_mountain_range::MMR::new(*mmr_size, &store); let root = cost_return_on_error!( &mut cost, - mmr.get_root() + mmr.get_root(grove_version) .map_err(|e| Error::CorruptedData(format!("MMR get_root failed: {}", e))) ); Ok(root.hash()).wrap_with_cost(cost) diff --git a/grovedb/src/tests/mmr_tree_tests.rs b/grovedb/src/tests/mmr_tree_tests.rs index 89be16ebb..78503319c 100644 --- a/grovedb/src/tests/mmr_tree_tests.rs +++ b/grovedb/src/tests/mmr_tree_tests.rs @@ -56,12 +56,12 @@ fn expected_mmr_root(values: &[Vec]) -> [u8; 32] { let store = MemStore::new(); let mut mmr = MMR::new(0, &store); for v in values { - mmr.push(MmrNode::leaf(v.clone())) + mmr.push(MmrNode::leaf(v.clone()), GroveVersion::latest()) .unwrap() .expect("push should succeed"); } mmr.commit().unwrap().expect("commit should succeed"); - mmr.get_root() + mmr.get_root(GroveVersion::latest()) .unwrap() .expect("root hash should succeed") .hash() From f6e76c9848ae1d871ffb325dd49ff166f47b42b8 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 20 Aug 2026 14:47:32 +0700 Subject: [PATCH 19/19] test(pds): cover the empty-at-creation guard on total_count MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `validate_private_document_store_creation` takes `total_count` to reject an element that CLAIMS entries it has no data for. That guard had no test, so this adds one — and verifies it is load-bearing rather than assuming it. With the check removed, the direct insert returns `Ok(())` and commits the element: `Element::PrivateDocumentStore` is a public variant with public fields, so `total_count` need not come from `Element::empty_private_document_store`, and one can also arrive by deserialization. The committed count would then have no backing chunks or buffer entries — the state root is derived as if empty, so the tree still verifies as intact while reads of the claimed positions fail. The test asserts both the direct and batch paths refuse it and that no element is left at the key. It mirrors the guard the generic tree insert already has ("a tree should be empty at the moment of insertion when not using batches"). Full CI gate set green (4823 tests). Co-Authored-By: Claude Fable 5 --- .../src/tests/private_document_store_tests.rs | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/grovedb/src/tests/private_document_store_tests.rs b/grovedb/src/tests/private_document_store_tests.rs index 9e93b68ed..226fd9c84 100644 --- a/grovedb/src/tests/private_document_store_tests.rs +++ b/grovedb/src/tests/private_document_store_tests.rs @@ -1641,3 +1641,75 @@ fn test_private_document_store_creation_bills_its_two_hashes() { bulk_cost ); } + +/// A store element that CLAIMS entries it has no data for must be refused at +/// creation, on every path. +/// +/// `Element::PrivateDocumentStore` is a public variant with public fields, so +/// `total_count` is not guaranteed to come from +/// `Element::empty_private_document_store` — a caller can construct one +/// directly, and one can arrive by deserialization. Accepting it would commit +/// an element whose committed count has no backing chunks or buffer entries: +/// the state root would be derived as if empty, and reads of the claimed +/// positions would fail against a tree that verifies as intact. +#[test] +fn test_private_document_store_rejects_nonempty_total_count_at_creation() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + db.insert( + EMPTY_PATH, + b"root", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert root tree"); + + let claims_five = Element::PrivateDocumentStore(5, TEST_ENTRY_SIZE, TEST_CHUNK_POWER, None); + + // Direct insert. + let direct = db + .insert( + &[b"root"], + b"store", + claims_five.clone(), + None, + None, + grove_version, + ) + .unwrap(); + assert!( + direct.is_err(), + "direct insert must refuse a store claiming entries it has no data \ + for, got {:?}", + direct + ); + + // Batch insert. + let batch = vec![QualifiedGroveDbOp::insert_or_replace_op( + vec![b"root".to_vec()], + b"store".to_vec(), + claims_five, + )]; + let batched = db.apply_batch(batch, None, None, grove_version).unwrap(); + assert!( + batched.is_err(), + "batch insert must refuse it too, got {:?}", + batched + ); + + // Nothing was created either way. + assert!( + db.get_raw( + [b"root".as_slice()].as_ref().into(), + b"store", + None, + grove_version + ) + .unwrap() + .is_err(), + "no element should exist at the key" + ); +}