feat: add PrivateDocumentStore element type, gated to GROVE_V4

Cargo.toml

@@ -17,6 +17,7 @@ members = [
     "grovedb-bulk-append-tree",
     "grovedb-dense-fixed-sized-merkle-tree",
     "grovedb-query",
+    "grovedb-private-document-store",
 ]
 
 [workspace.dependencies]

grovedb-bulk-append-tree/Cargo.toml

@@ -18,6 +18,9 @@ storage = [
     "grovedb-dense-fixed-sized-merkle-tree/storage",
 ]
 mem_store = ["grovedb-merkle-mountain-range/mem_store"]
+# Exposes `test_utils::MemStorageContext` to other crates' test builds so the
+# in-memory StorageContext harness lives in exactly one place.
+test-utils = ["storage"]
 
 [dependencies]
 grovedb-merkle-mountain-range = { version = "5.0.1", path = "../grovedb-merkle-mountain-range", default-features = false }

grovedb-bulk-append-tree/src/lib.rs

@@ -13,8 +13,8 @@ mod error;
 pub mod proof;
 mod tree;
 
-#[cfg(all(test, feature = "storage"))]
-pub(crate) mod test_utils;
+#[cfg(all(feature = "storage", any(test, feature = "test-utils")))]
+pub mod test_utils;
 
 // Re-export main types
 pub use chunk::{deserialize_chunk_blob, serialize_chunk_blob};

grovedb-bulk-append-tree/src/test_utils.rs

@@ -14,7 +14,7 @@ use grovedb_storage::{Batch, RawIterator, StorageContext};
 /// (data storage) have real implementations; all other `StorageContext`
 /// methods panic if called.
 #[derive(Default)]
-pub(crate) struct MemStorageContext {
+pub struct MemStorageContext {
     pub data: RefCell<HashMap<Vec<u8>, Vec<u8>>>,
 }
 
@@ -141,7 +141,7 @@ impl<'db> StorageContext<'db> for MemStorageContext {
 // ── Batch and RawIterator stubs ───────────────────────────────────────
 
 /// No-op batch (never used — MemStorageContext does immediate writes).
-pub(crate) struct MemBatch;
+pub struct MemBatch;
 
 impl Batch for MemBatch {
     fn put<K: AsRef<[u8]>>(
@@ -186,7 +186,7 @@ impl Batch for MemBatch {
 }
 
 /// Stub iterator (never used by the bulk append tree).
-pub(crate) struct MemRawIterator;
+pub struct MemRawIterator;
 
 impl RawIterator for MemRawIterator {
     fn seek_to_first(&mut self) -> CostContext<()> {

grovedb-bulk-append-tree/src/tree/append.rs

@@ -1,5 +1,6 @@
 //! Append and compaction logic for BulkAppendTree.
 
+use grovedb_costs::{CostResult, CostsExt, OperationCost};
 use grovedb_merkle_mountain_range::{
     hash_count_for_push, mmr_size_to_leaf_count, MmrKeySize, MmrNode, MmrStore, MMR,
 };
@@ -128,6 +129,61 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree<S> {
         })
     }
 
+    /// Append a value deferring **both** the dense-tree root and the
+    /// state root.
+    ///
+    /// Storage effect is identical to [`append`](Self::append), but the
+    /// per-insert `compute_root_hash` walk over the dense buffer is skipped.
+    /// [`append_no_state_root`](Self::append_no_state_root) still pays that
+    /// walk on every call (via `try_insert`), which makes a run of N appends
+    /// O(N^2) in hash calls — 65,535 entries at `height = 16` costs ~4.3
+    /// billion. This variant is O(N) plus one final root computation.
+    ///
+    /// The caller MUST recover the state root once at the end via
+    /// [`compute_current_state_root`](Self::compute_current_state_root);
+    /// until then the dense root is stale in-memory only (it is always
+    /// recomputed from stored values, never cached).
+    ///
+    /// Compaction still happens inline when the buffer fills, because the
+    /// chunk blob is built from the stored values, not from the root.
+    pub fn append_deferred_roots(
+        &mut self,
+        value: &[u8],
+    ) -> Result<AppendNoStateRootResult, BulkAppendError> {
+        let mut hash_count: u32 = 0;
+        let global_position = self.total_count;
+
+        let try_result = self
+            .dense_tree
+            .try_insert_no_root(value)
+            .unwrap()
+            .map_err(|e| {
+                BulkAppendError::StorageError(format!("dense tree insert failed: {}", e))
+            })?;
+
+        let compacted = match try_result {
+            // Inserted into the buffer; no root walk, so no hashes yet.
+            Some(_position) => false,
+            None => {
+                // Buffer full — compact existing entries plus this value.
+                // Must run before incrementing total_count so self.mmr_size()
+                // reflects the pre-compaction state.
+                let (compact_hashes, mmr_root) = self.compact_with_value(value)?;
+                hash_count += compact_hashes;
+                self.last_mmr_root = Some(mmr_root);
+                true
+            }
+        };
+
+        self.total_count += 1;
+
+        Ok(AppendNoStateRootResult {
+            global_position,
+            hash_count,
+            compacted,
+        })
+    }
+
     /// Compute the current state root without modifying the tree.
     ///
     /// Uses the cached MMR root when available, so this is O(1) on the
@@ -145,6 +201,42 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree<S> {
         Ok(compute_state_root(&mmr_root, &dense_root))
     }
 
+    /// Cost-propagating variant of
+    /// [`compute_current_state_root`](Self::compute_current_state_root).
+    ///
+    /// Identical result; the difference is that the dense-tree root walk's
+    /// storage reads reach the caller instead of being discarded, and the
+    /// hash calls it performs (two per filled buffer position, plus the
+    /// state-root blake3) are charged. Callers that bill work — anything
+    /// returning a `CostResult` — should prefer this.
+    pub fn compute_current_state_root_with_cost(&self) -> CostResult<[u8; 32], BulkAppendError> {
+        let mut cost = OperationCost::default();
+        let mmr_root = match self.last_mmr_root {
+            Some(r) => r,
+            None => match self.get_mmr_root() {
+                Ok(r) => r,
+                Err(e) => return Err(e).wrap_with_cost(cost),
+            },
+        };
+        let dense_root = match self.dense_tree.root_hash().unwrap_add_cost(&mut cost) {
+            Ok(r) => r,
+            Err(e) => {
+                return Err(BulkAppendError::StorageError(format!(
+                    "dense tree root_hash failed: {}",
+                    e
+                )))
+                .wrap_with_cost(cost);
+            }
+        };
+        // The root walk hashes every filled position twice, then one blake3
+        // combines the MMR and dense roots into the state root.
+        cost.hash_node_calls = cost
+            .hash_node_calls
+            .saturating_add(self.dense_tree.count() as u32 * 2)
+            .saturating_add(1);
+        Ok(compute_state_root(&mmr_root, &dense_root)).wrap_with_cost(cost)
+    }
+
     /// Compact all dense tree entries plus a new value into a chunk blob
     /// and append to the chunk MMR. Resets the dense tree.
     /// Returns `(hash_count, mmr_root)`.

grovedb-bulk-append-tree/src/tree/fetch.rs

@@ -1,5 +1,6 @@
 //! Read operations for BulkAppendTree.
 
+use grovedb_costs::{CostResult, CostsExt, OperationCost};
 use grovedb_dense_fixed_sized_merkle_tree::DenseTreeProof;
 use grovedb_merkle_mountain_range::{leaf_to_pos, MmrKeySize, MmrStore, MMR};
 use grovedb_query::Query;
@@ -38,12 +39,34 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree<S> {
     /// from completed chunks. The position is relative to the current buffer
     /// cycle (0-based).
     pub fn get_buffer_value(&self, position: u16) -> Result<Option<Vec<u8>>, BulkAppendError> {
+        self.get_buffer_value_with_cost(position).unwrap()
+    }
+
+    /// Cost-propagating variant of
+    /// [`get_buffer_value`](Self::get_buffer_value).
+    ///
+    /// Identical behavior; the difference is that the dense-tree read's
+    /// `OperationCost` (seek count and loaded bytes) reaches the caller
+    /// instead of being discarded. Callers that bill reads — anything
+    /// returning a `CostResult` to GroveDB — must use this variant, or the
+    /// read is served for free.
+    pub fn get_buffer_value_with_cost(
+        &self,
+        position: u16,
+    ) -> CostResult<Option<Vec<u8>>, BulkAppendError> {
+        let mut cost = OperationCost::default();
         if position >= self.buffer_count() {
-            return Ok(None);
+            return Ok(None).wrap_with_cost(cost);
         }
-        self.dense_tree.get(position).unwrap().map_err(|e| {
-            BulkAppendError::StorageError(format!("dense tree get at {} failed: {}", position, e))
-        })
+        let result = self.dense_tree.get(position).unwrap_add_cost(&mut cost);
+        result
+            .map_err(|e| {
+                BulkAppendError::StorageError(format!(
+                    "dense tree get at {} failed: {}",
+                    position, e
+                ))
+            })
+            .wrap_with_cost(cost)
     }
 
     /// Query the buffer using a dense tree query.
@@ -73,28 +96,45 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree<S> {
     /// Uses the MMR overlay to find nodes that were pushed during this session
     /// but not yet committed to storage.
     pub fn get_chunk_value(&self, chunk_index: u64) -> Result<Option<Vec<u8>>, BulkAppendError> {
+        self.get_chunk_value_with_cost(chunk_index).unwrap()
+    }
+
+    /// Cost-propagating variant of
+    /// [`get_chunk_value`](Self::get_chunk_value). See
+    /// [`get_buffer_value_with_cost`](Self::get_buffer_value_with_cost) for
+    /// why the cost-bearing form exists.
+    pub fn get_chunk_value_with_cost(
+        &self,
+        chunk_index: u64,
+    ) -> CostResult<Option<Vec<u8>>, BulkAppendError> {
+        let mut cost = OperationCost::default();
         if chunk_index >= self.chunk_count() {
-            return Ok(None);
+            return Ok(None).wrap_with_cost(cost);
         }
         let mmr_pos = leaf_to_pos(chunk_index);
         let mmr_store = MmrStore::with_key_size(&self.dense_tree.storage, MmrKeySize::U32);
         let mmr = MMR::new_with_overlay(self.mmr_size(), &mmr_store, self.mmr_overlay.clone());
-        let node = mmr
+        let node = match mmr
             .batch
             .element_at_position(mmr_pos)
-            .unwrap()
-            .map_err(|e| {
-                BulkAppendError::MmrError(format!(
+            .unwrap_add_cost(&mut cost)
+        {
+            Ok(n) => n,
+            Err(e) => {
+                return Err(BulkAppendError::MmrError(format!(
                     "failed to read MMR node for chunk {}: {}",
                     chunk_index, e
-                ))
-            })?;
+                )))
+                .wrap_with_cost(cost);
+            }
+        };
         match node {
-            Some(n) => Ok(n.into_value()),
+            Some(n) => Ok(n.into_value()).wrap_with_cost(cost),
             None => Err(BulkAppendError::CorruptedData(format!(
                 "missing MMR leaf for chunk {}",
                 chunk_index
-            ))),
+            )))
+            .wrap_with_cost(cost),
         }
     }
 

grovedb-dense-fixed-sized-merkle-tree/src/tree.rs

@@ -187,6 +187,35 @@ impl<'db, S: StorageContext<'db>> DenseFixedSizedMerkleTree<S> {
         }
     }
 
+    /// Insert a value **without** recomputing the root hash.
+    ///
+    /// Same storage effect as [`try_insert`](Self::try_insert) — the value is
+    /// written at the next free position and `count` is incremented — but the
+    /// O(count) `compute_root_hash` walk is skipped. Returns the position, or
+    /// `None` when the tree is already full.
+    ///
+    /// Use this when inserting a run of values and only the FINAL root
+    /// matters: calling [`try_insert`](Self::try_insert) in a loop is
+    /// O(n^2) in hash calls, because every insert re-walks every filled
+    /// position. Recover the root once at the end with
+    /// [`root_hash`](Self::root_hash).
+    pub fn try_insert_no_root(
+        &mut self,
+        value: &[u8],
+    ) -> CostResult<Option<u16>, DenseMerkleError> {
+        let mut cost = OperationCost::default();
+
+        if self.count >= self.capacity() {
+            return Ok(None).wrap_with_cost(cost);
+        }
+
+        let position = self.count;
+        cost_return_on_error!(cost, self.put_value(position, value));
+        self.count += 1;
+
+        Ok(Some(position)).wrap_with_cost(cost)
+    }
+
     /// Get a value by position.
     ///
     /// Returns `None` if position >= count. Returns an error if position <

grovedb-element/src/element/constructor.rs

@@ -500,6 +500,70 @@ impl Element {
         Element::DenseAppendOnlyFixedSizeTree(count, height, flags)
     }
 
+    /// Set element to an empty private document store.
+    ///
+    /// Returns `InvalidInput` unless `entry_size >= 1` and `chunk_power` is
+    /// in `1..=16` (the underlying `BulkAppendTree` dense-buffer height
+    /// range). Unlike `empty_commitment_tree` / `empty_bulk_append_tree`,
+    /// the constraints are enforced eagerly here: the configuration is
+    /// committed into the state root, so an unusable config must never be
+    /// constructible.
+    pub fn empty_private_document_store(
+        entry_size: u32,
+        chunk_power: u8,
+    ) -> Result<Self, ElementError> {
+        Self::empty_private_document_store_with_flags(entry_size, chunk_power, None)
+    }
+
+    /// Set element to an empty private document store with flags.
+    ///
+    /// Same validation as [`Element::empty_private_document_store`].
+    pub fn empty_private_document_store_with_flags(
+        entry_size: u32,
+        chunk_power: u8,
+        flags: Option<ElementFlags>,
+    ) -> Result<Self, ElementError> {
+        if entry_size == 0 || entry_size > u16::MAX as u32 {
+            // Upper bound keeps `2^16 * entry_size` (the worst-case
+            // compaction blob) inside the u32 `added_bytes` field, so the
+            // worst-case storage estimate stays a real bound. An entry
+            // larger than 64 KiB is outside this type's design envelope
+            // anyway.
+            return Err(ElementError::InvalidInput(
+                "private document store entry_size must be in 1..=65535",
+            ));
+        }
+        if !(1..=16).contains(&chunk_power) {
+            return Err(ElementError::InvalidInput(
+                "private document store chunk_power must be between 1 and 16",
+            ));
+        }
+        Ok(Element::PrivateDocumentStore(
+            0,
+            entry_size,
+            chunk_power,
+            flags,
+        ))
+    }
+
+    /// Set element to a private document store with all fields.
+    ///
+    /// Restoration constructor: unchecked, mirroring `new_commitment_tree` /
+    /// `new_bulk_append_tree` — it rebuilds an element from already-validated
+    /// state (stored bytes, batch metadata). Invalid configurations are
+    /// rejected at every real ingress: the `empty_*` constructors, the direct
+    /// and batch insert paths, and both (de)serialization codecs
+    /// (`Element::serialize` / `Element::deserialize` / serde) via
+    /// [`Element::validate_private_document_store_config`].
+    pub fn new_private_document_store(
+        total_count: u64,
+        entry_size: u32,
+        chunk_power: u8,
+        flags: Option<ElementFlags>,
+    ) -> Self {
+        Element::PrivateDocumentStore(total_count, entry_size, chunk_power, flags)
+    }
+
     /// Set element to an empty provable sum-indexed tree without flags.
     pub fn empty_provable_sum_indexed_tree() -> Self {
         Element::ProvableSumIndexedTree(None, None, 0, None)