From 56926538240df3217be163ac2d709d3c0fd00f5d Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Mon, 3 Aug 2026 07:05:30 +0700 Subject: [PATCH 1/9] feat(bulk-append-tree): chunk-aligned paginated range reads with proof helpers Add the position-range read seam at the shared BulkAppendTree layer so every BulkAppendTree-backed element type (CommitmentTree today, the planned PrivateDocumentStore / DataCommitmentTree) inherits it: - BulkAppendTree::get_range(start, limit): fetch entries for [start, start + limit) clamped to total_count, returned as a RangePage (entries + total_count). Chunk-aligned: each completed chunk overlapping the range is read and deserialized exactly once, so a page costs O(chunks touched) blob reads instead of O(entries) random reads. - position_range_query(start, limit): the canonical Query (8-byte big-endian position keys) shared by prover and verifier. - BulkAppendTreeProof::generate_for_range / verify_range: paginated proof round-trip over the existing chunk-MMR + dense-buffer proof, with completeness enforced; absence past the end falls out of the authenticated total_count (position >= count), not per-position absence proofs. - CommitmentTree::get_range pass-through for the shielded-pool scanning path. Part of work-plan item 4 of dashpay/grovedb#784. Co-Authored-By: Claude Fable 5 --- grovedb-bulk-append-tree/src/lib.rs | 4 +- grovedb-bulk-append-tree/src/proof/mod.rs | 67 +++++++ grovedb-bulk-append-tree/src/proof/tests.rs | 166 ++++++++++++++++++ grovedb-bulk-append-tree/src/tree/fetch.rs | 71 +++++++- grovedb-bulk-append-tree/src/tree/mod.rs | 15 ++ grovedb-bulk-append-tree/src/tree/tests.rs | 139 +++++++++++++++ .../src/commitment_tree/mod.rs | 16 +- grovedb-commitment-tree/src/lib.rs | 2 +- 8 files changed, 475 insertions(+), 5 deletions(-) diff --git a/grovedb-bulk-append-tree/src/lib.rs b/grovedb-bulk-append-tree/src/lib.rs index f46436b85..36a18f7b1 100644 --- a/grovedb-bulk-append-tree/src/lib.rs +++ b/grovedb-bulk-append-tree/src/lib.rs @@ -22,7 +22,7 @@ pub use error::BulkAppendError; pub use grovedb_dense_fixed_sized_merkle_tree::{DenseFixedSizedMerkleTree, DenseTreeProof}; #[cfg(feature = "storage")] pub use grovedb_merkle_mountain_range::{MmrKeySize, MmrStore}; -pub use proof::{BulkAppendTreeProof, BulkAppendTreeProofResult}; -pub use tree::{hash::compute_state_root, leaf_count_to_mmr_size, BulkAppendTree}; +pub use proof::{position_range_query, BulkAppendTreeProof, BulkAppendTreeProofResult}; +pub use tree::{hash::compute_state_root, leaf_count_to_mmr_size, BulkAppendTree, RangePage}; #[cfg(feature = "storage")] pub use tree::{AppendResult, BufferQueryResult, ChunkQueryResult}; diff --git a/grovedb-bulk-append-tree/src/proof/mod.rs b/grovedb-bulk-append-tree/src/proof/mod.rs index a8dd6186d..4dbcab404 100644 --- a/grovedb-bulk-append-tree/src/proof/mod.rs +++ b/grovedb-bulk-append-tree/src/proof/mod.rs @@ -176,6 +176,25 @@ fn query_to_ranges(query: &Query, total_count: u64) -> Result, B Ok(merged) } +/// Build the canonical [`Query`] selecting the position range +/// `[start, start + limit)`, with positions encoded as 8-byte big-endian +/// keys. +/// +/// This is the query shape used by the paginated-scan pattern: prover and +/// verifier both derive it from `(start, limit)`, so a client only needs its +/// cursor and page size. `start + limit` saturates at `u64::MAX`, and +/// verification clamps the range to the tree's provable total count. +pub fn position_range_query(start: u64, limit: u16) -> Query { + let end = start.saturating_add(limit as u64); + Query { + items: vec![QueryItem::Range( + start.to_be_bytes().to_vec()..end.to_be_bytes().to_vec(), + )], + left_to_right: true, + ..Query::default() + } +} + /// Check whether `pos` falls inside any of the sorted, non-overlapping ranges. fn in_ranges(pos: u64, ranges: &[(u64, u64)]) -> bool { ranges @@ -325,6 +344,54 @@ impl BulkAppendTreeProof { }) } + /// Generate a proof for the paginated position range + /// `[start, start + limit)`. + /// + /// Convenience wrapper over [`generate`](Self::generate) using the + /// canonical [`position_range_query`]. The proof is chunk-aligned: it + /// carries each completed chunk blob overlapping the range plus the + /// buffer entries in range, so proof size is O(chunks touched). + /// + /// Ranges past the end of the tree are valid and produce a proof of the + /// (empty) result: absence of positions `>= total_count` falls out of + /// the authenticated element's total count, not out of per-position + /// absence proofs. + #[cfg(feature = "storage")] + pub fn generate_for_range<'db, S: StorageContext<'db>>( + tree: &BulkAppendTree, + start: u64, + limit: u16, + ) -> Result { + Self::generate(&position_range_query(start, limit), tree) + } + + /// Verify this proof against the paginated position range + /// `[start, start + limit)`. + /// + /// Convenience wrapper over + /// [`verify_against_query`](Self::verify_against_query) using the + /// canonical [`position_range_query`]. Returns the `(global_position, + /// value)` pairs in the range, ascending and contiguous, clamped to + /// `total_count`. Completeness is enforced: a proof missing any + /// requested position below `total_count` is rejected. Positions + /// `>= total_count` are provably absent by `total_count` itself, which + /// callers must take from the authenticated BulkAppendTree element. + pub fn verify_range( + &self, + expected_state_root: &[u8; 32], + height: u8, + total_count: u64, + start: u64, + limit: u16, + ) -> Result)>, BulkAppendError> { + self.verify_against_query( + expected_state_root, + height, + total_count, + &position_range_query(start, limit), + ) + } + /// Verify this proof against an expected state root. /// /// `height` and `total_count` come from the authenticated BulkAppendTree diff --git a/grovedb-bulk-append-tree/src/proof/tests.rs b/grovedb-bulk-append-tree/src/proof/tests.rs index 83b6f30d3..ec6a77e44 100644 --- a/grovedb-bulk-append-tree/src/proof/tests.rs +++ b/grovedb-bulk-append-tree/src/proof/tests.rs @@ -940,4 +940,170 @@ mod proof_tests { ); } } + + // ── generate_for_range / verify_range (paginated scan pattern) ─────── + + /// Helper: build a tree of `n` values "val_0".."val_{n-1}" and return + /// (state_root, tree). + fn build_indexed_tree(height: u8, n: u32) -> ([u8; 32], BulkAppendTree) { + let values: Vec> = (0..n).map(|i| format!("val_{}", i).into_bytes()).collect(); + build_test_tree(height, &values) + } + + /// Helper: round-trip a range proof and assert the returned page is + /// exactly positions `expected_start..expected_end`. + fn assert_range_roundtrip( + state_root: &[u8; 32], + tree: &BulkAppendTree, + start: u64, + limit: u16, + expected_start: u64, + expected_end: u64, + ) { + let proof = + BulkAppendTreeProof::generate_for_range(tree, start, limit).expect("generate range"); + + // Wire round-trip: encode + decode like a real client + let bytes = proof.encode_to_vec().expect("encode"); + let decoded = BulkAppendTreeProof::decode_from_slice(&bytes).expect("decode"); + + let entries = decoded + .verify_range(state_root, tree.height(), tree.total_count, start, limit) + .expect("verify range"); + + assert_eq!(entries.len(), (expected_end - expected_start) as usize); + for (i, (pos, value)) in entries.iter().enumerate() { + assert_eq!(*pos, expected_start + i as u64); + assert_eq!(value, format!("val_{}", pos).as_bytes()); + } + } + + #[test] + fn test_range_roundtrip_buffer_only() { + // height=3, capacity=7: 5 values all in buffer + let (root, tree) = build_indexed_tree(3, 5); + assert_range_roundtrip(&root, &tree, 1, 3, 1, 4); + } + + #[test] + fn test_range_roundtrip_across_chunk_boundary() { + // height=2, epoch_size=4: 10 values = 2 chunks + 2 buffered + let (root, tree) = build_indexed_tree(2, 10); + // spans chunk 0 / chunk 1 + assert_range_roundtrip(&root, &tree, 3, 3, 3, 6); + // spans chunk 1 / buffer + assert_range_roundtrip(&root, &tree, 6, 4, 6, 10); + } + + #[test] + fn test_range_roundtrip_single_entry_pages() { + let (root, tree) = build_indexed_tree(2, 10); + for pos in 0..10u64 { + assert_range_roundtrip(&root, &tree, pos, 1, pos, pos + 1); + } + } + + #[test] + fn test_range_roundtrip_empty_range() { + let (root, tree) = build_indexed_tree(2, 10); + // limit 0: proof still verifies against the root, returns nothing + assert_range_roundtrip(&root, &tree, 3, 0, 3, 3); + } + + #[test] + fn test_range_roundtrip_past_end() { + let (root, tree) = build_indexed_tree(2, 10); + // starts exactly at total_count + assert_range_roundtrip(&root, &tree, 10, 5, 10, 10); + // starts far past total_count + assert_range_roundtrip(&root, &tree, 1000, 5, 1000, 1000); + // clamped at the end + assert_range_roundtrip(&root, &tree, 8, 100, 8, 10); + } + + #[test] + fn test_range_roundtrip_large_multi_chunk_page() { + // height=4, epoch_size=16: 100 values = 6 chunks + 4 buffered. + // One page covering everything touches all chunks and the buffer. + let (root, tree) = build_indexed_tree(4, 100); + assert_range_roundtrip(&root, &tree, 0, 100, 0, 100); + // A large page crossing several chunk boundaries mid-tree + assert_range_roundtrip(&root, &tree, 10, 70, 10, 80); + } + + #[test] + fn test_range_roundtrip_empty_tree() { + let (_, tree) = build_indexed_tree(2, 0); + // For an empty tree the state root is blake3("bulk_state" || 0*32 || 0*32) + let root = crate::compute_state_root(&[0u8; 32], &[0u8; 32]); + assert_range_roundtrip(&root, &tree, 0, 10, 0, 0); + } + + #[test] + fn test_range_paged_scan_covers_everything() { + // The client scan pattern: page through the whole tree with + // limit=7 (deliberately not aligned to epoch_size=4). + let (root, tree) = build_indexed_tree(2, 30); + let mut cursor = 0u64; + let mut seen = Vec::new(); + while cursor < tree.total_count { + let proof = + BulkAppendTreeProof::generate_for_range(&tree, cursor, 7).expect("generate page"); + let entries = proof + .verify_range(&root, tree.height(), tree.total_count, cursor, 7) + .expect("verify page"); + assert!(!entries.is_empty()); + cursor += entries.len() as u64; + seen.extend(entries); + } + assert_eq!(seen.len(), 30); + for (i, (pos, value)) in seen.iter().enumerate() { + assert_eq!(*pos, i as u64); + assert_eq!(value, format!("val_{}", i).as_bytes()); + } + } + + #[test] + fn test_range_proof_wrong_root_rejected() { + let (root, tree) = build_indexed_tree(2, 10); + let proof = BulkAppendTreeProof::generate_for_range(&tree, 0, 5).expect("generate"); + let mut bad_root = root; + bad_root[0] ^= 1; + proof + .verify_range(&bad_root, tree.height(), tree.total_count, 0, 5) + .expect_err("tampered root must be rejected"); + } + + #[test] + fn test_range_proof_missing_chunk_rejected() { + // Proof generated for [0, 2) (chunk 0 only) must not verify a + // request for [0, 6) which also needs chunk 1. + let (root, tree) = build_indexed_tree(2, 10); + let narrow = BulkAppendTreeProof::generate_for_range(&tree, 0, 2).expect("generate"); + narrow + .verify_range(&root, tree.height(), tree.total_count, 0, 6) + .expect_err("proof missing chunk 1 must be rejected for the wider range"); + } + + #[test] + fn test_position_range_query_shape() { + let q = super::super::position_range_query(5, 3); + assert_eq!(q.items.len(), 1); + match &q.items[0] { + QueryItem::Range(r) => { + assert_eq!(r.start, 5u64.to_be_bytes().to_vec()); + assert_eq!(r.end, 8u64.to_be_bytes().to_vec()); + } + other => panic!("expected Range item, got {:?}", other), + } + + // start + limit saturates instead of wrapping + let q = super::super::position_range_query(u64::MAX - 1, 100); + match &q.items[0] { + QueryItem::Range(r) => { + assert_eq!(r.end, u64::MAX.to_be_bytes().to_vec()); + } + other => panic!("expected Range item, got {:?}", other), + } + } } diff --git a/grovedb-bulk-append-tree/src/tree/fetch.rs b/grovedb-bulk-append-tree/src/tree/fetch.rs index 3ad8bed63..4d9ea342d 100644 --- a/grovedb-bulk-append-tree/src/tree/fetch.rs +++ b/grovedb-bulk-append-tree/src/tree/fetch.rs @@ -5,7 +5,7 @@ use grovedb_merkle_mountain_range::{leaf_to_pos, MmrKeySize, MmrStore, MMR}; use grovedb_query::Query; use grovedb_storage::StorageContext; -use super::BulkAppendTree; +use super::{BulkAppendTree, RangePage}; use crate::{chunk::deserialize_chunk_blob, BulkAppendError}; /// Result of querying the dense tree buffer. @@ -63,6 +63,75 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { Ok(BufferQueryResult { entries, proof }) } + // ── Range operations (chunks + buffer) ─────────────────────────── + + /// Fetch entries for the position range `[start, start + limit)`, + /// clamped to the tree's total count. + /// + /// This is the paginated-scan read path: clients walking "all entries + /// since my cursor" call it with their cursor as `start` and advance by + /// `entries.len()`. The read is chunk-aligned — each completed chunk + /// overlapping the range is read and deserialized exactly once, so a + /// page costs O(chunks touched) blob reads plus one read per buffer + /// entry, not O(entries) random reads. + /// + /// Absence needs no lookup: positions `>= total_count` do not exist, so + /// a page shorter than `limit` means the end of the tree was reached. + pub fn get_range(&self, start: u64, limit: u16) -> Result { + let total_count = self.total_count; + let end = start.saturating_add(limit as u64).min(total_count); + if start >= end { + return Ok(RangePage { + entries: Vec::new(), + total_count, + }); + } + let mut entries = Vec::with_capacity((end - start) as usize); + + let epoch_size = self.epoch_size(); + let buffer_start = self.chunk_count() * epoch_size; + + // Completed chunks overlapping [start, min(end, buffer_start)) + let chunk_end = end.min(buffer_start); + if start < chunk_end { + let first_chunk = start / epoch_size; + let last_chunk = (chunk_end - 1) / epoch_size; + for chunk_idx in first_chunk..=last_chunk { + let blob = self.get_chunk_value(chunk_idx)?.ok_or_else(|| { + BulkAppendError::CorruptedData(format!( + "missing chunk blob for index {}", + chunk_idx + )) + })?; + let chunk_entries = deserialize_chunk_blob(&blob)?; + let chunk_start = chunk_idx * epoch_size; + for (i, value) in chunk_entries.into_iter().enumerate() { + let pos = chunk_start + i as u64; + if pos >= start && pos < chunk_end { + entries.push((pos, value)); + } + } + } + } + + // Buffer tail: positions in [max(start, buffer_start), end) + for pos in start.max(buffer_start)..end { + let buffer_pos = (pos - buffer_start) as u16; + let value = self.get_buffer_value(buffer_pos)?.ok_or_else(|| { + BulkAppendError::CorruptedData(format!( + "missing buffer value at position {}", + buffer_pos + )) + })?; + entries.push((pos, value)); + } + + Ok(RangePage { + entries, + total_count, + }) + } + // ── Chunk operations (MMR) ─────────────────────────────────────── /// Get a single completed chunk's raw blob by chunk index. diff --git a/grovedb-bulk-append-tree/src/tree/mod.rs b/grovedb-bulk-append-tree/src/tree/mod.rs index bcea9a521..e41f2e6b8 100644 --- a/grovedb-bulk-append-tree/src/tree/mod.rs +++ b/grovedb-bulk-append-tree/src/tree/mod.rs @@ -57,6 +57,21 @@ pub struct AppendNoStateRootResult { pub compacted: bool, } +/// A contiguous page of entries returned by a position-range read. +/// +/// Produced by [`BulkAppendTree::get_range`], which fetches the entries for +/// `[start, start + limit)` clamped to the tree's total count. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RangePage { + /// `(global_position, value)` pairs, ascending and contiguous from the + /// requested start position (clamped to `total_count`). + pub entries: Vec<(u64, Vec)>, + /// Total number of entries in the tree at read time. Positions + /// `>= total_count` do not exist, so a page that ends before + /// `start + limit` is complete — there is nothing further to fetch. + pub total_count: u64, +} + /// Compute MMR size from leaf count: `2 * n - popcount(n)`. /// /// This is a well-known MMR property: the total number of nodes (leaves + diff --git a/grovedb-bulk-append-tree/src/tree/tests.rs b/grovedb-bulk-append-tree/src/tree/tests.rs index 3c3f2e490..e5987e8d2 100644 --- a/grovedb-bulk-append-tree/src/tree/tests.rs +++ b/grovedb-bulk-append-tree/src/tree/tests.rs @@ -390,3 +390,142 @@ fn query_chunks_empty_indices_returns_empty_proof() { assert!(result.mmr_proof_items.is_empty()); assert_eq!(result.mmr_root, [0u8; 32]); } + +// ── get_range (paginated position-range reads) ─────────────────────── + +/// Helper: build a tree with `n` single-byte values `[0], [1], ...`. +fn build_range_tree(height: u8, n: u8) -> BulkAppendTree { + let mut tree = BulkAppendTree::new(height, MemStorageContext::new()).expect("create tree"); + for i in 0..n { + tree.append(&[i]).expect("append"); + } + tree +} + +/// Helper: assert a page holds exactly positions `start..end` with value +/// `[pos as u8]` at each. +fn assert_page(page: &super::RangePage, start: u64, end: u64, total_count: u64) { + assert_eq!(page.total_count, total_count); + assert_eq!(page.entries.len(), (end - start) as usize); + for (i, (pos, value)) in page.entries.iter().enumerate() { + assert_eq!(*pos, start + i as u64); + assert_eq!(value, &vec![*pos as u8]); + } +} + +#[test] +fn get_range_buffer_only() { + // height=3, capacity=7: 5 values all in buffer + let tree = build_range_tree(3, 5); + assert_eq!(tree.chunk_count(), 0); + + let page = tree.get_range(1, 3).expect("get range"); + assert_page(&page, 1, 4, 5); +} + +#[test] +fn get_range_single_chunk() { + // height=2, epoch_size=4: 8 values = 2 full chunks + let tree = build_range_tree(2, 8); + assert_eq!(tree.chunk_count(), 2); + assert_eq!(tree.buffer_count(), 0); + + // Page entirely inside chunk 0 + let page = tree.get_range(1, 2).expect("get range"); + assert_page(&page, 1, 3, 8); +} + +#[test] +fn get_range_across_chunk_boundary() { + // height=2, epoch_size=4: 10 values = 2 chunks + 2 buffered + let tree = build_range_tree(2, 10); + assert_eq!(tree.chunk_count(), 2); + assert_eq!(tree.buffer_count(), 2); + + // Page [3, 6) spans the chunk 0 / chunk 1 boundary + let page = tree.get_range(3, 3).expect("get range"); + assert_page(&page, 3, 6, 10); + + // Page [6, 10) spans the chunk 1 / buffer boundary + let page = tree.get_range(6, 4).expect("get range"); + assert_page(&page, 6, 10, 10); +} + +#[test] +fn get_range_whole_tree() { + let tree = build_range_tree(2, 10); + let page = tree.get_range(0, 100).expect("get range"); + assert_page(&page, 0, 10, 10); +} + +#[test] +fn get_range_empty_limit() { + let tree = build_range_tree(2, 10); + let page = tree.get_range(3, 0).expect("get range"); + assert_page(&page, 3, 3, 10); +} + +#[test] +fn get_range_past_end() { + let tree = build_range_tree(2, 10); + + // Start exactly at total_count + let page = tree.get_range(10, 5).expect("get range"); + assert!(page.entries.is_empty()); + assert_eq!(page.total_count, 10); + + // Start far past total_count + let page = tree.get_range(1000, 5).expect("get range"); + assert!(page.entries.is_empty()); + assert_eq!(page.total_count, 10); + + // Range that starts inside but extends past the end is clamped + let page = tree.get_range(8, 100).expect("get range"); + assert_page(&page, 8, 10, 10); +} + +#[test] +fn get_range_single_entry() { + let tree = build_range_tree(2, 10); + let page = tree.get_range(7, 1).expect("get range"); + assert_page(&page, 7, 8, 10); +} + +#[test] +fn get_range_empty_tree() { + let tree = build_range_tree(2, 0); + let page = tree.get_range(0, 10).expect("get range"); + assert!(page.entries.is_empty()); + assert_eq!(page.total_count, 0); +} + +#[test] +fn get_range_start_saturating_overflow() { + let tree = build_range_tree(2, 10); + let page = tree.get_range(u64::MAX, u16::MAX).expect("get range"); + assert!(page.entries.is_empty()); + assert_eq!(page.total_count, 10); +} + +#[test] +fn get_range_paged_scan_covers_everything() { + // The scanning pattern: walk the whole tree in pages of 3 and check the + // concatenation matches per-position reads. + let tree = build_range_tree(2, 11); // 2 chunks + 3 buffered + let mut cursor = 0u64; + let mut seen = Vec::new(); + loop { + let page = tree.get_range(cursor, 3).expect("get range"); + if page.entries.is_empty() { + assert!(cursor >= page.total_count, "empty page only at the end"); + break; + } + cursor += page.entries.len() as u64; + seen.extend(page.entries); + } + assert_eq!(seen.len(), 11); + for (i, (pos, value)) in seen.iter().enumerate() { + assert_eq!(*pos, i as u64); + assert_eq!(value, &vec![i as u8]); + } +} diff --git a/grovedb-commitment-tree/src/commitment_tree/mod.rs b/grovedb-commitment-tree/src/commitment_tree/mod.rs index 8649211d0..4521207d4 100644 --- a/grovedb-commitment-tree/src/commitment_tree/mod.rs +++ b/grovedb-commitment-tree/src/commitment_tree/mod.rs @@ -11,7 +11,7 @@ use std::marker::PhantomData; -use grovedb_bulk_append_tree::BulkAppendTree; +use grovedb_bulk_append_tree::{BulkAppendTree, RangePage}; use grovedb_costs::{CostResult, CostsExt, OperationCost}; use grovedb_storage::StorageContext; use orchard::{ @@ -636,6 +636,20 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { .map_err(|e| CommitmentTreeError::InvalidData(format!("chunk value: {}", e))) } + /// Fetch entries for the position range `[start, start + limit)`, + /// clamped to the tree's total count. + /// + /// This is the shielded-pool scanning read path: each returned value is + /// the raw `cmx || rho || cv_net || payload` bytes at that position. + /// Delegates to [`BulkAppendTree::get_range`], so the read is + /// chunk-aligned — O(chunks touched) blob reads, not O(entries) random + /// reads. + pub fn get_range(&self, start: u64, limit: u16) -> Result { + self.bulk_tree + .get_range(start, limit) + .map_err(|e| CommitmentTreeError::InvalidData(format!("range read: {}", e))) + } + /// The number of entries per completed chunk (epoch). pub fn epoch_size(&self) -> u64 { self.bulk_tree.epoch_size() diff --git a/grovedb-commitment-tree/src/lib.rs b/grovedb-commitment-tree/src/lib.rs index 2792576b2..f94285c9c 100644 --- a/grovedb-commitment-tree/src/lib.rs +++ b/grovedb-commitment-tree/src/lib.rs @@ -71,7 +71,7 @@ pub use commitment_tree::{ pub use error::CommitmentTreeError; #[cfg(feature = "server")] pub use grovedb_bulk_append_tree::{ - deserialize_chunk_blob, serialize_chunk_blob, BulkAppendError, BulkAppendTree, + deserialize_chunk_blob, serialize_chunk_blob, BulkAppendError, BulkAppendTree, RangePage, }; pub use grovedb_costs::{self}; pub use incrementalmerkletree::{Hashable, Level, Position, Retention}; From b4734315004c91f42013e535ecdb93fda431f7d0 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Mon, 3 Aug 2026 07:09:48 +0700 Subject: [PATCH 2/9] fix(grovedb): chain empty BulkAppendTree V1 lower layers through NULL_HASH A V1 proof descending into an empty BulkAppendTree could never verify: insert commits the element with a NULL_HASH child hash (there is no bulk state until the first append) and verify_grovedb's integrity walk mirrors that, but verify_bulk_append_lower_layer returned the domain-tagged empty state root blake3("bulk_state" || 0*32 || 0*32), so the combine_hash chain check always failed. Return NULL_HASH for a zero-count BulkAppendTree element instead, matching what the writer commits. This is sound because verify_and_compute_root has already rejected any proof carrying chunk or buffer data for a zero-count tree. CommitmentTree keeps the computed value: its insert commits EMPTY_COMMITMENT_TREE_STATE_ROOT, which folds in the computed empty bulk root. Verification-only change on an element type not reachable in any shipped grove version's state; no wire format or state root changes, so no GROVE_V* gate. Co-Authored-By: Claude Fable 5 --- grovedb/src/operations/proof/verify.rs | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/grovedb/src/operations/proof/verify.rs b/grovedb/src/operations/proof/verify.rs index ab3db34cd..a20f12d19 100644 --- a/grovedb/src/operations/proof/verify.rs +++ b/grovedb/src/operations/proof/verify.rs @@ -1645,11 +1645,27 @@ impl GroveDb { .verify_and_compute_root(element_height, element_total_count) .map_err(|e| Error::InvalidProof(query.clone(), format!("{}", e)))?; + // An empty `BulkAppendTree` element chains through NULL_HASH, not the + // domain-tagged empty state root: insert commits the element with a + // NULL_HASH child hash (there is no bulk state until the first + // append), and `verify_grovedb`'s integrity walk mirrors that. This + // is sound because `verify_and_compute_root` above already rejected + // any proof carrying data for a zero-count tree. `CommitmentTree` is + // different — its insert commits `EMPTY_COMMITMENT_TREE_STATE_ROOT`, + // which folds in the *computed* empty bulk root, so the CT wrapper + // (our caller) keeps the computed value. + let child_hash = + if element_total_count == 0 && matches!(element, Element::BulkAppendTree(..)) { + NULL_HASH + } else { + bulk_state_root + }; + // Root only: the caller is binding the parent element and does not // report this layer's entries, so there is no query at this path to // extract a position range from. if !report_contents { - return Ok(bulk_state_root); + return Ok(child_hash); } // Get the query range from the path query to extract matching values @@ -1722,8 +1738,8 @@ impl GroveDb { } } - // Return computed state_root as child Merk hash - Ok(bulk_state_root) + // Return the derived child Merk hash (see the empty-tree note above) + Ok(child_hash) } /// Verify a CommitmentTree lower layer proof and add results. From 9ae0377ed42fd5674e4f8eae3aa07fd145d6b562 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Mon, 3 Aug 2026 07:10:10 +0700 Subject: [PATCH 3/9] feat(grovedb): paginated position-range reads and proofs for append-only trees MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire the BulkAppendTree range-read seam through GroveDB, per work-plan item 4 of dashpay/grovedb#784 — clients of append-only stores walk "all entries since my cursor" in pages: - GroveDb::bulk_get_range / GroveDb::commitment_tree_get_range: chunk-aligned page fetch returning a RangePage (entries + total_count), O(chunks touched) instead of O(entries) random reads. - PathQuery::new_bulk_position_range: the canonical query shape for a page — element key plus an 8-byte big-endian position-range subquery — derived identically by prover and verifier from (start, limit). - GroveDb::prove_bulk_position_range: proves one page via the existing V1 proof layering (ProofBytes::BulkAppendTree / CommitmentTree); no wire-format change, so no new GROVE_V* gate. - GroveDb::verify_bulk_position_range_proof: verifies the page entries (ascending, contiguous, complete) and extracts the authenticated total_count from the same proof bytes by subset-verifying the element itself. Absence beyond the end falls out of the provable count (position >= total_count) — no per-position absence proofs. The seam lives at the shared BulkAppendTree layer and dispatches on the element type, so the planned PrivateDocumentStore (#784) and DataCommitmentTree (#783) pick it up by adding their element variants to the prove/verify match arms. Tests cover round-trips across chunk boundaries, empty ranges, ranges past the end, single-entry pages, large multi-chunk pages, empty trees, nested paths, wrong-element errors, narrower-proof rejection, and the cursor-walk scan pattern for both BulkAppendTree and CommitmentTree. Co-Authored-By: Claude Fable 5 --- grovedb/src/lib.rs | 2 + grovedb/src/operations/bulk_append_tree.rs | 66 +++- grovedb/src/operations/commitment_tree.rs | 67 +++- grovedb/src/operations/proof/generate.rs | 56 +++ grovedb/src/operations/proof/verify.rs | 138 ++++++++ grovedb/src/query/mod.rs | 42 +++ grovedb/src/tests/bulk_append_tree_tests.rs | 365 ++++++++++++++++++++ grovedb/src/tests/commitment_tree_tests.rs | 241 +++++++++++++ 8 files changed, 975 insertions(+), 2 deletions(-) diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index a3f509a8a..ec3690ab1 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -180,6 +180,8 @@ pub use element::aggregate_sum_query::{AggregateSumQueryOptions, AggregateSumQue pub use element::Element; #[cfg(any(feature = "minimal", feature = "verify"))] pub use element::ElementFlags; +#[cfg(any(feature = "minimal", feature = "verify"))] +pub use grovedb_bulk_append_tree::RangePage; #[cfg(feature = "minimal")] use grovedb_costs::cost_return_on_error_into; #[cfg(feature = "minimal")] diff --git a/grovedb/src/operations/bulk_append_tree.rs b/grovedb/src/operations/bulk_append_tree.rs index 50d13cccb..e87ef35fd 100644 --- a/grovedb/src/operations/bulk_append_tree.rs +++ b/grovedb/src/operations/bulk_append_tree.rs @@ -6,7 +6,7 @@ use std::collections::HashMap; -use grovedb_bulk_append_tree::{deserialize_chunk_blob, BulkAppendTree}; +use grovedb_bulk_append_tree::{deserialize_chunk_blob, BulkAppendTree, RangePage}; use grovedb_costs::{ cost_return_on_error, cost_return_on_error_into, cost_return_on_error_no_add, CostResult, CostsExt, OperationCost, @@ -250,6 +250,70 @@ impl GroveDb { } } + /// Fetch entries for the position range `[start, start + limit)` from a + /// BulkAppendTree, clamped to its total count. + /// + /// This is the paginated-scan read path: clients walking "all entries + /// since my cursor" call it with their cursor as `start` and advance by + /// `entries.len()`. The read is chunk-aligned — each completed chunk + /// overlapping the range is read and deserialized exactly once, so a + /// page costs O(chunks touched) blob reads plus one read per buffer + /// entry, not O(entries) random reads. + /// + /// The returned [`RangePage`] also carries the tree's `total_count`: + /// positions `>= total_count` do not exist, so a page shorter than + /// `limit` means the end of the tree was reached. + pub fn bulk_get_range<'b, B, P>( + &self, + path: P, + key: &[u8], + start: u64, + limit: u16, + transaction: TransactionArg, + grove_version: &GroveVersion, + ) -> CostResult + where + B: AsRef<[u8]> + 'b, + P: Into>, + { + let path: SubtreePath = path.into(); + let mut cost = OperationCost::default(); + let tx = TxRef::new(&self.db, transaction); + + let element = cost_return_on_error!( + &mut cost, + self.get_raw_caching_optional(path.clone(), key, true, transaction, grove_version) + ); + + // Look through NonCounted: a wrapped BulkAppendTree is still one. + let (total_count, chunk_power) = match element.underlying() { + Element::BulkAppendTree(tc, cp, _) => (*tc, *cp), + _ => { + return Err(Error::InvalidInput("element is not a BulkAppendTree")) + .wrap_with_cost(cost); + } + }; + + let subtree_path_vec = self.build_subtree_path_for_bulk(&path, key); + let subtree_path_refs: Vec<&[u8]> = subtree_path_vec.iter().map(|v| v.as_slice()).collect(); + let subtree_path = SubtreePath::from(subtree_path_refs.as_slice()); + + let storage_ctx = self + .db + .get_transactional_storage_context(subtree_path, None, tx.as_ref()) + .unwrap_add_cost(&mut cost); + + let tree = cost_return_on_error_no_add!( + cost, + BulkAppendTree::from_state(total_count, chunk_power, storage_ctx).map_err(map_bulk_err) + ); + + let page = + cost_return_on_error_no_add!(cost, tree.get_range(start, limit).map_err(map_bulk_err)); + + Ok(page).wrap_with_cost(cost) + } + /// Get a completed chunk blob from a BulkAppendTree. /// /// Returns the raw serialized blob (length-prefixed entries) for the given diff --git a/grovedb/src/operations/commitment_tree.rs b/grovedb/src/operations/commitment_tree.rs index b55a2f4e9..a8e08aa78 100644 --- a/grovedb/src/operations/commitment_tree.rs +++ b/grovedb/src/operations/commitment_tree.rs @@ -17,7 +17,7 @@ use std::collections::HashMap; use grovedb_commitment_tree::{ deserialize_chunk_blob, serialize_ciphertext, Anchor, CommitmentTree, DashMemo, MemoSize, - TransmittedNoteCiphertext, + RangePage, TransmittedNoteCiphertext, }; use grovedb_costs::{ cost_return_on_error, cost_return_on_error_into, cost_return_on_error_no_add, CostResult, @@ -388,6 +388,71 @@ impl GroveDb { } } + /// Fetch entries for the position range `[start, start + limit)` from a + /// CommitmentTree, clamped to its total count. + /// + /// This is the shielded-pool scanning read path: clients walk "all notes + /// since my cursor" in pages, trial-decrypting each returned + /// `cmx || rho || cv_net || payload` value. The read is chunk-aligned — + /// each completed chunk overlapping the range is read and deserialized + /// exactly once, so a page costs O(chunks touched) blob reads plus one + /// read per buffer entry, not O(entries) random reads. + /// + /// The returned [`RangePage`] also carries the tree's `total_count`: + /// positions `>= total_count` do not exist, so a page shorter than + /// `limit` means the scan caught up with the tip. + pub fn commitment_tree_get_range<'b, B, P>( + &self, + path: P, + key: &[u8], + start: u64, + limit: u16, + transaction: TransactionArg, + grove_version: &GroveVersion, + ) -> CostResult + where + B: AsRef<[u8]> + 'b, + P: Into>, + { + let path: SubtreePath = path.into(); + let mut cost = OperationCost::default(); + let tx = TxRef::new(&self.db, transaction); + + let element = cost_return_on_error!( + &mut cost, + self.get_raw_caching_optional(path.clone(), key, true, transaction, grove_version) + ); + + // Look through NonCounted: a wrapped CommitmentTree is still one. + let (total_count, chunk_power) = match element.underlying() { + Element::CommitmentTree(tc, cp, _) => (*tc, *cp), + _ => { + return Err(Error::InvalidInput("element is not a commitment tree")) + .wrap_with_cost(cost); + } + }; + + let ct_path_vec = self.build_ct_path(&path, key); + let ct_path_refs: Vec<&[u8]> = ct_path_vec.iter().map(|v| v.as_slice()).collect(); + let ct_path = SubtreePath::from(ct_path_refs.as_slice()); + + let storage_ctx = self + .db + .get_transactional_storage_context(ct_path, None, tx.as_ref()) + .unwrap_add_cost(&mut cost); + + let ct = cost_return_on_error!( + &mut cost, + CommitmentTree::<_, DashMemo>::open(total_count, chunk_power, storage_ctx) + .map(|r| r.map_err(map_ct_err)) + ); + + let page = + cost_return_on_error_no_add!(cost, ct.get_range(start, limit).map_err(map_ct_err)); + + Ok(page).wrap_with_cost(cost) + } + /// Get the total count of items in a CommitmentTree. pub fn commitment_tree_count<'b, B, P>( &self, diff --git a/grovedb/src/operations/proof/generate.rs b/grovedb/src/operations/proof/generate.rs index b095ee0f5..6f982564d 100644 --- a/grovedb/src/operations/proof/generate.rs +++ b/grovedb/src/operations/proof/generate.rs @@ -205,6 +205,62 @@ impl GroveDb { } } + /// Prove the paginated position range `[start, start + limit)` of the + /// append-only, BulkAppendTree-backed element at `path`/`key` + /// (`Element::BulkAppendTree` or `Element::CommitmentTree`). + /// + /// This is the scanning hot path: clients walking "all entries since my + /// cursor" prove one page per call, passing their cursor as `start`. The + /// proof reuses the existing V1 layering (`ProofBytes::BulkAppendTree` / + /// `ProofBytes::CommitmentTree`) over the canonical + /// [`PathQuery::new_bulk_position_range`] query, so it is chunk-aligned: + /// it carries each completed chunk blob overlapping the range plus the + /// in-range buffer entries — O(chunks touched), not O(entries). It also + /// binds the element itself, whose authenticated `total_count` makes + /// absence beyond the end provable (`position >= total_count`), so + /// ranges past the end need no per-position absence proofs. + /// + /// Verify with [`GroveDb::verify_bulk_position_range_proof`], which + /// derives the same canonical query from `(path, key, start, limit)`. + pub fn prove_bulk_position_range( + &self, + path: Vec>, + key: &[u8], + start: u64, + limit: u16, + prove_options: Option, + grove_version: &GroveVersion, + ) -> CostResult, Error> { + let mut cost = OperationCost::default(); + + // Fail fast with a clear error when the target is not an append-only + // store — a generic proof for some other element type would only be + // rejected later, at verification time. + let path_refs: Vec<&[u8]> = path.iter().map(|p| p.as_slice()).collect(); + let subtree_path = grovedb_path::SubtreePath::from(path_refs.as_slice()); + let element = cost_return_on_error!( + &mut cost, + self.get_raw_caching_optional(subtree_path, key, true, None, grove_version) + ); + match element.underlying() { + Element::BulkAppendTree(..) | Element::CommitmentTree(..) => {} + _ => { + return Err(Error::InvalidInput( + "prove_bulk_position_range requires a BulkAppendTree or CommitmentTree \ + element", + )) + .wrap_with_cost(cost); + } + } + + let path_query = PathQuery::new_bulk_position_range(path, key.to_vec(), start, limit); + let proof = cost_return_on_error!( + &mut cost, + self.prove_query(&path_query, prove_options, grove_version) + ); + Ok(proof).wrap_with_cost(cost) + } + /// Helper for the top-level count-offset gate in /// `prove_query_non_serialized_v{0,1}`. Opens the merk at /// `path_query.path` and confirms its `tree_type` is one of the diff --git a/grovedb/src/operations/proof/verify.rs b/grovedb/src/operations/proof/verify.rs index a20f12d19..bf0a562d3 100644 --- a/grovedb/src/operations/proof/verify.rs +++ b/grovedb/src/operations/proof/verify.rs @@ -2596,6 +2596,144 @@ impl GroveDb { ) } + /// Verify a proof produced by [`GroveDb::prove_bulk_position_range`], + /// returning `(root_hash, page)` for the position range + /// `[start, start + limit)` of the append-only, BulkAppendTree-backed + /// element at `path`/`key` (`Element::BulkAppendTree` or + /// `Element::CommitmentTree`). + /// + /// The page's entries are the `(position, value)` pairs, ascending and + /// contiguous from `start`, clamped to the element's authenticated + /// `total_count` (returned in + /// [`RangePage::total_count`](grovedb_bulk_append_tree::RangePage::total_count)). + /// Completeness is enforced: a proof missing any requested position below + /// `total_count` is rejected. Absence beyond the end falls out of the + /// provable count — positions `>= total_count` do not exist — so a page + /// shorter than `limit` means the scan caught up with the tip; no + /// per-position absence proofs are involved. + /// + /// Both the entries and `total_count` are extracted from the same proof + /// bytes: the entries by verifying the canonical + /// [`PathQuery::new_bulk_position_range`] query, and `total_count` by + /// subset-verifying the element itself, whose serialized bytes are bound + /// to the parent Merk (and through it to `root_hash`). + pub fn verify_bulk_position_range_proof( + proof: &[u8], + path: Vec>, + key: &[u8], + start: u64, + limit: u16, + grove_version: &GroveVersion, + ) -> Result<(CryptoHash, grovedb_bulk_append_tree::RangePage), Error> { + // 1. Verify the range entries against the canonical range query. + // Succinctness cannot be required: range proofs are chunk-aligned + // and intentionally carry whole chunk blobs — a superset of the + // queried positions. + let range_query = + PathQuery::new_bulk_position_range(path.clone(), key.to_vec(), start, limit); + let (root_hash, results) = Self::verify_query_with_options( + proof, + &range_query, + VerifyOptions { + absence_proofs_for_non_existing_searched_keys: false, + verify_proof_succinctness: false, + include_empty_trees_in_result: false, + }, + grove_version, + )?; + + // 2. Extract the element's authenticated total_count from the same + // proof bytes via a single-key subset query on the element itself. + let element_query = PathQuery::new_single_key(path, key.to_vec()); + let (element_root_hash, element_results) = + Self::verify_subset_query(proof, &element_query, grove_version)?; + if element_root_hash != root_hash { + return Err(Error::InvalidProof( + range_query, + "range and element sub-proofs derived different root hashes".to_string(), + )); + } + let total_count = match element_results.as_slice() { + [(_, element_key, Some(element))] if element_key.as_slice() == key => { + match element.underlying() { + Element::BulkAppendTree(total_count, _, _) + | Element::CommitmentTree(total_count, _, _) => *total_count, + _ => { + return Err(Error::InvalidProof( + range_query, + "element at path/key is not a BulkAppendTree or CommitmentTree" + .to_string(), + )); + } + } + } + _ => { + return Err(Error::InvalidProof( + range_query, + "proof does not bind the append-only element at path/key".to_string(), + )); + } + }; + + // 3. Map the results to (position, value) entries and require the + // page to be exactly [start, min(start + limit, total_count)) — + // ascending, contiguous, complete. The lower-layer verification + // already enforces completeness; this re-check keeps the helper's + // guarantee independent of that layer's internals. + let end = start + .saturating_add(limit as u64) + .min(total_count) + .max(start); + let mut entries = Vec::with_capacity((end - start) as usize); + let mut expected_position = start; + for (_, position_key, element) in results { + let position_bytes: [u8; 8] = position_key.as_slice().try_into().map_err(|_| { + Error::InvalidProof( + range_query.clone(), + "range entry key is not an 8-byte big-endian position".to_string(), + ) + })?; + let position = u64::from_be_bytes(position_bytes); + let value = match element { + Some(Element::Item(value, _)) => value, + _ => { + return Err(Error::InvalidProof( + range_query, + format!("range entry at position {} is not an item", position), + )); + } + }; + if position != expected_position { + return Err(Error::InvalidProof( + range_query, + format!( + "range entries not contiguous: expected position {}, got {}", + expected_position, position + ), + )); + } + expected_position += 1; + entries.push((position, value)); + } + if expected_position != end { + return Err(Error::InvalidProof( + range_query, + format!( + "range proof incomplete: expected positions [{}, {}), got up to {}", + start, end, expected_position + ), + )); + } + + Ok(( + root_hash, + grovedb_bulk_append_tree::RangePage { + entries, + total_count, + }, + )) + } + /// The point of this query is to get the parent tree information which will /// be present because we are querying in a subtree pub fn verify_query_get_parent_tree_info( diff --git a/grovedb/src/query/mod.rs b/grovedb/src/query/mod.rs index 7d102b581..2a1dee7fa 100644 --- a/grovedb/src/query/mod.rs +++ b/grovedb/src/query/mod.rs @@ -526,6 +526,48 @@ impl PathQuery { Self { path, query } } + /// Canonical `PathQuery` for a paginated position-range read of an + /// append-only, BulkAppendTree-backed element (`BulkAppendTree` or + /// `CommitmentTree`) at `path`/`key`. + /// + /// Selects the element at `key` and subqueries positions + /// `[start, start + limit)`, encoded as 8-byte big-endian keys + /// (`start + limit` saturates at `u64::MAX`). Prover and verifier both + /// derive this query from `(start, limit)`, so a scanning client only + /// needs its cursor and page size — see + /// [`GroveDb::prove_bulk_position_range`] and + /// [`GroveDb::verify_bulk_position_range_proof`]. + /// + /// [`GroveDb::prove_bulk_position_range`]: + /// crate::GroveDb::prove_bulk_position_range + /// [`GroveDb::verify_bulk_position_range_proof`]: + /// crate::GroveDb::verify_bulk_position_range_proof + pub fn new_bulk_position_range( + path: Vec>, + key: Vec, + start: u64, + limit: u16, + ) -> Self { + let position_query = grovedb_bulk_append_tree::position_range_query(start, limit); + Self { + path, + query: SizedQuery { + query: Query { + items: vec![QueryItem::Key(key)], + default_subquery_branch: SubqueryBranch { + subquery_path: None, + subquery: Some(position_query.into()), + }, + left_to_right: true, + conditional_subquery_branches: None, + add_parent_tree_on_subquery: false, + }, + limit: None, + offset: None, + }, + } + } + /// Construct a `PathQuery` for an aggregate-count-on-range query against /// the subtree at `path`. `range` is the inner `QueryItem` describing the /// keys to count over; see [`Query::new_aggregate_count_on_range`] for the diff --git a/grovedb/src/tests/bulk_append_tree_tests.rs b/grovedb/src/tests/bulk_append_tree_tests.rs index 7028e8e0d..2e7cfcd38 100644 --- a/grovedb/src/tests/bulk_append_tree_tests.rs +++ b/grovedb/src/tests/bulk_append_tree_tests.rs @@ -2125,3 +2125,368 @@ fn test_bulk_batch_multi_compaction_transaction_rollback() { "bulk tree should be empty after multi-compaction tx rollback" ); } + +// =========================================================================== +// Paginated position-range reads (get_range) and range proofs +// =========================================================================== + +/// Helper: create a DB with a BulkAppendTree at root key `b"bulk"` holding +/// `n` values `b"value_0"`, `b"value_1"`, ... (chunk_power = 2 → chunk size +/// 4). +fn make_bulk_db_with_values(n: u32) -> crate::tests::TempGroveDb { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + db.insert( + EMPTY_PATH, + b"bulk", + Element::empty_bulk_append_tree(TEST_CHUNK_POWER).expect("valid chunk_power"), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert bulk append tree"); + + for i in 0..n { + db.bulk_append( + EMPTY_PATH, + b"bulk", + format!("value_{}", i).into_bytes(), + None, + grove_version, + ) + .unwrap() + .expect("bulk append"); + } + + db +} + +/// Helper: assert a page holds exactly positions `start..end` with the +/// values written by [`make_bulk_db_with_values`]. +fn assert_bulk_page(page: &crate::RangePage, start: u64, end: u64, total_count: u64) { + assert_eq!(page.total_count, total_count, "page total_count"); + assert_eq!(page.entries.len(), (end - start) as usize, "page length"); + for (i, (pos, value)) in page.entries.iter().enumerate() { + assert_eq!(*pos, start + i as u64, "page position"); + assert_eq!(value, format!("value_{}", pos).as_bytes(), "page value"); + } +} + +#[test] +fn test_bulk_get_range_matches_get_value() { + let grove_version = GroveVersion::latest(); + // 11 values = 2 full chunks (8) + 3 buffered + let db = make_bulk_db_with_values(11); + + // Every possible page of size 4 must match per-position reads + for start in 0..12u64 { + let page = db + .bulk_get_range(EMPTY_PATH, b"bulk", start, 4, None, grove_version) + .unwrap() + .expect("bulk get range"); + assert_eq!(page.total_count, 11); + + let end = (start + 4).min(11); + let expected_len = end.saturating_sub(start.min(end)); + assert_eq!(page.entries.len(), expected_len as usize); + + for (pos, value) in &page.entries { + let expected = db + .bulk_get_value(EMPTY_PATH, b"bulk", *pos, None, grove_version) + .unwrap() + .expect("bulk get value") + .expect("value exists"); + assert_eq!(value, &expected); + } + } +} + +#[test] +fn test_bulk_get_range_empty_and_past_end() { + let grove_version = GroveVersion::latest(); + let db = make_bulk_db_with_values(10); + + // limit = 0 + let page = db + .bulk_get_range(EMPTY_PATH, b"bulk", 3, 0, None, grove_version) + .unwrap() + .expect("bulk get range"); + assert_bulk_page(&page, 3, 3, 10); + + // start exactly at total_count + let page = db + .bulk_get_range(EMPTY_PATH, b"bulk", 10, 5, None, grove_version) + .unwrap() + .expect("bulk get range"); + assert_bulk_page(&page, 10, 10, 10); + + // start far past total_count + let page = db + .bulk_get_range(EMPTY_PATH, b"bulk", 1000, 5, None, grove_version) + .unwrap() + .expect("bulk get range"); + assert!(page.entries.is_empty()); + assert_eq!(page.total_count, 10); + + // clamped at the end + let page = db + .bulk_get_range(EMPTY_PATH, b"bulk", 8, 100, None, grove_version) + .unwrap() + .expect("bulk get range"); + assert_bulk_page(&page, 8, 10, 10); +} + +#[test] +fn test_bulk_get_range_wrong_element_type() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + db.insert( + EMPTY_PATH, + b"normal", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert normal tree"); + + let result = db + .bulk_get_range(EMPTY_PATH, b"normal", 0, 4, None, grove_version) + .unwrap(); + assert!(matches!(result, Err(Error::InvalidInput(_)))); +} + +/// Helper: prove and verify a position range page, asserting the returned +/// root hash matches the database root hash, and return the page. +fn roundtrip_bulk_range_proof( + db: &crate::tests::TempGroveDb, + start: u64, + limit: u16, +) -> crate::RangePage { + let grove_version = GroveVersion::latest(); + + let proof = db + .prove_bulk_position_range(vec![], b"bulk", start, limit, None, grove_version) + .unwrap() + .expect("prove bulk position range"); + + let (root_hash, page) = crate::GroveDb::verify_bulk_position_range_proof( + &proof, + vec![], + b"bulk", + start, + limit, + grove_version, + ) + .expect("verify bulk position range proof"); + + let expected_root = db.root_hash(None, grove_version).unwrap().unwrap(); + assert_eq!(root_hash, expected_root, "proof root must match db root"); + + page +} + +#[test] +fn test_bulk_position_range_proof_across_chunk_boundary() { + // 10 values = 2 chunks (8) + 2 buffered, chunk size 4 + let db = make_bulk_db_with_values(10); + + // Page [3, 6) spans the chunk 0 / chunk 1 boundary + let page = roundtrip_bulk_range_proof(&db, 3, 3); + assert_bulk_page(&page, 3, 6, 10); + + // Page [6, 10) spans the chunk 1 / buffer boundary + let page = roundtrip_bulk_range_proof(&db, 6, 4); + assert_bulk_page(&page, 6, 10, 10); +} + +#[test] +fn test_bulk_position_range_proof_empty_range() { + let db = make_bulk_db_with_values(10); + let page = roundtrip_bulk_range_proof(&db, 3, 0); + assert_bulk_page(&page, 3, 3, 10); +} + +#[test] +fn test_bulk_position_range_proof_past_end() { + let db = make_bulk_db_with_values(10); + + // Start exactly at total_count: provably nothing there — the verified + // total_count is the absence proof. + let page = roundtrip_bulk_range_proof(&db, 10, 5); + assert!(page.entries.is_empty()); + assert_eq!(page.total_count, 10); + + // Start far past total_count + let page = roundtrip_bulk_range_proof(&db, 1000, 5); + assert!(page.entries.is_empty()); + assert_eq!(page.total_count, 10); + + // Range starting inside but extending past the end is clamped + let page = roundtrip_bulk_range_proof(&db, 8, 100); + assert_bulk_page(&page, 8, 10, 10); +} + +#[test] +fn test_bulk_position_range_proof_single_entry_pages() { + let db = make_bulk_db_with_values(10); + for pos in 0..10u64 { + let page = roundtrip_bulk_range_proof(&db, pos, 1); + assert_bulk_page(&page, pos, pos + 1, 10); + } +} + +#[test] +fn test_bulk_position_range_proof_large_multi_chunk_page() { + // 30 values = 7 full chunks (28) + 2 buffered, chunk size 4. + let db = make_bulk_db_with_values(30); + + // One page covering everything touches all chunks and the buffer + let page = roundtrip_bulk_range_proof(&db, 0, 30); + assert_bulk_page(&page, 0, 30, 30); + + // A large page crossing several chunk boundaries mid-tree + let page = roundtrip_bulk_range_proof(&db, 5, 20); + assert_bulk_page(&page, 5, 25, 30); +} + +#[test] +fn test_bulk_position_range_proof_empty_tree() { + let db = make_bulk_db_with_values(0); + let page = roundtrip_bulk_range_proof(&db, 0, 10); + assert!(page.entries.is_empty()); + assert_eq!(page.total_count, 0); +} + +#[test] +fn test_bulk_position_range_paged_scan() { + // The scanning pattern: walk the whole tree in proved pages of 7 + // (deliberately not aligned to the chunk size of 4). + let db = make_bulk_db_with_values(30); + + let mut cursor = 0u64; + let mut seen = Vec::new(); + loop { + let page = roundtrip_bulk_range_proof(&db, cursor, 7); + if page.entries.is_empty() { + assert!(cursor >= page.total_count, "empty page only at the end"); + break; + } + cursor += page.entries.len() as u64; + seen.extend(page.entries); + } + assert_eq!(seen.len(), 30); + for (i, (pos, value)) in seen.iter().enumerate() { + assert_eq!(*pos, i as u64); + assert_eq!(value, format!("value_{}", i).as_bytes()); + } +} + +#[test] +fn test_bulk_position_range_prove_wrong_element_type() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + db.insert( + EMPTY_PATH, + b"normal", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert normal tree"); + + let result = db + .prove_bulk_position_range(vec![], b"normal", 0, 4, None, grove_version) + .unwrap(); + assert!(matches!(result, Err(Error::InvalidInput(_)))); +} + +#[test] +fn test_bulk_position_range_proof_wrong_range_rejected() { + let grove_version = GroveVersion::latest(); + let db = make_bulk_db_with_values(10); + + // Proof generated for [0, 2) (inside chunk 0) must not verify a request + // for [0, 6), which also needs chunk 1. + let narrow_proof = db + .prove_bulk_position_range(vec![], b"bulk", 0, 2, None, grove_version) + .unwrap() + .expect("prove narrow range"); + + crate::GroveDb::verify_bulk_position_range_proof( + &narrow_proof, + vec![], + b"bulk", + 0, + 6, + grove_version, + ) + .expect_err("proof for a narrower range must be rejected"); +} + +#[test] +fn test_bulk_position_range_proof_deep_path() { + // Same flow but with the BulkAppendTree nested under a normal tree. + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + db.insert( + EMPTY_PATH, + b"deep", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert parent tree"); + + db.insert( + &[b"deep"], + b"bulk", + Element::empty_bulk_append_tree(TEST_CHUNK_POWER).expect("valid chunk_power"), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert bulk append tree"); + + for i in 0..10u32 { + db.bulk_append( + &[b"deep"], + b"bulk", + format!("value_{}", i).into_bytes(), + None, + grove_version, + ) + .unwrap() + .expect("bulk append"); + } + + let proof = db + .prove_bulk_position_range(vec![b"deep".to_vec()], b"bulk", 2, 5, None, grove_version) + .unwrap() + .expect("prove nested bulk position range"); + + let (root_hash, page) = crate::GroveDb::verify_bulk_position_range_proof( + &proof, + vec![b"deep".to_vec()], + b"bulk", + 2, + 5, + grove_version, + ) + .expect("verify nested bulk position range proof"); + + let expected_root = db.root_hash(None, grove_version).unwrap().unwrap(); + assert_eq!(root_hash, expected_root); + assert_bulk_page(&page, 2, 7, 10); +} diff --git a/grovedb/src/tests/commitment_tree_tests.rs b/grovedb/src/tests/commitment_tree_tests.rs index 2d76f08b4..660ad12d7 100644 --- a/grovedb/src/tests/commitment_tree_tests.rs +++ b/grovedb/src/tests/commitment_tree_tests.rs @@ -2858,3 +2858,244 @@ fn test_commitment_tree_element_count_subset_query_against_note_fetch_proof() { other => panic!("expected CommitmentTree element, got {:?}", other), } } + +// =========================================================================== +// Paginated position-range reads (get_range) and range proofs +// =========================================================================== + +/// Helper: create a DB with a CommitmentTree at [b"root"]/b"pool" +/// (chunk_power = 2 → chunk size 4) holding `n` notes, returning the +/// expected `cmx || rho || cv_net || payload` bytes per position. +fn make_ct_db_with_notes(n: u8) -> (crate::tests::TempGroveDb, Vec>) { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + db.insert( + EMPTY_PATH, + b"root", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert root tree"); + + db.insert( + &[b"root"], + b"pool", + Element::empty_commitment_tree(2).expect("valid chunk_power"), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert commitment tree"); + + let mut expected_values = Vec::new(); + for i in 0..n { + let cmx = test_cmx(i); + let rho = test_rho(i); + let cv_net = test_cv_net(i); + let payload = serialize_ciphertext(&test_ciphertext(i)); + db.commitment_tree_insert( + &[b"root"], + b"pool", + cmx, + rho, + cv_net, + test_ciphertext(i), + None, + grove_version, + ) + .unwrap() + .expect("commitment tree insert"); + + let mut expected = Vec::with_capacity(32 + 32 + 32 + payload.len()); + expected.extend_from_slice(&cmx); + expected.extend_from_slice(&rho); + expected.extend_from_slice(&cv_net); + expected.extend_from_slice(&payload); + expected_values.push(expected); + } + + (db, expected_values) +} + +#[test] +fn test_commitment_tree_get_range_matches_get_value() { + let grove_version = GroveVersion::latest(); + // 10 notes = 2 full chunks (8) + 2 buffered, chunk size 4 + let (db, expected_values) = make_ct_db_with_notes(10); + + for start in 0..11u64 { + let page = db + .commitment_tree_get_range(&[b"root"], b"pool", start, 4, None, grove_version) + .unwrap() + .expect("commitment tree get range"); + assert_eq!(page.total_count, 10); + + let end = (start + 4).min(10); + let expected_len = end.saturating_sub(start.min(end)); + assert_eq!(page.entries.len(), expected_len as usize); + + for (i, (pos, value)) in page.entries.iter().enumerate() { + assert_eq!(*pos, start + i as u64); + assert_eq!(value, &expected_values[*pos as usize]); + + let per_position = db + .commitment_tree_get_value(&[b"root"], b"pool", *pos, None, grove_version) + .unwrap() + .expect("commitment tree get value") + .expect("value exists"); + assert_eq!(value, &per_position); + } + } +} + +#[test] +fn test_commitment_tree_get_range_wrong_element_type() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + db.insert( + EMPTY_PATH, + b"normal", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert normal tree"); + + let result = db + .commitment_tree_get_range(EMPTY_PATH, b"normal", 0, 4, None, grove_version) + .unwrap(); + assert!(matches!(result, Err(Error::InvalidInput(_)))); +} + +/// Helper: prove and verify a CommitmentTree position range page, asserting +/// the returned root hash matches the database root hash. +fn roundtrip_ct_range_proof( + db: &crate::tests::TempGroveDb, + start: u64, + limit: u16, +) -> crate::RangePage { + let grove_version = GroveVersion::latest(); + + let proof = db + .prove_bulk_position_range( + vec![b"root".to_vec()], + b"pool", + start, + limit, + None, + grove_version, + ) + .unwrap() + .expect("prove commitment tree position range"); + + let (root_hash, page) = GroveDb::verify_bulk_position_range_proof( + &proof, + vec![b"root".to_vec()], + b"pool", + start, + limit, + grove_version, + ) + .expect("verify commitment tree position range proof"); + + let expected_root = db.root_hash(None, grove_version).unwrap().unwrap(); + assert_eq!(root_hash, expected_root, "proof root must match db root"); + + page +} + +#[test] +fn test_commitment_tree_position_range_proof_across_chunk_boundary() { + let (db, expected_values) = make_ct_db_with_notes(10); + + // Page [3, 7) spans the chunk 0 / chunk 1 boundary + let page = roundtrip_ct_range_proof(&db, 3, 4); + assert_eq!(page.total_count, 10); + assert_eq!(page.entries.len(), 4); + for (i, (pos, value)) in page.entries.iter().enumerate() { + assert_eq!(*pos, 3 + i as u64); + assert_eq!(value, &expected_values[*pos as usize]); + } + + // Page [6, 10) spans the chunk 1 / buffer boundary + let page = roundtrip_ct_range_proof(&db, 6, 4); + assert_eq!(page.entries.len(), 4); + for (i, (pos, value)) in page.entries.iter().enumerate() { + assert_eq!(*pos, 6 + i as u64); + assert_eq!(value, &expected_values[*pos as usize]); + } +} + +#[test] +fn test_commitment_tree_position_range_proof_empty_and_past_end() { + let (db, _) = make_ct_db_with_notes(10); + + // Empty range + let page = roundtrip_ct_range_proof(&db, 3, 0); + assert!(page.entries.is_empty()); + assert_eq!(page.total_count, 10); + + // Past the end: the verified total_count is the absence proof for + // positions >= 10. + let page = roundtrip_ct_range_proof(&db, 10, 5); + assert!(page.entries.is_empty()); + assert_eq!(page.total_count, 10); + + // Clamped at the end + let page = roundtrip_ct_range_proof(&db, 8, 100); + assert_eq!(page.entries.len(), 2); + assert_eq!(page.total_count, 10); +} + +#[test] +fn test_commitment_tree_position_range_proof_single_entry_pages() { + let (db, expected_values) = make_ct_db_with_notes(6); + + for pos in 0..6u64 { + let page = roundtrip_ct_range_proof(&db, pos, 1); + assert_eq!(page.total_count, 6); + assert_eq!(page.entries.len(), 1); + assert_eq!(page.entries[0].0, pos); + assert_eq!(page.entries[0].1, expected_values[pos as usize]); + } +} + +#[test] +fn test_commitment_tree_position_range_paged_scan() { + // The shielded-pool scanning pattern: walk all notes since cursor 0 in + // proved pages of 3 (not aligned to the chunk size of 4). + let (db, expected_values) = make_ct_db_with_notes(10); + + let mut cursor = 0u64; + let mut seen = Vec::new(); + loop { + let page = roundtrip_ct_range_proof(&db, cursor, 3); + if page.entries.is_empty() { + assert!(cursor >= page.total_count, "empty page only at the end"); + break; + } + cursor += page.entries.len() as u64; + seen.extend(page.entries); + } + assert_eq!(seen.len(), 10); + for (i, (pos, value)) in seen.iter().enumerate() { + assert_eq!(*pos, i as u64); + assert_eq!(value, &expected_values[i]); + } +} + +#[test] +fn test_commitment_tree_position_range_proof_empty_tree() { + let (db, _) = make_ct_db_with_notes(0); + let page = roundtrip_ct_range_proof(&db, 0, 10); + assert!(page.entries.is_empty()); + assert_eq!(page.total_count, 0); +} From 3f3efe2e0ee9ac15b572b6e994237a2be11a9dfd Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Mon, 3 Aug 2026 07:10:28 +0700 Subject: [PATCH 4/9] test(grovedb): add paged-scan benchmark for bulk position-range reads Benchmark the append-only scanning hot path on a 4096-entry BulkAppendTree (chunk size 64, 96-byte entries): single-page reads, per-page proof generation and verification at page sizes 16/256/1024, and a full proved cursor-walk of the tree at page size 256. Co-Authored-By: Claude Fable 5 --- grovedb/Cargo.toml | 4 + grovedb/benches/bulk_range_scan_benchmark.rs | 203 +++++++++++++++++++ 2 files changed, 207 insertions(+) create mode 100644 grovedb/benches/bulk_range_scan_benchmark.rs diff --git a/grovedb/Cargo.toml b/grovedb/Cargo.toml index 6358fbba8..6097a721d 100644 --- a/grovedb/Cargo.toml +++ b/grovedb/Cargo.toml @@ -80,6 +80,10 @@ harness = false name = "cidx_benchmark" harness = false +[[bench]] +name = "bulk_range_scan_benchmark" +harness = false + [features] default = ["full", "estimated_costs"] proof_debug = ["grovedb-merk/proof_debug"] diff --git a/grovedb/benches/bulk_range_scan_benchmark.rs b/grovedb/benches/bulk_range_scan_benchmark.rs new file mode 100644 index 000000000..0d3206852 --- /dev/null +++ b/grovedb/benches/bulk_range_scan_benchmark.rs @@ -0,0 +1,203 @@ +//! Benchmark for the BulkAppendTree paged-scan pattern. +//! +//! Clients of append-only stores walk "all entries since my cursor" in +//! pages. This benchmark measures that pattern end to end on a +//! BulkAppendTree populated with fixed-size entries: +//! +//! 1. **Paged read** (`bulk_get_range`): fetching one page of entries, +//! chunk-aligned, at various page sizes. +//! 2. **Paged proof generation** (`prove_bulk_position_range`): proving one +//! page. +//! 3. **Paged proof verification** (`verify_bulk_position_range_proof`): +//! verifying one page. +//! 4. **Full scan**: walking the entire tree page by page with proofs, the +//! shielded-pool-style sync flow. + +#[cfg(feature = "minimal")] +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; +#[cfg(feature = "minimal")] +use grovedb::{Element, GroveDb}; +#[cfg(feature = "minimal")] +use grovedb_version::version::GroveVersion; +#[cfg(feature = "minimal")] +use tempfile::TempDir; + +/// Total number of entries appended to the tree. +#[cfg(feature = "minimal")] +const N_ENTRIES: u32 = 4096; + +/// Chunk power of the tree: chunk size = 2^6 = 64 entries, so the tree holds +/// 64 completed chunks with the buffer empty. +#[cfg(feature = "minimal")] +const CHUNK_POWER: u8 = 6; + +/// Size of each entry in bytes (a 32-byte commitment plus a small payload). +#[cfg(feature = "minimal")] +const ENTRY_SIZE: usize = 96; + +#[cfg(feature = "minimal")] +fn setup_db() -> (TempDir, GroveDb) { + let grove_version = GroveVersion::latest(); + let dir = TempDir::new().expect("cannot create temp dir"); + let db = GroveDb::open(dir.path()).expect("cannot open grovedb"); + + db.insert( + &[] as &[&[u8]], + b"bulk", + Element::empty_bulk_append_tree(CHUNK_POWER).expect("valid chunk_power"), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert bulk append tree"); + + for i in 0..N_ENTRIES { + let mut entry = vec![0u8; ENTRY_SIZE]; + entry[..4].copy_from_slice(&i.to_be_bytes()); + db.bulk_append(&[] as &[&[u8]], b"bulk", entry, None, grove_version) + .unwrap() + .expect("bulk append"); + } + + (dir, db) +} + +/// Read one page of entries at various page sizes, starting mid-tree so the +/// page is not chunk-aligned. +#[cfg(feature = "minimal")] +pub fn paged_read(c: &mut Criterion) { + let grove_version = GroveVersion::latest(); + let (_dir, db) = setup_db(); + let mut group = c.benchmark_group("bulk_range_paged_read"); + + for &page_size in &[16u16, 256, 1024] { + group.bench_function(BenchmarkId::from_parameter(page_size), |b| { + b.iter(|| { + let page = db + .bulk_get_range( + &[] as &[&[u8]], + b"bulk", + (N_ENTRIES / 3) as u64, + page_size, + None, + grove_version, + ) + .unwrap() + .expect("bulk get range"); + assert_eq!(page.entries.len(), page_size as usize); + }); + }); + } + group.finish(); +} + +/// Prove one page at various page sizes. +#[cfg(feature = "minimal")] +pub fn paged_proof_generation(c: &mut Criterion) { + let grove_version = GroveVersion::latest(); + let (_dir, db) = setup_db(); + let mut group = c.benchmark_group("bulk_range_paged_prove"); + + for &page_size in &[16u16, 256, 1024] { + group.bench_function(BenchmarkId::from_parameter(page_size), |b| { + b.iter(|| { + let _proof = db + .prove_bulk_position_range( + vec![], + b"bulk", + (N_ENTRIES / 3) as u64, + page_size, + None, + grove_version, + ) + .unwrap() + .expect("prove bulk position range"); + }); + }); + } + group.finish(); +} + +/// Verify one page proof at various page sizes. +#[cfg(feature = "minimal")] +pub fn paged_proof_verification(c: &mut Criterion) { + let grove_version = GroveVersion::latest(); + let (_dir, db) = setup_db(); + let mut group = c.benchmark_group("bulk_range_paged_verify"); + + for &page_size in &[16u16, 256, 1024] { + let start = (N_ENTRIES / 3) as u64; + let proof = db + .prove_bulk_position_range(vec![], b"bulk", start, page_size, None, grove_version) + .unwrap() + .expect("prove bulk position range"); + + group.bench_function(BenchmarkId::from_parameter(page_size), |b| { + b.iter(|| { + let (_root, page) = GroveDb::verify_bulk_position_range_proof( + &proof, + vec![], + b"bulk", + start, + page_size, + grove_version, + ) + .expect("verify bulk position range proof"); + assert_eq!(page.entries.len(), page_size as usize); + }); + }); + } + group.finish(); +} + +/// Walk the entire tree with proved pages of 256 — the client sync flow. +#[cfg(feature = "minimal")] +pub fn full_paged_scan_with_proofs(c: &mut Criterion) { + let grove_version = GroveVersion::latest(); + let (_dir, db) = setup_db(); + const PAGE: u16 = 256; + + c.bench_function("bulk_range_full_scan_with_proofs", |b| { + b.iter(|| { + let mut cursor = 0u64; + let mut total = 0usize; + loop { + let proof = db + .prove_bulk_position_range(vec![], b"bulk", cursor, PAGE, None, grove_version) + .unwrap() + .expect("prove page"); + let (_root, page) = GroveDb::verify_bulk_position_range_proof( + &proof, + vec![], + b"bulk", + cursor, + PAGE, + grove_version, + ) + .expect("verify page"); + if page.entries.is_empty() { + break; + } + cursor += page.entries.len() as u64; + total += page.entries.len(); + } + assert_eq!(total, N_ENTRIES as usize); + }); + }); +} + +#[cfg(feature = "minimal")] +criterion_group!( + name = benches; + config = Criterion::default().sample_size(10); + targets = paged_read, + paged_proof_generation, + paged_proof_verification, + full_paged_scan_with_proofs +); +#[cfg(feature = "minimal")] +criterion_main!(benches); + +#[cfg(not(feature = "minimal"))] +fn main() {} From ad8c8a7520aebbff6b8b175d8718ab7155dcf25b Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Mon, 3 Aug 2026 07:40:27 +0700 Subject: [PATCH 5/9] refactor: address CodeRabbit review on paginated range reads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Build the chunk MMR once per get_range call instead of once per chunk: going through get_chunk_value re-cloned the MMR overlay for every chunk in the page, exactly the repeated work chunk-alignment is meant to avoid. Reuses the same single-MMR pattern as BulkAppendTreeProof::generate. - Add explicit version gates (prove_bulk_position_range, verify_bulk_position_range_proof) matching the other public proof entry points; 0 across GROVE_V1..V4, so no behavior change — the delegated prove_query/verify_query gates still apply transitively. - Pin the shared-chunk disjoint-range semantics with a test: a proof generated for [0, 2) verifies a request for [2, 4) inside the same chunk, because chunk-aligned proofs carry the whole authenticated blob. Rejection of ranges needing an unproved chunk is unchanged. - Add the CommitmentTree-envelope wrong-range rejection test mirroring the BulkAppendTree one (different proof envelope and child-hash derivation, so the bulk test did not cover it). Co-Authored-By: Claude Fable 5 --- grovedb-bulk-append-tree/src/tree/fetch.rs | 19 ++++++++++++-- .../src/version/grovedb_versions.rs | 2 ++ grovedb-version/src/version/v1.rs | 2 ++ grovedb-version/src/version/v2.rs | 2 ++ grovedb-version/src/version/v3.rs | 2 ++ grovedb-version/src/version/v4.rs | 2 ++ grovedb/src/operations/proof/generate.rs | 8 ++++++ grovedb/src/operations/proof/verify.rs | 8 ++++++ grovedb/src/tests/bulk_append_tree_tests.rs | 18 +++++++++++++ grovedb/src/tests/commitment_tree_tests.rs | 26 +++++++++++++++++++ 10 files changed, 87 insertions(+), 2 deletions(-) diff --git a/grovedb-bulk-append-tree/src/tree/fetch.rs b/grovedb-bulk-append-tree/src/tree/fetch.rs index 4d9ea342d..1f85b89a0 100644 --- a/grovedb-bulk-append-tree/src/tree/fetch.rs +++ b/grovedb-bulk-append-tree/src/tree/fetch.rs @@ -91,13 +91,28 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { let epoch_size = self.epoch_size(); let buffer_start = self.chunk_count() * epoch_size; - // Completed chunks overlapping [start, min(end, buffer_start)) + // Completed chunks overlapping [start, min(end, buffer_start)). + // The MMR (with its overlay clone) is built once and reused for every + // chunk in the page — going through `get_chunk_value` would rebuild + // it, and re-clone the overlay, per chunk. let chunk_end = end.min(buffer_start); if start < chunk_end { let first_chunk = start / epoch_size; let last_chunk = (chunk_end - 1) / epoch_size; + let mmr_store = MmrStore::with_key_size(&self.dense_tree.storage, MmrKeySize::U32); + let mmr = MMR::new_with_overlay(self.mmr_size(), &mmr_store, self.mmr_overlay.clone()); for chunk_idx in first_chunk..=last_chunk { - let blob = self.get_chunk_value(chunk_idx)?.ok_or_else(|| { + let node = mmr + .batch + .element_at_position(leaf_to_pos(chunk_idx)) + .unwrap() + .map_err(|e| { + BulkAppendError::MmrError(format!( + "failed to read MMR node for chunk {}: {}", + chunk_idx, e + )) + })?; + let blob = node.and_then(|n| n.into_value()).ok_or_else(|| { BulkAppendError::CorruptedData(format!( "missing chunk blob for index {}", chunk_idx diff --git a/grovedb-version/src/version/grovedb_versions.rs b/grovedb-version/src/version/grovedb_versions.rs index b63af3694..119492680 100644 --- a/grovedb-version/src/version/grovedb_versions.rs +++ b/grovedb-version/src/version/grovedb_versions.rs @@ -130,6 +130,8 @@ pub struct GroveDBOperationsProofVersions { pub prove_trunk_chunk_non_serialized: FeatureVersion, pub prove_branch_chunk: FeatureVersion, pub prove_branch_chunk_non_serialized: FeatureVersion, + pub prove_bulk_position_range: FeatureVersion, + pub verify_bulk_position_range_proof: FeatureVersion, pub verify_query_with_options: FeatureVersion, pub verify_query_raw: FeatureVersion, pub verify_layer_proof: FeatureVersion, diff --git a/grovedb-version/src/version/v1.rs b/grovedb-version/src/version/v1.rs index be2b1d472..945c44d58 100644 --- a/grovedb-version/src/version/v1.rs +++ b/grovedb-version/src/version/v1.rs @@ -157,6 +157,8 @@ pub const GROVE_V1: GroveVersion = GroveVersion { prove_trunk_chunk_non_serialized: 0, prove_branch_chunk: 0, prove_branch_chunk_non_serialized: 0, + prove_bulk_position_range: 0, + verify_bulk_position_range_proof: 0, verify_query_with_options: 0, verify_query_raw: 0, verify_layer_proof: 0, diff --git a/grovedb-version/src/version/v2.rs b/grovedb-version/src/version/v2.rs index 14f8d0936..677918d7c 100644 --- a/grovedb-version/src/version/v2.rs +++ b/grovedb-version/src/version/v2.rs @@ -157,6 +157,8 @@ pub const GROVE_V2: GroveVersion = GroveVersion { prove_trunk_chunk_non_serialized: 0, prove_branch_chunk: 0, prove_branch_chunk_non_serialized: 0, + prove_bulk_position_range: 0, + verify_bulk_position_range_proof: 0, verify_query_with_options: 0, verify_query_raw: 0, verify_layer_proof: 0, diff --git a/grovedb-version/src/version/v3.rs b/grovedb-version/src/version/v3.rs index c65b8518f..4e2d9b974 100644 --- a/grovedb-version/src/version/v3.rs +++ b/grovedb-version/src/version/v3.rs @@ -161,6 +161,8 @@ pub const GROVE_V3: GroveVersion = GroveVersion { prove_trunk_chunk_non_serialized: 1, prove_branch_chunk: 0, prove_branch_chunk_non_serialized: 0, + prove_bulk_position_range: 0, + verify_bulk_position_range_proof: 0, verify_query_with_options: 0, verify_query_raw: 0, verify_layer_proof: 0, diff --git a/grovedb-version/src/version/v4.rs b/grovedb-version/src/version/v4.rs index 7b53a2033..f8e74454c 100644 --- a/grovedb-version/src/version/v4.rs +++ b/grovedb-version/src/version/v4.rs @@ -207,6 +207,8 @@ pub const GROVE_V4: GroveVersion = GroveVersion { prove_trunk_chunk_non_serialized: 1, prove_branch_chunk: 0, prove_branch_chunk_non_serialized: 0, + prove_bulk_position_range: 0, + verify_bulk_position_range_proof: 0, verify_query_with_options: 0, verify_query_raw: 0, verify_layer_proof: 0, diff --git a/grovedb/src/operations/proof/generate.rs b/grovedb/src/operations/proof/generate.rs index 6f982564d..6fa88c4f5 100644 --- a/grovedb/src/operations/proof/generate.rs +++ b/grovedb/src/operations/proof/generate.rs @@ -231,6 +231,14 @@ impl GroveDb { prove_options: Option, grove_version: &GroveVersion, ) -> CostResult, Error> { + check_grovedb_v0_with_cost!( + "prove_bulk_position_range", + grove_version + .grovedb_versions + .operations + .proof + .prove_bulk_position_range + ); let mut cost = OperationCost::default(); // Fail fast with a clear error when the target is not an append-only diff --git a/grovedb/src/operations/proof/verify.rs b/grovedb/src/operations/proof/verify.rs index bf0a562d3..cfaf54a06 100644 --- a/grovedb/src/operations/proof/verify.rs +++ b/grovedb/src/operations/proof/verify.rs @@ -2625,6 +2625,14 @@ impl GroveDb { limit: u16, grove_version: &GroveVersion, ) -> Result<(CryptoHash, grovedb_bulk_append_tree::RangePage), Error> { + check_grovedb_v0!( + "verify_bulk_position_range_proof", + grove_version + .grovedb_versions + .operations + .proof + .verify_bulk_position_range_proof + ); // 1. Verify the range entries against the canonical range query. // Succinctness cannot be required: range proofs are chunk-aligned // and intentionally carry whole chunk blobs — a superset of the diff --git a/grovedb/src/tests/bulk_append_tree_tests.rs b/grovedb/src/tests/bulk_append_tree_tests.rs index 2e7cfcd38..00e92e046 100644 --- a/grovedb/src/tests/bulk_append_tree_tests.rs +++ b/grovedb/src/tests/bulk_append_tree_tests.rs @@ -2429,6 +2429,24 @@ fn test_bulk_position_range_proof_wrong_range_rejected() { grove_version, ) .expect_err("proof for a narrower range must be rejected"); + + // Intended semantics pin: a DISJOINT range within the SAME chunk + // verifies. The proof for [0, 2) carries the whole chunk 0 blob + // (positions 0..4) because proofs are chunk-aligned, and every entry in + // it is authenticated against the root — so a verifier asking [2, 4) + // legitimately gets those entries from the same bytes. + let (root_hash, page) = crate::GroveDb::verify_bulk_position_range_proof( + &narrow_proof, + vec![], + b"bulk", + 2, + 2, + grove_version, + ) + .expect("disjoint range within the proved chunk must verify"); + let expected_root = db.root_hash(None, grove_version).unwrap().unwrap(); + assert_eq!(root_hash, expected_root); + assert_bulk_page(&page, 2, 4, 10); } #[test] diff --git a/grovedb/src/tests/commitment_tree_tests.rs b/grovedb/src/tests/commitment_tree_tests.rs index 660ad12d7..f4732dd9d 100644 --- a/grovedb/src/tests/commitment_tree_tests.rs +++ b/grovedb/src/tests/commitment_tree_tests.rs @@ -3099,3 +3099,29 @@ fn test_commitment_tree_position_range_proof_empty_tree() { assert!(page.entries.is_empty()); assert_eq!(page.total_count, 0); } + +/// The CommitmentTree envelope (`ProofBytes::CommitmentTree`, sinsemilla +/// prefix + bulk proof) must reject a wider range than was proved, same as +/// the plain BulkAppendTree envelope. +#[test] +fn test_commitment_tree_position_range_proof_wrong_range_rejected() { + let grove_version = GroveVersion::latest(); + let (db, _) = make_ct_db_with_notes(10); + + // Proof generated for [0, 2) (inside chunk 0) must not verify a request + // for [0, 6), which also needs chunk 1. + let narrow_proof = db + .prove_bulk_position_range(vec![b"root".to_vec()], b"pool", 0, 2, None, grove_version) + .unwrap() + .expect("prove narrow range"); + + GroveDb::verify_bulk_position_range_proof( + &narrow_proof, + vec![b"root".to_vec()], + b"pool", + 0, + 6, + grove_version, + ) + .expect_err("proof for a narrower range must be rejected"); +} From 9326581f1d996e61133487c38f1bb2186939e688 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Mon, 3 Aug 2026 07:51:39 +0700 Subject: [PATCH 6/9] test: cover reachable error paths in paginated range reads codecov/patch flagged the new code at 80.55% against the 90% bar. Cover the arms that are reachable without contrived proofs: - get_range corruption paths: storage claiming chunks / buffered entries it does not hold must error, not silently skip entries. - Version-gate rejections for prove_bulk_position_range and verify_bulk_position_range_proof under an unknown feature version. - verify_bulk_position_range_proof adversarial inputs: a canonical range proof over a plain Tree with 8-byte item keys (element type must be rejected when extracting total_count) and over a nonexistent key (a proof binding no element must not invent a total_count). The remaining uncovered lines are defense-in-depth InvalidProof arms (root-mismatch between sub-proofs of the same bytes, non-8-byte / non-item / non-contiguous rows under a genuine bulk element) that cannot be reached through any proof that survives the lower layer's own checks. Co-Authored-By: Claude Fable 5 --- grovedb-bulk-append-tree/src/tree/tests.rs | 23 ++++ grovedb/src/tests/bulk_append_tree_tests.rs | 110 ++++++++++++++++++++ 2 files changed, 133 insertions(+) diff --git a/grovedb-bulk-append-tree/src/tree/tests.rs b/grovedb-bulk-append-tree/src/tree/tests.rs index e5987e8d2..1a7413386 100644 --- a/grovedb-bulk-append-tree/src/tree/tests.rs +++ b/grovedb-bulk-append-tree/src/tree/tests.rs @@ -529,3 +529,26 @@ fn get_range_paged_scan_covers_everything() { assert_eq!(value, &vec![i as u8]); } } + +#[test] +fn get_range_missing_chunk_is_corruption() { + // Storage claims 2 completed chunks (via from_state) but holds no data: + // the chunk MMR leaf lookup comes back empty and the read must surface + // corruption, not silently skip entries. + let tree = BulkAppendTree::from_state(4, 1, MemStorageContext::new()).expect("from_state"); + assert_eq!(tree.chunk_count(), 2); + let err = tree + .get_range(0, 4) + .expect_err("missing chunk blob must error"); + assert!(matches!(err, crate::BulkAppendError::CorruptedData(_))); +} + +#[test] +fn get_range_missing_buffer_value_is_corruption() { + // Storage claims 1 buffered entry (via from_state) but holds no data: + // the buffer read must surface an error, not silently skip entries. + let tree = BulkAppendTree::from_state(1, 2, MemStorageContext::new()).expect("from_state"); + assert_eq!(tree.buffer_count(), 1); + tree.get_range(0, 1) + .expect_err("missing buffer value must error"); +} diff --git a/grovedb/src/tests/bulk_append_tree_tests.rs b/grovedb/src/tests/bulk_append_tree_tests.rs index 00e92e046..e4a69742c 100644 --- a/grovedb/src/tests/bulk_append_tree_tests.rs +++ b/grovedb/src/tests/bulk_append_tree_tests.rs @@ -2508,3 +2508,113 @@ fn test_bulk_position_range_proof_deep_path() { assert_eq!(root_hash, expected_root); assert_bulk_page(&page, 2, 7, 10); } + +#[test] +fn test_bulk_position_range_version_gates() { + let grove_version = GroveVersion::latest(); + let db = make_bulk_db_with_values(4); + let proof = db + .prove_bulk_position_range(vec![], b"bulk", 0, 4, None, grove_version) + .unwrap() + .expect("prove under latest version"); + + // Unknown prove version → VersionError before any work happens + let mut gated = grove_version.clone(); + gated + .grovedb_versions + .operations + .proof + .prove_bulk_position_range = 99; + let result = db + .prove_bulk_position_range(vec![], b"bulk", 0, 4, None, &gated) + .unwrap(); + assert!(matches!(result, Err(Error::VersionError(_)))); + + // Unknown verify version → VersionError before any work happens + let mut gated = grove_version.clone(); + gated + .grovedb_versions + .operations + .proof + .verify_bulk_position_range_proof = 99; + let result = + crate::GroveDb::verify_bulk_position_range_proof(&proof, vec![], b"bulk", 0, 4, &gated); + assert!(matches!(result, Err(Error::VersionError(_)))); +} + +/// A proof over a NORMAL tree holding items at 8-byte keys: the range +/// entries verify as plain Merk rows, but total_count extraction must +/// reject the element type instead of trusting a non-append-only element. +#[test] +fn test_bulk_position_range_verify_rejects_non_append_element() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + db.insert( + EMPTY_PATH, + b"plain", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert normal tree"); + for i in 0..4u64 { + db.insert( + &[b"plain"], + &i.to_be_bytes(), + Element::new_item(vec![i as u8]), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert item"); + } + + // Bypass prove_bulk_position_range's own element check by proving the + // canonical query directly, as a crafted prover would. + let query = crate::PathQuery::new_bulk_position_range(vec![], b"plain".to_vec(), 0, 4); + let proof = db + .prove_query(&query, None, grove_version) + .unwrap() + .expect("prove canonical query over normal tree"); + + let err = crate::GroveDb::verify_bulk_position_range_proof( + &proof, + vec![], + b"plain", + 0, + 4, + grove_version, + ) + .expect_err("normal tree element must be rejected"); + assert!(matches!(err, Error::InvalidProof(..))); +} + +/// A proof for a key that does not exist: the range query verifies (as an +/// absence), but the element subset query binds nothing, so the verifier +/// must refuse to invent a total_count. +#[test] +fn test_bulk_position_range_verify_rejects_unbound_element() { + let grove_version = GroveVersion::latest(); + let db = make_bulk_db_with_values(4); + + let query = crate::PathQuery::new_bulk_position_range(vec![], b"ghost".to_vec(), 0, 4); + let proof = db + .prove_query(&query, None, grove_version) + .unwrap() + .expect("prove canonical query for missing key"); + + let err = crate::GroveDb::verify_bulk_position_range_proof( + &proof, + vec![], + b"ghost", + 0, + 4, + grove_version, + ) + .expect_err("proof that binds no element must be rejected"); + assert!(matches!(err, Error::InvalidProof(..))); +} From eac3480b8b8f4c75c7b009a7f87bd9ad957b0b8b Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Mon, 3 Aug 2026 08:04:58 +0700 Subject: [PATCH 7/9] test: cover the MMR storage-error path in get_range MemStorageContext (codecov-ignored test util) grows a fail_gets switch that makes every read error, so the chunk-lookup MmrError arm in get_range is exercised: a broken backing store must surface as an error, not a panic or a silent empty page. The buffer-side ok_or_else arm stays uncovered by design: the dense tree's get() errors (never returns Ok(None)) for a missing value below its count, so that arm is unreachable defensive depth, same as the remaining InvalidProof arms in verify_bulk_position_range_proof. Co-Authored-By: Claude Fable 5 --- grovedb-bulk-append-tree/src/test_utils.rs | 9 +++++++++ grovedb-bulk-append-tree/src/tree/tests.rs | 14 ++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/grovedb-bulk-append-tree/src/test_utils.rs b/grovedb-bulk-append-tree/src/test_utils.rs index fbf79a715..732dd934d 100644 --- a/grovedb-bulk-append-tree/src/test_utils.rs +++ b/grovedb-bulk-append-tree/src/test_utils.rs @@ -16,6 +16,9 @@ use grovedb_storage::{Batch, RawIterator, StorageContext}; #[derive(Default)] pub(crate) struct MemStorageContext { pub data: RefCell, Vec>>, + /// When set, every `get` fails — simulates a broken backing store for + /// exercising storage-error paths. + pub fail_gets: std::cell::Cell, } impl MemStorageContext { @@ -29,6 +32,12 @@ impl<'db> StorageContext<'db> for MemStorageContext { type RawIterator = MemRawIterator; fn get>(&self, key: K) -> CostResult>, grovedb_storage::Error> { + if self.fail_gets.get() { + return Err(grovedb_storage::Error::StorageError( + "simulated read failure".to_string(), + )) + .wrap_with_cost(OperationCost::default()); + } Ok(self.data.borrow().get(key.as_ref()).cloned()).wrap_with_cost(OperationCost::default()) } diff --git a/grovedb-bulk-append-tree/src/tree/tests.rs b/grovedb-bulk-append-tree/src/tree/tests.rs index 1a7413386..648ec2fa1 100644 --- a/grovedb-bulk-append-tree/src/tree/tests.rs +++ b/grovedb-bulk-append-tree/src/tree/tests.rs @@ -552,3 +552,17 @@ fn get_range_missing_buffer_value_is_corruption() { tree.get_range(0, 1) .expect_err("missing buffer value must error"); } + +#[test] +fn get_range_storage_read_failure_is_mmr_error() { + // A backing store that fails reads must surface as an MMR error from the + // chunk lookup, not a panic or a silent empty page. + let ctx = MemStorageContext::new(); + ctx.fail_gets.set(true); + let tree = BulkAppendTree::from_state(4, 1, ctx).expect("from_state"); + assert_eq!(tree.chunk_count(), 2); + let err = tree + .get_range(0, 4) + .expect_err("failing storage must error"); + assert!(matches!(err, crate::BulkAppendError::MmrError(_))); +} From acb99bdbdfea5508f09ece4bfe4f0433a4da3863 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Mon, 3 Aug 2026 08:54:37 +0700 Subject: [PATCH 8/9] fix: charge range-read storage costs and validate completed chunk length MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address review on paginated range reads: - get_range now returns a CostResult: the chunk-MMR node reads and each dense-buffer read charge their seeks and loaded bytes, aggregated through CommitmentTree::get_range, bulk_get_range, and commitment_tree_get_range, so cost limits reflect a page's actual work instead of treating up to 65,535 buffer reads as free. The buffer reads go through dense_tree.get directly so their costs are captured. - get_range rejects a completed chunk as corrupted unless it holds exactly epoch_size entries: a short blob would silently omit positions and an oversized one would overlap the next chunk, breaking the contiguous-page contract and stalling cursor scans. Unlike proof verification — where chunk bytes are bound to the state root and a length check is redundant (see the NOTE in proof/mod.rs) — this raw read path has no root comparison backing it. Tests: a tampered-overlay chunk with the wrong entry count (short and oversized) is rejected with CorruptedData, and bulk_get_range reports nonzero seek/loaded-byte costs that grow with page size. Co-Authored-By: Claude Fable 5 --- grovedb-bulk-append-tree/src/tree/fetch.rs | 85 ++++++++++++++----- grovedb-bulk-append-tree/src/tree/tests.rs | 70 ++++++++++++--- .../src/commitment_tree/mod.rs | 6 +- grovedb/src/operations/bulk_append_tree.rs | 7 +- grovedb/src/operations/commitment_tree.rs | 6 +- grovedb/src/tests/bulk_append_tree_tests.rs | 37 ++++++++ 6 files changed, 171 insertions(+), 40 deletions(-) diff --git a/grovedb-bulk-append-tree/src/tree/fetch.rs b/grovedb-bulk-append-tree/src/tree/fetch.rs index 1f85b89a0..d1974e4fc 100644 --- a/grovedb-bulk-append-tree/src/tree/fetch.rs +++ b/grovedb-bulk-append-tree/src/tree/fetch.rs @@ -1,5 +1,6 @@ //! Read operations for BulkAppendTree. +use grovedb_costs::{CostResult, CostsExt, OperationCost}; use grovedb_dense_fixed_sized_merkle_tree::DenseTreeProof; use grovedb_merkle_mountain_range::{leaf_to_pos, MmrKeySize, MmrStore, MMR}; use grovedb_query::Query; @@ -77,14 +78,20 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { /// /// Absence needs no lookup: positions `>= total_count` do not exist, so /// a page shorter than `limit` means the end of the tree was reached. - pub fn get_range(&self, start: u64, limit: u16) -> Result { + /// + /// Returns a [`CostResult`] so callers can charge the page's actual + /// storage work (chunk MMR seeks and buffer reads) against cost limits. + pub fn get_range(&self, start: u64, limit: u16) -> CostResult { + let mut cost = OperationCost::default(); + let total_count = self.total_count; let end = start.saturating_add(limit as u64).min(total_count); if start >= end { return Ok(RangePage { entries: Vec::new(), total_count, - }); + }) + .wrap_with_cost(cost); } let mut entries = Vec::with_capacity((end - start) as usize); @@ -102,23 +109,48 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { let mmr_store = MmrStore::with_key_size(&self.dense_tree.storage, MmrKeySize::U32); let mmr = MMR::new_with_overlay(self.mmr_size(), &mmr_store, self.mmr_overlay.clone()); for chunk_idx in first_chunk..=last_chunk { - let node = mmr + let node = match mmr .batch .element_at_position(leaf_to_pos(chunk_idx)) - .unwrap() - .map_err(|e| { - BulkAppendError::MmrError(format!( + .unwrap_add_cost(&mut cost) + { + Ok(node) => node, + Err(e) => { + return Err(BulkAppendError::MmrError(format!( "failed to read MMR node for chunk {}: {}", chunk_idx, e - )) - })?; - let blob = node.and_then(|n| n.into_value()).ok_or_else(|| { - BulkAppendError::CorruptedData(format!( + ))) + .wrap_with_cost(cost); + } + }; + let Some(blob) = node.and_then(|n| n.into_value()) else { + return Err(BulkAppendError::CorruptedData(format!( "missing chunk blob for index {}", chunk_idx - )) - })?; - let chunk_entries = deserialize_chunk_blob(&blob)?; + ))) + .wrap_with_cost(cost); + }; + let chunk_entries = match deserialize_chunk_blob(&blob) { + Ok(chunk_entries) => chunk_entries, + Err(e) => return Err(e).wrap_with_cost(cost), + }; + // A completed chunk holds exactly `epoch_size` entries — a + // short blob would silently omit positions and an oversized + // one would overlap the next chunk, breaking the contiguous + // page contract. Unlike proof verification (where chunk + // bytes are bound to the state root and a length check is + // redundant — see the NOTE in proof/mod.rs), this raw read + // path has no root comparison backing it, so the length is + // validated here. + if chunk_entries.len() as u64 != epoch_size { + return Err(BulkAppendError::CorruptedData(format!( + "chunk {} holds {} entries, expected {}", + chunk_idx, + chunk_entries.len(), + epoch_size + ))) + .wrap_with_cost(cost); + } let chunk_start = chunk_idx * epoch_size; for (i, value) in chunk_entries.into_iter().enumerate() { let pos = chunk_start + i as u64; @@ -129,15 +161,27 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { } } - // Buffer tail: positions in [max(start, buffer_start), end) + // Buffer tail: positions in [max(start, buffer_start), end). Read + // through the dense tree directly so each read's cost is charged. for pos in start.max(buffer_start)..end { let buffer_pos = (pos - buffer_start) as u16; - let value = self.get_buffer_value(buffer_pos)?.ok_or_else(|| { - BulkAppendError::CorruptedData(format!( - "missing buffer value at position {}", - buffer_pos - )) - })?; + let value = match self.dense_tree.get(buffer_pos).unwrap_add_cost(&mut cost) { + Ok(Some(value)) => value, + Ok(None) => { + return Err(BulkAppendError::CorruptedData(format!( + "missing buffer value at position {}", + buffer_pos + ))) + .wrap_with_cost(cost); + } + Err(e) => { + return Err(BulkAppendError::StorageError(format!( + "dense tree get at {} failed: {}", + buffer_pos, e + ))) + .wrap_with_cost(cost); + } + }; entries.push((pos, value)); } @@ -145,6 +189,7 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { entries, total_count, }) + .wrap_with_cost(cost) } // ── Chunk operations (MMR) ─────────────────────────────────────── diff --git a/grovedb-bulk-append-tree/src/tree/tests.rs b/grovedb-bulk-append-tree/src/tree/tests.rs index 648ec2fa1..0f5c7c847 100644 --- a/grovedb-bulk-append-tree/src/tree/tests.rs +++ b/grovedb-bulk-append-tree/src/tree/tests.rs @@ -419,7 +419,7 @@ fn get_range_buffer_only() { let tree = build_range_tree(3, 5); assert_eq!(tree.chunk_count(), 0); - let page = tree.get_range(1, 3).expect("get range"); + let page = tree.get_range(1, 3).unwrap().expect("get range"); assert_page(&page, 1, 4, 5); } @@ -431,7 +431,7 @@ fn get_range_single_chunk() { assert_eq!(tree.buffer_count(), 0); // Page entirely inside chunk 0 - let page = tree.get_range(1, 2).expect("get range"); + let page = tree.get_range(1, 2).unwrap().expect("get range"); assert_page(&page, 1, 3, 8); } @@ -443,25 +443,25 @@ fn get_range_across_chunk_boundary() { assert_eq!(tree.buffer_count(), 2); // Page [3, 6) spans the chunk 0 / chunk 1 boundary - let page = tree.get_range(3, 3).expect("get range"); + let page = tree.get_range(3, 3).unwrap().expect("get range"); assert_page(&page, 3, 6, 10); // Page [6, 10) spans the chunk 1 / buffer boundary - let page = tree.get_range(6, 4).expect("get range"); + let page = tree.get_range(6, 4).unwrap().expect("get range"); assert_page(&page, 6, 10, 10); } #[test] fn get_range_whole_tree() { let tree = build_range_tree(2, 10); - let page = tree.get_range(0, 100).expect("get range"); + let page = tree.get_range(0, 100).unwrap().expect("get range"); assert_page(&page, 0, 10, 10); } #[test] fn get_range_empty_limit() { let tree = build_range_tree(2, 10); - let page = tree.get_range(3, 0).expect("get range"); + let page = tree.get_range(3, 0).unwrap().expect("get range"); assert_page(&page, 3, 3, 10); } @@ -470,31 +470,31 @@ fn get_range_past_end() { let tree = build_range_tree(2, 10); // Start exactly at total_count - let page = tree.get_range(10, 5).expect("get range"); + let page = tree.get_range(10, 5).unwrap().expect("get range"); assert!(page.entries.is_empty()); assert_eq!(page.total_count, 10); // Start far past total_count - let page = tree.get_range(1000, 5).expect("get range"); + let page = tree.get_range(1000, 5).unwrap().expect("get range"); assert!(page.entries.is_empty()); assert_eq!(page.total_count, 10); // Range that starts inside but extends past the end is clamped - let page = tree.get_range(8, 100).expect("get range"); + let page = tree.get_range(8, 100).unwrap().expect("get range"); assert_page(&page, 8, 10, 10); } #[test] fn get_range_single_entry() { let tree = build_range_tree(2, 10); - let page = tree.get_range(7, 1).expect("get range"); + let page = tree.get_range(7, 1).unwrap().expect("get range"); assert_page(&page, 7, 8, 10); } #[test] fn get_range_empty_tree() { let tree = build_range_tree(2, 0); - let page = tree.get_range(0, 10).expect("get range"); + let page = tree.get_range(0, 10).unwrap().expect("get range"); assert!(page.entries.is_empty()); assert_eq!(page.total_count, 0); } @@ -502,7 +502,10 @@ fn get_range_empty_tree() { #[test] fn get_range_start_saturating_overflow() { let tree = build_range_tree(2, 10); - let page = tree.get_range(u64::MAX, u16::MAX).expect("get range"); + let page = tree + .get_range(u64::MAX, u16::MAX) + .unwrap() + .expect("get range"); assert!(page.entries.is_empty()); assert_eq!(page.total_count, 10); } @@ -515,7 +518,7 @@ fn get_range_paged_scan_covers_everything() { let mut cursor = 0u64; let mut seen = Vec::new(); loop { - let page = tree.get_range(cursor, 3).expect("get range"); + let page = tree.get_range(cursor, 3).unwrap().expect("get range"); if page.entries.is_empty() { assert!(cursor >= page.total_count, "empty page only at the end"); break; @@ -539,6 +542,7 @@ fn get_range_missing_chunk_is_corruption() { assert_eq!(tree.chunk_count(), 2); let err = tree .get_range(0, 4) + .unwrap() .expect_err("missing chunk blob must error"); assert!(matches!(err, crate::BulkAppendError::CorruptedData(_))); } @@ -550,6 +554,7 @@ fn get_range_missing_buffer_value_is_corruption() { let tree = BulkAppendTree::from_state(1, 2, MemStorageContext::new()).expect("from_state"); assert_eq!(tree.buffer_count(), 1); tree.get_range(0, 1) + .unwrap() .expect_err("missing buffer value must error"); } @@ -563,6 +568,45 @@ fn get_range_storage_read_failure_is_mmr_error() { assert_eq!(tree.chunk_count(), 2); let err = tree .get_range(0, 4) + .unwrap() .expect_err("failing storage must error"); assert!(matches!(err, crate::BulkAppendError::MmrError(_))); } + +#[test] +fn get_range_wrong_chunk_entry_count_is_corruption() { + // A completed chunk must hold exactly epoch_size entries: a short blob + // would silently omit positions and an oversized one would overlap the + // next chunk. Tamper the MMR overlay so chunk 0's blob deserializes to + // the wrong entry count and verify the read rejects it. + for bad_count in [1usize, 3] { + // epoch_size = 2: append 2 values to complete one genuine chunk. + let mut tree = BulkAppendTree::new(1u8, MemStorageContext::new()).expect("create tree"); + tree.append(&[0]).expect("append"); + tree.append(&[1]).expect("append"); + assert_eq!(tree.chunk_count(), 1); + + let bad_blob = + crate::serialize_chunk_blob(&(0..bad_count).map(|i| vec![i as u8]).collect::>()) + .expect("serialize bad blob"); + tree.mmr_overlay = vec![( + 0, + vec![grovedb_merkle_mountain_range::MmrNode::leaf(bad_blob)], + )]; + + let err = tree + .get_range(0, 2) + .unwrap() + .expect_err("wrong chunk entry count must error"); + match err { + crate::BulkAppendError::CorruptedData(msg) => { + assert!( + msg.contains(&format!("holds {} entries, expected 2", bad_count)), + "unexpected message: {}", + msg + ); + } + other => panic!("expected CorruptedData, got {:?}", other), + } + } +} diff --git a/grovedb-commitment-tree/src/commitment_tree/mod.rs b/grovedb-commitment-tree/src/commitment_tree/mod.rs index 4521207d4..04c15b128 100644 --- a/grovedb-commitment-tree/src/commitment_tree/mod.rs +++ b/grovedb-commitment-tree/src/commitment_tree/mod.rs @@ -643,11 +643,11 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { /// the raw `cmx || rho || cv_net || payload` bytes at that position. /// Delegates to [`BulkAppendTree::get_range`], so the read is /// chunk-aligned — O(chunks touched) blob reads, not O(entries) random - /// reads. - pub fn get_range(&self, start: u64, limit: u16) -> Result { + /// reads — and carries the page's storage costs. + pub fn get_range(&self, start: u64, limit: u16) -> CostResult { self.bulk_tree .get_range(start, limit) - .map_err(|e| CommitmentTreeError::InvalidData(format!("range read: {}", e))) + .map(|r| r.map_err(|e| CommitmentTreeError::InvalidData(format!("range read: {}", e)))) } /// The number of entries per completed chunk (epoch). diff --git a/grovedb/src/operations/bulk_append_tree.rs b/grovedb/src/operations/bulk_append_tree.rs index e87ef35fd..955c27ef4 100644 --- a/grovedb/src/operations/bulk_append_tree.rs +++ b/grovedb/src/operations/bulk_append_tree.rs @@ -308,8 +308,11 @@ impl GroveDb { BulkAppendTree::from_state(total_count, chunk_power, storage_ctx).map_err(map_bulk_err) ); - let page = - cost_return_on_error_no_add!(cost, tree.get_range(start, limit).map_err(map_bulk_err)); + let page = cost_return_on_error!( + &mut cost, + tree.get_range(start, limit) + .map(|r| r.map_err(map_bulk_err)) + ); Ok(page).wrap_with_cost(cost) } diff --git a/grovedb/src/operations/commitment_tree.rs b/grovedb/src/operations/commitment_tree.rs index a8e08aa78..6fa337406 100644 --- a/grovedb/src/operations/commitment_tree.rs +++ b/grovedb/src/operations/commitment_tree.rs @@ -447,8 +447,10 @@ impl GroveDb { .map(|r| r.map_err(map_ct_err)) ); - let page = - cost_return_on_error_no_add!(cost, ct.get_range(start, limit).map_err(map_ct_err)); + let page = cost_return_on_error!( + &mut cost, + ct.get_range(start, limit).map(|r| r.map_err(map_ct_err)) + ); Ok(page).wrap_with_cost(cost) } diff --git a/grovedb/src/tests/bulk_append_tree_tests.rs b/grovedb/src/tests/bulk_append_tree_tests.rs index e4a69742c..bc27a1123 100644 --- a/grovedb/src/tests/bulk_append_tree_tests.rs +++ b/grovedb/src/tests/bulk_append_tree_tests.rs @@ -2618,3 +2618,40 @@ fn test_bulk_position_range_verify_rejects_unbound_element() { .expect_err("proof that binds no element must be rejected"); assert!(matches!(err, Error::InvalidProof(..))); } + +/// Range reads must charge their storage work: the page's chunk-MMR seeks +/// and buffer reads all show up in the returned cost, so cost limits +/// reflect the actual work instead of treating a 65k-entry page as free. +#[test] +fn test_bulk_get_range_reports_storage_costs() { + let grove_version = GroveVersion::latest(); + // 11 values = 2 full chunks + 3 buffered (chunk size 4) + let db = make_bulk_db_with_values(11); + + // A page spanning both chunks and the buffer + let result = db.bulk_get_range(EMPTY_PATH, b"bulk", 0, 11, None, grove_version); + let cost = result.cost.clone(); + let page = result.unwrap().expect("bulk get range"); + assert_eq!(page.entries.len(), 11); + assert!( + cost.storage_loaded_bytes > 0, + "range read must charge loaded bytes, got {:?}", + cost + ); + assert!( + cost.seek_count > 0, + "range read must charge seeks, got {:?}", + cost + ); + + // A wider page must not cost less than a narrower one + let narrow_cost = db + .bulk_get_range(EMPTY_PATH, b"bulk", 8, 1, None, grove_version) + .cost; + assert!( + cost.storage_loaded_bytes > narrow_cost.storage_loaded_bytes, + "an 11-entry page must load more bytes than a 1-entry page ({} vs {})", + cost.storage_loaded_bytes, + narrow_cost.storage_loaded_bytes + ); +} From 7c038fc48826d2952b0990dab83740f76f886be9 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Sat, 22 Aug 2026 15:17:41 +0700 Subject: [PATCH 9/9] refactor: route get_range buffer reads through get_buffer_value_with_cost; document range reads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit get_range hand-rolled its cost-charging dense-tree read because, when it was written, get_buffer_value discarded the read's cost. develop (#826) has since added get_buffer_value_with_cost for exactly that purpose, so the buffer tail of the page now goes through it — same cost accounting, same CorruptedData on a lost value, less duplication. The chunk loop keeps its hoisted single MMR (the per-chunk rebuild get_chunk_value_with_cost would do is what CodeRabbit asked to avoid). Book: add bulk_get_range / commitment_tree_get_range to the operation tables and describe the paginated position-range proof entry points. Co-Authored-By: Claude Fable 5 --- docs/book/src/bulk-append-tree.md | 15 ++++++++++++++- docs/book/src/commitment-tree.md | 9 ++++++++- grovedb-bulk-append-tree/src/tree/fetch.rs | 19 +++++++++---------- 3 files changed, 31 insertions(+), 12 deletions(-) diff --git a/docs/book/src/bulk-append-tree.md b/docs/book/src/bulk-append-tree.md index 440275a4a..a8a1e9a1f 100644 --- a/docs/book/src/bulk-append-tree.md +++ b/docs/book/src/bulk-append-tree.md @@ -245,7 +245,7 @@ The buffer IS a `DenseFixedSizedMerkleTree` — its root hash is `dense_tree_roo ## GroveDB Operations -The BulkAppendTree integrates with GroveDB through six operations defined in +The BulkAppendTree integrates with GroveDB through seven operations defined in `grovedb/src/operations/bulk_append_tree.rs`: ### bulk_append @@ -272,6 +272,7 @@ append operation are added to `cost.hash_node_calls`. |---|---|---| | `bulk_get_value(path, key, position)` | Value at global position | Yes — reads from chunk blob or buffer | | `bulk_get_chunk(path, key, chunk_index)` | Raw chunk blob | Yes — reads chunk key | +| `bulk_get_range(path, key, start, limit)` | `RangePage { entries, total_count }` — positions `[start, start+limit)` clipped to the tree | Yes — each overlapping chunk blob read once, plus one read per buffer entry | | `bulk_get_buffer(path, key)` | All current buffer entries | Yes — reads buffer keys | | `bulk_count(path, key)` | Total count (u64) | No — reads from element | | `bulk_chunk_count(path, key)` | Completed chunks (u64) | No — computed from element | @@ -427,6 +428,18 @@ After verification succeeds, the `BulkAppendTreeProofResult` provides a `values_in_range(start, end)` method that extracts specific values from the verified chunk blobs and buffer entries. +### Paginated position-range proofs + +`GroveDb::prove_bulk_position_range(path, key, start, limit)` proves one page of +a cursor scan; `GroveDb::verify_bulk_position_range_proof` verifies it and +returns the page entries (ascending, contiguous, complete) together with the +authenticated `total_count` from the same proof bytes. Both sides derive the +query from `(start, limit)` via `PathQuery::new_bulk_position_range`, so a +scanning client only needs its cursor and page size. Absence beyond the end +falls out of the proved count (`position >= total_count` does not exist), so a +page shorter than `limit` means the scan caught up with the tip. The same entry +points serve `CommitmentTree` elements. + ## How It Ties to the GroveDB Root Hash The BulkAppendTree is a **non-Merk tree** — it stores data in the data namespace, diff --git a/docs/book/src/commitment-tree.md b/docs/book/src/commitment-tree.md index bf62fa224..c7125e9e7 100644 --- a/docs/book/src/commitment-tree.md +++ b/docs/book/src/commitment-tree.md @@ -306,7 +306,7 @@ sub-merk recursion for these types. ## GroveDB Operations -CommitmentTree provides four operations. The insert operation is generic over +CommitmentTree provides five operations. The insert operation is generic over `M: MemoSize` (from the `orchard` crate), which controls ciphertext payload size validation. The default `M = DashMemo` gives a 216-byte payload (32 epk + 104 enc + 80 out). @@ -325,6 +325,9 @@ db.commitment_tree_anchor(path, key, tx, version) // Retrieve a value by global position db.commitment_tree_get_value(path, key, position, tx, version) +// Retrieve a page of values: positions [start, start + limit), plus total_count +db.commitment_tree_get_range(path, key, start, limit, tx, version) + // Get the current item count db.commitment_tree_count(path, key, tx, version) ``` @@ -903,6 +906,10 @@ Individual items (cmx || rho || cv_net || payload) can be queried by position an V1 proofs (§9.6), the same mechanism used by standalone BulkAppendTree. The V1 proof includes the BulkAppendTree authentication path for the requested position, chained to the parent Merk proof for the CommitmentTree element. +Paginated scans use `GroveDb::prove_bulk_position_range` / +`GroveDb::verify_bulk_position_range_proof`, which dispatch on the element type +and so serve CommitmentTree pages through the same `ProofBytes::CommitmentTree` +envelope (see the BulkAppendTree chapter). ## Cost Tracking diff --git a/grovedb-bulk-append-tree/src/tree/fetch.rs b/grovedb-bulk-append-tree/src/tree/fetch.rs index d8393df20..4a2920858 100644 --- a/grovedb-bulk-append-tree/src/tree/fetch.rs +++ b/grovedb-bulk-append-tree/src/tree/fetch.rs @@ -184,11 +184,16 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { } } - // Buffer tail: positions in [max(start, buffer_start), end). Read - // through the dense tree directly so each read's cost is charged. + // Buffer tail: positions in [max(start, buffer_start), end). Every + // position here is below `total_count`, so the buffer must hold it; + // `get_buffer_value_with_cost` charges each read's seek and bytes, + // and a `None` from it can only mean the backing store lost a value. for pos in start.max(buffer_start)..end { let buffer_pos = (pos - buffer_start) as u16; - let value = match self.dense_tree.get(buffer_pos).unwrap_add_cost(&mut cost) { + let value = match self + .get_buffer_value_with_cost(buffer_pos) + .unwrap_add_cost(&mut cost) + { Ok(Some(value)) => value, Ok(None) => { return Err(BulkAppendError::CorruptedData(format!( @@ -197,13 +202,7 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { ))) .wrap_with_cost(cost); } - Err(e) => { - return Err(BulkAppendError::StorageError(format!( - "dense tree get at {} failed: {}", - buffer_pos, e - ))) - .wrap_with_cost(cost); - } + Err(e) => return Err(e).wrap_with_cost(cost), }; entries.push((pos, value)); }