diff --git a/docs/book/src/bulk-append-tree.md b/docs/book/src/bulk-append-tree.md index 440275a4a..a8a1e9a1f 100644 --- a/docs/book/src/bulk-append-tree.md +++ b/docs/book/src/bulk-append-tree.md @@ -245,7 +245,7 @@ The buffer IS a `DenseFixedSizedMerkleTree` — its root hash is `dense_tree_roo ## GroveDB Operations -The BulkAppendTree integrates with GroveDB through six operations defined in +The BulkAppendTree integrates with GroveDB through seven operations defined in `grovedb/src/operations/bulk_append_tree.rs`: ### bulk_append @@ -272,6 +272,7 @@ append operation are added to `cost.hash_node_calls`. |---|---|---| | `bulk_get_value(path, key, position)` | Value at global position | Yes — reads from chunk blob or buffer | | `bulk_get_chunk(path, key, chunk_index)` | Raw chunk blob | Yes — reads chunk key | +| `bulk_get_range(path, key, start, limit)` | `RangePage { entries, total_count }` — positions `[start, start+limit)` clipped to the tree | Yes — each overlapping chunk blob read once, plus one read per buffer entry | | `bulk_get_buffer(path, key)` | All current buffer entries | Yes — reads buffer keys | | `bulk_count(path, key)` | Total count (u64) | No — reads from element | | `bulk_chunk_count(path, key)` | Completed chunks (u64) | No — computed from element | @@ -427,6 +428,18 @@ After verification succeeds, the `BulkAppendTreeProofResult` provides a `values_in_range(start, end)` method that extracts specific values from the verified chunk blobs and buffer entries. +### Paginated position-range proofs + +`GroveDb::prove_bulk_position_range(path, key, start, limit)` proves one page of +a cursor scan; `GroveDb::verify_bulk_position_range_proof` verifies it and +returns the page entries (ascending, contiguous, complete) together with the +authenticated `total_count` from the same proof bytes. Both sides derive the +query from `(start, limit)` via `PathQuery::new_bulk_position_range`, so a +scanning client only needs its cursor and page size. Absence beyond the end +falls out of the proved count (`position >= total_count` does not exist), so a +page shorter than `limit` means the scan caught up with the tip. The same entry +points serve `CommitmentTree` elements. + ## How It Ties to the GroveDB Root Hash The BulkAppendTree is a **non-Merk tree** — it stores data in the data namespace, diff --git a/docs/book/src/commitment-tree.md b/docs/book/src/commitment-tree.md index bf62fa224..c7125e9e7 100644 --- a/docs/book/src/commitment-tree.md +++ b/docs/book/src/commitment-tree.md @@ -306,7 +306,7 @@ sub-merk recursion for these types. ## GroveDB Operations -CommitmentTree provides four operations. The insert operation is generic over +CommitmentTree provides five operations. The insert operation is generic over `M: MemoSize` (from the `orchard` crate), which controls ciphertext payload size validation. The default `M = DashMemo` gives a 216-byte payload (32 epk + 104 enc + 80 out). @@ -325,6 +325,9 @@ db.commitment_tree_anchor(path, key, tx, version) // Retrieve a value by global position db.commitment_tree_get_value(path, key, position, tx, version) +// Retrieve a page of values: positions [start, start + limit), plus total_count +db.commitment_tree_get_range(path, key, start, limit, tx, version) + // Get the current item count db.commitment_tree_count(path, key, tx, version) ``` @@ -903,6 +906,10 @@ Individual items (cmx || rho || cv_net || payload) can be queried by position an V1 proofs (§9.6), the same mechanism used by standalone BulkAppendTree. The V1 proof includes the BulkAppendTree authentication path for the requested position, chained to the parent Merk proof for the CommitmentTree element. +Paginated scans use `GroveDb::prove_bulk_position_range` / +`GroveDb::verify_bulk_position_range_proof`, which dispatch on the element type +and so serve CommitmentTree pages through the same `ProofBytes::CommitmentTree` +envelope (see the BulkAppendTree chapter). ## Cost Tracking diff --git a/grovedb-bulk-append-tree/src/lib.rs b/grovedb-bulk-append-tree/src/lib.rs index 8a81f7654..c5b203070 100644 --- a/grovedb-bulk-append-tree/src/lib.rs +++ b/grovedb-bulk-append-tree/src/lib.rs @@ -23,7 +23,7 @@ pub use error::BulkAppendError; pub use grovedb_dense_fixed_sized_merkle_tree::{DenseFixedSizedMerkleTree, DenseTreeProof}; #[cfg(feature = "storage")] pub use grovedb_merkle_mountain_range::{MmrKeySize, MmrStore}; -pub use proof::{BulkAppendTreeProof, BulkAppendTreeProofResult}; -pub use tree::{hash::compute_state_root, leaf_count_to_mmr_size, BulkAppendTree}; +pub use proof::{position_range_query, BulkAppendTreeProof, BulkAppendTreeProofResult}; +pub use tree::{hash::compute_state_root, leaf_count_to_mmr_size, BulkAppendTree, RangePage}; #[cfg(feature = "storage")] pub use tree::{AppendNoStateRootResult, AppendResult, BufferQueryResult, ChunkQueryResult}; diff --git a/grovedb-bulk-append-tree/src/proof/mod.rs b/grovedb-bulk-append-tree/src/proof/mod.rs index a8dd6186d..4dbcab404 100644 --- a/grovedb-bulk-append-tree/src/proof/mod.rs +++ b/grovedb-bulk-append-tree/src/proof/mod.rs @@ -176,6 +176,25 @@ fn query_to_ranges(query: &Query, total_count: u64) -> Result, B Ok(merged) } +/// Build the canonical [`Query`] selecting the position range +/// `[start, start + limit)`, with positions encoded as 8-byte big-endian +/// keys. +/// +/// This is the query shape used by the paginated-scan pattern: prover and +/// verifier both derive it from `(start, limit)`, so a client only needs its +/// cursor and page size. `start + limit` saturates at `u64::MAX`, and +/// verification clamps the range to the tree's provable total count. +pub fn position_range_query(start: u64, limit: u16) -> Query { + let end = start.saturating_add(limit as u64); + Query { + items: vec![QueryItem::Range( + start.to_be_bytes().to_vec()..end.to_be_bytes().to_vec(), + )], + left_to_right: true, + ..Query::default() + } +} + /// Check whether `pos` falls inside any of the sorted, non-overlapping ranges. fn in_ranges(pos: u64, ranges: &[(u64, u64)]) -> bool { ranges @@ -325,6 +344,54 @@ impl BulkAppendTreeProof { }) } + /// Generate a proof for the paginated position range + /// `[start, start + limit)`. + /// + /// Convenience wrapper over [`generate`](Self::generate) using the + /// canonical [`position_range_query`]. The proof is chunk-aligned: it + /// carries each completed chunk blob overlapping the range plus the + /// buffer entries in range, so proof size is O(chunks touched). + /// + /// Ranges past the end of the tree are valid and produce a proof of the + /// (empty) result: absence of positions `>= total_count` falls out of + /// the authenticated element's total count, not out of per-position + /// absence proofs. + #[cfg(feature = "storage")] + pub fn generate_for_range<'db, S: StorageContext<'db>>( + tree: &BulkAppendTree, + start: u64, + limit: u16, + ) -> Result { + Self::generate(&position_range_query(start, limit), tree) + } + + /// Verify this proof against the paginated position range + /// `[start, start + limit)`. + /// + /// Convenience wrapper over + /// [`verify_against_query`](Self::verify_against_query) using the + /// canonical [`position_range_query`]. Returns the `(global_position, + /// value)` pairs in the range, ascending and contiguous, clamped to + /// `total_count`. Completeness is enforced: a proof missing any + /// requested position below `total_count` is rejected. Positions + /// `>= total_count` are provably absent by `total_count` itself, which + /// callers must take from the authenticated BulkAppendTree element. + pub fn verify_range( + &self, + expected_state_root: &[u8; 32], + height: u8, + total_count: u64, + start: u64, + limit: u16, + ) -> Result)>, BulkAppendError> { + self.verify_against_query( + expected_state_root, + height, + total_count, + &position_range_query(start, limit), + ) + } + /// Verify this proof against an expected state root. /// /// `height` and `total_count` come from the authenticated BulkAppendTree diff --git a/grovedb-bulk-append-tree/src/proof/tests.rs b/grovedb-bulk-append-tree/src/proof/tests.rs index d51452025..cbf22a875 100644 --- a/grovedb-bulk-append-tree/src/proof/tests.rs +++ b/grovedb-bulk-append-tree/src/proof/tests.rs @@ -943,4 +943,170 @@ mod proof_tests { ); } } + + // ── generate_for_range / verify_range (paginated scan pattern) ─────── + + /// Helper: build a tree of `n` values "val_0".."val_{n-1}" and return + /// (state_root, tree). + fn build_indexed_tree(height: u8, n: u32) -> ([u8; 32], BulkAppendTree) { + let values: Vec> = (0..n).map(|i| format!("val_{}", i).into_bytes()).collect(); + build_test_tree(height, &values) + } + + /// Helper: round-trip a range proof and assert the returned page is + /// exactly positions `expected_start..expected_end`. + fn assert_range_roundtrip( + state_root: &[u8; 32], + tree: &BulkAppendTree, + start: u64, + limit: u16, + expected_start: u64, + expected_end: u64, + ) { + let proof = + BulkAppendTreeProof::generate_for_range(tree, start, limit).expect("generate range"); + + // Wire round-trip: encode + decode like a real client + let bytes = proof.encode_to_vec().expect("encode"); + let decoded = BulkAppendTreeProof::decode_from_slice(&bytes).expect("decode"); + + let entries = decoded + .verify_range(state_root, tree.height(), tree.total_count, start, limit) + .expect("verify range"); + + assert_eq!(entries.len(), (expected_end - expected_start) as usize); + for (i, (pos, value)) in entries.iter().enumerate() { + assert_eq!(*pos, expected_start + i as u64); + assert_eq!(value, format!("val_{}", pos).as_bytes()); + } + } + + #[test] + fn test_range_roundtrip_buffer_only() { + // height=3, capacity=7: 5 values all in buffer + let (root, tree) = build_indexed_tree(3, 5); + assert_range_roundtrip(&root, &tree, 1, 3, 1, 4); + } + + #[test] + fn test_range_roundtrip_across_chunk_boundary() { + // height=2, epoch_size=4: 10 values = 2 chunks + 2 buffered + let (root, tree) = build_indexed_tree(2, 10); + // spans chunk 0 / chunk 1 + assert_range_roundtrip(&root, &tree, 3, 3, 3, 6); + // spans chunk 1 / buffer + assert_range_roundtrip(&root, &tree, 6, 4, 6, 10); + } + + #[test] + fn test_range_roundtrip_single_entry_pages() { + let (root, tree) = build_indexed_tree(2, 10); + for pos in 0..10u64 { + assert_range_roundtrip(&root, &tree, pos, 1, pos, pos + 1); + } + } + + #[test] + fn test_range_roundtrip_empty_range() { + let (root, tree) = build_indexed_tree(2, 10); + // limit 0: proof still verifies against the root, returns nothing + assert_range_roundtrip(&root, &tree, 3, 0, 3, 3); + } + + #[test] + fn test_range_roundtrip_past_end() { + let (root, tree) = build_indexed_tree(2, 10); + // starts exactly at total_count + assert_range_roundtrip(&root, &tree, 10, 5, 10, 10); + // starts far past total_count + assert_range_roundtrip(&root, &tree, 1000, 5, 1000, 1000); + // clamped at the end + assert_range_roundtrip(&root, &tree, 8, 100, 8, 10); + } + + #[test] + fn test_range_roundtrip_large_multi_chunk_page() { + // height=4, epoch_size=16: 100 values = 6 chunks + 4 buffered. + // One page covering everything touches all chunks and the buffer. + let (root, tree) = build_indexed_tree(4, 100); + assert_range_roundtrip(&root, &tree, 0, 100, 0, 100); + // A large page crossing several chunk boundaries mid-tree + assert_range_roundtrip(&root, &tree, 10, 70, 10, 80); + } + + #[test] + fn test_range_roundtrip_empty_tree() { + let (_, tree) = build_indexed_tree(2, 0); + // For an empty tree the state root is blake3("bulk_state" || 0*32 || 0*32) + let root = crate::compute_state_root(&[0u8; 32], &[0u8; 32]); + assert_range_roundtrip(&root, &tree, 0, 10, 0, 0); + } + + #[test] + fn test_range_paged_scan_covers_everything() { + // The client scan pattern: page through the whole tree with + // limit=7 (deliberately not aligned to epoch_size=4). + let (root, tree) = build_indexed_tree(2, 30); + let mut cursor = 0u64; + let mut seen = Vec::new(); + while cursor < tree.total_count { + let proof = + BulkAppendTreeProof::generate_for_range(&tree, cursor, 7).expect("generate page"); + let entries = proof + .verify_range(&root, tree.height(), tree.total_count, cursor, 7) + .expect("verify page"); + assert!(!entries.is_empty()); + cursor += entries.len() as u64; + seen.extend(entries); + } + assert_eq!(seen.len(), 30); + for (i, (pos, value)) in seen.iter().enumerate() { + assert_eq!(*pos, i as u64); + assert_eq!(value, format!("val_{}", i).as_bytes()); + } + } + + #[test] + fn test_range_proof_wrong_root_rejected() { + let (root, tree) = build_indexed_tree(2, 10); + let proof = BulkAppendTreeProof::generate_for_range(&tree, 0, 5).expect("generate"); + let mut bad_root = root; + bad_root[0] ^= 1; + proof + .verify_range(&bad_root, tree.height(), tree.total_count, 0, 5) + .expect_err("tampered root must be rejected"); + } + + #[test] + fn test_range_proof_missing_chunk_rejected() { + // Proof generated for [0, 2) (chunk 0 only) must not verify a + // request for [0, 6) which also needs chunk 1. + let (root, tree) = build_indexed_tree(2, 10); + let narrow = BulkAppendTreeProof::generate_for_range(&tree, 0, 2).expect("generate"); + narrow + .verify_range(&root, tree.height(), tree.total_count, 0, 6) + .expect_err("proof missing chunk 1 must be rejected for the wider range"); + } + + #[test] + fn test_position_range_query_shape() { + let q = super::super::position_range_query(5, 3); + assert_eq!(q.items.len(), 1); + match &q.items[0] { + QueryItem::Range(r) => { + assert_eq!(r.start, 5u64.to_be_bytes().to_vec()); + assert_eq!(r.end, 8u64.to_be_bytes().to_vec()); + } + other => panic!("expected Range item, got {:?}", other), + } + + // start + limit saturates instead of wrapping + let q = super::super::position_range_query(u64::MAX - 1, 100); + match &q.items[0] { + QueryItem::Range(r) => { + assert_eq!(r.end, u64::MAX.to_be_bytes().to_vec()); + } + other => panic!("expected Range item, got {:?}", other), + } + } } diff --git a/grovedb-bulk-append-tree/src/tree/fetch.rs b/grovedb-bulk-append-tree/src/tree/fetch.rs index 7a3baaea5..4a2920858 100644 --- a/grovedb-bulk-append-tree/src/tree/fetch.rs +++ b/grovedb-bulk-append-tree/src/tree/fetch.rs @@ -6,7 +6,7 @@ use grovedb_merkle_mountain_range::{leaf_to_pos, MmrKeySize, MmrStore, MMR}; use grovedb_query::Query; use grovedb_storage::StorageContext; -use super::BulkAppendTree; +use super::{BulkAppendTree, RangePage}; use crate::{chunk::deserialize_chunk_blob, BulkAppendError}; use grovedb_version::version::GroveVersion; @@ -87,6 +87,133 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree { Ok(BufferQueryResult { entries, proof }) } + // ── Range operations (chunks + buffer) ─────────────────────────── + + /// Fetch entries for the position range `[start, start + limit)`, + /// clamped to the tree's total count. + /// + /// This is the paginated-scan read path: clients walking "all entries + /// since my cursor" call it with their cursor as `start` and advance by + /// `entries.len()`. The read is chunk-aligned — each completed chunk + /// overlapping the range is read and deserialized exactly once, so a + /// page costs O(chunks touched) blob reads plus one read per buffer + /// entry, not O(entries) random reads. + /// + /// Absence needs no lookup: positions `>= total_count` do not exist, so + /// a page shorter than `limit` means the end of the tree was reached. + /// + /// Returns a [`CostResult`] so callers can charge the page's actual + /// storage work (chunk MMR seeks and buffer reads) against cost limits. + pub fn get_range(&self, start: u64, limit: u16) -> CostResult { + let mut cost = OperationCost::default(); + + let total_count = self.total_count; + let end = start.saturating_add(limit as u64).min(total_count); + if start >= end { + return Ok(RangePage { + entries: Vec::new(), + total_count, + }) + .wrap_with_cost(cost); + } + let mut entries = Vec::with_capacity((end - start) as usize); + + let epoch_size = self.epoch_size(); + let buffer_start = self.chunk_count() * epoch_size; + + // Completed chunks overlapping [start, min(end, buffer_start)). + // The MMR (with its overlay clone) is built once and reused for every + // chunk in the page — going through `get_chunk_value` would rebuild + // it, and re-clone the overlay, per chunk. + let chunk_end = end.min(buffer_start); + if start < chunk_end { + let first_chunk = start / epoch_size; + let last_chunk = (chunk_end - 1) / epoch_size; + let mmr_store = MmrStore::with_key_size(&self.dense_tree.storage, MmrKeySize::U32); + let mmr = MMR::new_with_overlay(self.mmr_size(), &mmr_store, self.mmr_overlay.clone()); + for chunk_idx in first_chunk..=last_chunk { + let node = match mmr + .batch + .element_at_position(leaf_to_pos(chunk_idx)) + .unwrap_add_cost(&mut cost) + { + Ok(node) => node, + Err(e) => { + return Err(BulkAppendError::MmrError(format!( + "failed to read MMR node for chunk {}: {}", + chunk_idx, e + ))) + .wrap_with_cost(cost); + } + }; + let Some(blob) = node.and_then(|n| n.into_value()) else { + return Err(BulkAppendError::CorruptedData(format!( + "missing chunk blob for index {}", + chunk_idx + ))) + .wrap_with_cost(cost); + }; + let chunk_entries = match deserialize_chunk_blob(&blob) { + Ok(chunk_entries) => chunk_entries, + Err(e) => return Err(e).wrap_with_cost(cost), + }; + // A completed chunk holds exactly `epoch_size` entries — a + // short blob would silently omit positions and an oversized + // one would overlap the next chunk, breaking the contiguous + // page contract. Unlike proof verification (where chunk + // bytes are bound to the state root and a length check is + // redundant — see the NOTE in proof/mod.rs), this raw read + // path has no root comparison backing it, so the length is + // validated here. + if chunk_entries.len() as u64 != epoch_size { + return Err(BulkAppendError::CorruptedData(format!( + "chunk {} holds {} entries, expected {}", + chunk_idx, + chunk_entries.len(), + epoch_size + ))) + .wrap_with_cost(cost); + } + let chunk_start = chunk_idx * epoch_size; + for (i, value) in chunk_entries.into_iter().enumerate() { + let pos = chunk_start + i as u64; + if pos >= start && pos < chunk_end { + entries.push((pos, value)); + } + } + } + } + + // Buffer tail: positions in [max(start, buffer_start), end). Every + // position here is below `total_count`, so the buffer must hold it; + // `get_buffer_value_with_cost` charges each read's seek and bytes, + // and a `None` from it can only mean the backing store lost a value. + for pos in start.max(buffer_start)..end { + let buffer_pos = (pos - buffer_start) as u16; + let value = match self + .get_buffer_value_with_cost(buffer_pos) + .unwrap_add_cost(&mut cost) + { + Ok(Some(value)) => value, + Ok(None) => { + return Err(BulkAppendError::CorruptedData(format!( + "missing buffer value at position {}", + buffer_pos + ))) + .wrap_with_cost(cost); + } + Err(e) => return Err(e).wrap_with_cost(cost), + }; + entries.push((pos, value)); + } + + Ok(RangePage { + entries, + total_count, + }) + .wrap_with_cost(cost) + } + // ── Chunk operations (MMR) ─────────────────────────────────────── /// Get a single completed chunk's raw blob by chunk index. diff --git a/grovedb-bulk-append-tree/src/tree/mod.rs b/grovedb-bulk-append-tree/src/tree/mod.rs index 01130febe..33ccc794c 100644 --- a/grovedb-bulk-append-tree/src/tree/mod.rs +++ b/grovedb-bulk-append-tree/src/tree/mod.rs @@ -86,6 +86,21 @@ pub struct AppendNoStateRootResult { pub storage_accounting_cost: OperationCost, } +/// A contiguous page of entries returned by a position-range read. +/// +/// Produced by [`BulkAppendTree::get_range`], which fetches the entries for +/// `[start, start + limit)` clamped to the tree's total count. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RangePage { + /// `(global_position, value)` pairs, ascending and contiguous from the + /// requested start position (clamped to `total_count`). + pub entries: Vec<(u64, Vec)>, + /// Total number of entries in the tree at read time. Positions + /// `>= total_count` do not exist, so a page that ends before + /// `start + limit` is complete — there is nothing further to fetch. + pub total_count: u64, +} + /// Compute MMR size from leaf count: `2 * n - popcount(n)`. /// /// This is a well-known MMR property: the total number of nodes (leaves + diff --git a/grovedb-bulk-append-tree/src/tree/tests.rs b/grovedb-bulk-append-tree/src/tree/tests.rs index f2bb87c72..c2e932674 100644 --- a/grovedb-bulk-append-tree/src/tree/tests.rs +++ b/grovedb-bulk-append-tree/src/tree/tests.rs @@ -408,3 +408,223 @@ fn query_chunks_empty_indices_returns_empty_proof() { assert!(result.mmr_proof_items.is_empty()); assert_eq!(result.mmr_root, [0u8; 32]); } + +// ── get_range (paginated position-range reads) ─────────────────────── + +/// Helper: build a tree with `n` single-byte values `[0], [1], ...`. +fn build_range_tree(height: u8, n: u8) -> BulkAppendTree { + let mut tree = BulkAppendTree::new(height, MemStorageContext::new()).expect("create tree"); + for i in 0..n { + tree.append(&[i], GroveVersion::latest()).expect("append"); + } + tree +} + +/// Helper: assert a page holds exactly positions `start..end` with value +/// `[pos as u8]` at each. +fn assert_page(page: &super::RangePage, start: u64, end: u64, total_count: u64) { + assert_eq!(page.total_count, total_count); + assert_eq!(page.entries.len(), (end - start) as usize); + for (i, (pos, value)) in page.entries.iter().enumerate() { + assert_eq!(*pos, start + i as u64); + assert_eq!(value, &vec![*pos as u8]); + } +} + +#[test] +fn get_range_buffer_only() { + // height=3, capacity=7: 5 values all in buffer + let tree = build_range_tree(3, 5); + assert_eq!(tree.chunk_count(), 0); + + let page = tree.get_range(1, 3).unwrap().expect("get range"); + assert_page(&page, 1, 4, 5); +} + +#[test] +fn get_range_single_chunk() { + // height=2, epoch_size=4: 8 values = 2 full chunks + let tree = build_range_tree(2, 8); + assert_eq!(tree.chunk_count(), 2); + assert_eq!(tree.buffer_count(), 0); + + // Page entirely inside chunk 0 + let page = tree.get_range(1, 2).unwrap().expect("get range"); + assert_page(&page, 1, 3, 8); +} + +#[test] +fn get_range_across_chunk_boundary() { + // height=2, epoch_size=4: 10 values = 2 chunks + 2 buffered + let tree = build_range_tree(2, 10); + assert_eq!(tree.chunk_count(), 2); + assert_eq!(tree.buffer_count(), 2); + + // Page [3, 6) spans the chunk 0 / chunk 1 boundary + let page = tree.get_range(3, 3).unwrap().expect("get range"); + assert_page(&page, 3, 6, 10); + + // Page [6, 10) spans the chunk 1 / buffer boundary + let page = tree.get_range(6, 4).unwrap().expect("get range"); + assert_page(&page, 6, 10, 10); +} + +#[test] +fn get_range_whole_tree() { + let tree = build_range_tree(2, 10); + let page = tree.get_range(0, 100).unwrap().expect("get range"); + assert_page(&page, 0, 10, 10); +} + +#[test] +fn get_range_empty_limit() { + let tree = build_range_tree(2, 10); + let page = tree.get_range(3, 0).unwrap().expect("get range"); + assert_page(&page, 3, 3, 10); +} + +#[test] +fn get_range_past_end() { + let tree = build_range_tree(2, 10); + + // Start exactly at total_count + let page = tree.get_range(10, 5).unwrap().expect("get range"); + assert!(page.entries.is_empty()); + assert_eq!(page.total_count, 10); + + // Start far past total_count + let page = tree.get_range(1000, 5).unwrap().expect("get range"); + assert!(page.entries.is_empty()); + assert_eq!(page.total_count, 10); + + // Range that starts inside but extends past the end is clamped + let page = tree.get_range(8, 100).unwrap().expect("get range"); + assert_page(&page, 8, 10, 10); +} + +#[test] +fn get_range_single_entry() { + let tree = build_range_tree(2, 10); + let page = tree.get_range(7, 1).unwrap().expect("get range"); + assert_page(&page, 7, 8, 10); +} + +#[test] +fn get_range_empty_tree() { + let tree = build_range_tree(2, 0); + let page = tree.get_range(0, 10).unwrap().expect("get range"); + assert!(page.entries.is_empty()); + assert_eq!(page.total_count, 0); +} + +#[test] +fn get_range_start_saturating_overflow() { + let tree = build_range_tree(2, 10); + let page = tree + .get_range(u64::MAX, u16::MAX) + .unwrap() + .expect("get range"); + assert!(page.entries.is_empty()); + assert_eq!(page.total_count, 10); +} + +#[test] +fn get_range_paged_scan_covers_everything() { + // The scanning pattern: walk the whole tree in pages of 3 and check the + // concatenation matches per-position reads. + let tree = build_range_tree(2, 11); // 2 chunks + 3 buffered + let mut cursor = 0u64; + let mut seen = Vec::new(); + loop { + let page = tree.get_range(cursor, 3).unwrap().expect("get range"); + if page.entries.is_empty() { + assert!(cursor >= page.total_count, "empty page only at the end"); + break; + } + cursor += page.entries.len() as u64; + seen.extend(page.entries); + } + assert_eq!(seen.len(), 11); + for (i, (pos, value)) in seen.iter().enumerate() { + assert_eq!(*pos, i as u64); + assert_eq!(value, &vec![i as u8]); + } +} + +#[test] +fn get_range_missing_chunk_is_corruption() { + // Storage claims 2 completed chunks (via from_state) but holds no data: + // the chunk MMR leaf lookup comes back empty and the read must surface + // corruption, not silently skip entries. + let tree = BulkAppendTree::from_state(4, 1, MemStorageContext::new()).expect("from_state"); + assert_eq!(tree.chunk_count(), 2); + let err = tree + .get_range(0, 4) + .unwrap() + .expect_err("missing chunk blob must error"); + assert!(matches!(err, crate::BulkAppendError::CorruptedData(_))); +} + +#[test] +fn get_range_missing_buffer_value_is_corruption() { + // Storage claims 1 buffered entry (via from_state) but holds no data: + // the buffer read must surface an error, not silently skip entries. + let tree = BulkAppendTree::from_state(1, 2, MemStorageContext::new()).expect("from_state"); + assert_eq!(tree.buffer_count(), 1); + tree.get_range(0, 1) + .unwrap() + .expect_err("missing buffer value must error"); +} + +#[test] +fn get_range_storage_read_failure_is_mmr_error() { + // A backing store that fails reads must surface as an MMR error from the + // chunk lookup, not a panic or a silent empty page. + let ctx = MemStorageContext::new(); + ctx.fail_get.set(true); + let tree = BulkAppendTree::from_state(4, 1, ctx).expect("from_state"); + assert_eq!(tree.chunk_count(), 2); + let err = tree + .get_range(0, 4) + .unwrap() + .expect_err("failing storage must error"); + assert!(matches!(err, crate::BulkAppendError::MmrError(_))); +} + +#[test] +fn get_range_wrong_chunk_entry_count_is_corruption() { + // A completed chunk must hold exactly epoch_size entries: a short blob + // would silently omit positions and an oversized one would overlap the + // next chunk. Tamper the MMR overlay so chunk 0's blob deserializes to + // the wrong entry count and verify the read rejects it. + for bad_count in [1usize, 3] { + // epoch_size = 2: append 2 values to complete one genuine chunk. + let mut tree = BulkAppendTree::new(1u8, MemStorageContext::new()).expect("create tree"); + tree.append(&[0], GroveVersion::latest()).expect("append"); + tree.append(&[1], GroveVersion::latest()).expect("append"); + assert_eq!(tree.chunk_count(), 1); + + let bad_blob = + crate::serialize_chunk_blob(&(0..bad_count).map(|i| vec![i as u8]).collect::>()) + .expect("serialize bad blob"); + tree.mmr_overlay = vec![( + 0, + vec![grovedb_merkle_mountain_range::MmrNode::leaf(bad_blob)], + )]; + + let err = tree + .get_range(0, 2) + .unwrap() + .expect_err("wrong chunk entry count must error"); + match err { + crate::BulkAppendError::CorruptedData(msg) => { + assert!( + msg.contains(&format!("holds {} entries, expected 2", bad_count)), + "unexpected message: {}", + msg + ); + } + other => panic!("expected CorruptedData, got {:?}", other), + } + } +} diff --git a/grovedb-commitment-tree/src/commitment_tree/mod.rs b/grovedb-commitment-tree/src/commitment_tree/mod.rs index 1079a3e2e..1f96fe680 100644 --- a/grovedb-commitment-tree/src/commitment_tree/mod.rs +++ b/grovedb-commitment-tree/src/commitment_tree/mod.rs @@ -11,7 +11,7 @@ use std::marker::PhantomData; -use grovedb_bulk_append_tree::BulkAppendTree; +use grovedb_bulk_append_tree::{BulkAppendTree, RangePage}; use grovedb_costs::{CostResult, CostsExt, OperationCost}; use grovedb_storage::StorageContext; use grovedb_version::version::GroveVersion; @@ -680,6 +680,20 @@ impl<'db, S: StorageContext<'db>, M: MemoSize> CommitmentTree { .map_err(|e| CommitmentTreeError::InvalidData(format!("chunk value: {}", e))) } + /// Fetch entries for the position range `[start, start + limit)`, + /// clamped to the tree's total count. + /// + /// This is the shielded-pool scanning read path: each returned value is + /// the raw `cmx || rho || cv_net || payload` bytes at that position. + /// Delegates to [`BulkAppendTree::get_range`], so the read is + /// chunk-aligned — O(chunks touched) blob reads, not O(entries) random + /// reads — and carries the page's storage costs. + pub fn get_range(&self, start: u64, limit: u16) -> CostResult { + self.bulk_tree + .get_range(start, limit) + .map(|r| r.map_err(|e| CommitmentTreeError::InvalidData(format!("range read: {}", e)))) + } + /// The number of entries per completed chunk (epoch). pub fn epoch_size(&self) -> u64 { self.bulk_tree.epoch_size() diff --git a/grovedb-commitment-tree/src/lib.rs b/grovedb-commitment-tree/src/lib.rs index 2792576b2..f94285c9c 100644 --- a/grovedb-commitment-tree/src/lib.rs +++ b/grovedb-commitment-tree/src/lib.rs @@ -71,7 +71,7 @@ pub use commitment_tree::{ pub use error::CommitmentTreeError; #[cfg(feature = "server")] pub use grovedb_bulk_append_tree::{ - deserialize_chunk_blob, serialize_chunk_blob, BulkAppendError, BulkAppendTree, + deserialize_chunk_blob, serialize_chunk_blob, BulkAppendError, BulkAppendTree, RangePage, }; pub use grovedb_costs::{self}; pub use incrementalmerkletree::{Hashable, Level, Position, Retention}; diff --git a/grovedb-version/src/version/grovedb_versions.rs b/grovedb-version/src/version/grovedb_versions.rs index d1c418637..11b10c676 100644 --- a/grovedb-version/src/version/grovedb_versions.rs +++ b/grovedb-version/src/version/grovedb_versions.rs @@ -222,6 +222,8 @@ pub struct GroveDBOperationsProofVersions { pub prove_trunk_chunk_non_serialized: FeatureVersion, pub prove_branch_chunk: FeatureVersion, pub prove_branch_chunk_non_serialized: FeatureVersion, + pub prove_bulk_position_range: FeatureVersion, + pub verify_bulk_position_range_proof: FeatureVersion, pub verify_query_with_options: FeatureVersion, pub verify_query_raw: FeatureVersion, pub verify_layer_proof: FeatureVersion, diff --git a/grovedb-version/src/version/v1.rs b/grovedb-version/src/version/v1.rs index 72bf4b887..6d8ecd6fc 100644 --- a/grovedb-version/src/version/v1.rs +++ b/grovedb-version/src/version/v1.rs @@ -168,6 +168,8 @@ pub const GROVE_V1: GroveVersion = GroveVersion { prove_trunk_chunk_non_serialized: 0, prove_branch_chunk: 0, prove_branch_chunk_non_serialized: 0, + prove_bulk_position_range: 0, + verify_bulk_position_range_proof: 0, verify_query_with_options: 0, verify_query_raw: 0, verify_layer_proof: 0, diff --git a/grovedb-version/src/version/v2.rs b/grovedb-version/src/version/v2.rs index 05372dc63..103f76d04 100644 --- a/grovedb-version/src/version/v2.rs +++ b/grovedb-version/src/version/v2.rs @@ -168,6 +168,8 @@ pub const GROVE_V2: GroveVersion = GroveVersion { prove_trunk_chunk_non_serialized: 0, prove_branch_chunk: 0, prove_branch_chunk_non_serialized: 0, + prove_bulk_position_range: 0, + verify_bulk_position_range_proof: 0, verify_query_with_options: 0, verify_query_raw: 0, verify_layer_proof: 0, diff --git a/grovedb-version/src/version/v3.rs b/grovedb-version/src/version/v3.rs index ad5721c31..969ca6134 100644 --- a/grovedb-version/src/version/v3.rs +++ b/grovedb-version/src/version/v3.rs @@ -172,6 +172,8 @@ pub const GROVE_V3: GroveVersion = GroveVersion { prove_trunk_chunk_non_serialized: 1, prove_branch_chunk: 0, prove_branch_chunk_non_serialized: 0, + prove_bulk_position_range: 0, + verify_bulk_position_range_proof: 0, verify_query_with_options: 0, verify_query_raw: 0, verify_layer_proof: 0, diff --git a/grovedb-version/src/version/v4.rs b/grovedb-version/src/version/v4.rs index e275ea3e3..409a184d2 100644 --- a/grovedb-version/src/version/v4.rs +++ b/grovedb-version/src/version/v4.rs @@ -315,6 +315,8 @@ pub const GROVE_V4: GroveVersion = GroveVersion { prove_trunk_chunk_non_serialized: 1, prove_branch_chunk: 0, prove_branch_chunk_non_serialized: 0, + prove_bulk_position_range: 0, + verify_bulk_position_range_proof: 0, verify_query_with_options: 0, verify_query_raw: 0, verify_layer_proof: 0, diff --git a/grovedb/Cargo.toml b/grovedb/Cargo.toml index a3b527f45..0f3af7ff3 100644 --- a/grovedb/Cargo.toml +++ b/grovedb/Cargo.toml @@ -81,6 +81,10 @@ harness = false name = "cidx_benchmark" harness = false +[[bench]] +name = "bulk_range_scan_benchmark" +harness = false + [features] default = ["full", "estimated_costs"] proof_debug = ["grovedb-merk/proof_debug"] diff --git a/grovedb/benches/bulk_range_scan_benchmark.rs b/grovedb/benches/bulk_range_scan_benchmark.rs new file mode 100644 index 000000000..0d3206852 --- /dev/null +++ b/grovedb/benches/bulk_range_scan_benchmark.rs @@ -0,0 +1,203 @@ +//! Benchmark for the BulkAppendTree paged-scan pattern. +//! +//! Clients of append-only stores walk "all entries since my cursor" in +//! pages. This benchmark measures that pattern end to end on a +//! BulkAppendTree populated with fixed-size entries: +//! +//! 1. **Paged read** (`bulk_get_range`): fetching one page of entries, +//! chunk-aligned, at various page sizes. +//! 2. **Paged proof generation** (`prove_bulk_position_range`): proving one +//! page. +//! 3. **Paged proof verification** (`verify_bulk_position_range_proof`): +//! verifying one page. +//! 4. **Full scan**: walking the entire tree page by page with proofs, the +//! shielded-pool-style sync flow. + +#[cfg(feature = "minimal")] +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; +#[cfg(feature = "minimal")] +use grovedb::{Element, GroveDb}; +#[cfg(feature = "minimal")] +use grovedb_version::version::GroveVersion; +#[cfg(feature = "minimal")] +use tempfile::TempDir; + +/// Total number of entries appended to the tree. +#[cfg(feature = "minimal")] +const N_ENTRIES: u32 = 4096; + +/// Chunk power of the tree: chunk size = 2^6 = 64 entries, so the tree holds +/// 64 completed chunks with the buffer empty. +#[cfg(feature = "minimal")] +const CHUNK_POWER: u8 = 6; + +/// Size of each entry in bytes (a 32-byte commitment plus a small payload). +#[cfg(feature = "minimal")] +const ENTRY_SIZE: usize = 96; + +#[cfg(feature = "minimal")] +fn setup_db() -> (TempDir, GroveDb) { + let grove_version = GroveVersion::latest(); + let dir = TempDir::new().expect("cannot create temp dir"); + let db = GroveDb::open(dir.path()).expect("cannot open grovedb"); + + db.insert( + &[] as &[&[u8]], + b"bulk", + Element::empty_bulk_append_tree(CHUNK_POWER).expect("valid chunk_power"), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert bulk append tree"); + + for i in 0..N_ENTRIES { + let mut entry = vec![0u8; ENTRY_SIZE]; + entry[..4].copy_from_slice(&i.to_be_bytes()); + db.bulk_append(&[] as &[&[u8]], b"bulk", entry, None, grove_version) + .unwrap() + .expect("bulk append"); + } + + (dir, db) +} + +/// Read one page of entries at various page sizes, starting mid-tree so the +/// page is not chunk-aligned. +#[cfg(feature = "minimal")] +pub fn paged_read(c: &mut Criterion) { + let grove_version = GroveVersion::latest(); + let (_dir, db) = setup_db(); + let mut group = c.benchmark_group("bulk_range_paged_read"); + + for &page_size in &[16u16, 256, 1024] { + group.bench_function(BenchmarkId::from_parameter(page_size), |b| { + b.iter(|| { + let page = db + .bulk_get_range( + &[] as &[&[u8]], + b"bulk", + (N_ENTRIES / 3) as u64, + page_size, + None, + grove_version, + ) + .unwrap() + .expect("bulk get range"); + assert_eq!(page.entries.len(), page_size as usize); + }); + }); + } + group.finish(); +} + +/// Prove one page at various page sizes. +#[cfg(feature = "minimal")] +pub fn paged_proof_generation(c: &mut Criterion) { + let grove_version = GroveVersion::latest(); + let (_dir, db) = setup_db(); + let mut group = c.benchmark_group("bulk_range_paged_prove"); + + for &page_size in &[16u16, 256, 1024] { + group.bench_function(BenchmarkId::from_parameter(page_size), |b| { + b.iter(|| { + let _proof = db + .prove_bulk_position_range( + vec![], + b"bulk", + (N_ENTRIES / 3) as u64, + page_size, + None, + grove_version, + ) + .unwrap() + .expect("prove bulk position range"); + }); + }); + } + group.finish(); +} + +/// Verify one page proof at various page sizes. +#[cfg(feature = "minimal")] +pub fn paged_proof_verification(c: &mut Criterion) { + let grove_version = GroveVersion::latest(); + let (_dir, db) = setup_db(); + let mut group = c.benchmark_group("bulk_range_paged_verify"); + + for &page_size in &[16u16, 256, 1024] { + let start = (N_ENTRIES / 3) as u64; + let proof = db + .prove_bulk_position_range(vec![], b"bulk", start, page_size, None, grove_version) + .unwrap() + .expect("prove bulk position range"); + + group.bench_function(BenchmarkId::from_parameter(page_size), |b| { + b.iter(|| { + let (_root, page) = GroveDb::verify_bulk_position_range_proof( + &proof, + vec![], + b"bulk", + start, + page_size, + grove_version, + ) + .expect("verify bulk position range proof"); + assert_eq!(page.entries.len(), page_size as usize); + }); + }); + } + group.finish(); +} + +/// Walk the entire tree with proved pages of 256 — the client sync flow. +#[cfg(feature = "minimal")] +pub fn full_paged_scan_with_proofs(c: &mut Criterion) { + let grove_version = GroveVersion::latest(); + let (_dir, db) = setup_db(); + const PAGE: u16 = 256; + + c.bench_function("bulk_range_full_scan_with_proofs", |b| { + b.iter(|| { + let mut cursor = 0u64; + let mut total = 0usize; + loop { + let proof = db + .prove_bulk_position_range(vec![], b"bulk", cursor, PAGE, None, grove_version) + .unwrap() + .expect("prove page"); + let (_root, page) = GroveDb::verify_bulk_position_range_proof( + &proof, + vec![], + b"bulk", + cursor, + PAGE, + grove_version, + ) + .expect("verify page"); + if page.entries.is_empty() { + break; + } + cursor += page.entries.len() as u64; + total += page.entries.len(); + } + assert_eq!(total, N_ENTRIES as usize); + }); + }); +} + +#[cfg(feature = "minimal")] +criterion_group!( + name = benches; + config = Criterion::default().sample_size(10); + targets = paged_read, + paged_proof_generation, + paged_proof_verification, + full_paged_scan_with_proofs +); +#[cfg(feature = "minimal")] +criterion_main!(benches); + +#[cfg(not(feature = "minimal"))] +fn main() {} diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index f6a9fb501..9f3509a6a 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -180,6 +180,8 @@ pub use element::aggregate_sum_query::{AggregateSumQueryOptions, AggregateSumQue pub use element::Element; #[cfg(any(feature = "minimal", feature = "verify"))] pub use element::ElementFlags; +#[cfg(any(feature = "minimal", feature = "verify"))] +pub use grovedb_bulk_append_tree::RangePage; #[cfg(feature = "minimal")] use grovedb_costs::cost_return_on_error_into; #[cfg(feature = "minimal")] diff --git a/grovedb/src/operations/bulk_append_tree.rs b/grovedb/src/operations/bulk_append_tree.rs index 847c17cb2..d8cc92945 100644 --- a/grovedb/src/operations/bulk_append_tree.rs +++ b/grovedb/src/operations/bulk_append_tree.rs @@ -6,7 +6,7 @@ use std::collections::HashMap; -use grovedb_bulk_append_tree::{deserialize_chunk_blob, BulkAppendTree}; +use grovedb_bulk_append_tree::{deserialize_chunk_blob, BulkAppendTree, RangePage}; use grovedb_costs::{ cost_return_on_error, cost_return_on_error_into, cost_return_on_error_no_add, CostResult, CostsExt, OperationCost, @@ -269,6 +269,73 @@ impl GroveDb { } } + /// Fetch entries for the position range `[start, start + limit)` from a + /// BulkAppendTree, clamped to its total count. + /// + /// This is the paginated-scan read path: clients walking "all entries + /// since my cursor" call it with their cursor as `start` and advance by + /// `entries.len()`. The read is chunk-aligned — each completed chunk + /// overlapping the range is read and deserialized exactly once, so a + /// page costs O(chunks touched) blob reads plus one read per buffer + /// entry, not O(entries) random reads. + /// + /// The returned [`RangePage`] also carries the tree's `total_count`: + /// positions `>= total_count` do not exist, so a page shorter than + /// `limit` means the end of the tree was reached. + pub fn bulk_get_range<'b, B, P>( + &self, + path: P, + key: &[u8], + start: u64, + limit: u16, + transaction: TransactionArg, + grove_version: &GroveVersion, + ) -> CostResult + where + B: AsRef<[u8]> + 'b, + P: Into>, + { + let path: SubtreePath = path.into(); + let mut cost = OperationCost::default(); + let tx = TxRef::new(&self.db, transaction); + + let element = cost_return_on_error!( + &mut cost, + self.get_raw_caching_optional(path.clone(), key, true, transaction, grove_version) + ); + + // Look through NonCounted: a wrapped BulkAppendTree is still one. + let (total_count, chunk_power) = match element.underlying() { + Element::BulkAppendTree(tc, cp, _) => (*tc, *cp), + _ => { + return Err(Error::InvalidInput("element is not a BulkAppendTree")) + .wrap_with_cost(cost); + } + }; + + let subtree_path_vec = crate::util::subtree_path_with_key(&path, key); + let subtree_path_refs: Vec<&[u8]> = subtree_path_vec.iter().map(|v| v.as_slice()).collect(); + let subtree_path = SubtreePath::from(subtree_path_refs.as_slice()); + + let storage_ctx = self + .db + .get_transactional_storage_context(subtree_path, None, tx.as_ref()) + .unwrap_add_cost(&mut cost); + + let tree = cost_return_on_error_no_add!( + cost, + BulkAppendTree::from_state(total_count, chunk_power, storage_ctx).map_err(map_bulk_err) + ); + + let page = cost_return_on_error!( + &mut cost, + tree.get_range(start, limit) + .map(|r| r.map_err(map_bulk_err)) + ); + + Ok(page).wrap_with_cost(cost) + } + /// Get a completed chunk blob from a BulkAppendTree. /// /// Returns the raw serialized blob (length-prefixed entries) for the given diff --git a/grovedb/src/operations/commitment_tree.rs b/grovedb/src/operations/commitment_tree.rs index 815166050..d5434f447 100644 --- a/grovedb/src/operations/commitment_tree.rs +++ b/grovedb/src/operations/commitment_tree.rs @@ -17,7 +17,7 @@ use std::collections::HashMap; use grovedb_commitment_tree::{ deserialize_chunk_blob, serialize_ciphertext, Anchor, CommitmentTree, DashMemo, MemoSize, - TransmittedNoteCiphertext, + RangePage, TransmittedNoteCiphertext, }; use grovedb_costs::{ cost_return_on_error, cost_return_on_error_into, cost_return_on_error_no_add, CostResult, @@ -402,6 +402,73 @@ impl GroveDb { } } + /// Fetch entries for the position range `[start, start + limit)` from a + /// CommitmentTree, clamped to its total count. + /// + /// This is the shielded-pool scanning read path: clients walk "all notes + /// since my cursor" in pages, trial-decrypting each returned + /// `cmx || rho || cv_net || payload` value. The read is chunk-aligned — + /// each completed chunk overlapping the range is read and deserialized + /// exactly once, so a page costs O(chunks touched) blob reads plus one + /// read per buffer entry, not O(entries) random reads. + /// + /// The returned [`RangePage`] also carries the tree's `total_count`: + /// positions `>= total_count` do not exist, so a page shorter than + /// `limit` means the scan caught up with the tip. + pub fn commitment_tree_get_range<'b, B, P>( + &self, + path: P, + key: &[u8], + start: u64, + limit: u16, + transaction: TransactionArg, + grove_version: &GroveVersion, + ) -> CostResult + where + B: AsRef<[u8]> + 'b, + P: Into>, + { + let path: SubtreePath = path.into(); + let mut cost = OperationCost::default(); + let tx = TxRef::new(&self.db, transaction); + + let element = cost_return_on_error!( + &mut cost, + self.get_raw_caching_optional(path.clone(), key, true, transaction, grove_version) + ); + + // Look through NonCounted: a wrapped CommitmentTree is still one. + let (total_count, chunk_power) = match element.underlying() { + Element::CommitmentTree(tc, cp, _) => (*tc, *cp), + _ => { + return Err(Error::InvalidInput("element is not a commitment tree")) + .wrap_with_cost(cost); + } + }; + + let ct_path_vec = crate::util::subtree_path_with_key(&path, key); + let ct_path_refs: Vec<&[u8]> = ct_path_vec.iter().map(|v| v.as_slice()).collect(); + let ct_path = SubtreePath::from(ct_path_refs.as_slice()); + + let storage_ctx = self + .db + .get_transactional_storage_context(ct_path, None, tx.as_ref()) + .unwrap_add_cost(&mut cost); + + let ct = cost_return_on_error!( + &mut cost, + CommitmentTree::<_, DashMemo>::open(total_count, chunk_power, storage_ctx) + .map(|r| r.map_err(map_ct_err)) + ); + + let page = cost_return_on_error!( + &mut cost, + ct.get_range(start, limit).map(|r| r.map_err(map_ct_err)) + ); + + Ok(page).wrap_with_cost(cost) + } + /// Get the total count of items in a CommitmentTree. pub fn commitment_tree_count<'b, B, P>( &self, diff --git a/grovedb/src/operations/proof/generate.rs b/grovedb/src/operations/proof/generate.rs index 24bcf6a6e..beddd417a 100644 --- a/grovedb/src/operations/proof/generate.rs +++ b/grovedb/src/operations/proof/generate.rs @@ -283,6 +283,70 @@ impl GroveDb { } } + /// Prove the paginated position range `[start, start + limit)` of the + /// append-only, BulkAppendTree-backed element at `path`/`key` + /// (`Element::BulkAppendTree` or `Element::CommitmentTree`). + /// + /// This is the scanning hot path: clients walking "all entries since my + /// cursor" prove one page per call, passing their cursor as `start`. The + /// proof reuses the existing V1 layering (`ProofBytes::BulkAppendTree` / + /// `ProofBytes::CommitmentTree`) over the canonical + /// [`PathQuery::new_bulk_position_range`] query, so it is chunk-aligned: + /// it carries each completed chunk blob overlapping the range plus the + /// in-range buffer entries — O(chunks touched), not O(entries). It also + /// binds the element itself, whose authenticated `total_count` makes + /// absence beyond the end provable (`position >= total_count`), so + /// ranges past the end need no per-position absence proofs. + /// + /// Verify with [`GroveDb::verify_bulk_position_range_proof`], which + /// derives the same canonical query from `(path, key, start, limit)`. + pub fn prove_bulk_position_range( + &self, + path: Vec>, + key: &[u8], + start: u64, + limit: u16, + prove_options: Option, + grove_version: &GroveVersion, + ) -> CostResult, Error> { + check_grovedb_v0_with_cost!( + "prove_bulk_position_range", + grove_version + .grovedb_versions + .operations + .proof + .prove_bulk_position_range + ); + let mut cost = OperationCost::default(); + + // Fail fast with a clear error when the target is not an append-only + // store — a generic proof for some other element type would only be + // rejected later, at verification time. + let path_refs: Vec<&[u8]> = path.iter().map(|p| p.as_slice()).collect(); + let subtree_path = grovedb_path::SubtreePath::from(path_refs.as_slice()); + let element = cost_return_on_error!( + &mut cost, + self.get_raw_caching_optional(subtree_path, key, true, None, grove_version) + ); + match element.underlying() { + Element::BulkAppendTree(..) | Element::CommitmentTree(..) => {} + _ => { + return Err(Error::InvalidInput( + "prove_bulk_position_range requires a BulkAppendTree or CommitmentTree \ + element", + )) + .wrap_with_cost(cost); + } + } + + let path_query = PathQuery::new_bulk_position_range(path, key.to_vec(), start, limit); + let proof = cost_return_on_error!( + &mut cost, + self.prove_query(&path_query, prove_options, grove_version) + ); + Ok(proof).wrap_with_cost(cost) + } + /// Helper for the top-level count-offset gate in /// `prove_query_non_serialized_v{0,1}`. Opens the merk at /// `path_query.path` and confirms its `tree_type` is one of the diff --git a/grovedb/src/operations/proof/verify.rs b/grovedb/src/operations/proof/verify.rs index c2afb9f40..fd042e02a 100644 --- a/grovedb/src/operations/proof/verify.rs +++ b/grovedb/src/operations/proof/verify.rs @@ -2453,11 +2453,27 @@ impl GroveDb { .verify_and_compute_root(element_height, element_total_count) .map_err(|e| Error::InvalidProof(query.clone(), format!("{}", e)))?; + // An empty `BulkAppendTree` element chains through NULL_HASH, not the + // domain-tagged empty state root: insert commits the element with a + // NULL_HASH child hash (there is no bulk state until the first + // append), and `verify_grovedb`'s integrity walk mirrors that. This + // is sound because `verify_and_compute_root` above already rejected + // any proof carrying data for a zero-count tree. `CommitmentTree` is + // different — its insert commits `EMPTY_COMMITMENT_TREE_STATE_ROOT`, + // which folds in the *computed* empty bulk root, so the CT wrapper + // (our caller) keeps the computed value. + let child_hash = + if element_total_count == 0 && matches!(element, Element::BulkAppendTree(..)) { + NULL_HASH + } else { + bulk_state_root + }; + // Root only: the caller is binding the parent element and does not // report this layer's entries, so there is no query at this path to // extract a position range from. if !report_contents { - return Ok(bulk_state_root); + return Ok(child_hash); } // Get the query range from the path query to extract matching values @@ -2530,8 +2546,8 @@ impl GroveDb { } } - // Return computed state_root as child Merk hash - Ok(bulk_state_root) + // Return the derived child Merk hash (see the empty-tree note above) + Ok(child_hash) } /// Verify a CommitmentTree lower layer proof and add results. @@ -3389,6 +3405,152 @@ impl GroveDb { ) } + /// Verify a proof produced by [`GroveDb::prove_bulk_position_range`], + /// returning `(root_hash, page)` for the position range + /// `[start, start + limit)` of the append-only, BulkAppendTree-backed + /// element at `path`/`key` (`Element::BulkAppendTree` or + /// `Element::CommitmentTree`). + /// + /// The page's entries are the `(position, value)` pairs, ascending and + /// contiguous from `start`, clamped to the element's authenticated + /// `total_count` (returned in + /// [`RangePage::total_count`](grovedb_bulk_append_tree::RangePage::total_count)). + /// Completeness is enforced: a proof missing any requested position below + /// `total_count` is rejected. Absence beyond the end falls out of the + /// provable count — positions `>= total_count` do not exist — so a page + /// shorter than `limit` means the scan caught up with the tip; no + /// per-position absence proofs are involved. + /// + /// Both the entries and `total_count` are extracted from the same proof + /// bytes: the entries by verifying the canonical + /// [`PathQuery::new_bulk_position_range`] query, and `total_count` by + /// subset-verifying the element itself, whose serialized bytes are bound + /// to the parent Merk (and through it to `root_hash`). + pub fn verify_bulk_position_range_proof( + proof: &[u8], + path: Vec>, + key: &[u8], + start: u64, + limit: u16, + grove_version: &GroveVersion, + ) -> Result<(CryptoHash, grovedb_bulk_append_tree::RangePage), Error> { + check_grovedb_v0!( + "verify_bulk_position_range_proof", + grove_version + .grovedb_versions + .operations + .proof + .verify_bulk_position_range_proof + ); + // 1. Verify the range entries against the canonical range query. + // Succinctness cannot be required: range proofs are chunk-aligned + // and intentionally carry whole chunk blobs — a superset of the + // queried positions. + let range_query = + PathQuery::new_bulk_position_range(path.clone(), key.to_vec(), start, limit); + let (root_hash, results) = Self::verify_query_with_options( + proof, + &range_query, + VerifyOptions { + absence_proofs_for_non_existing_searched_keys: false, + verify_proof_succinctness: false, + include_empty_trees_in_result: false, + }, + grove_version, + )?; + + // 2. Extract the element's authenticated total_count from the same + // proof bytes via a single-key subset query on the element itself. + let element_query = PathQuery::new_single_key(path, key.to_vec()); + let (element_root_hash, element_results) = + Self::verify_subset_query(proof, &element_query, grove_version)?; + if element_root_hash != root_hash { + return Err(Error::InvalidProof( + range_query, + "range and element sub-proofs derived different root hashes".to_string(), + )); + } + let total_count = match element_results.as_slice() { + [(_, element_key, Some(element))] if element_key.as_slice() == key => { + match element.underlying() { + Element::BulkAppendTree(total_count, _, _) + | Element::CommitmentTree(total_count, _, _) => *total_count, + _ => { + return Err(Error::InvalidProof( + range_query, + "element at path/key is not a BulkAppendTree or CommitmentTree" + .to_string(), + )); + } + } + } + _ => { + return Err(Error::InvalidProof( + range_query, + "proof does not bind the append-only element at path/key".to_string(), + )); + } + }; + + // 3. Map the results to (position, value) entries and require the + // page to be exactly [start, min(start + limit, total_count)) — + // ascending, contiguous, complete. The lower-layer verification + // already enforces completeness; this re-check keeps the helper's + // guarantee independent of that layer's internals. + let end = start + .saturating_add(limit as u64) + .min(total_count) + .max(start); + let mut entries = Vec::with_capacity((end - start) as usize); + let mut expected_position = start; + for (_, position_key, element) in results { + let position_bytes: [u8; 8] = position_key.as_slice().try_into().map_err(|_| { + Error::InvalidProof( + range_query.clone(), + "range entry key is not an 8-byte big-endian position".to_string(), + ) + })?; + let position = u64::from_be_bytes(position_bytes); + let value = match element { + Some(Element::Item(value, _)) => value, + _ => { + return Err(Error::InvalidProof( + range_query, + format!("range entry at position {} is not an item", position), + )); + } + }; + if position != expected_position { + return Err(Error::InvalidProof( + range_query, + format!( + "range entries not contiguous: expected position {}, got {}", + expected_position, position + ), + )); + } + expected_position += 1; + entries.push((position, value)); + } + if expected_position != end { + return Err(Error::InvalidProof( + range_query, + format!( + "range proof incomplete: expected positions [{}, {}), got up to {}", + start, end, expected_position + ), + )); + } + + Ok(( + root_hash, + grovedb_bulk_append_tree::RangePage { + entries, + total_count, + }, + )) + } + /// The point of this query is to get the parent tree information which will /// be present because we are querying in a subtree pub fn verify_query_get_parent_tree_info( diff --git a/grovedb/src/query/mod.rs b/grovedb/src/query/mod.rs index 0585ab8e1..20892e867 100644 --- a/grovedb/src/query/mod.rs +++ b/grovedb/src/query/mod.rs @@ -532,6 +532,49 @@ impl PathQuery { Self { path, query } } + /// Canonical `PathQuery` for a paginated position-range read of an + /// append-only, BulkAppendTree-backed element (`BulkAppendTree` or + /// `CommitmentTree`) at `path`/`key`. + /// + /// Selects the element at `key` and subqueries positions + /// `[start, start + limit)`, encoded as 8-byte big-endian keys + /// (`start + limit` saturates at `u64::MAX`). Prover and verifier both + /// derive this query from `(start, limit)`, so a scanning client only + /// needs its cursor and page size — see + /// [`GroveDb::prove_bulk_position_range`] and + /// [`GroveDb::verify_bulk_position_range_proof`]. + /// + /// [`GroveDb::prove_bulk_position_range`]: + /// crate::GroveDb::prove_bulk_position_range + /// [`GroveDb::verify_bulk_position_range_proof`]: + /// crate::GroveDb::verify_bulk_position_range_proof + pub fn new_bulk_position_range( + path: Vec>, + key: Vec, + start: u64, + limit: u16, + ) -> Self { + let position_query = grovedb_bulk_append_tree::position_range_query(start, limit); + Self { + path, + query: SizedQuery { + query: Query { + items: vec![QueryItem::Key(key)], + default_subquery_branch: SubqueryBranch { + subquery_path: None, + subquery: Some(position_query.into()), + }, + left_to_right: true, + conditional_subquery_branches: None, + add_parent_tree_on_subquery: false, + read_mode: None, + }, + limit: None, + offset: None, + }, + } + } + /// Construct a `PathQuery` for an aggregate-count-on-range query against /// the subtree at `path`. `range` is the inner `QueryItem` describing the /// keys to count over; see [`Query::new_aggregate_count_on_range`] for the diff --git a/grovedb/src/tests/bulk_append_tree_tests.rs b/grovedb/src/tests/bulk_append_tree_tests.rs index 7028e8e0d..bc27a1123 100644 --- a/grovedb/src/tests/bulk_append_tree_tests.rs +++ b/grovedb/src/tests/bulk_append_tree_tests.rs @@ -2125,3 +2125,533 @@ fn test_bulk_batch_multi_compaction_transaction_rollback() { "bulk tree should be empty after multi-compaction tx rollback" ); } + +// =========================================================================== +// Paginated position-range reads (get_range) and range proofs +// =========================================================================== + +/// Helper: create a DB with a BulkAppendTree at root key `b"bulk"` holding +/// `n` values `b"value_0"`, `b"value_1"`, ... (chunk_power = 2 → chunk size +/// 4). +fn make_bulk_db_with_values(n: u32) -> crate::tests::TempGroveDb { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + db.insert( + EMPTY_PATH, + b"bulk", + Element::empty_bulk_append_tree(TEST_CHUNK_POWER).expect("valid chunk_power"), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert bulk append tree"); + + for i in 0..n { + db.bulk_append( + EMPTY_PATH, + b"bulk", + format!("value_{}", i).into_bytes(), + None, + grove_version, + ) + .unwrap() + .expect("bulk append"); + } + + db +} + +/// Helper: assert a page holds exactly positions `start..end` with the +/// values written by [`make_bulk_db_with_values`]. +fn assert_bulk_page(page: &crate::RangePage, start: u64, end: u64, total_count: u64) { + assert_eq!(page.total_count, total_count, "page total_count"); + assert_eq!(page.entries.len(), (end - start) as usize, "page length"); + for (i, (pos, value)) in page.entries.iter().enumerate() { + assert_eq!(*pos, start + i as u64, "page position"); + assert_eq!(value, format!("value_{}", pos).as_bytes(), "page value"); + } +} + +#[test] +fn test_bulk_get_range_matches_get_value() { + let grove_version = GroveVersion::latest(); + // 11 values = 2 full chunks (8) + 3 buffered + let db = make_bulk_db_with_values(11); + + // Every possible page of size 4 must match per-position reads + for start in 0..12u64 { + let page = db + .bulk_get_range(EMPTY_PATH, b"bulk", start, 4, None, grove_version) + .unwrap() + .expect("bulk get range"); + assert_eq!(page.total_count, 11); + + let end = (start + 4).min(11); + let expected_len = end.saturating_sub(start.min(end)); + assert_eq!(page.entries.len(), expected_len as usize); + + for (pos, value) in &page.entries { + let expected = db + .bulk_get_value(EMPTY_PATH, b"bulk", *pos, None, grove_version) + .unwrap() + .expect("bulk get value") + .expect("value exists"); + assert_eq!(value, &expected); + } + } +} + +#[test] +fn test_bulk_get_range_empty_and_past_end() { + let grove_version = GroveVersion::latest(); + let db = make_bulk_db_with_values(10); + + // limit = 0 + let page = db + .bulk_get_range(EMPTY_PATH, b"bulk", 3, 0, None, grove_version) + .unwrap() + .expect("bulk get range"); + assert_bulk_page(&page, 3, 3, 10); + + // start exactly at total_count + let page = db + .bulk_get_range(EMPTY_PATH, b"bulk", 10, 5, None, grove_version) + .unwrap() + .expect("bulk get range"); + assert_bulk_page(&page, 10, 10, 10); + + // start far past total_count + let page = db + .bulk_get_range(EMPTY_PATH, b"bulk", 1000, 5, None, grove_version) + .unwrap() + .expect("bulk get range"); + assert!(page.entries.is_empty()); + assert_eq!(page.total_count, 10); + + // clamped at the end + let page = db + .bulk_get_range(EMPTY_PATH, b"bulk", 8, 100, None, grove_version) + .unwrap() + .expect("bulk get range"); + assert_bulk_page(&page, 8, 10, 10); +} + +#[test] +fn test_bulk_get_range_wrong_element_type() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + db.insert( + EMPTY_PATH, + b"normal", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert normal tree"); + + let result = db + .bulk_get_range(EMPTY_PATH, b"normal", 0, 4, None, grove_version) + .unwrap(); + assert!(matches!(result, Err(Error::InvalidInput(_)))); +} + +/// Helper: prove and verify a position range page, asserting the returned +/// root hash matches the database root hash, and return the page. +fn roundtrip_bulk_range_proof( + db: &crate::tests::TempGroveDb, + start: u64, + limit: u16, +) -> crate::RangePage { + let grove_version = GroveVersion::latest(); + + let proof = db + .prove_bulk_position_range(vec![], b"bulk", start, limit, None, grove_version) + .unwrap() + .expect("prove bulk position range"); + + let (root_hash, page) = crate::GroveDb::verify_bulk_position_range_proof( + &proof, + vec![], + b"bulk", + start, + limit, + grove_version, + ) + .expect("verify bulk position range proof"); + + let expected_root = db.root_hash(None, grove_version).unwrap().unwrap(); + assert_eq!(root_hash, expected_root, "proof root must match db root"); + + page +} + +#[test] +fn test_bulk_position_range_proof_across_chunk_boundary() { + // 10 values = 2 chunks (8) + 2 buffered, chunk size 4 + let db = make_bulk_db_with_values(10); + + // Page [3, 6) spans the chunk 0 / chunk 1 boundary + let page = roundtrip_bulk_range_proof(&db, 3, 3); + assert_bulk_page(&page, 3, 6, 10); + + // Page [6, 10) spans the chunk 1 / buffer boundary + let page = roundtrip_bulk_range_proof(&db, 6, 4); + assert_bulk_page(&page, 6, 10, 10); +} + +#[test] +fn test_bulk_position_range_proof_empty_range() { + let db = make_bulk_db_with_values(10); + let page = roundtrip_bulk_range_proof(&db, 3, 0); + assert_bulk_page(&page, 3, 3, 10); +} + +#[test] +fn test_bulk_position_range_proof_past_end() { + let db = make_bulk_db_with_values(10); + + // Start exactly at total_count: provably nothing there — the verified + // total_count is the absence proof. + let page = roundtrip_bulk_range_proof(&db, 10, 5); + assert!(page.entries.is_empty()); + assert_eq!(page.total_count, 10); + + // Start far past total_count + let page = roundtrip_bulk_range_proof(&db, 1000, 5); + assert!(page.entries.is_empty()); + assert_eq!(page.total_count, 10); + + // Range starting inside but extending past the end is clamped + let page = roundtrip_bulk_range_proof(&db, 8, 100); + assert_bulk_page(&page, 8, 10, 10); +} + +#[test] +fn test_bulk_position_range_proof_single_entry_pages() { + let db = make_bulk_db_with_values(10); + for pos in 0..10u64 { + let page = roundtrip_bulk_range_proof(&db, pos, 1); + assert_bulk_page(&page, pos, pos + 1, 10); + } +} + +#[test] +fn test_bulk_position_range_proof_large_multi_chunk_page() { + // 30 values = 7 full chunks (28) + 2 buffered, chunk size 4. + let db = make_bulk_db_with_values(30); + + // One page covering everything touches all chunks and the buffer + let page = roundtrip_bulk_range_proof(&db, 0, 30); + assert_bulk_page(&page, 0, 30, 30); + + // A large page crossing several chunk boundaries mid-tree + let page = roundtrip_bulk_range_proof(&db, 5, 20); + assert_bulk_page(&page, 5, 25, 30); +} + +#[test] +fn test_bulk_position_range_proof_empty_tree() { + let db = make_bulk_db_with_values(0); + let page = roundtrip_bulk_range_proof(&db, 0, 10); + assert!(page.entries.is_empty()); + assert_eq!(page.total_count, 0); +} + +#[test] +fn test_bulk_position_range_paged_scan() { + // The scanning pattern: walk the whole tree in proved pages of 7 + // (deliberately not aligned to the chunk size of 4). + let db = make_bulk_db_with_values(30); + + let mut cursor = 0u64; + let mut seen = Vec::new(); + loop { + let page = roundtrip_bulk_range_proof(&db, cursor, 7); + if page.entries.is_empty() { + assert!(cursor >= page.total_count, "empty page only at the end"); + break; + } + cursor += page.entries.len() as u64; + seen.extend(page.entries); + } + assert_eq!(seen.len(), 30); + for (i, (pos, value)) in seen.iter().enumerate() { + assert_eq!(*pos, i as u64); + assert_eq!(value, format!("value_{}", i).as_bytes()); + } +} + +#[test] +fn test_bulk_position_range_prove_wrong_element_type() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + db.insert( + EMPTY_PATH, + b"normal", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert normal tree"); + + let result = db + .prove_bulk_position_range(vec![], b"normal", 0, 4, None, grove_version) + .unwrap(); + assert!(matches!(result, Err(Error::InvalidInput(_)))); +} + +#[test] +fn test_bulk_position_range_proof_wrong_range_rejected() { + let grove_version = GroveVersion::latest(); + let db = make_bulk_db_with_values(10); + + // Proof generated for [0, 2) (inside chunk 0) must not verify a request + // for [0, 6), which also needs chunk 1. + let narrow_proof = db + .prove_bulk_position_range(vec![], b"bulk", 0, 2, None, grove_version) + .unwrap() + .expect("prove narrow range"); + + crate::GroveDb::verify_bulk_position_range_proof( + &narrow_proof, + vec![], + b"bulk", + 0, + 6, + grove_version, + ) + .expect_err("proof for a narrower range must be rejected"); + + // Intended semantics pin: a DISJOINT range within the SAME chunk + // verifies. The proof for [0, 2) carries the whole chunk 0 blob + // (positions 0..4) because proofs are chunk-aligned, and every entry in + // it is authenticated against the root — so a verifier asking [2, 4) + // legitimately gets those entries from the same bytes. + let (root_hash, page) = crate::GroveDb::verify_bulk_position_range_proof( + &narrow_proof, + vec![], + b"bulk", + 2, + 2, + grove_version, + ) + .expect("disjoint range within the proved chunk must verify"); + let expected_root = db.root_hash(None, grove_version).unwrap().unwrap(); + assert_eq!(root_hash, expected_root); + assert_bulk_page(&page, 2, 4, 10); +} + +#[test] +fn test_bulk_position_range_proof_deep_path() { + // Same flow but with the BulkAppendTree nested under a normal tree. + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + db.insert( + EMPTY_PATH, + b"deep", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert parent tree"); + + db.insert( + &[b"deep"], + b"bulk", + Element::empty_bulk_append_tree(TEST_CHUNK_POWER).expect("valid chunk_power"), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert bulk append tree"); + + for i in 0..10u32 { + db.bulk_append( + &[b"deep"], + b"bulk", + format!("value_{}", i).into_bytes(), + None, + grove_version, + ) + .unwrap() + .expect("bulk append"); + } + + let proof = db + .prove_bulk_position_range(vec![b"deep".to_vec()], b"bulk", 2, 5, None, grove_version) + .unwrap() + .expect("prove nested bulk position range"); + + let (root_hash, page) = crate::GroveDb::verify_bulk_position_range_proof( + &proof, + vec![b"deep".to_vec()], + b"bulk", + 2, + 5, + grove_version, + ) + .expect("verify nested bulk position range proof"); + + let expected_root = db.root_hash(None, grove_version).unwrap().unwrap(); + assert_eq!(root_hash, expected_root); + assert_bulk_page(&page, 2, 7, 10); +} + +#[test] +fn test_bulk_position_range_version_gates() { + let grove_version = GroveVersion::latest(); + let db = make_bulk_db_with_values(4); + let proof = db + .prove_bulk_position_range(vec![], b"bulk", 0, 4, None, grove_version) + .unwrap() + .expect("prove under latest version"); + + // Unknown prove version → VersionError before any work happens + let mut gated = grove_version.clone(); + gated + .grovedb_versions + .operations + .proof + .prove_bulk_position_range = 99; + let result = db + .prove_bulk_position_range(vec![], b"bulk", 0, 4, None, &gated) + .unwrap(); + assert!(matches!(result, Err(Error::VersionError(_)))); + + // Unknown verify version → VersionError before any work happens + let mut gated = grove_version.clone(); + gated + .grovedb_versions + .operations + .proof + .verify_bulk_position_range_proof = 99; + let result = + crate::GroveDb::verify_bulk_position_range_proof(&proof, vec![], b"bulk", 0, 4, &gated); + assert!(matches!(result, Err(Error::VersionError(_)))); +} + +/// A proof over a NORMAL tree holding items at 8-byte keys: the range +/// entries verify as plain Merk rows, but total_count extraction must +/// reject the element type instead of trusting a non-append-only element. +#[test] +fn test_bulk_position_range_verify_rejects_non_append_element() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + db.insert( + EMPTY_PATH, + b"plain", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert normal tree"); + for i in 0..4u64 { + db.insert( + &[b"plain"], + &i.to_be_bytes(), + Element::new_item(vec![i as u8]), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert item"); + } + + // Bypass prove_bulk_position_range's own element check by proving the + // canonical query directly, as a crafted prover would. + let query = crate::PathQuery::new_bulk_position_range(vec![], b"plain".to_vec(), 0, 4); + let proof = db + .prove_query(&query, None, grove_version) + .unwrap() + .expect("prove canonical query over normal tree"); + + let err = crate::GroveDb::verify_bulk_position_range_proof( + &proof, + vec![], + b"plain", + 0, + 4, + grove_version, + ) + .expect_err("normal tree element must be rejected"); + assert!(matches!(err, Error::InvalidProof(..))); +} + +/// A proof for a key that does not exist: the range query verifies (as an +/// absence), but the element subset query binds nothing, so the verifier +/// must refuse to invent a total_count. +#[test] +fn test_bulk_position_range_verify_rejects_unbound_element() { + let grove_version = GroveVersion::latest(); + let db = make_bulk_db_with_values(4); + + let query = crate::PathQuery::new_bulk_position_range(vec![], b"ghost".to_vec(), 0, 4); + let proof = db + .prove_query(&query, None, grove_version) + .unwrap() + .expect("prove canonical query for missing key"); + + let err = crate::GroveDb::verify_bulk_position_range_proof( + &proof, + vec![], + b"ghost", + 0, + 4, + grove_version, + ) + .expect_err("proof that binds no element must be rejected"); + assert!(matches!(err, Error::InvalidProof(..))); +} + +/// Range reads must charge their storage work: the page's chunk-MMR seeks +/// and buffer reads all show up in the returned cost, so cost limits +/// reflect the actual work instead of treating a 65k-entry page as free. +#[test] +fn test_bulk_get_range_reports_storage_costs() { + let grove_version = GroveVersion::latest(); + // 11 values = 2 full chunks + 3 buffered (chunk size 4) + let db = make_bulk_db_with_values(11); + + // A page spanning both chunks and the buffer + let result = db.bulk_get_range(EMPTY_PATH, b"bulk", 0, 11, None, grove_version); + let cost = result.cost.clone(); + let page = result.unwrap().expect("bulk get range"); + assert_eq!(page.entries.len(), 11); + assert!( + cost.storage_loaded_bytes > 0, + "range read must charge loaded bytes, got {:?}", + cost + ); + assert!( + cost.seek_count > 0, + "range read must charge seeks, got {:?}", + cost + ); + + // A wider page must not cost less than a narrower one + let narrow_cost = db + .bulk_get_range(EMPTY_PATH, b"bulk", 8, 1, None, grove_version) + .cost; + assert!( + cost.storage_loaded_bytes > narrow_cost.storage_loaded_bytes, + "an 11-entry page must load more bytes than a 1-entry page ({} vs {})", + cost.storage_loaded_bytes, + narrow_cost.storage_loaded_bytes + ); +} diff --git a/grovedb/src/tests/commitment_tree_tests.rs b/grovedb/src/tests/commitment_tree_tests.rs index c71c8f301..f93bd591b 100644 --- a/grovedb/src/tests/commitment_tree_tests.rs +++ b/grovedb/src/tests/commitment_tree_tests.rs @@ -2863,3 +2863,270 @@ fn test_commitment_tree_element_count_subset_query_against_note_fetch_proof() { other => panic!("expected CommitmentTree element, got {:?}", other), } } + +// =========================================================================== +// Paginated position-range reads (get_range) and range proofs +// =========================================================================== + +/// Helper: create a DB with a CommitmentTree at [b"root"]/b"pool" +/// (chunk_power = 2 → chunk size 4) holding `n` notes, returning the +/// expected `cmx || rho || cv_net || payload` bytes per position. +fn make_ct_db_with_notes(n: u8) -> (crate::tests::TempGroveDb, Vec>) { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + db.insert( + EMPTY_PATH, + b"root", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert root tree"); + + db.insert( + &[b"root"], + b"pool", + Element::empty_commitment_tree(2).expect("valid chunk_power"), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert commitment tree"); + + let mut expected_values = Vec::new(); + for i in 0..n { + let cmx = test_cmx(i); + let rho = test_rho(i); + let cv_net = test_cv_net(i); + let payload = serialize_ciphertext(&test_ciphertext(i)); + db.commitment_tree_insert( + &[b"root"], + b"pool", + cmx, + rho, + cv_net, + test_ciphertext(i), + None, + grove_version, + ) + .unwrap() + .expect("commitment tree insert"); + + let mut expected = Vec::with_capacity(32 + 32 + 32 + payload.len()); + expected.extend_from_slice(&cmx); + expected.extend_from_slice(&rho); + expected.extend_from_slice(&cv_net); + expected.extend_from_slice(&payload); + expected_values.push(expected); + } + + (db, expected_values) +} + +#[test] +fn test_commitment_tree_get_range_matches_get_value() { + let grove_version = GroveVersion::latest(); + // 10 notes = 2 full chunks (8) + 2 buffered, chunk size 4 + let (db, expected_values) = make_ct_db_with_notes(10); + + for start in 0..11u64 { + let page = db + .commitment_tree_get_range(&[b"root"], b"pool", start, 4, None, grove_version) + .unwrap() + .expect("commitment tree get range"); + assert_eq!(page.total_count, 10); + + let end = (start + 4).min(10); + let expected_len = end.saturating_sub(start.min(end)); + assert_eq!(page.entries.len(), expected_len as usize); + + for (i, (pos, value)) in page.entries.iter().enumerate() { + assert_eq!(*pos, start + i as u64); + assert_eq!(value, &expected_values[*pos as usize]); + + let per_position = db + .commitment_tree_get_value(&[b"root"], b"pool", *pos, None, grove_version) + .unwrap() + .expect("commitment tree get value") + .expect("value exists"); + assert_eq!(value, &per_position); + } + } +} + +#[test] +fn test_commitment_tree_get_range_wrong_element_type() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + db.insert( + EMPTY_PATH, + b"normal", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert normal tree"); + + let result = db + .commitment_tree_get_range(EMPTY_PATH, b"normal", 0, 4, None, grove_version) + .unwrap(); + assert!(matches!(result, Err(Error::InvalidInput(_)))); +} + +/// Helper: prove and verify a CommitmentTree position range page, asserting +/// the returned root hash matches the database root hash. +fn roundtrip_ct_range_proof( + db: &crate::tests::TempGroveDb, + start: u64, + limit: u16, +) -> crate::RangePage { + let grove_version = GroveVersion::latest(); + + let proof = db + .prove_bulk_position_range( + vec![b"root".to_vec()], + b"pool", + start, + limit, + None, + grove_version, + ) + .unwrap() + .expect("prove commitment tree position range"); + + let (root_hash, page) = GroveDb::verify_bulk_position_range_proof( + &proof, + vec![b"root".to_vec()], + b"pool", + start, + limit, + grove_version, + ) + .expect("verify commitment tree position range proof"); + + let expected_root = db.root_hash(None, grove_version).unwrap().unwrap(); + assert_eq!(root_hash, expected_root, "proof root must match db root"); + + page +} + +#[test] +fn test_commitment_tree_position_range_proof_across_chunk_boundary() { + let (db, expected_values) = make_ct_db_with_notes(10); + + // Page [3, 7) spans the chunk 0 / chunk 1 boundary + let page = roundtrip_ct_range_proof(&db, 3, 4); + assert_eq!(page.total_count, 10); + assert_eq!(page.entries.len(), 4); + for (i, (pos, value)) in page.entries.iter().enumerate() { + assert_eq!(*pos, 3 + i as u64); + assert_eq!(value, &expected_values[*pos as usize]); + } + + // Page [6, 10) spans the chunk 1 / buffer boundary + let page = roundtrip_ct_range_proof(&db, 6, 4); + assert_eq!(page.entries.len(), 4); + for (i, (pos, value)) in page.entries.iter().enumerate() { + assert_eq!(*pos, 6 + i as u64); + assert_eq!(value, &expected_values[*pos as usize]); + } +} + +#[test] +fn test_commitment_tree_position_range_proof_empty_and_past_end() { + let (db, _) = make_ct_db_with_notes(10); + + // Empty range + let page = roundtrip_ct_range_proof(&db, 3, 0); + assert!(page.entries.is_empty()); + assert_eq!(page.total_count, 10); + + // Past the end: the verified total_count is the absence proof for + // positions >= 10. + let page = roundtrip_ct_range_proof(&db, 10, 5); + assert!(page.entries.is_empty()); + assert_eq!(page.total_count, 10); + + // Clamped at the end + let page = roundtrip_ct_range_proof(&db, 8, 100); + assert_eq!(page.entries.len(), 2); + assert_eq!(page.total_count, 10); +} + +#[test] +fn test_commitment_tree_position_range_proof_single_entry_pages() { + let (db, expected_values) = make_ct_db_with_notes(6); + + for pos in 0..6u64 { + let page = roundtrip_ct_range_proof(&db, pos, 1); + assert_eq!(page.total_count, 6); + assert_eq!(page.entries.len(), 1); + assert_eq!(page.entries[0].0, pos); + assert_eq!(page.entries[0].1, expected_values[pos as usize]); + } +} + +#[test] +fn test_commitment_tree_position_range_paged_scan() { + // The shielded-pool scanning pattern: walk all notes since cursor 0 in + // proved pages of 3 (not aligned to the chunk size of 4). + let (db, expected_values) = make_ct_db_with_notes(10); + + let mut cursor = 0u64; + let mut seen = Vec::new(); + loop { + let page = roundtrip_ct_range_proof(&db, cursor, 3); + if page.entries.is_empty() { + assert!(cursor >= page.total_count, "empty page only at the end"); + break; + } + cursor += page.entries.len() as u64; + seen.extend(page.entries); + } + assert_eq!(seen.len(), 10); + for (i, (pos, value)) in seen.iter().enumerate() { + assert_eq!(*pos, i as u64); + assert_eq!(value, &expected_values[i]); + } +} + +#[test] +fn test_commitment_tree_position_range_proof_empty_tree() { + let (db, _) = make_ct_db_with_notes(0); + let page = roundtrip_ct_range_proof(&db, 0, 10); + assert!(page.entries.is_empty()); + assert_eq!(page.total_count, 0); +} + +/// The CommitmentTree envelope (`ProofBytes::CommitmentTree`, sinsemilla +/// prefix + bulk proof) must reject a wider range than was proved, same as +/// the plain BulkAppendTree envelope. +#[test] +fn test_commitment_tree_position_range_proof_wrong_range_rejected() { + let grove_version = GroveVersion::latest(); + let (db, _) = make_ct_db_with_notes(10); + + // Proof generated for [0, 2) (inside chunk 0) must not verify a request + // for [0, 6), which also needs chunk 1. + let narrow_proof = db + .prove_bulk_position_range(vec![b"root".to_vec()], b"pool", 0, 2, None, grove_version) + .unwrap() + .expect("prove narrow range"); + + GroveDb::verify_bulk_position_range_proof( + &narrow_proof, + vec![b"root".to_vec()], + b"pool", + 0, + 6, + grove_version, + ) + .expect_err("proof for a narrower range must be rejected"); +}