feat: paginated position-range reads with proofs at the BulkAppendTree layer

grovedb-bulk-append-tree/src/lib.rs

@@ -22,7 +22,7 @@ pub use error::BulkAppendError;
 pub use grovedb_dense_fixed_sized_merkle_tree::{DenseFixedSizedMerkleTree, DenseTreeProof};
 #[cfg(feature = "storage")]
 pub use grovedb_merkle_mountain_range::{MmrKeySize, MmrStore};
-pub use proof::{BulkAppendTreeProof, BulkAppendTreeProofResult};
-pub use tree::{hash::compute_state_root, leaf_count_to_mmr_size, BulkAppendTree};
+pub use proof::{position_range_query, BulkAppendTreeProof, BulkAppendTreeProofResult};
+pub use tree::{hash::compute_state_root, leaf_count_to_mmr_size, BulkAppendTree, RangePage};
 #[cfg(feature = "storage")]
 pub use tree::{AppendResult, BufferQueryResult, ChunkQueryResult};

grovedb-bulk-append-tree/src/proof/mod.rs

@@ -176,6 +176,25 @@ fn query_to_ranges(query: &Query, total_count: u64) -> Result<Vec<(u64, u64)>, B
     Ok(merged)
 }
 
+/// Build the canonical [`Query`] selecting the position range
+/// `[start, start + limit)`, with positions encoded as 8-byte big-endian
+/// keys.
+///
+/// This is the query shape used by the paginated-scan pattern: prover and
+/// verifier both derive it from `(start, limit)`, so a client only needs its
+/// cursor and page size. `start + limit` saturates at `u64::MAX`, and
+/// verification clamps the range to the tree's provable total count.
+pub fn position_range_query(start: u64, limit: u16) -> Query {
+    let end = start.saturating_add(limit as u64);
+    Query {
+        items: vec![QueryItem::Range(
+            start.to_be_bytes().to_vec()..end.to_be_bytes().to_vec(),
+        )],
+        left_to_right: true,
+        ..Query::default()
+    }
+}
+
 /// Check whether `pos` falls inside any of the sorted, non-overlapping ranges.
 fn in_ranges(pos: u64, ranges: &[(u64, u64)]) -> bool {
     ranges
@@ -325,6 +344,54 @@ impl BulkAppendTreeProof {
         })
     }
 
+    /// Generate a proof for the paginated position range
+    /// `[start, start + limit)`.
+    ///
+    /// Convenience wrapper over [`generate`](Self::generate) using the
+    /// canonical [`position_range_query`]. The proof is chunk-aligned: it
+    /// carries each completed chunk blob overlapping the range plus the
+    /// buffer entries in range, so proof size is O(chunks touched).
+    ///
+    /// Ranges past the end of the tree are valid and produce a proof of the
+    /// (empty) result: absence of positions `>= total_count` falls out of
+    /// the authenticated element's total count, not out of per-position
+    /// absence proofs.
+    #[cfg(feature = "storage")]
+    pub fn generate_for_range<'db, S: StorageContext<'db>>(
+        tree: &BulkAppendTree<S>,
+        start: u64,
+        limit: u16,
+    ) -> Result<Self, BulkAppendError> {
+        Self::generate(&position_range_query(start, limit), tree)
+    }
+
+    /// Verify this proof against the paginated position range
+    /// `[start, start + limit)`.
+    ///
+    /// Convenience wrapper over
+    /// [`verify_against_query`](Self::verify_against_query) using the
+    /// canonical [`position_range_query`]. Returns the `(global_position,
+    /// value)` pairs in the range, ascending and contiguous, clamped to
+    /// `total_count`. Completeness is enforced: a proof missing any
+    /// requested position below `total_count` is rejected. Positions
+    /// `>= total_count` are provably absent by `total_count` itself, which
+    /// callers must take from the authenticated BulkAppendTree element.
+    pub fn verify_range(
+        &self,
+        expected_state_root: &[u8; 32],
+        height: u8,
+        total_count: u64,
+        start: u64,
+        limit: u16,
+    ) -> Result<Vec<(u64, Vec<u8>)>, BulkAppendError> {
+        self.verify_against_query(
+            expected_state_root,
+            height,
+            total_count,
+            &position_range_query(start, limit),
+        )
+    }
+
     /// Verify this proof against an expected state root.
     ///
     /// `height` and `total_count` come from the authenticated BulkAppendTree

grovedb-bulk-append-tree/src/proof/tests.rs

@@ -940,4 +940,170 @@ mod proof_tests {
             );
         }
     }
+
+    // ── generate_for_range / verify_range (paginated scan pattern) ───────
+
+    /// Helper: build a tree of `n` values "val_0".."val_{n-1}" and return
+    /// (state_root, tree).
+    fn build_indexed_tree(height: u8, n: u32) -> ([u8; 32], BulkAppendTree<MemStorageContext>) {
+        let values: Vec<Vec<u8>> = (0..n).map(|i| format!("val_{}", i).into_bytes()).collect();
+        build_test_tree(height, &values)
+    }
+
+    /// Helper: round-trip a range proof and assert the returned page is
+    /// exactly positions `expected_start..expected_end`.
+    fn assert_range_roundtrip(
+        state_root: &[u8; 32],
+        tree: &BulkAppendTree<MemStorageContext>,
+        start: u64,
+        limit: u16,
+        expected_start: u64,
+        expected_end: u64,
+    ) {
+        let proof =
+            BulkAppendTreeProof::generate_for_range(tree, start, limit).expect("generate range");
+
+        // Wire round-trip: encode + decode like a real client
+        let bytes = proof.encode_to_vec().expect("encode");
+        let decoded = BulkAppendTreeProof::decode_from_slice(&bytes).expect("decode");
+
+        let entries = decoded
+            .verify_range(state_root, tree.height(), tree.total_count, start, limit)
+            .expect("verify range");
+
+        assert_eq!(entries.len(), (expected_end - expected_start) as usize);
+        for (i, (pos, value)) in entries.iter().enumerate() {
+            assert_eq!(*pos, expected_start + i as u64);
+            assert_eq!(value, format!("val_{}", pos).as_bytes());
+        }
+    }
+
+    #[test]
+    fn test_range_roundtrip_buffer_only() {
+        // height=3, capacity=7: 5 values all in buffer
+        let (root, tree) = build_indexed_tree(3, 5);
+        assert_range_roundtrip(&root, &tree, 1, 3, 1, 4);
+    }
+
+    #[test]
+    fn test_range_roundtrip_across_chunk_boundary() {
+        // height=2, epoch_size=4: 10 values = 2 chunks + 2 buffered
+        let (root, tree) = build_indexed_tree(2, 10);
+        // spans chunk 0 / chunk 1
+        assert_range_roundtrip(&root, &tree, 3, 3, 3, 6);
+        // spans chunk 1 / buffer
+        assert_range_roundtrip(&root, &tree, 6, 4, 6, 10);
+    }
+
+    #[test]
+    fn test_range_roundtrip_single_entry_pages() {
+        let (root, tree) = build_indexed_tree(2, 10);
+        for pos in 0..10u64 {
+            assert_range_roundtrip(&root, &tree, pos, 1, pos, pos + 1);
+        }
+    }
+
+    #[test]
+    fn test_range_roundtrip_empty_range() {
+        let (root, tree) = build_indexed_tree(2, 10);
+        // limit 0: proof still verifies against the root, returns nothing
+        assert_range_roundtrip(&root, &tree, 3, 0, 3, 3);
+    }
+
+    #[test]
+    fn test_range_roundtrip_past_end() {
+        let (root, tree) = build_indexed_tree(2, 10);
+        // starts exactly at total_count
+        assert_range_roundtrip(&root, &tree, 10, 5, 10, 10);
+        // starts far past total_count
+        assert_range_roundtrip(&root, &tree, 1000, 5, 1000, 1000);
+        // clamped at the end
+        assert_range_roundtrip(&root, &tree, 8, 100, 8, 10);
+    }
+
+    #[test]
+    fn test_range_roundtrip_large_multi_chunk_page() {
+        // height=4, epoch_size=16: 100 values = 6 chunks + 4 buffered.
+        // One page covering everything touches all chunks and the buffer.
+        let (root, tree) = build_indexed_tree(4, 100);
+        assert_range_roundtrip(&root, &tree, 0, 100, 0, 100);
+        // A large page crossing several chunk boundaries mid-tree
+        assert_range_roundtrip(&root, &tree, 10, 70, 10, 80);
+    }
+
+    #[test]
+    fn test_range_roundtrip_empty_tree() {
+        let (_, tree) = build_indexed_tree(2, 0);
+        // For an empty tree the state root is blake3("bulk_state" || 0*32 || 0*32)
+        let root = crate::compute_state_root(&[0u8; 32], &[0u8; 32]);
+        assert_range_roundtrip(&root, &tree, 0, 10, 0, 0);
+    }
+
+    #[test]
+    fn test_range_paged_scan_covers_everything() {
+        // The client scan pattern: page through the whole tree with
+        // limit=7 (deliberately not aligned to epoch_size=4).
+        let (root, tree) = build_indexed_tree(2, 30);
+        let mut cursor = 0u64;
+        let mut seen = Vec::new();
+        while cursor < tree.total_count {
+            let proof =
+                BulkAppendTreeProof::generate_for_range(&tree, cursor, 7).expect("generate page");
+            let entries = proof
+                .verify_range(&root, tree.height(), tree.total_count, cursor, 7)
+                .expect("verify page");
+            assert!(!entries.is_empty());
+            cursor += entries.len() as u64;
+            seen.extend(entries);
+        }
+        assert_eq!(seen.len(), 30);
+        for (i, (pos, value)) in seen.iter().enumerate() {
+            assert_eq!(*pos, i as u64);
+            assert_eq!(value, format!("val_{}", i).as_bytes());
+        }
+    }
+
+    #[test]
+    fn test_range_proof_wrong_root_rejected() {
+        let (root, tree) = build_indexed_tree(2, 10);
+        let proof = BulkAppendTreeProof::generate_for_range(&tree, 0, 5).expect("generate");
+        let mut bad_root = root;
+        bad_root[0] ^= 1;
+        proof
+            .verify_range(&bad_root, tree.height(), tree.total_count, 0, 5)
+            .expect_err("tampered root must be rejected");
+    }
+
+    #[test]
+    fn test_range_proof_missing_chunk_rejected() {
+        // Proof generated for [0, 2) (chunk 0 only) must not verify a
+        // request for [0, 6) which also needs chunk 1.
+        let (root, tree) = build_indexed_tree(2, 10);
+        let narrow = BulkAppendTreeProof::generate_for_range(&tree, 0, 2).expect("generate");
+        narrow
+            .verify_range(&root, tree.height(), tree.total_count, 0, 6)
+            .expect_err("proof missing chunk 1 must be rejected for the wider range");
+    }
+
+    #[test]
+    fn test_position_range_query_shape() {
+        let q = super::super::position_range_query(5, 3);
+        assert_eq!(q.items.len(), 1);
+        match &q.items[0] {
+            QueryItem::Range(r) => {
+                assert_eq!(r.start, 5u64.to_be_bytes().to_vec());
+                assert_eq!(r.end, 8u64.to_be_bytes().to_vec());
+            }
+            other => panic!("expected Range item, got {:?}", other),
+        }
+
+        // start + limit saturates instead of wrapping
+        let q = super::super::position_range_query(u64::MAX - 1, 100);
+        match &q.items[0] {
+            QueryItem::Range(r) => {
+                assert_eq!(r.end, u64::MAX.to_be_bytes().to_vec());
+            }
+            other => panic!("expected Range item, got {:?}", other),
+        }
+    }
 }

grovedb-bulk-append-tree/src/test_utils.rs

@@ -16,6 +16,9 @@ use grovedb_storage::{Batch, RawIterator, StorageContext};
 #[derive(Default)]
 pub(crate) struct MemStorageContext {
     pub data: RefCell<HashMap<Vec<u8>, Vec<u8>>>,
+    /// When set, every `get` fails — simulates a broken backing store for
+    /// exercising storage-error paths.
+    pub fail_gets: std::cell::Cell<bool>,
 }
 
 impl MemStorageContext {
@@ -29,6 +32,12 @@ impl<'db> StorageContext<'db> for MemStorageContext {
     type RawIterator = MemRawIterator;
 
     fn get<K: AsRef<[u8]>>(&self, key: K) -> CostResult<Option<Vec<u8>>, grovedb_storage::Error> {
+        if self.fail_gets.get() {
+            return Err(grovedb_storage::Error::StorageError(
+                "simulated read failure".to_string(),
+            ))
+            .wrap_with_cost(OperationCost::default());
+        }
         Ok(self.data.borrow().get(key.as_ref()).cloned()).wrap_with_cost(OperationCost::default())
     }
 

grovedb-bulk-append-tree/src/tree/fetch.rs

@@ -5,7 +5,7 @@ use grovedb_merkle_mountain_range::{leaf_to_pos, MmrKeySize, MmrStore, MMR};
 use grovedb_query::Query;
 use grovedb_storage::StorageContext;
 
-use super::BulkAppendTree;
+use super::{BulkAppendTree, RangePage};
 use crate::{chunk::deserialize_chunk_blob, BulkAppendError};
 
 /// Result of querying the dense tree buffer.
@@ -63,6 +63,90 @@ impl<'db, S: StorageContext<'db>> BulkAppendTree<S> {
         Ok(BufferQueryResult { entries, proof })
     }
 
+    // ── Range operations (chunks + buffer) ───────────────────────────
+
+    /// Fetch entries for the position range `[start, start + limit)`,
+    /// clamped to the tree's total count.
+    ///
+    /// This is the paginated-scan read path: clients walking "all entries
+    /// since my cursor" call it with their cursor as `start` and advance by
+    /// `entries.len()`. The read is chunk-aligned — each completed chunk
+    /// overlapping the range is read and deserialized exactly once, so a
+    /// page costs O(chunks touched) blob reads plus one read per buffer
+    /// entry, not O(entries) random reads.
+    ///
+    /// Absence needs no lookup: positions `>= total_count` do not exist, so
+    /// a page shorter than `limit` means the end of the tree was reached.
+    pub fn get_range(&self, start: u64, limit: u16) -> Result<RangePage, BulkAppendError> {
+        let total_count = self.total_count;
+        let end = start.saturating_add(limit as u64).min(total_count);
+        if start >= end {
+            return Ok(RangePage {
+                entries: Vec::new(),
+                total_count,
+            });
+        }
+        let mut entries = Vec::with_capacity((end - start) as usize);
+
+        let epoch_size = self.epoch_size();
+        let buffer_start = self.chunk_count() * epoch_size;
+
+        // Completed chunks overlapping [start, min(end, buffer_start)).
+        // The MMR (with its overlay clone) is built once and reused for every
+        // chunk in the page — going through `get_chunk_value` would rebuild
+        // it, and re-clone the overlay, per chunk.
+        let chunk_end = end.min(buffer_start);
+        if start < chunk_end {
+            let first_chunk = start / epoch_size;
+            let last_chunk = (chunk_end - 1) / epoch_size;
+            let mmr_store = MmrStore::with_key_size(&self.dense_tree.storage, MmrKeySize::U32);
+            let mmr = MMR::new_with_overlay(self.mmr_size(), &mmr_store, self.mmr_overlay.clone());
+            for chunk_idx in first_chunk..=last_chunk {
+                let node = mmr
+                    .batch
+                    .element_at_position(leaf_to_pos(chunk_idx))
+                    .unwrap()
+                    .map_err(|e| {
+                        BulkAppendError::MmrError(format!(
+                            "failed to read MMR node for chunk {}: {}",
+                            chunk_idx, e
+                        ))
+                    })?;
+                let blob = node.and_then(|n| n.into_value()).ok_or_else(|| {
+                    BulkAppendError::CorruptedData(format!(
+                        "missing chunk blob for index {}",
+                        chunk_idx
+                    ))
+                })?;
+                let chunk_entries = deserialize_chunk_blob(&blob)?;
+                let chunk_start = chunk_idx * epoch_size;
+                for (i, value) in chunk_entries.into_iter().enumerate() {
+                    let pos = chunk_start + i as u64;
+                    if pos >= start && pos < chunk_end {
+                        entries.push((pos, value));
+                    }
+                }
+            }
+        }
+
+        // Buffer tail: positions in [max(start, buffer_start), end)
+        for pos in start.max(buffer_start)..end {
+            let buffer_pos = (pos - buffer_start) as u16;
+            let value = self.get_buffer_value(buffer_pos)?.ok_or_else(|| {
+                BulkAppendError::CorruptedData(format!(
+                    "missing buffer value at position {}",
+                    buffer_pos
+                ))
+            })?;
+            entries.push((pos, value));
+        }
+
+        Ok(RangePage {
+            entries,
+            total_count,
+        })
+    }
+
     // ── Chunk operations (MMR) ───────────────────────────────────────
 
     /// Get a single completed chunk's raw blob by chunk index.