feat: paginated position-range reads with proofs at the BulkAppendTree layer

docs/book/src/bulk-append-tree.md

@@ -245,7 +245,7 @@ The buffer IS a `DenseFixedSizedMerkleTree` — its root hash is `dense_tree_roo
 
 ## GroveDB Operations
 
-The BulkAppendTree integrates with GroveDB through six operations defined in
+The BulkAppendTree integrates with GroveDB through seven operations defined in
 `grovedb/src/operations/bulk_append_tree.rs`:
 
 ### bulk_append
@@ -272,6 +272,7 @@ append operation are added to `cost.hash_node_calls`.
 |---|---|---|
 | `bulk_get_value(path, key, position)` | Value at global position | Yes — reads from chunk blob or buffer |
 | `bulk_get_chunk(path, key, chunk_index)` | Raw chunk blob | Yes — reads chunk key |
+| `bulk_get_range(path, key, start, limit)` | `RangePage { entries, total_count }` — positions `[start, start+limit)` clipped to the tree | Yes — each overlapping chunk blob read once, plus one read per buffer entry |
 | `bulk_get_buffer(path, key)` | All current buffer entries | Yes — reads buffer keys |
 | `bulk_count(path, key)` | Total count (u64) | No — reads from element |
 | `bulk_chunk_count(path, key)` | Completed chunks (u64) | No — computed from element |
@@ -427,6 +428,18 @@ After verification succeeds, the `BulkAppendTreeProofResult` provides a
 `values_in_range(start, end)` method that extracts specific values from the verified
 chunk blobs and buffer entries.
 
+### Paginated position-range proofs
+
+`GroveDb::prove_bulk_position_range(path, key, start, limit)` proves one page of
+a cursor scan; `GroveDb::verify_bulk_position_range_proof` verifies it and
+returns the page entries (ascending, contiguous, complete) together with the
+authenticated `total_count` from the same proof bytes. Both sides derive the
+query from `(start, limit)` via `PathQuery::new_bulk_position_range`, so a
+scanning client only needs its cursor and page size. Absence beyond the end
+falls out of the proved count (`position >= total_count` does not exist), so a
+page shorter than `limit` means the scan caught up with the tip. The same entry
+points serve `CommitmentTree` elements.
+
 ## How It Ties to the GroveDB Root Hash
 
 The BulkAppendTree is a **non-Merk tree** — it stores data in the data namespace,

docs/book/src/commitment-tree.md

@@ -306,7 +306,7 @@ sub-merk recursion for these types.
 
 ## GroveDB Operations
 
-CommitmentTree provides four operations. The insert operation is generic over
+CommitmentTree provides five operations. The insert operation is generic over
 `M: MemoSize` (from the `orchard` crate), which controls ciphertext payload
 size validation. The default `M = DashMemo` gives a 216-byte payload
 (32 epk + 104 enc + 80 out).
@@ -325,6 +325,9 @@ db.commitment_tree_anchor(path, key, tx, version)
 // Retrieve a value by global position
 db.commitment_tree_get_value(path, key, position, tx, version)
 
+// Retrieve a page of values: positions [start, start + limit), plus total_count
+db.commitment_tree_get_range(path, key, start, limit, tx, version)
+
 // Get the current item count
 db.commitment_tree_count(path, key, tx, version)
 ```
@@ -903,6 +906,10 @@ Individual items (cmx || rho || cv_net || payload) can be queried by position an
 V1 proofs (§9.6), the same mechanism used by standalone BulkAppendTree. The
 V1 proof includes the BulkAppendTree authentication path for the requested
 position, chained to the parent Merk proof for the CommitmentTree element.
+Paginated scans use `GroveDb::prove_bulk_position_range` /
+`GroveDb::verify_bulk_position_range_proof`, which dispatch on the element type
+and so serve CommitmentTree pages through the same `ProofBytes::CommitmentTree`
+envelope (see the BulkAppendTree chapter).
 
 ## Cost Tracking
 

grovedb-bulk-append-tree/src/lib.rs

@@ -23,7 +23,7 @@ pub use error::BulkAppendError;
 pub use grovedb_dense_fixed_sized_merkle_tree::{DenseFixedSizedMerkleTree, DenseTreeProof};
 #[cfg(feature = "storage")]
 pub use grovedb_merkle_mountain_range::{MmrKeySize, MmrStore};
-pub use proof::{BulkAppendTreeProof, BulkAppendTreeProofResult};
-pub use tree::{hash::compute_state_root, leaf_count_to_mmr_size, BulkAppendTree};
+pub use proof::{position_range_query, BulkAppendTreeProof, BulkAppendTreeProofResult};
+pub use tree::{hash::compute_state_root, leaf_count_to_mmr_size, BulkAppendTree, RangePage};
 #[cfg(feature = "storage")]
 pub use tree::{AppendNoStateRootResult, AppendResult, BufferQueryResult, ChunkQueryResult};

grovedb-bulk-append-tree/src/proof/mod.rs

@@ -176,6 +176,25 @@ fn query_to_ranges(query: &Query, total_count: u64) -> Result<Vec<(u64, u64)>, B
     Ok(merged)
 }
 
+/// Build the canonical [`Query`] selecting the position range
+/// `[start, start + limit)`, with positions encoded as 8-byte big-endian
+/// keys.
+///
+/// This is the query shape used by the paginated-scan pattern: prover and
+/// verifier both derive it from `(start, limit)`, so a client only needs its
+/// cursor and page size. `start + limit` saturates at `u64::MAX`, and
+/// verification clamps the range to the tree's provable total count.
+pub fn position_range_query(start: u64, limit: u16) -> Query {
+    let end = start.saturating_add(limit as u64);
+    Query {
+        items: vec![QueryItem::Range(
+            start.to_be_bytes().to_vec()..end.to_be_bytes().to_vec(),
+        )],
+        left_to_right: true,
+        ..Query::default()
+    }
+}
+
 /// Check whether `pos` falls inside any of the sorted, non-overlapping ranges.
 fn in_ranges(pos: u64, ranges: &[(u64, u64)]) -> bool {
     ranges
@@ -325,6 +344,54 @@ impl BulkAppendTreeProof {
         })
     }
 
+    /// Generate a proof for the paginated position range
+    /// `[start, start + limit)`.
+    ///
+    /// Convenience wrapper over [`generate`](Self::generate) using the
+    /// canonical [`position_range_query`]. The proof is chunk-aligned: it
+    /// carries each completed chunk blob overlapping the range plus the
+    /// buffer entries in range, so proof size is O(chunks touched).
+    ///
+    /// Ranges past the end of the tree are valid and produce a proof of the
+    /// (empty) result: absence of positions `>= total_count` falls out of
+    /// the authenticated element's total count, not out of per-position
+    /// absence proofs.
+    #[cfg(feature = "storage")]
+    pub fn generate_for_range<'db, S: StorageContext<'db>>(
+        tree: &BulkAppendTree<S>,
+        start: u64,
+        limit: u16,
+    ) -> Result<Self, BulkAppendError> {
+        Self::generate(&position_range_query(start, limit), tree)
+    }
+
+    /// Verify this proof against the paginated position range
+    /// `[start, start + limit)`.
+    ///
+    /// Convenience wrapper over
+    /// [`verify_against_query`](Self::verify_against_query) using the
+    /// canonical [`position_range_query`]. Returns the `(global_position,
+    /// value)` pairs in the range, ascending and contiguous, clamped to
+    /// `total_count`. Completeness is enforced: a proof missing any
+    /// requested position below `total_count` is rejected. Positions
+    /// `>= total_count` are provably absent by `total_count` itself, which
+    /// callers must take from the authenticated BulkAppendTree element.
+    pub fn verify_range(
+        &self,
+        expected_state_root: &[u8; 32],
+        height: u8,
+        total_count: u64,
+        start: u64,
+        limit: u16,
+    ) -> Result<Vec<(u64, Vec<u8>)>, BulkAppendError> {
+        self.verify_against_query(
+            expected_state_root,
+            height,
+            total_count,
+            &position_range_query(start, limit),
+        )
+    }
+
     /// Verify this proof against an expected state root.
     ///
     /// `height` and `total_count` come from the authenticated BulkAppendTree

grovedb-bulk-append-tree/src/proof/tests.rs

@@ -943,4 +943,170 @@ mod proof_tests {
             );
         }
     }
+
+    // ── generate_for_range / verify_range (paginated scan pattern) ───────
+
+    /// Helper: build a tree of `n` values "val_0".."val_{n-1}" and return
+    /// (state_root, tree).
+    fn build_indexed_tree(height: u8, n: u32) -> ([u8; 32], BulkAppendTree<MemStorageContext>) {
+        let values: Vec<Vec<u8>> = (0..n).map(|i| format!("val_{}", i).into_bytes()).collect();
+        build_test_tree(height, &values)
+    }
+
+    /// Helper: round-trip a range proof and assert the returned page is
+    /// exactly positions `expected_start..expected_end`.
+    fn assert_range_roundtrip(
+        state_root: &[u8; 32],
+        tree: &BulkAppendTree<MemStorageContext>,
+        start: u64,
+        limit: u16,
+        expected_start: u64,
+        expected_end: u64,
+    ) {
+        let proof =
+            BulkAppendTreeProof::generate_for_range(tree, start, limit).expect("generate range");
+
+        // Wire round-trip: encode + decode like a real client
+        let bytes = proof.encode_to_vec().expect("encode");
+        let decoded = BulkAppendTreeProof::decode_from_slice(&bytes).expect("decode");
+
+        let entries = decoded
+            .verify_range(state_root, tree.height(), tree.total_count, start, limit)
+            .expect("verify range");
+
+        assert_eq!(entries.len(), (expected_end - expected_start) as usize);
+        for (i, (pos, value)) in entries.iter().enumerate() {
+            assert_eq!(*pos, expected_start + i as u64);
+            assert_eq!(value, format!("val_{}", pos).as_bytes());
+        }
+    }
+
+    #[test]
+    fn test_range_roundtrip_buffer_only() {
+        // height=3, capacity=7: 5 values all in buffer
+        let (root, tree) = build_indexed_tree(3, 5);
+        assert_range_roundtrip(&root, &tree, 1, 3, 1, 4);
+    }
+
+    #[test]
+    fn test_range_roundtrip_across_chunk_boundary() {
+        // height=2, epoch_size=4: 10 values = 2 chunks + 2 buffered
+        let (root, tree) = build_indexed_tree(2, 10);
+        // spans chunk 0 / chunk 1
+        assert_range_roundtrip(&root, &tree, 3, 3, 3, 6);
+        // spans chunk 1 / buffer
+        assert_range_roundtrip(&root, &tree, 6, 4, 6, 10);
+    }
+
+    #[test]
+    fn test_range_roundtrip_single_entry_pages() {
+        let (root, tree) = build_indexed_tree(2, 10);
+        for pos in 0..10u64 {
+            assert_range_roundtrip(&root, &tree, pos, 1, pos, pos + 1);
+        }
+    }
+
+    #[test]
+    fn test_range_roundtrip_empty_range() {
+        let (root, tree) = build_indexed_tree(2, 10);
+        // limit 0: proof still verifies against the root, returns nothing
+        assert_range_roundtrip(&root, &tree, 3, 0, 3, 3);
+    }
+
+    #[test]
+    fn test_range_roundtrip_past_end() {
+        let (root, tree) = build_indexed_tree(2, 10);
+        // starts exactly at total_count
+        assert_range_roundtrip(&root, &tree, 10, 5, 10, 10);
+        // starts far past total_count
+        assert_range_roundtrip(&root, &tree, 1000, 5, 1000, 1000);
+        // clamped at the end
+        assert_range_roundtrip(&root, &tree, 8, 100, 8, 10);
+    }
+
+    #[test]
+    fn test_range_roundtrip_large_multi_chunk_page() {
+        // height=4, epoch_size=16: 100 values = 6 chunks + 4 buffered.
+        // One page covering everything touches all chunks and the buffer.
+        let (root, tree) = build_indexed_tree(4, 100);
+        assert_range_roundtrip(&root, &tree, 0, 100, 0, 100);
+        // A large page crossing several chunk boundaries mid-tree
+        assert_range_roundtrip(&root, &tree, 10, 70, 10, 80);
+    }
+
+    #[test]
+    fn test_range_roundtrip_empty_tree() {
+        let (_, tree) = build_indexed_tree(2, 0);
+        // For an empty tree the state root is blake3("bulk_state" || 0*32 || 0*32)
+        let root = crate::compute_state_root(&[0u8; 32], &[0u8; 32]);
+        assert_range_roundtrip(&root, &tree, 0, 10, 0, 0);
+    }
+
+    #[test]
+    fn test_range_paged_scan_covers_everything() {
+        // The client scan pattern: page through the whole tree with
+        // limit=7 (deliberately not aligned to epoch_size=4).
+        let (root, tree) = build_indexed_tree(2, 30);
+        let mut cursor = 0u64;
+        let mut seen = Vec::new();
+        while cursor < tree.total_count {
+            let proof =
+                BulkAppendTreeProof::generate_for_range(&tree, cursor, 7).expect("generate page");
+            let entries = proof
+                .verify_range(&root, tree.height(), tree.total_count, cursor, 7)
+                .expect("verify page");
+            assert!(!entries.is_empty());
+            cursor += entries.len() as u64;
+            seen.extend(entries);
+        }
+        assert_eq!(seen.len(), 30);
+        for (i, (pos, value)) in seen.iter().enumerate() {
+            assert_eq!(*pos, i as u64);
+            assert_eq!(value, format!("val_{}", i).as_bytes());
+        }
+    }
+
+    #[test]
+    fn test_range_proof_wrong_root_rejected() {
+        let (root, tree) = build_indexed_tree(2, 10);
+        let proof = BulkAppendTreeProof::generate_for_range(&tree, 0, 5).expect("generate");
+        let mut bad_root = root;
+        bad_root[0] ^= 1;
+        proof
+            .verify_range(&bad_root, tree.height(), tree.total_count, 0, 5)
+            .expect_err("tampered root must be rejected");
+    }
+
+    #[test]
+    fn test_range_proof_missing_chunk_rejected() {
+        // Proof generated for [0, 2) (chunk 0 only) must not verify a
+        // request for [0, 6) which also needs chunk 1.
+        let (root, tree) = build_indexed_tree(2, 10);
+        let narrow = BulkAppendTreeProof::generate_for_range(&tree, 0, 2).expect("generate");
+        narrow
+            .verify_range(&root, tree.height(), tree.total_count, 0, 6)
+            .expect_err("proof missing chunk 1 must be rejected for the wider range");
+    }
+
+    #[test]
+    fn test_position_range_query_shape() {
+        let q = super::super::position_range_query(5, 3);
+        assert_eq!(q.items.len(), 1);
+        match &q.items[0] {
+            QueryItem::Range(r) => {
+                assert_eq!(r.start, 5u64.to_be_bytes().to_vec());
+                assert_eq!(r.end, 8u64.to_be_bytes().to_vec());
+            }
+            other => panic!("expected Range item, got {:?}", other),
+        }
+
+        // start + limit saturates instead of wrapping
+        let q = super::super::position_range_query(u64::MAX - 1, 100);
+        match &q.items[0] {
+            QueryItem::Range(r) => {
+                assert_eq!(r.end, u64::MAX.to_be_bytes().to_vec());
+            }
+            other => panic!("expected Range item, got {:?}", other),
+        }
+    }
 }