feat(replication): state sync support for the append-only tree family (fixes #785)

grovedb/src/lib.rs

@@ -326,12 +326,18 @@ type VerificationIssues = HashMap<Vec<Vec<u8>>, (CryptoHash, CryptoHash, CryptoH
 /// It represents a tuple containing:
 /// - A `Merk` instance with a prefixed RocksDB immediate storage context.
 /// - An optional `root_key`, represented as a vector of bytes.
-/// - A boolean indicating whether the Merk is a sum tree.
+/// - The `TreeType` of the subtree (with its parameters, e.g. chunk power,
+///   preserved from the parent element).
+/// - The parent-declared `Element` for this subtree, or `None` when opening
+///   the root subtree (which has no parent element). Replication needs the
+///   full element for non-Merk append-only trees, whose entry counts and
+///   parameters drive the raw-replay restore path.
 #[cfg(feature = "minimal")]
 type OpenedMerkForReplication<'tx> = (
     Merk<PrefixedRocksDbImmediateStorageContext<'tx>>,
     Option<Vec<u8>>,
     TreeType,
+    Option<Element>,
 );
 
 /// Verify that an indexed tree's secondary projection is exactly derivable
@@ -564,7 +570,7 @@ impl GroveDb {
                     ))
                 })
                 .unwrap()?;
-            if let Some((root_key, tree_type)) = element.root_key_and_tree_type_owned() {
+            if let Some((root_key, tree_type)) = element.clone().root_key_and_tree_type_owned() {
                 Ok((
                     Merk::open_layered_with_root_key(
                         storage,
@@ -581,6 +587,7 @@ impl GroveDb {
                     .unwrap()?,
                     root_key,
                     tree_type,
+                    Some(element),
                 ))
             } else {
                 Err(Error::CorruptedPath(
@@ -599,6 +606,7 @@ impl GroveDb {
                 .unwrap()?,
                 None,
                 TreeType::NormalTree,
+                None,
             ))
         }
     }
@@ -3021,6 +3029,135 @@ impl GroveDb {
             _ => merk_root_hash,
         }
     }
+
+    /// Strict variant of [`Self::compute_non_merk_child_hash`] for callers
+    /// that must not silently fall back when the payload is unreadable —
+    /// notably state-sync restore, where a missing or corrupt payload has to
+    /// reject the subtree rather than slip through as a hash that may
+    /// coincidentally match.
+    ///
+    /// Recomputes the tree-type-specific state root from the payload stored
+    /// in the subtree's data namespace:
+    /// - `CommitmentTree`: `blake3("ct_state" || sinsemilla_root ||
+    ///   bulk_state_root)` (reads the frontier and the bulk store)
+    /// - `BulkAppendTree`: `blake3("bulk_state" || mmr_root || dense_root)`
+    /// - `MmrTree`: the MMR root hash
+    /// - `DenseAppendOnlyFixedSizeTree`: the dense tree root hash
+    ///
+    /// For empty trees this returns the same conventions the insert path
+    /// binds into the parent: `EMPTY_COMMITMENT_TREE_STATE_ROOT` for an
+    /// empty commitment tree, `NULL_HASH` (the empty Merk root) for the
+    /// other three types.
+    ///
+    /// Returns an error if `element` is not a non-Merk data tree, or if the
+    /// payload cannot be read back as a consistent tree of the declared
+    /// size.
+    pub(crate) fn compute_non_merk_state_root<'b, B: AsRef<[u8]>>(
+        &self,
+        element: &Element,
+        subtree_path: SubtreePath<'b, B>,
+        transaction: &Transaction,
+        grove_version: &GroveVersion,
+    ) -> Result<CryptoHash, Error> {
+        use grovedb_merk::tree::hash::NULL_HASH;
+        match element.underlying() {
+            Element::CommitmentTree(total_count, chunk_power, _) => {
+                if *total_count == 0 {
+                    return Ok(grovedb_commitment_tree::EMPTY_COMMITMENT_TREE_STATE_ROOT);
+                }
+                let storage_ctx = self
+                    .db
+                    .get_transactional_storage_context(subtree_path, None, transaction)
+                    .unwrap();
+                let ct = grovedb_commitment_tree::CommitmentTree::<_>::open(
+                    *total_count,
+                    *chunk_power,
+                    storage_ctx,
+                )
+                .value
+                .map_err(|e| {
+                    Error::CorruptedData(format!(
+                        "cannot open commitment tree of {total_count} entries from payload: {e}"
+                    ))
+                })?;
+                ct.compute_current_state_root().map_err(|e| {
+                    Error::CorruptedData(format!(
+                        "cannot compute commitment tree state root from payload: {e}"
+                    ))
+                })
+            }
+            Element::BulkAppendTree(total_count, chunk_power, _) => {
+                if *total_count == 0 {
+                    return Ok(NULL_HASH);
+                }
+                let storage_ctx = self
+                    .db
+                    .get_transactional_storage_context(subtree_path, None, transaction)
+                    .unwrap();
+                let tree = grovedb_bulk_append_tree::BulkAppendTree::from_state(
+                    *total_count,
+                    *chunk_power,
+                    storage_ctx,
+                )
+                .map_err(|e| {
+                    Error::CorruptedData(format!(
+                        "cannot open bulk append tree of {total_count} entries from payload: {e}"
+                    ))
+                })?;
+                tree.compute_current_state_root().map_err(|e| {
+                    Error::CorruptedData(format!(
+                        "cannot compute bulk append tree state root from payload: {e}"
+                    ))
+                })
+            }
+            Element::MmrTree(mmr_size, _) => {
+                if *mmr_size == 0 {
+                    return Ok(NULL_HASH);
+                }
+                let storage_ctx = self
+                    .db
+                    .get_transactional_storage_context(subtree_path, None, transaction)
+                    .unwrap();
+                let store = grovedb_merkle_mountain_range::MmrStore::new(&storage_ctx);
+                let mmr = grovedb_merkle_mountain_range::MMR::new(*mmr_size, &store);
+                mmr.get_root(grove_version)
+                    .value
+                    .map(|root| root.hash())
+                    .map_err(|e| {
+                        Error::CorruptedData(format!(
+                            "cannot compute MMR root of size {mmr_size} from payload: {e}"
+                        ))
+                    })
+            }
+            Element::DenseAppendOnlyFixedSizeTree(count, height, _) => {
+                if *count == 0 {
+                    return Ok(NULL_HASH);
+                }
+                let storage_ctx = self
+                    .db
+                    .get_transactional_storage_context(subtree_path, None, transaction)
+                    .unwrap();
+                use grovedb_dense_fixed_sized_merkle_tree::DenseFixedSizedMerkleTree;
+                DenseFixedSizedMerkleTree::from_state(*height, *count, storage_ctx)
+                    .map_err(|e| {
+                        Error::CorruptedData(format!(
+                            "cannot open dense tree of {count} entries from payload: {e}"
+                        ))
+                    })?
+                    .root_hash()
+                    .unwrap()
+                    .map_err(|e| {
+                        Error::CorruptedData(format!(
+                            "cannot compute dense tree root from payload: {e}"
+                        ))
+                    })
+            }
+            _ => Err(Error::InternalError(format!(
+                "compute_non_merk_state_root called on a non append-only element: {}",
+                element.type_str()
+            ))),
+        }
+    }
 }
 
 /// Inspect a tree-bearing Element together with the actual aggregate data of

grovedb/src/replication.rs

@@ -1,3 +1,4 @@
+pub(crate) mod non_merk_sync;
 mod state_sync_session;
 
 use std::pin::Pin;
@@ -93,6 +94,13 @@ impl GroveDb {
     /// - The function opens a `Merk` tree for each chunk and retrieves the
     ///   associated data.
     /// - Empty trees return an empty byte vector.
+    /// - Non-Merk append-only subtrees (`CommitmentTree`, `MmrTree`,
+    ///   `BulkAppendTree`, `DenseAppendOnlyFixedSizeTree`) are served as
+    ///   cursor-based entry pages instead of Merk chunks. A request for one
+    ///   of these subtrees without a page cursor returns
+    ///   `Error::NotSupported`.
+    /// - Indexed-tree requests and populated `PrivateDocumentStore`
+    ///   requests return `Error::NotSupported`.
     pub fn fetch_chunk(
         &self,
         packed_global_chunk_id: &[u8],
@@ -143,6 +151,37 @@ impl GroveDb {
                 ));
             }
 
+            // Non-Merk append-only trees (CommitmentTree / MmrTree /
+            // BulkAppendTree / DenseAppendOnlyFixedSizeTree) have no Merk
+            // nodes to chunk — their payload is served as target-driven
+            // entry pages instead. The target encodes a page cursor into
+            // every local chunk id; a request without one comes from a
+            // peer speaking the pre-#785 protocol, which cannot sync
+            // these subtrees. (Other non-Merk types without a replay arm
+            // — PrivateDocumentStore — fall through to the Merk path,
+            // which serves them empty or rejects them populated below.)
+            if non_merk_sync::supports_entry_replay(tree_type) {
+                if nested_chunk_ids.is_empty() {
+                    return Err(Error::NotSupported(
+                        "append-only subtree chunk request is missing its page \
+                         cursor — the requesting peer does not support state \
+                         sync of append-only trees (see issue #785)"
+                            .to_string(),
+                    ));
+                }
+                let mut local_chunk_bytes: Vec<Vec<u8>> = vec![];
+                for chunk_id in &nested_chunk_ids {
+                    local_chunk_bytes.push(self.fetch_non_merk_page(
+                        chunk_prefix,
+                        tree_type,
+                        chunk_id,
+                        tx.as_ref(),
+                    )?);
+                }
+                global_chunk_bytes.push(pack_nested_bytes(local_chunk_bytes)?);
+                continue;
+            }
+
             let mut local_chunk_bytes: Vec<Vec<u8>> = vec![];
 
             let merk = self
@@ -165,6 +204,16 @@ impl GroveDb {
             if merk.is_empty_tree().unwrap() {
                 local_chunk_bytes.push(vec![]);
             } else {
+                // A non-Merk data tree whose namespace is populated but that
+                // has no entry-replay arm (PrivateDocumentStore, see issues
+                // #783 / #784): there are no Merk nodes to chunk, so fail
+                // descriptively instead of dying in the chunk producer.
+                if tree_type.uses_non_merk_data_storage() {
+                    return Err(Error::NotSupported(format!(
+                        "state sync does not yet support populated {tree_type} subtrees \
+                         (non-Merk data storage without an entry-replay arm)"
+                    )));
+                }
                 let mut chunk_producer = ChunkProducer::new(&merk).map_err(|e| {
                     Error::CorruptedData(format!(
                         "failed to create chunk producer by prefix tx:{} with:{}",