fix: CommitmentTreeInsert under-costed in estimated-cost paths (issue #812)

grovedb-element/src/element/constructor.rs

@@ -2,7 +2,10 @@
 //! Functions for setting an element's type
 
 use crate::{
-    element::{BigSumValue, CountValue, Element, ElementFlags, MaxReferenceHop, SumValue},
+    element::{
+        BigSumValue, CountValue, Element, ElementFlags, MaxReferenceHop, SumValue,
+        MAX_COMMITMENT_TREE_CHUNK_POWER,
+    },
     error::ElementError,
     reference_path::ReferencePathType,
 };
@@ -408,23 +411,28 @@ impl Element {
 
     /// Set element to an empty commitment tree.
     ///
-    /// Returns `InvalidInput` if `chunk_power > 31`.
+    /// Returns `InvalidInput` if `chunk_power >
+    /// MAX_COMMITMENT_TREE_CHUNK_POWER` (11) — the estimated-cost model
+    /// only covers epochs up to that size, and an estimate that is not
+    /// an upper bound is an admission-control bypass for consumers.
     pub fn empty_commitment_tree(chunk_power: u8) -> Result<Self, ElementError> {
-        if chunk_power > 31 {
-            return Err(ElementError::InvalidInput("chunk_power must be <= 31"));
+        if chunk_power > MAX_COMMITMENT_TREE_CHUNK_POWER {
+            return Err(ElementError::InvalidInput("chunk_power must be <= 11"));
         }
         Ok(Element::CommitmentTree(0, chunk_power, None))
     }
 
     /// Set element to an empty commitment tree with flags.
     ///
-    /// Returns `InvalidInput` if `chunk_power > 31`.
+    /// Returns `InvalidInput` if `chunk_power >
+    /// MAX_COMMITMENT_TREE_CHUNK_POWER` (11) — see
+    /// [`empty_commitment_tree`](Self::empty_commitment_tree).
     pub fn empty_commitment_tree_with_flags(
         chunk_power: u8,
         flags: Option<ElementFlags>,
     ) -> Result<Self, ElementError> {
-        if chunk_power > 31 {
-            return Err(ElementError::InvalidInput("chunk_power must be <= 31"));
+        if chunk_power > MAX_COMMITMENT_TREE_CHUNK_POWER {
+            return Err(ElementError::InvalidInput("chunk_power must be <= 11"));
         }
         Ok(Element::CommitmentTree(0, chunk_power, flags))
     }

grovedb-element/src/element/mod.rs

@@ -17,6 +17,19 @@ use bincode::{Decode, Encode};
 
 use crate::{element_type::ElementType, reference_path::ReferencePathType};
 
+/// Largest `chunk_power` accepted when creating a commitment tree
+/// (2^11 = 2048-entry epochs, matching the largest deployed value — Dash
+/// Platform's shielded notes pool).
+///
+/// This is the authoritative cap that GroveDB's estimated-cost model for
+/// `CommitmentTreeInsert` covers when the actual chunk power is not
+/// declared in the estimation layer information: the per-append dense
+/// buffer recompute and the epoch-compaction blob both scale with
+/// `2^chunk_power`, so a creatable tree must never exceed what the
+/// estimator charges (issue #812). Raising this constant loosens that
+/// fallback estimate proportionally.
+pub const MAX_COMMITMENT_TREE_CHUNK_POWER: u8 = 11;
+
 /// Optional meta-data to be stored per element
 pub type ElementFlags = Vec<u8>;
 

grovedb/src/batch/batch_structure.rs

@@ -11,6 +11,8 @@ use grovedb_costs::{
 };
 use grovedb_merk::element::tree_type::ElementTreeTypeExtensions;
 #[cfg(feature = "minimal")]
+use grovedb_storage::worst_case_costs::WorstKeyLength;
+#[cfg(feature = "minimal")]
 use grovedb_visualize::{DebugByteVectors, DebugBytes};
 #[cfg(feature = "minimal")]
 use intmap::IntMap;
@@ -28,6 +30,35 @@ pub type OpsByPath = BTreeMap<KeyInfoPath, BTreeMap<KeyInfo, GroveOp>>;
 #[cfg(feature = "minimal")]
 pub type OpsByLevelPath = IntMap<u32, OpsByPath>;
 
+/// Build the synthetic key under which a keyless append-only op is filed.
+///
+/// The `MaxKeySize` variant sizes estimates with the real tree-key length,
+/// while the 8-byte big-endian op-index prefix in `unique_id` keeps several
+/// appends to the same tree from collapsing into a single `BTreeMap` entry
+/// (each append must be charged). [`keyless_op_tree_key`] is the inverse.
+#[cfg(feature = "minimal")]
+pub(in crate::batch) fn keyless_op_synthetic_key(op_index: usize, tree_key: &KeyInfo) -> KeyInfo {
+    let mut unique_id = (op_index as u64).to_be_bytes().to_vec();
+    unique_id.extend_from_slice(tree_key.as_slice());
+    KeyInfo::MaxKeySize {
+        unique_id,
+        max_size: tree_key.max_length(),
+    }
+}
+
+/// Recover the real tree-key bytes from a [`keyless_op_synthetic_key`].
+///
+/// Only meaningful for keys of ops that arrive keyless (the append-only tree
+/// ops) — a user-supplied `MaxKeySize` key on a keyed op has no such
+/// structure, so callers must check the op type before trusting the result.
+#[cfg(feature = "minimal")]
+pub(in crate::batch) fn keyless_op_tree_key(key: &KeyInfo) -> Option<&[u8]> {
+    match key {
+        KeyInfo::MaxKeySize { unique_id, .. } => unique_id.get(8..),
+        KeyInfo::KnownKey(_) => None,
+    }
+}
+
 /// Batch structure
 #[cfg(feature = "minimal")]
 pub(super) struct BatchStructure<C, F, SR> {
@@ -117,18 +148,42 @@ where
         // qualified paths meaning path + key
         let mut ops_by_qualified_paths: BTreeMap<Vec<Vec<u8>>, GroveOp> = BTreeMap::new();
 
-        for op in ops.into_iter() {
+        for (op_index, op) in ops.into_iter().enumerate() {
             let QualifiedGroveDbOp {
                 path: op_path,
                 key: op_key,
                 op: grove_op,
             } = op;
 
-            // Keyless ops (append-only tree ops) are handled by preprocessing.
-            // In estimated-cost paths they have no cost model yet — skip.
-            let key = match op_key {
-                Some(k) => k,
-                None => continue,
+            // Keyless ops (append-only tree ops: CommitmentTreeInsert,
+            // MmrTreeAppend, BulkAppend, DenseTreeInsert) carry the tree key
+            // as the last segment of `path`. In the apply path they are
+            // rewritten into keyed ops by preprocessing before reaching here;
+            // in the estimated-cost paths there is no preprocessing, so split
+            // the tree key off the path and let the op flow to the cost
+            // dispatch. Silently dropping them here (as this code used to do)
+            // made every append estimate as free — see issue #812.
+            //
+            // The synthetic key (see `keyless_op_synthetic_key`) sizes
+            // estimates with the real tree-key length while keeping one map
+            // entry per op, so each append is charged. If such an op ever
+            // reaches real execution, `execute_ops_on_path` rejects it with
+            // "should have been preprocessed" — a loud failure instead of a
+            // silent drop.
+            let (op_path, key, is_keyless_append) = match op_key {
+                Some(k) => (op_path, k, false),
+                None => {
+                    let mut path = op_path;
+                    let Some(tree_key) = path.0.pop() else {
+                        return Err(Error::InvalidBatchOperation(
+                            "keyless append-only op must have the tree key as its path's last \
+                             segment",
+                        ))
+                        .wrap_with_cost(cost);
+                    };
+                    let key = keyless_op_synthetic_key(op_index, &tree_key);
+                    (path, key, true)
+                }
             };
 
             // Validate key length: Merk link encoding stores key length as a
@@ -141,10 +196,15 @@ where
                     .wrap_with_cost(cost);
             }
 
-            // Build qualified path (path + key) for reference lookups
-            let mut qualified_path = op_path.clone();
-            qualified_path.push(key.clone());
-            ops_by_qualified_paths.insert(qualified_path.to_path_consume(), grove_op.clone());
+            // Build qualified path (path + key) for reference lookups.
+            // Keyless append ops are skipped: they are not elements a
+            // reference can target, and their synthetic keys must not
+            // shadow the tree element itself.
+            if !is_keyless_append {
+                let mut qualified_path = op_path.clone();
+                qualified_path.push(key.clone());
+                ops_by_qualified_paths.insert(qualified_path.to_path_consume(), grove_op.clone());
+            }
 
             let op_cost = OperationCost::default();
             let op_result = match &grove_op {

grovedb/src/batch/estimated_costs/average_case_costs.rs

@@ -45,6 +45,11 @@ impl GroveOp {
         &self,
         key: &KeyInfo,
         layer_element_estimates: &EstimatedLayerInformation,
+        // The declared chunk power of the commitment tree a
+        // `CommitmentTreeInsert` op targets (from the tree's own layer in
+        // the estimation paths), or `None` to charge the constructor-
+        // enforced cap. Ignored by every other op type.
+        ct_chunk_power: Option<u8>,
         propagate: bool,
         grove_version: &GroveVersion,
     ) -> CostResult<(), Error> {
@@ -208,41 +213,25 @@ impl GroveOp {
                 grove_version,
             ),
             GroveOp::CommitmentTreeInsert { payload, .. } => {
-                // After preprocessing, CommitmentTreeInsert becomes
+                // In the apply path, preprocessing rewrites this op into
                 // ReplaceNonMerkTreeRoot. The base cost is a tree root key
-                // replacement in the parent Merk.
-                let item_cost = GroveDb::average_case_merk_replace_tree(
+                // replacement in the parent Merk; the append work itself
+                // (frontier I/O, Sinsemilla hashing, note write, epoch
+                // compaction) is charged by the shared upper-bound model —
+                // deliberately NOT an average, since the append cost is
+                // position-dependent and the position is adversary-chosen.
+                // See `commitment_tree_insert_op_cost`.
+                GroveDb::average_case_merk_replace_tree(
                     key,
                     layer_element_estimates,
-                    TreeType::CommitmentTree(0),
+                    TreeType::CommitmentTree(ct_chunk_power.unwrap_or(0)),
                     propagate,
                     grove_version,
-                );
-                use grovedb_costs::storage_cost::{removal::StorageRemovedBytes, StorageCost};
-                // Additional cost: frontier I/O (data storage load + save),
-                // buffer entry write, and Sinsemilla hashing.
-                //
-                // Average frontier size with ~16 ommers:
-                // 1 (flag) + 8 (position) + 32 (leaf) + 1 (count) + 16*32 = 554
-                const AVG_FRONTIER_SIZE: u32 = 554;
-                // Buffer entry: cmx (32) + rho (32) + cv_net (32) + payload
-                let buffer_entry_size = 96 + payload.len() as u32;
-                // Average Sinsemilla hashes per append:
-                // 32 (root computation) + 1 (avg ommer updates) = 33
-                const AVG_SINSEMILLA_HASHES: u32 = 33;
-                // Average blake3 hashes: 1 for running buffer hash
-                const AVG_BLAKE3_HASHES: u32 = 1;
-                item_cost.add_cost(OperationCost {
-                    seek_count: 3, // frontier load + frontier save + buffer write
-                    storage_cost: StorageCost {
-                        added_bytes: buffer_entry_size,
-                        replaced_bytes: AVG_FRONTIER_SIZE,
-                        removed_bytes: StorageRemovedBytes::NoStorageRemoval,
-                    },
-                    storage_loaded_bytes: AVG_FRONTIER_SIZE as u64,
-                    hash_node_calls: AVG_BLAKE3_HASHES,
-                    sinsemilla_hash_calls: AVG_SINSEMILLA_HASHES,
-                })
+                )
+                .add_cost(super::commitment_tree_insert_op_cost(
+                    payload.len() as u32,
+                    ct_chunk_power,
+                ))
             }
             GroveOp::MmrTreeAppend { value } => {
                 // Cost of updating parent element in the Merk
@@ -584,9 +573,33 @@ impl<G, SR> TreeCache<G, SR> for AverageCaseTreeCacheKnownPaths {
             .count();
 
         for (key, op) in ops_at_path_by_key.into_iter() {
+            // A CommitmentTreeInsert arrives under a synthetic key carrying
+            // the real tree key (see `keyless_op_synthetic_key`). When the
+            // caller declared the tree's own layer with
+            // `TreeType::CommitmentTree(chunk_power)` — as Dash Platform
+            // does — the estimate uses the tree's ACTUAL epoch scale
+            // instead of the constructor-enforced cap.
+            let ct_chunk_power = if matches!(op, GroveOp::CommitmentTreeInsert { .. }) {
+                crate::batch::batch_structure::keyless_op_tree_key(&key).and_then(|tree_key| {
+                    let mut tree_path = path.clone();
+                    tree_path.push(KeyInfo::KnownKey(tree_key.to_vec()));
+                    match self.paths.get(&tree_path).map(|layer| layer.tree_type) {
+                        Some(TreeType::CommitmentTree(chunk_power)) => Some(chunk_power),
+                        _ => None,
+                    }
+                })
+            } else {
+                None
+            };
             cost_return_on_error!(
                 &mut cost,
-                op.average_case_cost(&key, layer_element_estimates, false, grove_version)
+                op.average_case_cost(
+                    &key,
+                    layer_element_estimates,
+                    ct_chunk_power,
+                    false,
+                    grove_version
+                )
             );
         }
 
@@ -1644,7 +1657,7 @@ mod tests {
             estimated_layer_sizes: AllSubtrees(4, NoSumTrees, None),
         };
         let cost = op
-            .average_case_cost(&key, &layer_info, false, grove_version)
+            .average_case_cost(&key, &layer_info, None, false, grove_version)
             .cost_as_result()
             .expect("expected cost for commitment tree insert");
         // CommitmentTreeInsert includes frontier I/O and buffer writes plus
@@ -1691,7 +1704,7 @@ mod tests {
             estimated_layer_sizes: AllSubtrees(4, NoSumTrees, None),
         };
         let cost = op
-            .average_case_cost(&key, &layer_info, false, grove_version)
+            .average_case_cost(&key, &layer_info, None, false, grove_version)
             .cost_as_result()
             .expect("expected cost for mmr tree append");
         // MmrTreeAppend includes parent replace cost plus MMR node I/O.
@@ -1732,7 +1745,7 @@ mod tests {
             estimated_layer_sizes: AllSubtrees(4, NoSumTrees, None),
         };
         let cost = op
-            .average_case_cost(&key, &layer_info, false, grove_version)
+            .average_case_cost(&key, &layer_info, None, false, grove_version)
             .cost_as_result()
             .expect("expected cost for bulk append");
         // BulkAppend includes parent replace cost plus buffer write + running
@@ -1768,7 +1781,7 @@ mod tests {
             estimated_layer_sizes: AllSubtrees(4, NoSumTrees, None),
         };
         let cost = op
-            .average_case_cost(&key, &layer_info, false, grove_version)
+            .average_case_cost(&key, &layer_info, None, false, grove_version)
             .cost_as_result()
             .expect("expected cost for dense tree insert");
         // DenseTreeInsert includes parent replace cost plus value write and
@@ -1812,7 +1825,7 @@ mod tests {
             estimated_layer_sizes: AllSubtrees(4, NoSumTrees, None),
         };
         let cost = op
-            .average_case_cost(&key, &layer_info, false, grove_version)
+            .average_case_cost(&key, &layer_info, None, false, grove_version)
             .cost_as_result()
             .expect("expected cost for replace non-merk tree root");
         // ReplaceNonMerkTreeRoot delegates to average_case_merk_replace_tree.
@@ -1847,7 +1860,7 @@ mod tests {
             estimated_layer_sizes: AllSubtrees(4, NoSumTrees, None),
         };
         let cost = op
-            .average_case_cost(&key, &layer_info, false, grove_version)
+            .average_case_cost(&key, &layer_info, None, false, grove_version)
             .cost_as_result()
             .expect("expected cost for insert non-merk tree");
         // InsertNonMerkTree delegates to average_case_merk_insert_tree.
@@ -1896,7 +1909,7 @@ mod tests {
                 not_summed,
                 not_counted_or_summed,
             };
-            op.average_case_cost(&key, &layer_info, false, grove_version)
+            op.average_case_cost(&key, &layer_info, None, false, grove_version)
                 .cost_as_result()
                 .expect("expected cost for InsertTreeWithRootHash")
         };
@@ -1950,7 +1963,7 @@ mod tests {
                 meta: NonMerkTreeMeta::MmrTree { mmr_size: 50 },
                 non_counted,
             };
-            op.average_case_cost(&key, &layer_info, false, grove_version)
+            op.average_case_cost(&key, &layer_info, None, false, grove_version)
                 .cost_as_result()
                 .expect("expected cost for InsertNonMerkTree")
         };
@@ -1989,7 +2002,7 @@ mod tests {
             axes: vec![(0u8, [0xEFu8; 32], Some(b"srk".to_vec()))],
         };
         let cost_count = op_count
-            .average_case_cost(&key, &layer_info, false, grove_version)
+            .average_case_cost(&key, &layer_info, None, false, grove_version)
             .cost_as_result()
             .expect("expected average case cost for Count cidx replace");
         assert!(cost_count.seek_count > 0 || cost_count.hash_node_calls > 0);
@@ -2001,7 +2014,7 @@ mod tests {
             axes: vec![(0u8, [8u8; 32], None)],
         };
         let cost_pcount = op_pcount
-            .average_case_cost(&key, &layer_info, true, grove_version)
+            .average_case_cost(&key, &layer_info, None, true, grove_version)
             .cost_as_result()
             .expect("expected average case cost for ProvableCount cidx replace (propagate)");
         assert!(