fix: CommitmentTreeInsert under-costed in estimated-cost paths (issue #812)

grovedb-element/src/element/constructor.rs

@@ -2,7 +2,10 @@
 //! Functions for setting an element's type
 
 use crate::{
-    element::{BigSumValue, CountValue, Element, ElementFlags, MaxReferenceHop, SumValue},
+    element::{
+        BigSumValue, CountValue, Element, ElementFlags, MaxReferenceHop, SumValue,
+        MAX_COMMITMENT_TREE_CHUNK_POWER,
+    },
     error::ElementError,
     reference_path::ReferencePathType,
 };
@@ -408,23 +411,28 @@ impl Element {
 
     /// Set element to an empty commitment tree.
     ///
-    /// Returns `InvalidInput` if `chunk_power > 31`.
+    /// Returns `InvalidInput` if `chunk_power >
+    /// MAX_COMMITMENT_TREE_CHUNK_POWER` (11) — the estimated-cost model
+    /// only covers epochs up to that size, and an estimate that is not
+    /// an upper bound is an admission-control bypass for consumers.
     pub fn empty_commitment_tree(chunk_power: u8) -> Result<Self, ElementError> {
-        if chunk_power > 31 {
-            return Err(ElementError::InvalidInput("chunk_power must be <= 31"));
+        if chunk_power > MAX_COMMITMENT_TREE_CHUNK_POWER {
+            return Err(ElementError::InvalidInput("chunk_power must be <= 11"));
         }
         Ok(Element::CommitmentTree(0, chunk_power, None))
     }
 
     /// Set element to an empty commitment tree with flags.
     ///
-    /// Returns `InvalidInput` if `chunk_power > 31`.
+    /// Returns `InvalidInput` if `chunk_power >
+    /// MAX_COMMITMENT_TREE_CHUNK_POWER` (11) — see
+    /// [`empty_commitment_tree`](Self::empty_commitment_tree).
     pub fn empty_commitment_tree_with_flags(
         chunk_power: u8,
         flags: Option<ElementFlags>,
     ) -> Result<Self, ElementError> {
-        if chunk_power > 31 {
-            return Err(ElementError::InvalidInput("chunk_power must be <= 31"));
+        if chunk_power > MAX_COMMITMENT_TREE_CHUNK_POWER {
+            return Err(ElementError::InvalidInput("chunk_power must be <= 11"));
         }
         Ok(Element::CommitmentTree(0, chunk_power, flags))
     }

grovedb-element/src/element/mod.rs

@@ -17,6 +17,19 @@ use bincode::{Decode, Encode};
 
 use crate::{element_type::ElementType, reference_path::ReferencePathType};
 
+/// Largest `chunk_power` accepted when creating a commitment tree
+/// (2^11 = 2048-entry epochs, matching the largest deployed value — Dash
+/// Platform's shielded notes pool).
+///
+/// This is the authoritative cap that GroveDB's estimated-cost model for
+/// `CommitmentTreeInsert` covers when the actual chunk power is not
+/// declared in the estimation layer information: the per-append dense
+/// buffer recompute and the epoch-compaction blob both scale with
+/// `2^chunk_power`, so a creatable tree must never exceed what the
+/// estimator charges (issue #812). Raising this constant loosens that
+/// fallback estimate proportionally.
+pub const MAX_COMMITMENT_TREE_CHUNK_POWER: u8 = 11;
+
 /// Optional meta-data to be stored per element
 pub type ElementFlags = Vec<u8>;
 

grovedb/src/batch/batch_structure.rs

@@ -11,6 +11,8 @@ use grovedb_costs::{
 };
 use grovedb_merk::element::tree_type::ElementTreeTypeExtensions;
 #[cfg(feature = "minimal")]
+use grovedb_storage::worst_case_costs::WorstKeyLength;
+#[cfg(feature = "minimal")]
 use grovedb_visualize::{DebugByteVectors, DebugBytes};
 #[cfg(feature = "minimal")]
 use intmap::IntMap;
@@ -28,6 +30,35 @@ pub type OpsByPath = BTreeMap<KeyInfoPath, BTreeMap<KeyInfo, GroveOp>>;
 #[cfg(feature = "minimal")]
 pub type OpsByLevelPath = IntMap<u32, OpsByPath>;
 
+/// Build the synthetic key under which a keyless append-only op is filed.
+///
+/// The `MaxKeySize` variant sizes estimates with the real tree-key length,
+/// while the 8-byte big-endian op-index prefix in `unique_id` keeps several
+/// appends to the same tree from collapsing into a single `BTreeMap` entry
+/// (each append must be charged). [`keyless_op_tree_key`] is the inverse.
+#[cfg(feature = "minimal")]
+pub(in crate::batch) fn keyless_op_synthetic_key(op_index: usize, tree_key: &KeyInfo) -> KeyInfo {
+    let mut unique_id = (op_index as u64).to_be_bytes().to_vec();
+    unique_id.extend_from_slice(tree_key.as_slice());
+    KeyInfo::MaxKeySize {
+        unique_id,
+        max_size: tree_key.max_length(),
+    }
+}
+
+/// Recover the real tree-key bytes from a [`keyless_op_synthetic_key`].
+///
+/// Only meaningful for keys of ops that arrive keyless (the append-only tree
+/// ops) — a user-supplied `MaxKeySize` key on a keyed op has no such
+/// structure, so callers must check the op type before trusting the result.
+#[cfg(feature = "minimal")]
+pub(in crate::batch) fn keyless_op_tree_key(key: &KeyInfo) -> Option<&[u8]> {
+    match key {
+        KeyInfo::MaxKeySize { unique_id, .. } => unique_id.get(8..),
+        KeyInfo::KnownKey(_) => None,
+    }
+}
+
 /// Batch structure
 #[cfg(feature = "minimal")]
 pub(super) struct BatchStructure<C, F, SR> {
@@ -117,18 +148,42 @@ where
         // qualified paths meaning path + key
         let mut ops_by_qualified_paths: BTreeMap<Vec<Vec<u8>>, GroveOp> = BTreeMap::new();
 
-        for op in ops.into_iter() {
+        for (op_index, op) in ops.into_iter().enumerate() {
             let QualifiedGroveDbOp {
                 path: op_path,
                 key: op_key,
                 op: grove_op,
             } = op;
 
-            // Keyless ops (append-only tree ops) are handled by preprocessing.
-            // In estimated-cost paths they have no cost model yet — skip.
-            let key = match op_key {
-                Some(k) => k,
-                None => continue,
+            // Keyless ops (append-only tree ops: CommitmentTreeInsert,
+            // MmrTreeAppend, BulkAppend, DenseTreeInsert) carry the tree key
+            // as the last segment of `path`. In the apply path they are
+            // rewritten into keyed ops by preprocessing before reaching here;
+            // in the estimated-cost paths there is no preprocessing, so split
+            // the tree key off the path and let the op flow to the cost
+            // dispatch. Silently dropping them here (as this code used to do)
+            // made every append estimate as free — see issue #812.
+            //
+            // The synthetic key (see `keyless_op_synthetic_key`) sizes
+            // estimates with the real tree-key length while keeping one map
+            // entry per op, so each append is charged. If such an op ever
+            // reaches real execution, `execute_ops_on_path` rejects it with
+            // "should have been preprocessed" — a loud failure instead of a
+            // silent drop.
+            let (op_path, key, is_keyless_append) = match op_key {
+                Some(k) => (op_path, k, false),
+                None => {
+                    let mut path = op_path;
+                    let Some(tree_key) = path.0.pop() else {
+                        return Err(Error::InvalidBatchOperation(
+                            "keyless append-only op must have the tree key as its path's last \
+                             segment",
+                        ))
+                        .wrap_with_cost(cost);
+                    };
+                    let key = keyless_op_synthetic_key(op_index, &tree_key);
+                    (path, key, true)
+                }
             };
 
             // Validate key length: Merk link encoding stores key length as a
@@ -141,10 +196,15 @@ where
                     .wrap_with_cost(cost);
             }
 
-            // Build qualified path (path + key) for reference lookups
-            let mut qualified_path = op_path.clone();
-            qualified_path.push(key.clone());
-            ops_by_qualified_paths.insert(qualified_path.to_path_consume(), grove_op.clone());
+            // Build qualified path (path + key) for reference lookups.
+            // Keyless append ops are skipped: they are not elements a
+            // reference can target, and their synthetic keys must not
+            // shadow the tree element itself.
+            if !is_keyless_append {
+                let mut qualified_path = op_path.clone();
+                qualified_path.push(key.clone());
+                ops_by_qualified_paths.insert(qualified_path.to_path_consume(), grove_op.clone());
+            }
 
             let op_cost = OperationCost::default();
             let op_result = match &grove_op {