From db32f8fa5fc2fc207b65796534edd98fa32df474 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Mon, 17 Aug 2026 15:42:02 +0700 Subject: [PATCH 1/7] fix: CommitmentTreeInsert under-costed in estimated-cost paths (#812) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two independent defects made the estimated cost of CommitmentTreeInsert fall short of the actual cost, which downstream is an admission-control bypass (Dash Platform admits a transaction on the estimate, then fails mid-execution when actual exceeds it — two mainnet chain stalls on 2026-08-14/15). Defect 1 — keyless ops were dropped before cost dispatch. BatchStructure::continue_from_ops skipped every keyless append-only op (CommitmentTreeInsert, MmrTreeAppend, BulkAppend, DenseTreeInsert), so the cost arms that exist for them were unreachable during estimation and the append contributed zero. Keyless ops now have the tree key split off their path and flow to the cost dispatch, with a unique synthetic key per op so several appends to one tree don't collapse into a single map entry (each append must be charged). If such an op ever reaches real execution, execute_ops_on_path rejects it loudly instead of the old silent drop; in the apply path preprocessing still rewrites them into keyed ops before this code runs, so apply behavior is unchanged. Defect 2 — the model was average-case where an upper bound is required. The append cost is position-dependent and the position is adversary- chosen: the Sinsemilla ommer cascade is maximal exactly at positions 2^k - 1, and epoch compaction (which rewrites the whole chunk blob) fires exactly every 2^chunk_power-th append. Both estimators now share one upper-bound model (commitment_tree_insert_op_cost) with constants derived from Orchard's NOTE_COMMITMENT_TREE_DEPTH — max 64 Sinsemilla hashes, max 1066-byte frontier — and cover the previously unmodeled dense-buffer root recompute and full epoch compaction, capped at a documented MAX_ESTIMATED_CHUNK_POWER of 10. New property tests pin the invariant consumers rely on: for every cost dimension, both estimates dominate the actual apply cost across adversarial positions (2^k - 1 ommer cascades, compaction boundaries at chunk_power 4 and at the cap's 1024-entry epoch), for single- and multi-op batches, and every append in a batch is charged individually. Fixes #812 Co-Authored-By: Claude Fable 5 --- grovedb/src/batch/batch_structure.rs | 56 ++- .../estimated_costs/average_case_costs.rs | 39 +- grovedb/src/batch/estimated_costs/mod.rs | 120 ++++++ .../batch/estimated_costs/worst_case_costs.rs | 35 +- .../tests/commitment_tree_cost_bound_tests.rs | 352 ++++++++++++++++++ grovedb/src/tests/mod.rs | 1 + 6 files changed, 538 insertions(+), 65 deletions(-) create mode 100644 grovedb/src/tests/commitment_tree_cost_bound_tests.rs diff --git a/grovedb/src/batch/batch_structure.rs b/grovedb/src/batch/batch_structure.rs index 65132a301..dc1c76dd2 100644 --- a/grovedb/src/batch/batch_structure.rs +++ b/grovedb/src/batch/batch_structure.rs @@ -11,6 +11,8 @@ use grovedb_costs::{ }; use grovedb_merk::element::tree_type::ElementTreeTypeExtensions; #[cfg(feature = "minimal")] +use grovedb_storage::worst_case_costs::WorstKeyLength; +#[cfg(feature = "minimal")] use grovedb_visualize::{DebugByteVectors, DebugBytes}; #[cfg(feature = "minimal")] use intmap::IntMap; @@ -117,18 +119,47 @@ where // qualified paths meaning path + key let mut ops_by_qualified_paths: BTreeMap>, GroveOp> = BTreeMap::new(); - for op in ops.into_iter() { + for (op_index, op) in ops.into_iter().enumerate() { let QualifiedGroveDbOp { path: op_path, key: op_key, op: grove_op, } = op; - // Keyless ops (append-only tree ops) are handled by preprocessing. - // In estimated-cost paths they have no cost model yet — skip. - let key = match op_key { - Some(k) => k, - None => continue, + // Keyless ops (append-only tree ops: CommitmentTreeInsert, + // MmrTreeAppend, BulkAppend, DenseTreeInsert) carry the tree key + // as the last segment of `path`. In the apply path they are + // rewritten into keyed ops by preprocessing before reaching here; + // in the estimated-cost paths there is no preprocessing, so split + // the tree key off the path and let the op flow to the cost + // dispatch. Silently dropping them here (as this code used to do) + // made every append estimate as free — see issue #812. + // + // The synthetic `MaxKeySize` key sizes estimates with the real + // tree-key length while the op-index prefix keeps several appends + // to the same tree from collapsing into a single BTreeMap entry + // (each append must be charged). If such an op ever reaches real + // execution, `execute_ops_on_path` rejects it with "should have + // been preprocessed" — a loud failure instead of a silent drop. + let (op_path, key, is_keyless_append) = match op_key { + Some(k) => (op_path, k, false), + None => { + let mut path = op_path; + let Some(tree_key) = path.0.pop() else { + return Err(Error::InvalidBatchOperation( + "keyless append-only op must have the tree key as its path's last \ + segment", + )) + .wrap_with_cost(cost); + }; + let mut unique_id = (op_index as u64).to_be_bytes().to_vec(); + unique_id.extend_from_slice(tree_key.as_slice()); + let key = KeyInfo::MaxKeySize { + unique_id, + max_size: tree_key.max_length(), + }; + (path, key, true) + } }; // Validate key length: Merk link encoding stores key length as a @@ -141,10 +172,15 @@ where .wrap_with_cost(cost); } - // Build qualified path (path + key) for reference lookups - let mut qualified_path = op_path.clone(); - qualified_path.push(key.clone()); - ops_by_qualified_paths.insert(qualified_path.to_path_consume(), grove_op.clone()); + // Build qualified path (path + key) for reference lookups. + // Keyless append ops are skipped: they are not elements a + // reference can target, and their synthetic keys must not + // shadow the tree element itself. + if !is_keyless_append { + let mut qualified_path = op_path.clone(); + qualified_path.push(key.clone()); + ops_by_qualified_paths.insert(qualified_path.to_path_consume(), grove_op.clone()); + } let op_cost = OperationCost::default(); let op_result = match &grove_op { diff --git a/grovedb/src/batch/estimated_costs/average_case_costs.rs b/grovedb/src/batch/estimated_costs/average_case_costs.rs index b79da061a..c24ebfe08 100644 --- a/grovedb/src/batch/estimated_costs/average_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/average_case_costs.rs @@ -208,41 +208,22 @@ impl GroveOp { grove_version, ), GroveOp::CommitmentTreeInsert { payload, .. } => { - // After preprocessing, CommitmentTreeInsert becomes + // In the apply path, preprocessing rewrites this op into // ReplaceNonMerkTreeRoot. The base cost is a tree root key - // replacement in the parent Merk. - let item_cost = GroveDb::average_case_merk_replace_tree( + // replacement in the parent Merk; the append work itself + // (frontier I/O, Sinsemilla hashing, note write, epoch + // compaction) is charged by the shared upper-bound model — + // deliberately NOT an average, since the append cost is + // position-dependent and the position is adversary-chosen. + // See `commitment_tree_insert_op_cost`. + GroveDb::average_case_merk_replace_tree( key, layer_element_estimates, TreeType::CommitmentTree(0), propagate, grove_version, - ); - use grovedb_costs::storage_cost::{removal::StorageRemovedBytes, StorageCost}; - // Additional cost: frontier I/O (data storage load + save), - // buffer entry write, and Sinsemilla hashing. - // - // Average frontier size with ~16 ommers: - // 1 (flag) + 8 (position) + 32 (leaf) + 1 (count) + 16*32 = 554 - const AVG_FRONTIER_SIZE: u32 = 554; - // Buffer entry: cmx (32) + rho (32) + cv_net (32) + payload - let buffer_entry_size = 96 + payload.len() as u32; - // Average Sinsemilla hashes per append: - // 32 (root computation) + 1 (avg ommer updates) = 33 - const AVG_SINSEMILLA_HASHES: u32 = 33; - // Average blake3 hashes: 1 for running buffer hash - const AVG_BLAKE3_HASHES: u32 = 1; - item_cost.add_cost(OperationCost { - seek_count: 3, // frontier load + frontier save + buffer write - storage_cost: StorageCost { - added_bytes: buffer_entry_size, - replaced_bytes: AVG_FRONTIER_SIZE, - removed_bytes: StorageRemovedBytes::NoStorageRemoval, - }, - storage_loaded_bytes: AVG_FRONTIER_SIZE as u64, - hash_node_calls: AVG_BLAKE3_HASHES, - sinsemilla_hash_calls: AVG_SINSEMILLA_HASHES, - }) + ) + .add_cost(super::commitment_tree_insert_op_cost(payload.len() as u32)) } GroveOp::MmrTreeAppend { value } => { // Cost of updating parent element in the Merk diff --git a/grovedb/src/batch/estimated_costs/mod.rs b/grovedb/src/batch/estimated_costs/mod.rs index 2ce500c20..5706144cd 100644 --- a/grovedb/src/batch/estimated_costs/mod.rs +++ b/grovedb/src/batch/estimated_costs/mod.rs @@ -3,6 +3,11 @@ #[cfg(feature = "minimal")] use std::collections::HashMap; +#[cfg(feature = "minimal")] +use grovedb_costs::{ + storage_cost::{removal::StorageRemovedBytes, StorageCost}, + OperationCost, +}; #[cfg(feature = "minimal")] use grovedb_merk::estimated_costs::{ average_case_costs::EstimatedLayerInformation, worst_case_costs::WorstCaseLayerInformation, @@ -36,6 +41,121 @@ pub(in crate::batch) fn wrapper_overhead_for( } } +// ── CommitmentTreeInsert estimation model ─────────────────────────────── +// +// Every constant below is an UPPER BOUND, not an average. Downstream +// consumers (Dash Platform admission control) use the estimate as the +// bound that decides whether a transaction is adequately funded, then +// re-meter with the real cost during execution; `estimated >= actual` +// is the invariant they rely on. The expensive appends are not a rare +// tail: the Sinsemilla ommer cascade is maximal exactly at positions +// 2^k - 1 and epoch compaction fires exactly every 2^chunk_power-th +// append, both deterministic and cheaply reachable by an adversary +// choosing when to append. See issue #812. + +/// Depth of the Sinsemilla note-commitment frontier (Orchard's +/// `NOTE_COMMITMENT_TREE_DEPTH`, 32). All frontier-related bounds are +/// derived from this so a depth change cannot silently reintroduce an +/// estimation gap. +#[cfg(feature = "minimal")] +const FRONTIER_DEPTH: u32 = grovedb_commitment_tree::NOTE_COMMITMENT_TREE_DEPTH as u32; + +/// Upper bound on Sinsemilla hash calls for a single append: +/// `FRONTIER_DEPTH` for the leaf-to-root walk plus up to +/// `FRONTIER_DEPTH` ommer merges (`trailing_ones(position)`, maximal at +/// positions `2^k - 1`). +#[cfg(feature = "minimal")] +pub const MAX_SINSEMILLA_HASHES_PER_APPEND: u32 = FRONTIER_DEPTH + FRONTIER_DEPTH; + +/// Upper bound on the serialized frontier size: +/// 1 (flag) + 8 (position) + 32 (leaf) + 1 (ommer count) + 32 bytes per +/// ommer, with at most `FRONTIER_DEPTH` ommers. +#[cfg(feature = "minimal")] +pub const MAX_FRONTIER_SIZE: u32 = 1 + 8 + 32 + 1 + FRONTIER_DEPTH * 32; + +/// Largest `chunk_power` the CommitmentTreeInsert estimate covers +/// (2^10 = 1024-entry epochs, the recommended default). The dense +/// buffer's per-append root recompute and the epoch-compaction blob +/// both scale with `2^chunk_power`, which the op does not carry, so the +/// estimator charges this documented cap. Estimates for trees created +/// with a larger `chunk_power` are NOT upper bounds. +#[cfg(feature = "minimal")] +pub const MAX_ESTIMATED_CHUNK_POWER: u32 = 10; + +/// Epoch size implied by [`MAX_ESTIMATED_CHUNK_POWER`]. +#[cfg(feature = "minimal")] +const MAX_EPOCH_SIZE: u32 = 1 << MAX_ESTIMATED_CHUNK_POWER; + +/// Per-put storage overhead charged on data-storage writes: the 32-byte +/// blake3 path prefix, the logical key (dense positions, MMR indices, +/// `__ct_data__`), and the key/value varint length prefixes. +#[cfg(feature = "minimal")] +const PER_PUT_OVERHEAD: u32 = 50; + +/// Upper-bound cost of the append work a single `CommitmentTreeInsert` +/// performs outside the parent Merk (which is charged separately via +/// `average/worst_case_merk_replace_tree`): frontier I/O and Sinsemilla +/// hashing, the note write into the dense buffer (with its per-append +/// root recompute), and a full epoch compaction (chunk-blob write plus +/// MMR merge cascade). +/// +/// Used by BOTH the average-case and the worst-case estimators. The +/// append cost is position-dependent and the position is +/// adversary-controlled, so an "average" here is not a meaningful +/// bound; making the two estimators differ would make them silently +/// non-interchangeable, which is a consensus fault for admission +/// control (see issue #812). +#[cfg(feature = "minimal")] +pub(in crate::batch) fn commitment_tree_insert_op_cost(payload_len: u32) -> OperationCost { + // A stored note entry: cmx (32) || rho (32) || cv_net (32) || payload. + let entry_size = 96 + payload_len; + + // Chunk-blob serialization overhead per entry (length prefix) and + // per blob (entry count, MMR leaf node framing). + const CHUNK_ENTRY_OVERHEAD: u32 = 16; + const CHUNK_BLOB_OVERHEAD: u32 = 64; + // An MMR internal node: 1 (flag) + 32 (hash). + const MMR_INTERNAL_NODE_SIZE: u32 = 33; + + OperationCost { + // 2 reads (CommitmentTree element + frontier) and up to + // 3 + FRONTIER_DEPTH writes (note entry, frontier, chunk blob, + // MMR internal nodes). + seek_count: 5 + FRONTIER_DEPTH, + storage_cost: StorageCost { + // Data-storage writes are charged as added bytes (the + // commit path has no previous-size information for them, + // and dense/MMR keys are new within an epoch), so the whole + // write volume lands here: + // - the note entry into the dense buffer, + // - the re-serialized frontier (grows toward + // MAX_FRONTIER_SIZE), + // - on compaction: the epoch's chunk blob (every entry is + // re-written once into the blob) and the MMR merge + // cascade's internal nodes. + added_bytes: (entry_size + PER_PUT_OVERHEAD) + + (MAX_FRONTIER_SIZE + PER_PUT_OVERHEAD) + + (MAX_EPOCH_SIZE * (entry_size + CHUNK_ENTRY_OVERHEAD) + + CHUNK_BLOB_OVERHEAD + + PER_PUT_OVERHEAD) + + FRONTIER_DEPTH * (MMR_INTERNAL_NODE_SIZE + PER_PUT_OVERHEAD), + // The parent-Merk node replacement is charged by the + // replace_tree part; the append itself replaces nothing. + replaced_bytes: 0, + removed_bytes: StorageRemovedBytes::NoStorageRemoval, + }, + // Reads: the CommitmentTree element (generous margin for + // caller-supplied element flags) + the serialized frontier. + storage_loaded_bytes: (512 + MAX_FRONTIER_SIZE + PER_PUT_OVERHEAD) as u64, + // Blake3: the dense buffer's root recompute visits every filled + // slot (2 hashes each, up to a full buffer), plus on compaction + // the chunk-leaf hash and MMR merge cascade, plus the bulk + // state root and the ct_state binding hash. + hash_node_calls: 2 * (MAX_EPOCH_SIZE - 1) + FRONTIER_DEPTH + 4, + sinsemilla_hash_calls: MAX_SINSEMILLA_HASHES_PER_APPEND, + } +} + /// Estimated costs types #[cfg(feature = "minimal")] pub enum EstimatedCostsType { diff --git a/grovedb/src/batch/estimated_costs/worst_case_costs.rs b/grovedb/src/batch/estimated_costs/worst_case_costs.rs index 0689de98e..f67c7d683 100644 --- a/grovedb/src/batch/estimated_costs/worst_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/worst_case_costs.rs @@ -193,39 +193,22 @@ impl GroveOp { grove_version, ), GroveOp::CommitmentTreeInsert { payload, .. } => { - // After preprocessing, CommitmentTreeInsert becomes + // In the apply path, preprocessing rewrites this op into // ReplaceNonMerkTreeRoot. The base cost is a tree root key - // replacement in the parent Merk. - let item_cost = GroveDb::worst_case_merk_replace_tree( + // replacement in the parent Merk; the append work itself + // (frontier I/O, Sinsemilla hashing, note write, epoch + // compaction) is charged by the shared upper-bound model with + // constants derived from the frontier depth. See + // `commitment_tree_insert_op_cost`. + GroveDb::worst_case_merk_replace_tree( key, TreeType::CommitmentTree(0), in_parent_tree_type, worst_case_layer_element_estimates, propagate, grove_version, - ); - use grovedb_costs::storage_cost::{removal::StorageRemovedBytes, StorageCost}; - // Worst-case frontier size with 32 ommers (max depth): - // 1 (flag) + 8 (position) + 32 (leaf) + 1 (count) + 32*32 = 1066 - const MAX_FRONTIER_SIZE: u32 = 1066; - // Buffer entry: cmx (32) + rho (32) + cv_net (32) + payload - let buffer_entry_size = 96 + payload.len() as u32; - // Worst-case Sinsemilla hashes per append: - // 32 (root computation) + 32 (all ommers cascade) = 64 - const MAX_SINSEMILLA_HASHES: u32 = 64; - // 1 blake3 hash for running buffer hash - const MAX_BLAKE3_HASHES: u32 = 1; - item_cost.add_cost(OperationCost { - seek_count: 3, // frontier load + frontier save + buffer write - storage_cost: StorageCost { - added_bytes: buffer_entry_size, - replaced_bytes: MAX_FRONTIER_SIZE, - removed_bytes: StorageRemovedBytes::NoStorageRemoval, - }, - storage_loaded_bytes: MAX_FRONTIER_SIZE as u64, - hash_node_calls: MAX_BLAKE3_HASHES, - sinsemilla_hash_calls: MAX_SINSEMILLA_HASHES, - }) + ) + .add_cost(super::commitment_tree_insert_op_cost(payload.len() as u32)) } GroveOp::MmrTreeAppend { value } => { // Cost of updating parent element in the Merk diff --git a/grovedb/src/tests/commitment_tree_cost_bound_tests.rs b/grovedb/src/tests/commitment_tree_cost_bound_tests.rs new file mode 100644 index 000000000..ccbccf276 --- /dev/null +++ b/grovedb/src/tests/commitment_tree_cost_bound_tests.rs @@ -0,0 +1,352 @@ +//! `estimated >= actual` property tests for `CommitmentTreeInsert`. +//! +//! Downstream consumers (Dash Platform admission control) use the estimated +//! cost as the bound deciding whether a transaction is adequately funded, +//! then re-meter with the real cost during execution. If actual ever exceeds +//! estimated, an underfunded transaction is admitted and fails mid-execution +//! (issue #812 — two mainnet chain stalls). These tests pin the invariant the +//! consumers rely on: for every cost dimension, both the average-case and the +//! worst-case estimate of a single `CommitmentTreeInsert` dominate the actual +//! apply cost, across tree positions. +//! +//! Position coverage is deliberately adversarial, not random: +//! - positions `2^k - 1` maximize the Sinsemilla ommer cascade +//! (`trailing_ones(position)`), +//! - positions crossing a `2^chunk_power` boundary trigger epoch compaction +//! (the whole chunk blob is written by that single append). +//! +//! Both are deterministic and cheaply reachable by an adversary choosing when +//! to append, so they must be covered by the estimate, not treated as tail +//! cases. + +use std::collections::HashMap; + +use grovedb_commitment_tree::{DashMemo, NoteBytesData, TransmittedNoteCiphertext}; +use grovedb_costs::{storage_cost::removal::StorageRemovedBytes::NoStorageRemoval, OperationCost}; +use grovedb_merk::{ + estimated_costs::{ + average_case_costs::{ + EstimatedLayerCount::EstimatedLevel, EstimatedLayerInformation, + EstimatedLayerSizes::AllSubtrees, EstimatedSumTrees::NoSumTrees, + }, + worst_case_costs::WorstCaseLayerInformation::MaxElementsNumber, + }, + tree_type::TreeType, +}; +use grovedb_version::version::GroveVersion; + +use crate::{ + batch::{ + estimated_costs::EstimatedCostsType::{AverageCaseCostsType, WorstCaseCostsType}, + KeyInfoPath, QualifiedGroveDbOp, + }, + tests::{common::EMPTY_PATH, make_empty_grovedb}, + Element, GroveDb, +}; + +/// Deterministic valid Pallas field element from an index. +fn test_cmx(index: u32) -> [u8; 32] { + let mut bytes = [0u8; 32]; + bytes[..4].copy_from_slice(&index.to_le_bytes()); + // Clear the top bit so the bytes stay below the Pallas modulus. + bytes[31] &= 0x7f; + bytes +} + +/// Deterministic 216-byte DashMemo ciphertext from an index. +fn test_ciphertext(index: u32) -> TransmittedNoteCiphertext { + let mut epk_bytes = [0u8; 32]; + epk_bytes[..4].copy_from_slice(&index.to_le_bytes()); + let mut enc_data = [0u8; 104]; + enc_data[..4].copy_from_slice(&index.to_le_bytes()); + let mut out_ciphertext = [0u8; 80]; + out_ciphertext[..4].copy_from_slice(&index.to_le_bytes()); + TransmittedNoteCiphertext::from_parts(epk_bytes, NoteBytesData(enc_data), out_ciphertext) +} + +/// A single CommitmentTreeInsert op for the tree at root key `pool`. +fn ct_op(index: u32) -> QualifiedGroveDbOp { + let mut rho = [0u8; 32]; + rho[..4].copy_from_slice(&index.to_le_bytes()); + rho[4] = 0xAA; + let mut cv_net = [0u8; 32]; + cv_net[..4].copy_from_slice(&index.to_le_bytes()); + cv_net[4] = 0xCC; + QualifiedGroveDbOp::commitment_tree_insert_op_typed( + vec![b"pool".to_vec()], + test_cmx(index), + rho, + cv_net, + &test_ciphertext(index), + ) +} + +/// Average-case estimate for a batch of `ops` against a root layer holding a +/// handful of subtrees. +fn average_case_estimate( + ops: Vec, + grove_version: &GroveVersion, +) -> OperationCost { + let mut paths = HashMap::new(); + paths.insert( + KeyInfoPath(vec![]), + EstimatedLayerInformation { + tree_type: TreeType::NormalTree, + estimated_layer_count: EstimatedLevel(1, false), + estimated_layer_sizes: AllSubtrees(4, NoSumTrees, None), + }, + ); + GroveDb::estimated_case_operations_for_batch( + AverageCaseCostsType(paths), + ops, + None, + |_cost, _old_flags, _new_flags| Ok(false), + |_flags, _removed_key_bytes, _removed_value_bytes| Ok((NoStorageRemoval, NoStorageRemoval)), + grove_version, + ) + .cost_as_result() + .expect("expected to compute average case costs for CommitmentTreeInsert") +} + +/// Worst-case estimate for a batch of `ops` against a small root layer. +fn worst_case_estimate( + ops: Vec, + grove_version: &GroveVersion, +) -> OperationCost { + let mut paths = HashMap::new(); + paths.insert(KeyInfoPath(vec![]), MaxElementsNumber(2)); + GroveDb::estimated_case_operations_for_batch( + WorstCaseCostsType(paths), + ops, + None, + |_cost, _old_flags, _new_flags| Ok(false), + |_flags, _removed_key_bytes, _removed_value_bytes| Ok((NoStorageRemoval, NoStorageRemoval)), + grove_version, + ) + .cost_as_result() + .expect("expected to compute worst case costs for CommitmentTreeInsert") +} + +/// Assert both estimators dominate `actual` in every cost dimension. +fn assert_estimates_dominate( + position: u64, + chunk_power: u8, + average: &OperationCost, + worst: &OperationCost, + actual: &OperationCost, +) { + assert!( + average.worse_or_eq_than(actual), + "average-case estimate must dominate actual at position {} (chunk_power {});\nestimated \ + {:?}\nactual {:?}", + position, + chunk_power, + average, + actual, + ); + assert!( + worst.worse_or_eq_than(actual), + "worst-case estimate must dominate actual at position {} (chunk_power {});\nestimated \ + {:?}\nactual {:?}", + position, + chunk_power, + worst, + actual, + ); +} + +/// Sweep every position in `0..36` plus deeper `2^k - 1` / `2^k` pairs with a +/// small epoch (chunk_power 4), so the sweep crosses several compaction +/// boundaries (15, 31, 63, ...) that coincide with maximal ommer cascades. +#[test] +fn test_commitment_tree_insert_estimated_covers_actual_positions_chunk_power_4() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + const CHUNK_POWER: u8 = 4; + + db.insert( + EMPTY_PATH, + b"pool", + Element::empty_commitment_tree(CHUNK_POWER).expect("valid chunk_power"), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert commitment tree"); + + let mut targets: Vec = (0..36).collect(); + targets.extend([62, 63, 64, 65, 126, 127, 128, 254, 255, 256]); + targets.sort_unstable(); + targets.dedup(); + + let mut next_index: u32 = 0; + for &target in &targets { + // Seed the tree up to `target` (cheap bulk batch; correctness of the + // seeded appends is covered by the commitment tree tests). + if (next_index as u64) < target { + let seed_ops: Vec<_> = ((next_index as u64)..target) + .map(|i| ct_op(i as u32)) + .collect(); + next_index = target as u32; + db.apply_batch(seed_ops, None, None, grove_version) + .unwrap() + .expect("seeding appends should succeed"); + } + + let op = ct_op(next_index); + let average = average_case_estimate(vec![op.clone()], grove_version); + let worst = worst_case_estimate(vec![op.clone()], grove_version); + let actual = db.apply_batch(vec![op], None, None, grove_version).cost; + next_index += 1; + + assert_estimates_dominate(target, CHUNK_POWER, &average, &worst, &actual); + } +} + +/// Cross the epoch boundary at the estimator's chunk-power cap +/// (`MAX_ESTIMATED_CHUNK_POWER` = 10): position 1022 maximizes the dense +/// buffer's per-append root recompute, and position 1023 triggers compaction +/// of a full 1024-entry epoch — the single most expensive append a +/// cap-conforming tree can produce. +#[test] +fn test_commitment_tree_insert_estimated_covers_actual_epoch_boundary_chunk_power_10() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + const CHUNK_POWER: u8 = 10; + + db.insert( + EMPTY_PATH, + b"pool", + Element::empty_commitment_tree(CHUNK_POWER).expect("valid chunk_power"), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert commitment tree"); + + // Seed to position 1022 in one bulk batch. + let seed_ops: Vec<_> = (0..1022).map(ct_op).collect(); + db.apply_batch(seed_ops, None, None, grove_version) + .unwrap() + .expect("seeding appends should succeed"); + + for index in [1022u32, 1023, 1024] { + let op = ct_op(index); + let average = average_case_estimate(vec![op.clone()], grove_version); + let worst = worst_case_estimate(vec![op.clone()], grove_version); + let actual = db.apply_batch(vec![op], None, None, grove_version).cost; + + assert_estimates_dominate(index as u64, CHUNK_POWER, &average, &worst, &actual); + } +} + +/// A batch with several appends to the SAME tree must charge every append — +/// the ops share (path, key), and before the fix for issue #812 the batch +/// structure either dropped them entirely (keyless skip) or would have +/// collapsed them into a single map entry. The estimate for N ops must +/// therefore dominate N times the flat append cost, which it can only do if +/// each op is individually dispatched. +#[test] +fn test_commitment_tree_insert_estimate_charges_every_append_in_batch() { + let grove_version = GroveVersion::latest(); + + let one = average_case_estimate(vec![ct_op(0)], grove_version); + let three = average_case_estimate(vec![ct_op(0), ct_op(1), ct_op(2)], grove_version); + + // Each additional op must contribute at least the flat append cost's + // Sinsemilla component (the parent-node replacement may be shared). + assert!( + three.sinsemilla_hash_calls >= 3 * one.sinsemilla_hash_calls, + "3-op estimate must charge Sinsemilla for every append; one={:?} three={:?}", + one, + three, + ); + assert!( + three.storage_cost.added_bytes >= 2 * one.storage_cost.added_bytes, + "3-op estimate must charge storage for every append; one={:?} three={:?}", + one, + three, + ); + + let one_worst = worst_case_estimate(vec![ct_op(0)], grove_version); + let three_worst = worst_case_estimate(vec![ct_op(0), ct_op(1), ct_op(2)], grove_version); + assert!( + three_worst.sinsemilla_hash_calls >= 3 * one_worst.sinsemilla_hash_calls, + "3-op worst-case estimate must charge Sinsemilla for every append; one={:?} three={:?}", + one_worst, + three_worst, + ); +} + +/// A batch spanning several appends must still be dominated by the estimate +/// when applied for real — the end-to-end shape of the mainnet failure (a +/// multi-action shielded transaction). +#[test] +fn test_commitment_tree_insert_estimated_covers_actual_multi_op_batch() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + const CHUNK_POWER: u8 = 4; + + db.insert( + EMPTY_PATH, + b"pool", + Element::empty_commitment_tree(CHUNK_POWER).expect("valid chunk_power"), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert commitment tree"); + + // Seed so the batch below crosses the compaction boundary at 15. + let seed_ops: Vec<_> = (0..14).map(ct_op).collect(); + db.apply_batch(seed_ops, None, None, grove_version) + .unwrap() + .expect("seeding appends should succeed"); + + // A 4-op batch covering positions 14..=17 (compaction at 15). + let ops: Vec<_> = (14..18).map(ct_op).collect(); + let average = average_case_estimate(ops.clone(), grove_version); + let worst = worst_case_estimate(ops.clone(), grove_version); + let actual = db.apply_batch(ops, None, None, grove_version).cost; + + assert_estimates_dominate(14, CHUNK_POWER, &average, &worst, &actual); +} + +/// The keyless-op fix covers every append-only op type, not just +/// `CommitmentTreeInsert`: MMR, bulk-append, and dense-tree appends must also +/// reach their cost arms instead of estimating as free. +#[test] +fn test_other_keyless_append_ops_reach_estimation() { + let grove_version = GroveVersion::latest(); + + let ops_for = |op: QualifiedGroveDbOp| vec![op]; + + for (name, op) in [ + ( + "MmrTreeAppend", + QualifiedGroveDbOp::mmr_tree_append_op(vec![b"mmr".to_vec()], vec![1u8; 64]), + ), + ( + "BulkAppend", + QualifiedGroveDbOp::bulk_append_op(vec![b"bulk".to_vec()], vec![2u8; 64]), + ), + ( + "DenseTreeInsert", + QualifiedGroveDbOp::dense_tree_insert_op(vec![b"dense".to_vec()], vec![3u8; 64]), + ), + ] { + let average = average_case_estimate(ops_for(op.clone()), grove_version); + assert!( + average.seek_count > 0 && average.storage_cost.added_bytes > 0, + "{name} average-case estimate must be non-zero, got {average:?}", + ); + let worst = worst_case_estimate(ops_for(op), grove_version); + assert!( + worst.seek_count > 0 && worst.storage_cost.added_bytes > 0, + "{name} worst-case estimate must be non-zero, got {worst:?}", + ); + } +} diff --git a/grovedb/src/tests/mod.rs b/grovedb/src/tests/mod.rs index d125b1e92..bafe8cfa7 100644 --- a/grovedb/src/tests/mod.rs +++ b/grovedb/src/tests/mod.rs @@ -22,6 +22,7 @@ mod batch_unit_tests; mod bulk_append_tree_tests; mod checkpoint_tests; mod chunk_branch_proof_tests; +mod commitment_tree_cost_bound_tests; mod commitment_tree_tests; mod coverage_round7_tests; // NOTE: the former `count_indexed_tree_tests` (~12.3k LOC) was written From 47f5c8d09b735eb29f996c12a60f61d8014d8264 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Mon, 17 Aug 2026 16:01:38 +0700 Subject: [PATCH 2/7] fix: cover the deployed chunk_power in CommitmentTreeInsert estimates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses CodeRabbit review on #813. Dash Platform's shielded notes pool uses chunk_power 11 (2048-entry epochs), above the previous 2^10 estimation cap, and nothing stopped a tree from being created past the cap at all. - commitment_tree_insert_op_cost now takes the epoch scale. The average-case estimator reads the tree's ACTUAL chunk power from its declared layer (TreeType::CommitmentTree(chunk_power) in the estimation paths — the shape Platform already registers), recovering the tree key from the keyless op's synthetic key; it falls back to the cap when undeclared. The worst-case estimator keeps the cap, since WorstCaseLayerInformation carries no tree type to declare through. - The cap is now grovedb_element::MAX_COMMITMENT_TREE_CHUNK_POWER = 11, and the validated constructors (empty_commitment_tree{,_with_flags}) enforce it at creation (previously <= 31), so no creatable tree exceeds the fallback estimate. The unchecked new_commitment_tree used to rebuild metadata read from disk is untouched, so existing trees keep working. - Property tests: the epoch-boundary test now crosses the cap's 2048-entry compaction (the mainnet shape) and checks the declared-layer estimate as well; a new test shows a declared small chunk power tightens the estimate ~100x while still dominating the actual compaction append; apply_batch success is asserted before reading actual costs (review point 2). Co-Authored-By: Claude Fable 5 --- grovedb-element/src/element/constructor.rs | 22 ++- grovedb-element/src/element/mod.rs | 13 ++ grovedb/src/batch/batch_structure.rs | 48 +++++-- .../estimated_costs/average_case_costs.rs | 58 ++++++-- grovedb/src/batch/estimated_costs/mod.rs | 46 ++++-- .../batch/estimated_costs/worst_case_costs.rs | 10 +- .../tests/commitment_tree_cost_bound_tests.rs | 136 +++++++++++++++--- 7 files changed, 270 insertions(+), 63 deletions(-) diff --git a/grovedb-element/src/element/constructor.rs b/grovedb-element/src/element/constructor.rs index 8949ca1aa..71f975dd7 100644 --- a/grovedb-element/src/element/constructor.rs +++ b/grovedb-element/src/element/constructor.rs @@ -2,7 +2,10 @@ //! Functions for setting an element's type use crate::{ - element::{BigSumValue, CountValue, Element, ElementFlags, MaxReferenceHop, SumValue}, + element::{ + BigSumValue, CountValue, Element, ElementFlags, MaxReferenceHop, SumValue, + MAX_COMMITMENT_TREE_CHUNK_POWER, + }, error::ElementError, reference_path::ReferencePathType, }; @@ -408,23 +411,28 @@ impl Element { /// Set element to an empty commitment tree. /// - /// Returns `InvalidInput` if `chunk_power > 31`. + /// Returns `InvalidInput` if `chunk_power > + /// MAX_COMMITMENT_TREE_CHUNK_POWER` (11) — the estimated-cost model + /// only covers epochs up to that size, and an estimate that is not + /// an upper bound is an admission-control bypass for consumers. pub fn empty_commitment_tree(chunk_power: u8) -> Result { - if chunk_power > 31 { - return Err(ElementError::InvalidInput("chunk_power must be <= 31")); + if chunk_power > MAX_COMMITMENT_TREE_CHUNK_POWER { + return Err(ElementError::InvalidInput("chunk_power must be <= 11")); } Ok(Element::CommitmentTree(0, chunk_power, None)) } /// Set element to an empty commitment tree with flags. /// - /// Returns `InvalidInput` if `chunk_power > 31`. + /// Returns `InvalidInput` if `chunk_power > + /// MAX_COMMITMENT_TREE_CHUNK_POWER` (11) — see + /// [`empty_commitment_tree`](Self::empty_commitment_tree). pub fn empty_commitment_tree_with_flags( chunk_power: u8, flags: Option, ) -> Result { - if chunk_power > 31 { - return Err(ElementError::InvalidInput("chunk_power must be <= 31")); + if chunk_power > MAX_COMMITMENT_TREE_CHUNK_POWER { + return Err(ElementError::InvalidInput("chunk_power must be <= 11")); } Ok(Element::CommitmentTree(0, chunk_power, flags)) } diff --git a/grovedb-element/src/element/mod.rs b/grovedb-element/src/element/mod.rs index 3fc5e9022..129388a66 100644 --- a/grovedb-element/src/element/mod.rs +++ b/grovedb-element/src/element/mod.rs @@ -17,6 +17,19 @@ use bincode::{Decode, Encode}; use crate::{element_type::ElementType, reference_path::ReferencePathType}; +/// Largest `chunk_power` accepted when creating a commitment tree +/// (2^11 = 2048-entry epochs, matching the largest deployed value — Dash +/// Platform's shielded notes pool). +/// +/// This is the authoritative cap that GroveDB's estimated-cost model for +/// `CommitmentTreeInsert` covers when the actual chunk power is not +/// declared in the estimation layer information: the per-append dense +/// buffer recompute and the epoch-compaction blob both scale with +/// `2^chunk_power`, so a creatable tree must never exceed what the +/// estimator charges (issue #812). Raising this constant loosens that +/// fallback estimate proportionally. +pub const MAX_COMMITMENT_TREE_CHUNK_POWER: u8 = 11; + /// Optional meta-data to be stored per element pub type ElementFlags = Vec; diff --git a/grovedb/src/batch/batch_structure.rs b/grovedb/src/batch/batch_structure.rs index dc1c76dd2..ba0441fd0 100644 --- a/grovedb/src/batch/batch_structure.rs +++ b/grovedb/src/batch/batch_structure.rs @@ -30,6 +30,35 @@ pub type OpsByPath = BTreeMap>; #[cfg(feature = "minimal")] pub type OpsByLevelPath = IntMap; +/// Build the synthetic key under which a keyless append-only op is filed. +/// +/// The `MaxKeySize` variant sizes estimates with the real tree-key length, +/// while the 8-byte big-endian op-index prefix in `unique_id` keeps several +/// appends to the same tree from collapsing into a single `BTreeMap` entry +/// (each append must be charged). [`keyless_op_tree_key`] is the inverse. +#[cfg(feature = "minimal")] +pub(in crate::batch) fn keyless_op_synthetic_key(op_index: usize, tree_key: &KeyInfo) -> KeyInfo { + let mut unique_id = (op_index as u64).to_be_bytes().to_vec(); + unique_id.extend_from_slice(tree_key.as_slice()); + KeyInfo::MaxKeySize { + unique_id, + max_size: tree_key.max_length(), + } +} + +/// Recover the real tree-key bytes from a [`keyless_op_synthetic_key`]. +/// +/// Only meaningful for keys of ops that arrive keyless (the append-only tree +/// ops) — a user-supplied `MaxKeySize` key on a keyed op has no such +/// structure, so callers must check the op type before trusting the result. +#[cfg(feature = "minimal")] +pub(in crate::batch) fn keyless_op_tree_key(key: &KeyInfo) -> Option<&[u8]> { + match key { + KeyInfo::MaxKeySize { unique_id, .. } => unique_id.get(8..), + KeyInfo::KnownKey(_) => None, + } +} + /// Batch structure #[cfg(feature = "minimal")] pub(super) struct BatchStructure { @@ -135,12 +164,12 @@ where // dispatch. Silently dropping them here (as this code used to do) // made every append estimate as free — see issue #812. // - // The synthetic `MaxKeySize` key sizes estimates with the real - // tree-key length while the op-index prefix keeps several appends - // to the same tree from collapsing into a single BTreeMap entry - // (each append must be charged). If such an op ever reaches real - // execution, `execute_ops_on_path` rejects it with "should have - // been preprocessed" — a loud failure instead of a silent drop. + // The synthetic key (see `keyless_op_synthetic_key`) sizes + // estimates with the real tree-key length while keeping one map + // entry per op, so each append is charged. If such an op ever + // reaches real execution, `execute_ops_on_path` rejects it with + // "should have been preprocessed" — a loud failure instead of a + // silent drop. let (op_path, key, is_keyless_append) = match op_key { Some(k) => (op_path, k, false), None => { @@ -152,12 +181,7 @@ where )) .wrap_with_cost(cost); }; - let mut unique_id = (op_index as u64).to_be_bytes().to_vec(); - unique_id.extend_from_slice(tree_key.as_slice()); - let key = KeyInfo::MaxKeySize { - unique_id, - max_size: tree_key.max_length(), - }; + let key = keyless_op_synthetic_key(op_index, &tree_key); (path, key, true) } }; diff --git a/grovedb/src/batch/estimated_costs/average_case_costs.rs b/grovedb/src/batch/estimated_costs/average_case_costs.rs index c24ebfe08..d09111909 100644 --- a/grovedb/src/batch/estimated_costs/average_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/average_case_costs.rs @@ -45,6 +45,11 @@ impl GroveOp { &self, key: &KeyInfo, layer_element_estimates: &EstimatedLayerInformation, + // The declared chunk power of the commitment tree a + // `CommitmentTreeInsert` op targets (from the tree's own layer in + // the estimation paths), or `None` to charge the constructor- + // enforced cap. Ignored by every other op type. + ct_chunk_power: Option, propagate: bool, grove_version: &GroveVersion, ) -> CostResult<(), Error> { @@ -219,11 +224,14 @@ impl GroveOp { GroveDb::average_case_merk_replace_tree( key, layer_element_estimates, - TreeType::CommitmentTree(0), + TreeType::CommitmentTree(ct_chunk_power.unwrap_or(0)), propagate, grove_version, ) - .add_cost(super::commitment_tree_insert_op_cost(payload.len() as u32)) + .add_cost(super::commitment_tree_insert_op_cost( + payload.len() as u32, + ct_chunk_power, + )) } GroveOp::MmrTreeAppend { value } => { // Cost of updating parent element in the Merk @@ -565,9 +573,33 @@ impl TreeCache for AverageCaseTreeCacheKnownPaths { .count(); for (key, op) in ops_at_path_by_key.into_iter() { + // A CommitmentTreeInsert arrives under a synthetic key carrying + // the real tree key (see `keyless_op_synthetic_key`). When the + // caller declared the tree's own layer with + // `TreeType::CommitmentTree(chunk_power)` — as Dash Platform + // does — the estimate uses the tree's ACTUAL epoch scale + // instead of the constructor-enforced cap. + let ct_chunk_power = if matches!(op, GroveOp::CommitmentTreeInsert { .. }) { + crate::batch::batch_structure::keyless_op_tree_key(&key).and_then(|tree_key| { + let mut tree_path = path.clone(); + tree_path.push(KeyInfo::KnownKey(tree_key.to_vec())); + match self.paths.get(&tree_path).map(|layer| layer.tree_type) { + Some(TreeType::CommitmentTree(chunk_power)) => Some(chunk_power), + _ => None, + } + }) + } else { + None + }; cost_return_on_error!( &mut cost, - op.average_case_cost(&key, layer_element_estimates, false, grove_version) + op.average_case_cost( + &key, + layer_element_estimates, + ct_chunk_power, + false, + grove_version + ) ); } @@ -1625,7 +1657,7 @@ mod tests { estimated_layer_sizes: AllSubtrees(4, NoSumTrees, None), }; let cost = op - .average_case_cost(&key, &layer_info, false, grove_version) + .average_case_cost(&key, &layer_info, None, false, grove_version) .cost_as_result() .expect("expected cost for commitment tree insert"); // CommitmentTreeInsert includes frontier I/O and buffer writes plus @@ -1672,7 +1704,7 @@ mod tests { estimated_layer_sizes: AllSubtrees(4, NoSumTrees, None), }; let cost = op - .average_case_cost(&key, &layer_info, false, grove_version) + .average_case_cost(&key, &layer_info, None, false, grove_version) .cost_as_result() .expect("expected cost for mmr tree append"); // MmrTreeAppend includes parent replace cost plus MMR node I/O. @@ -1713,7 +1745,7 @@ mod tests { estimated_layer_sizes: AllSubtrees(4, NoSumTrees, None), }; let cost = op - .average_case_cost(&key, &layer_info, false, grove_version) + .average_case_cost(&key, &layer_info, None, false, grove_version) .cost_as_result() .expect("expected cost for bulk append"); // BulkAppend includes parent replace cost plus buffer write + running @@ -1749,7 +1781,7 @@ mod tests { estimated_layer_sizes: AllSubtrees(4, NoSumTrees, None), }; let cost = op - .average_case_cost(&key, &layer_info, false, grove_version) + .average_case_cost(&key, &layer_info, None, false, grove_version) .cost_as_result() .expect("expected cost for dense tree insert"); // DenseTreeInsert includes parent replace cost plus value write and @@ -1793,7 +1825,7 @@ mod tests { estimated_layer_sizes: AllSubtrees(4, NoSumTrees, None), }; let cost = op - .average_case_cost(&key, &layer_info, false, grove_version) + .average_case_cost(&key, &layer_info, None, false, grove_version) .cost_as_result() .expect("expected cost for replace non-merk tree root"); // ReplaceNonMerkTreeRoot delegates to average_case_merk_replace_tree. @@ -1828,7 +1860,7 @@ mod tests { estimated_layer_sizes: AllSubtrees(4, NoSumTrees, None), }; let cost = op - .average_case_cost(&key, &layer_info, false, grove_version) + .average_case_cost(&key, &layer_info, None, false, grove_version) .cost_as_result() .expect("expected cost for insert non-merk tree"); // InsertNonMerkTree delegates to average_case_merk_insert_tree. @@ -1877,7 +1909,7 @@ mod tests { not_summed, not_counted_or_summed, }; - op.average_case_cost(&key, &layer_info, false, grove_version) + op.average_case_cost(&key, &layer_info, None, false, grove_version) .cost_as_result() .expect("expected cost for InsertTreeWithRootHash") }; @@ -1931,7 +1963,7 @@ mod tests { meta: NonMerkTreeMeta::MmrTree { mmr_size: 50 }, non_counted, }; - op.average_case_cost(&key, &layer_info, false, grove_version) + op.average_case_cost(&key, &layer_info, None, false, grove_version) .cost_as_result() .expect("expected cost for InsertNonMerkTree") }; @@ -1970,7 +2002,7 @@ mod tests { axes: vec![(0u8, [0xEFu8; 32], Some(b"srk".to_vec()))], }; let cost_count = op_count - .average_case_cost(&key, &layer_info, false, grove_version) + .average_case_cost(&key, &layer_info, None, false, grove_version) .cost_as_result() .expect("expected average case cost for Count cidx replace"); assert!(cost_count.seek_count > 0 || cost_count.hash_node_calls > 0); @@ -1982,7 +2014,7 @@ mod tests { axes: vec![(0u8, [8u8; 32], None)], }; let cost_pcount = op_pcount - .average_case_cost(&key, &layer_info, true, grove_version) + .average_case_cost(&key, &layer_info, None, true, grove_version) .cost_as_result() .expect("expected average case cost for ProvableCount cidx replace (propagate)"); assert!( diff --git a/grovedb/src/batch/estimated_costs/mod.rs b/grovedb/src/batch/estimated_costs/mod.rs index 5706144cd..dbe578f92 100644 --- a/grovedb/src/batch/estimated_costs/mod.rs +++ b/grovedb/src/batch/estimated_costs/mod.rs @@ -73,18 +73,23 @@ pub const MAX_SINSEMILLA_HASHES_PER_APPEND: u32 = FRONTIER_DEPTH + FRONTIER_DEPT #[cfg(feature = "minimal")] pub const MAX_FRONTIER_SIZE: u32 = 1 + 8 + 32 + 1 + FRONTIER_DEPTH * 32; -/// Largest `chunk_power` the CommitmentTreeInsert estimate covers -/// (2^10 = 1024-entry epochs, the recommended default). The dense -/// buffer's per-append root recompute and the epoch-compaction blob -/// both scale with `2^chunk_power`, which the op does not carry, so the -/// estimator charges this documented cap. Estimates for trees created -/// with a larger `chunk_power` are NOT upper bounds. +/// Largest `chunk_power` the CommitmentTreeInsert estimate charges when +/// the actual value is not declared: the cap enforced by the validated +/// element constructors ([`grovedb_element::MAX_COMMITMENT_TREE_CHUNK_POWER`], +/// 2^11 = 2048-entry epochs), so no creatable tree exceeds the fallback +/// estimate. The average-case estimator uses the ACTUAL chunk power +/// instead when the caller declares the tree's own layer with +/// `TreeType::CommitmentTree(chunk_power)` in the estimation paths. #[cfg(feature = "minimal")] -pub const MAX_ESTIMATED_CHUNK_POWER: u32 = 10; +pub const MAX_ESTIMATED_CHUNK_POWER: u8 = grovedb_element::MAX_COMMITMENT_TREE_CHUNK_POWER; -/// Epoch size implied by [`MAX_ESTIMATED_CHUNK_POWER`]. +/// Physical ceiling on `chunk_power`: the dense buffer's `u16` count +/// limits the underlying tree height to 16, and `BulkAppendTree` +/// construction rejects anything larger, so no tree beyond this can +/// exist on disk. Declared chunk powers are clamped here to keep the +/// `1 << chunk_power` epoch arithmetic in range. #[cfg(feature = "minimal")] -const MAX_EPOCH_SIZE: u32 = 1 << MAX_ESTIMATED_CHUNK_POWER; +const PHYSICAL_MAX_CHUNK_POWER: u8 = 16; /// Per-put storage overhead charged on data-storage writes: the 32-byte /// blake3 path prefix, the logical key (dense positions, MMR indices, @@ -99,6 +104,12 @@ const PER_PUT_OVERHEAD: u32 = 50; /// root recompute), and a full epoch compaction (chunk-blob write plus /// MMR merge cascade). /// +/// `chunk_power` is the tree's declared epoch scale: the average-case +/// estimator reads it from the tree's own layer in the estimation paths +/// (`TreeType::CommitmentTree(chunk_power)`); pass `None` when it is +/// unknown — the [`MAX_ESTIMATED_CHUNK_POWER`] cap, which the validated +/// element constructors enforce at creation, is charged instead. +/// /// Used by BOTH the average-case and the worst-case estimators. The /// append cost is position-dependent and the position is /// adversary-controlled, so an "average" here is not a meaningful @@ -106,10 +117,21 @@ const PER_PUT_OVERHEAD: u32 = 50; /// non-interchangeable, which is a consensus fault for admission /// control (see issue #812). #[cfg(feature = "minimal")] -pub(in crate::batch) fn commitment_tree_insert_op_cost(payload_len: u32) -> OperationCost { +pub(in crate::batch) fn commitment_tree_insert_op_cost( + payload_len: u32, + chunk_power: Option, +) -> OperationCost { // A stored note entry: cmx (32) || rho (32) || cv_net (32) || payload. let entry_size = 96 + payload_len; + // Epoch size for the compaction and dense-recompute bounds. Clamped + // to the physical ceiling so hand-built layer information cannot + // overflow the shift. + let epoch_size: u32 = 1u32 + << chunk_power + .unwrap_or(MAX_ESTIMATED_CHUNK_POWER) + .min(PHYSICAL_MAX_CHUNK_POWER); + // Chunk-blob serialization overhead per entry (length prefix) and // per blob (entry count, MMR leaf node framing). const CHUNK_ENTRY_OVERHEAD: u32 = 16; @@ -135,7 +157,7 @@ pub(in crate::batch) fn commitment_tree_insert_op_cost(payload_len: u32) -> Oper // cascade's internal nodes. added_bytes: (entry_size + PER_PUT_OVERHEAD) + (MAX_FRONTIER_SIZE + PER_PUT_OVERHEAD) - + (MAX_EPOCH_SIZE * (entry_size + CHUNK_ENTRY_OVERHEAD) + + (epoch_size * (entry_size + CHUNK_ENTRY_OVERHEAD) + CHUNK_BLOB_OVERHEAD + PER_PUT_OVERHEAD) + FRONTIER_DEPTH * (MMR_INTERNAL_NODE_SIZE + PER_PUT_OVERHEAD), @@ -151,7 +173,7 @@ pub(in crate::batch) fn commitment_tree_insert_op_cost(payload_len: u32) -> Oper // slot (2 hashes each, up to a full buffer), plus on compaction // the chunk-leaf hash and MMR merge cascade, plus the bulk // state root and the ct_state binding hash. - hash_node_calls: 2 * (MAX_EPOCH_SIZE - 1) + FRONTIER_DEPTH + 4, + hash_node_calls: 2 * (epoch_size - 1) + FRONTIER_DEPTH + 4, sinsemilla_hash_calls: MAX_SINSEMILLA_HASHES_PER_APPEND, } } diff --git a/grovedb/src/batch/estimated_costs/worst_case_costs.rs b/grovedb/src/batch/estimated_costs/worst_case_costs.rs index f67c7d683..e26a04d02 100644 --- a/grovedb/src/batch/estimated_costs/worst_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/worst_case_costs.rs @@ -198,7 +198,10 @@ impl GroveOp { // replacement in the parent Merk; the append work itself // (frontier I/O, Sinsemilla hashing, note write, epoch // compaction) is charged by the shared upper-bound model with - // constants derived from the frontier depth. See + // constants derived from the frontier depth. The epoch scale + // is the constructor-enforced cap: unlike the average-case + // paths, `WorstCaseLayerInformation` carries no tree type, so + // the tree's actual chunk power cannot be declared here. See // `commitment_tree_insert_op_cost`. GroveDb::worst_case_merk_replace_tree( key, @@ -208,7 +211,10 @@ impl GroveOp { propagate, grove_version, ) - .add_cost(super::commitment_tree_insert_op_cost(payload.len() as u32)) + .add_cost(super::commitment_tree_insert_op_cost( + payload.len() as u32, + None, + )) } GroveOp::MmrTreeAppend { value } => { // Cost of updating parent element in the Merk diff --git a/grovedb/src/tests/commitment_tree_cost_bound_tests.rs b/grovedb/src/tests/commitment_tree_cost_bound_tests.rs index ccbccf276..9f9b5a056 100644 --- a/grovedb/src/tests/commitment_tree_cost_bound_tests.rs +++ b/grovedb/src/tests/commitment_tree_cost_bound_tests.rs @@ -22,12 +22,16 @@ use std::collections::HashMap; use grovedb_commitment_tree::{DashMemo, NoteBytesData, TransmittedNoteCiphertext}; -use grovedb_costs::{storage_cost::removal::StorageRemovedBytes::NoStorageRemoval, OperationCost}; +use grovedb_costs::{ + storage_cost::removal::StorageRemovedBytes::NoStorageRemoval, CostContext, OperationCost, +}; use grovedb_merk::{ estimated_costs::{ average_case_costs::{ - EstimatedLayerCount::EstimatedLevel, EstimatedLayerInformation, - EstimatedLayerSizes::AllSubtrees, EstimatedSumTrees::NoSumTrees, + EstimatedLayerCount::EstimatedLevel, + EstimatedLayerInformation, + EstimatedLayerSizes::{AllItems, AllSubtrees}, + EstimatedSumTrees::NoSumTrees, }, worst_case_costs::WorstCaseLayerInformation::MaxElementsNumber, }, @@ -82,9 +86,13 @@ fn ct_op(index: u32) -> QualifiedGroveDbOp { } /// Average-case estimate for a batch of `ops` against a root layer holding a -/// handful of subtrees. -fn average_case_estimate( +/// handful of subtrees. When `declared_chunk_power` is set, the commitment +/// tree's own layer is declared with `TreeType::CommitmentTree(chunk_power)` +/// — the shape Dash Platform registers — so the estimator charges the tree's +/// actual epoch scale instead of the constructor-enforced cap. +fn average_case_estimate_with_layers( ops: Vec, + declared_chunk_power: Option, grove_version: &GroveVersion, ) -> OperationCost { let mut paths = HashMap::new(); @@ -96,6 +104,16 @@ fn average_case_estimate( estimated_layer_sizes: AllSubtrees(4, NoSumTrees, None), }, ); + if let Some(chunk_power) = declared_chunk_power { + paths.insert( + KeyInfoPath::from_known_owned_path(vec![b"pool".to_vec()]), + EstimatedLayerInformation { + tree_type: TreeType::CommitmentTree(chunk_power), + estimated_layer_count: EstimatedLevel(16, false), + estimated_layer_sizes: AllItems(8, 312, None), + }, + ); + } GroveDb::estimated_case_operations_for_batch( AverageCaseCostsType(paths), ops, @@ -108,6 +126,15 @@ fn average_case_estimate( .expect("expected to compute average case costs for CommitmentTreeInsert") } +/// Average-case estimate without declaring the tree's own layer, so the +/// estimator falls back to the constructor-enforced chunk-power cap. +fn average_case_estimate( + ops: Vec, + grove_version: &GroveVersion, +) -> OperationCost { + average_case_estimate_with_layers(ops, None, grove_version) +} + /// Worst-case estimate for a batch of `ops` against a small root layer. fn worst_case_estimate( ops: Vec, @@ -197,7 +224,11 @@ fn test_commitment_tree_insert_estimated_covers_actual_positions_chunk_power_4() let op = ct_op(next_index); let average = average_case_estimate(vec![op.clone()], grove_version); let worst = worst_case_estimate(vec![op.clone()], grove_version); - let actual = db.apply_batch(vec![op], None, None, grove_version).cost; + let CostContext { + value, + cost: actual, + } = db.apply_batch(vec![op], None, None, grove_version); + value.expect("append should succeed"); next_index += 1; assert_estimates_dominate(target, CHUNK_POWER, &average, &worst, &actual); @@ -205,15 +236,17 @@ fn test_commitment_tree_insert_estimated_covers_actual_positions_chunk_power_4() } /// Cross the epoch boundary at the estimator's chunk-power cap -/// (`MAX_ESTIMATED_CHUNK_POWER` = 10): position 1022 maximizes the dense -/// buffer's per-append root recompute, and position 1023 triggers compaction -/// of a full 1024-entry epoch — the single most expensive append a -/// cap-conforming tree can produce. +/// (`MAX_COMMITMENT_TREE_CHUNK_POWER` = 11, the value Dash Platform's +/// shielded notes pool uses): position 2046 maximizes the dense buffer's +/// per-append root recompute, and position 2047 triggers compaction of a +/// full 2048-entry epoch — the single most expensive append a creatable +/// tree can produce. #[test] -fn test_commitment_tree_insert_estimated_covers_actual_epoch_boundary_chunk_power_10() { +fn test_commitment_tree_insert_estimated_covers_actual_epoch_boundary_at_cap() { let grove_version = GroveVersion::latest(); let db = make_empty_grovedb(); - const CHUNK_POWER: u8 = 10; + const CHUNK_POWER: u8 = grovedb_element::MAX_COMMITMENT_TREE_CHUNK_POWER; + const EPOCH: u32 = 1 << CHUNK_POWER as u32; db.insert( EMPTY_PATH, @@ -226,22 +259,87 @@ fn test_commitment_tree_insert_estimated_covers_actual_epoch_boundary_chunk_powe .unwrap() .expect("insert commitment tree"); - // Seed to position 1022 in one bulk batch. - let seed_ops: Vec<_> = (0..1022).map(ct_op).collect(); + // Seed to two positions before the compaction boundary in one bulk batch. + let seed_ops: Vec<_> = (0..EPOCH - 2).map(ct_op).collect(); db.apply_batch(seed_ops, None, None, grove_version) .unwrap() .expect("seeding appends should succeed"); - for index in [1022u32, 1023, 1024] { + for index in [EPOCH - 2, EPOCH - 1, EPOCH] { let op = ct_op(index); let average = average_case_estimate(vec![op.clone()], grove_version); + let declared = + average_case_estimate_with_layers(vec![op.clone()], Some(CHUNK_POWER), grove_version); let worst = worst_case_estimate(vec![op.clone()], grove_version); - let actual = db.apply_batch(vec![op], None, None, grove_version).cost; + let CostContext { + value, + cost: actual, + } = db.apply_batch(vec![op], None, None, grove_version); + value.expect("append should succeed"); assert_estimates_dominate(index as u64, CHUNK_POWER, &average, &worst, &actual); + // The declared-layer estimate (the shape Platform registers) must + // also dominate at the tree's own epoch scale. + assert!( + declared.worse_or_eq_than(&actual), + "declared-chunk-power estimate must dominate actual at position {index};\nestimated \ + {declared:?}\nactual {actual:?}", + ); } } +/// Declaring the tree's own layer (as Dash Platform does) makes the +/// average-case estimate use the tree's actual epoch scale: at a small +/// chunk power the declared estimate is far tighter than the cap-based +/// fallback, while still dominating the actual cost at the compaction +/// position — the most expensive append such a tree can produce. +#[test] +fn test_commitment_tree_insert_declared_chunk_power_tightens_estimate() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + const CHUNK_POWER: u8 = 4; + + db.insert( + EMPTY_PATH, + b"pool", + Element::empty_commitment_tree(CHUNK_POWER).expect("valid chunk_power"), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert commitment tree"); + + // Seed to one position before the compaction boundary at 15. + let seed_ops: Vec<_> = (0..15).map(ct_op).collect(); + db.apply_batch(seed_ops, None, None, grove_version) + .unwrap() + .expect("seeding appends should succeed"); + + let op = ct_op(15); + let declared = + average_case_estimate_with_layers(vec![op.clone()], Some(CHUNK_POWER), grove_version); + let fallback = average_case_estimate(vec![op.clone()], grove_version); + let CostContext { + value, + cost: actual, + } = db.apply_batch(vec![op], None, None, grove_version); + value.expect("compaction append should succeed"); + + // Tighter than the cap-based fallback (2^4 vs 2^11 epoch)... + assert!( + declared.storage_cost.added_bytes < fallback.storage_cost.added_bytes / 8, + "declared estimate should be far tighter than the fallback; declared {declared:?}\ + \nfallback {fallback:?}", + ); + // ...while still an upper bound of the compaction append. + assert!( + declared.worse_or_eq_than(&actual), + "declared-chunk-power estimate must dominate actual at the compaction \ + position;\nestimated {declared:?}\nactual {actual:?}", + ); +} + /// A batch with several appends to the SAME tree must charge every append — /// the ops share (path, key), and before the fix for issue #812 the batch /// structure either dropped them entirely (keyless skip) or would have @@ -310,7 +408,11 @@ fn test_commitment_tree_insert_estimated_covers_actual_multi_op_batch() { let ops: Vec<_> = (14..18).map(ct_op).collect(); let average = average_case_estimate(ops.clone(), grove_version); let worst = worst_case_estimate(ops.clone(), grove_version); - let actual = db.apply_batch(ops, None, None, grove_version).cost; + let CostContext { + value, + cost: actual, + } = db.apply_batch(ops, None, None, grove_version); + value.expect("multi-op batch should succeed"); assert_estimates_dominate(14, CHUNK_POWER, &average, &worst, &actual); } From 39ba4210420e10f4856d83fcbb9465ed9c66ff85 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Mon, 17 Aug 2026 16:18:00 +0700 Subject: [PATCH 3/7] test: pin the constructor-enforced chunk-power cap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Covers the rejection guard in empty_commitment_tree{,_with_flags} — the boundary the estimator's fallback relies on — so a revert to the old <= 31 bound fails a test. Co-Authored-By: Claude Fable 5 --- .../tests/commitment_tree_cost_bound_tests.rs | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/grovedb/src/tests/commitment_tree_cost_bound_tests.rs b/grovedb/src/tests/commitment_tree_cost_bound_tests.rs index 9f9b5a056..51d599b65 100644 --- a/grovedb/src/tests/commitment_tree_cost_bound_tests.rs +++ b/grovedb/src/tests/commitment_tree_cost_bound_tests.rs @@ -452,3 +452,23 @@ fn test_other_keyless_append_ops_reach_estimation() { ); } } + +/// The validated constructors enforce the chunk-power cap the estimator +/// charges as its fallback, so no creatable tree can exceed the estimate. +/// A revert to the old `<= 31` bound must fail here. +#[test] +fn test_commitment_tree_creation_rejects_chunk_power_above_estimator_cap() { + const CAP: u8 = grovedb_element::MAX_COMMITMENT_TREE_CHUNK_POWER; + + assert!(Element::empty_commitment_tree(CAP).is_ok()); + assert!(Element::empty_commitment_tree_with_flags(CAP, Some(vec![1])).is_ok()); + assert!( + Element::empty_commitment_tree(CAP + 1).is_err(), + "chunk_power above MAX_COMMITMENT_TREE_CHUNK_POWER must be rejected", + ); + assert!( + Element::empty_commitment_tree_with_flags(CAP + 1, Some(vec![1])).is_err(), + "chunk_power above MAX_COMMITMENT_TREE_CHUNK_POWER must be rejected", + ); + assert!(Element::empty_commitment_tree(31).is_err()); +} From 09711ae3f8ea365b163382a53e423dc06d408fd5 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Mon, 17 Aug 2026 16:30:32 +0700 Subject: [PATCH 4/7] fix: bound commitment-tree element flags in the load estimate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses CodeRabbit's follow-up on #813: the flat 512-byte allowance for the preprocessing read of the stored CommitmentTree element could be exceeded by caller-supplied flags, breaking estimated >= actual on storage_loaded_bytes. commitment_tree_insert_op_cost now takes an element-flags load bound: the average-case estimator derives it from the parent layer's declared flags size — the same metadata the parent-node replace already uses, so an undeclared flags size undercounts both consistently — and the worst-case estimator charges MERK_BIGGEST_VALUE_SIZE, consistent with the rest of the worst-case machinery. New property test appends to a tree carrying 2000-byte flags and asserts both estimates still dominate the actual apply cost. Co-Authored-By: Claude Fable 5 --- .../estimated_costs/average_case_costs.rs | 20 +++++++ grovedb/src/batch/estimated_costs/mod.rs | 25 +++++++- .../batch/estimated_costs/worst_case_costs.rs | 5 ++ .../tests/commitment_tree_cost_bound_tests.rs | 59 +++++++++++++++++++ 4 files changed, 106 insertions(+), 3 deletions(-) diff --git a/grovedb/src/batch/estimated_costs/average_case_costs.rs b/grovedb/src/batch/estimated_costs/average_case_costs.rs index d09111909..13bd8d887 100644 --- a/grovedb/src/batch/estimated_costs/average_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/average_case_costs.rs @@ -24,6 +24,8 @@ use grovedb_storage::rocksdb_storage::RocksDbStorage; use grovedb_storage::worst_case_costs::WorstKeyLength; use grovedb_version::version::GroveVersion; #[cfg(feature = "minimal")] +use integer_encoding::VarInt; +#[cfg(feature = "minimal")] use itertools::Itertools; use crate::Element; @@ -221,6 +223,23 @@ impl GroveOp { // deliberately NOT an average, since the append cost is // position-dependent and the position is adversary-chosen. // See `commitment_tree_insert_op_cost`. + // + // The preprocessing read of the stored element loads its + // caller-supplied flags too; bound them with the parent + // layer's declared flags size — the same metadata the + // parent-node replace below uses, so an undeclared flag + // size undercounts both consistently. + let element_flags_load_bound = match layer_element_estimates + .estimated_layer_sizes + .layered_flags_size() + { + Ok(flags_size) => flags_size + .map(|f| f + f.required_space() as u32) + .unwrap_or_default(), + Err(e) => { + return Err(Error::MerkError(e)).wrap_with_cost(OperationCost::default()) + } + }; GroveDb::average_case_merk_replace_tree( key, layer_element_estimates, @@ -231,6 +250,7 @@ impl GroveOp { .add_cost(super::commitment_tree_insert_op_cost( payload.len() as u32, ct_chunk_power, + element_flags_load_bound, )) } GroveOp::MmrTreeAppend { value } => { diff --git a/grovedb/src/batch/estimated_costs/mod.rs b/grovedb/src/batch/estimated_costs/mod.rs index dbe578f92..b473dd1ed 100644 --- a/grovedb/src/batch/estimated_costs/mod.rs +++ b/grovedb/src/batch/estimated_costs/mod.rs @@ -97,6 +97,14 @@ const PHYSICAL_MAX_CHUNK_POWER: u8 = 16; #[cfg(feature = "minimal")] const PER_PUT_OVERHEAD: u32 = 50; +/// Bytes loaded when reading the stored `CommitmentTree` element sans +/// flags: the serialized fields (variant, varint total count, chunk +/// power, flags option) plus the Merk node framing (hashes, key, +/// length prefixes) — measured ~87, with margin. Caller-supplied flags +/// are bounded separately via `element_flags_load_bound`. +#[cfg(feature = "minimal")] +const CT_ELEMENT_LOAD_BASE: u32 = 256; + /// Upper-bound cost of the append work a single `CommitmentTreeInsert` /// performs outside the parent Merk (which is charged separately via /// `average/worst_case_merk_replace_tree`): frontier I/O and Sinsemilla @@ -110,6 +118,12 @@ const PER_PUT_OVERHEAD: u32 = 50; /// unknown — the [`MAX_ESTIMATED_CHUNK_POWER`] cap, which the validated /// element constructors enforce at creation, is charged instead. /// +/// `element_flags_load_bound` bounds the caller-supplied flags on the +/// stored `CommitmentTree` element, which the preprocessing read loads: +/// the average-case estimator derives it from the parent layer's +/// declared flags size (the same metadata the parent-node replace +/// uses), the worst-case estimator passes the largest Merk value size. +/// /// Used by BOTH the average-case and the worst-case estimators. The /// append cost is position-dependent and the position is /// adversary-controlled, so an "average" here is not a meaningful @@ -120,6 +134,7 @@ const PER_PUT_OVERHEAD: u32 = 50; pub(in crate::batch) fn commitment_tree_insert_op_cost( payload_len: u32, chunk_power: Option, + element_flags_load_bound: u32, ) -> OperationCost { // A stored note entry: cmx (32) || rho (32) || cv_net (32) || payload. let entry_size = 96 + payload_len; @@ -166,9 +181,13 @@ pub(in crate::batch) fn commitment_tree_insert_op_cost( replaced_bytes: 0, removed_bytes: StorageRemovedBytes::NoStorageRemoval, }, - // Reads: the CommitmentTree element (generous margin for - // caller-supplied element flags) + the serialized frontier. - storage_loaded_bytes: (512 + MAX_FRONTIER_SIZE + PER_PUT_OVERHEAD) as u64, + // Reads: the stored CommitmentTree element (fixed serialized + // fields + Merk node framing, plus the caller-supplied flags + // bound) + the serialized frontier. + storage_loaded_bytes: (CT_ELEMENT_LOAD_BASE + + element_flags_load_bound + + MAX_FRONTIER_SIZE + + PER_PUT_OVERHEAD) as u64, // Blake3: the dense buffer's root recompute visits every filled // slot (2 hashes each, up to a full buffer), plus on compaction // the chunk-leaf hash and MMR merge cascade, plus the bulk diff --git a/grovedb/src/batch/estimated_costs/worst_case_costs.rs b/grovedb/src/batch/estimated_costs/worst_case_costs.rs index e26a04d02..17dd99231 100644 --- a/grovedb/src/batch/estimated_costs/worst_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/worst_case_costs.rs @@ -214,6 +214,11 @@ impl GroveOp { .add_cost(super::commitment_tree_insert_op_cost( payload.len() as u32, None, + // Caller-supplied element flags have no declared bound + // in the worst-case paths — charge the largest value a + // Merk node can store, consistent with the rest of the + // worst-case machinery. + MERK_BIGGEST_VALUE_SIZE, )) } GroveOp::MmrTreeAppend { value } => { diff --git a/grovedb/src/tests/commitment_tree_cost_bound_tests.rs b/grovedb/src/tests/commitment_tree_cost_bound_tests.rs index 51d599b65..eb6b0afe6 100644 --- a/grovedb/src/tests/commitment_tree_cost_bound_tests.rs +++ b/grovedb/src/tests/commitment_tree_cost_bound_tests.rs @@ -453,6 +453,65 @@ fn test_other_keyless_append_ops_reach_estimation() { } } +/// A commitment tree with large caller-supplied flags: the preprocessing +/// read loads the flags too, so the estimate's element-load bound must +/// cover them. The average-case estimator derives the bound from the +/// parent layer's declared flags size (the same metadata the parent-node +/// replace uses); the worst-case estimator assumes the largest Merk +/// value. Before this bound existed, flags above a fixed 512-byte +/// allowance broke `estimated >= actual` on `storage_loaded_bytes`. +#[test] +fn test_commitment_tree_insert_estimated_covers_actual_with_large_flags() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + const CHUNK_POWER: u8 = 4; + const FLAGS_LEN: usize = 2000; + + db.insert( + EMPTY_PATH, + b"pool", + Element::empty_commitment_tree_with_flags(CHUNK_POWER, Some(vec![7u8; FLAGS_LEN])) + .expect("valid chunk_power"), + None, + None, + grove_version, + ) + .unwrap() + .expect("insert commitment tree with flags"); + + let op = ct_op(0); + + // Average case with the flags size declared in the parent layer. + let mut paths = HashMap::new(); + paths.insert( + KeyInfoPath(vec![]), + EstimatedLayerInformation { + tree_type: TreeType::NormalTree, + estimated_layer_count: EstimatedLevel(1, false), + estimated_layer_sizes: AllSubtrees(4, NoSumTrees, Some(FLAGS_LEN as u32)), + }, + ); + let average = GroveDb::estimated_case_operations_for_batch( + AverageCaseCostsType(paths), + vec![op.clone()], + None, + |_cost, _old_flags, _new_flags| Ok(false), + |_flags, _removed_key_bytes, _removed_value_bytes| Ok((NoStorageRemoval, NoStorageRemoval)), + grove_version, + ) + .cost_as_result() + .expect("expected average case costs with declared flags size"); + + let worst = worst_case_estimate(vec![op.clone()], grove_version); + let CostContext { + value, + cost: actual, + } = db.apply_batch(vec![op], None, None, grove_version); + value.expect("append to flagged tree should succeed"); + + assert_estimates_dominate(0, CHUNK_POWER, &average, &worst, &actual); +} + /// The validated constructors enforce the chunk-power cap the estimator /// charges as its fallback, so no creatable tree can exceed the estimate. /// A revert to the old `<= 31` bound must fail here. From cbbb5f395487e0c3d189b6d55b90daa96a1ceee2 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Mon, 17 Aug 2026 21:42:21 +0700 Subject: [PATCH 5/7] fix: gate the #812 estimation changes behind GROVE_V4 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per review on #813: downstream the estimated cost is the admission bound in validate_fees_of_event, and a syncing platform node re-executes every historical block through full validation with the binary's current cost model. Raising the estimate ungated would make already-committed shield-family transitions — admitted under the old under-counting estimate with funding that covered actual but possibly not the new upper bound — re-validate as under-funded and permanently brick sync at that height. Historical blocks must evaluate identically under every future binary, so both halves of the fix are now version-gated: - apply_batch.keyless_op_cost_dispatch (0 on V1..V3, 1 on V4+): old versions keep silently skipping keyless append-only ops in the estimated-cost batch structure (the append estimates as free, exactly as historical admission decisions saw it); V4+ files them under synthetic keys so every append reaches the cost arms. - operations.{average,worst}_case.{average,worst}_case_commitment_tree_insert (0 on V1..V3, 1 on V4+): old versions keep the legacy CommitmentTreeInsert constants byte-for-byte (average 33 Sinsemilla / 554-byte frontier; worst 64 / 1066 with no compaction, frontier charged as replaced); V4+ uses the depth-derived upper-bound model. The apply path remains identical on every version (preprocessing rewrites keyless ops before the batch structure is built), and GroveVersion::latest() resolves to V4, so the cost-bound property tests exercise the new model unchanged. New companion tests pin the replay guarantee: under GROVE_V3 keyless append ops still estimate as exactly zero, and direct dispatch of both CommitmentTreeInsert arms reproduces the legacy outputs byte-for-byte (replace-tree part + pinned legacy flat constants), with the V4-only declared-chunk-power input ignored. Co-Authored-By: Claude Fable 5 --- .../src/version/grovedb_versions.rs | 38 ++++++ grovedb-version/src/version/v1.rs | 3 + grovedb-version/src/version/v2.rs | 3 + grovedb-version/src/version/v3.rs | 3 + grovedb-version/src/version/v4.rs | 24 ++++ grovedb/src/batch/batch_structure.rs | 18 ++- .../estimated_costs/average_case_costs.rs | 115 ++++++++++++++++- .../batch/estimated_costs/worst_case_costs.rs | 117 +++++++++++++++++- grovedb/src/batch/mod.rs | 12 +- .../tests/commitment_tree_cost_bound_tests.rs | 31 +++++ 10 files changed, 352 insertions(+), 12 deletions(-) diff --git a/grovedb-version/src/version/grovedb_versions.rs b/grovedb-version/src/version/grovedb_versions.rs index 630699109..5f3b2b8af 100644 --- a/grovedb-version/src/version/grovedb_versions.rs +++ b/grovedb-version/src/version/grovedb_versions.rs @@ -128,6 +128,23 @@ pub struct GroveDBApplyBatchVersions { /// not cost: a non-empty indexed replacement that would be accepted /// blind on V1..V3 is refused on V4+. pub overwrite_indexed_cleanup_inspection: FeatureVersion, + /// Whether keyless append-only ops (`CommitmentTreeInsert`, + /// `MmrTreeAppend`, `BulkAppend`, `DenseTreeInsert`) reach the cost + /// dispatch in the estimated-cost batch structure. + /// + /// - `0` (V1..V3): keyless ops are silently skipped when building the + /// batch structure, so in the estimated-cost paths the append + /// contributes ZERO — the under-estimate behind issue #812's + /// admission-control bypass. Preserved for replay: historical blocks + /// were admitted under this estimate and must evaluate identically. + /// - `1` (V4+): the tree key is split off the op's path and the op is + /// filed under a unique synthetic key, so every append reaches the + /// cost arms and is charged individually. + /// + /// The apply path is unaffected on every version: preprocessing + /// rewrites keyless ops into keyed ops before the batch structure is + /// built. + pub keyless_op_cost_dispatch: FeatureVersion, } #[derive(Clone, Debug, Default)] @@ -291,6 +308,16 @@ pub struct GroveDBOperationsAverageCaseVersions { pub add_average_case_get_raw_cost: FeatureVersion, pub add_average_case_get_raw_tree_cost: FeatureVersion, pub add_average_case_get_cost: FeatureVersion, + /// Cost model for the `CommitmentTreeInsert` estimation arm. + /// + /// - `0` (V1..V3): the legacy average-case constants (33 Sinsemilla + /// hashes, 554-byte frontier, 1 blake3, frontier charged as replaced + /// bytes). Preserved for replay of historical admission decisions. + /// - `1` (V4+): the depth-derived upper-bound model shared with the + /// worst-case arm (`commitment_tree_insert_op_cost`), covering the + /// full ommer cascade, dense-buffer recompute, and epoch compaction + /// (issue #812). + pub average_case_commitment_tree_insert: FeatureVersion, } #[derive(Clone, Debug, Default)] @@ -307,6 +334,17 @@ pub struct GroveDBOperationsWorstCaseVersions { pub add_worst_case_get_raw_tree_cost: FeatureVersion, pub add_worst_case_get_raw_cost: FeatureVersion, pub add_worst_case_get_cost: FeatureVersion, + /// Cost model for the `CommitmentTreeInsert` estimation arm. + /// + /// - `0` (V1..V3): the legacy flat model (64 Sinsemilla hashes and a + /// 1066-byte frontier, but only 3 seeks, 1 blake3, no dense-buffer + /// recompute or epoch compaction, frontier charged as replaced + /// bytes). Preserved for replay of historical admission decisions. + /// - `1` (V4+): the depth-derived upper-bound model shared with the + /// average-case arm (`commitment_tree_insert_op_cost`), covering the + /// full ommer cascade, dense-buffer recompute, and epoch compaction + /// (issue #812). + pub worst_case_commitment_tree_insert: FeatureVersion, } #[derive(Clone, Debug, Default)] diff --git a/grovedb-version/src/version/v1.rs b/grovedb-version/src/version/v1.rs index eb5a0a9a9..a1b1f6cdb 100644 --- a/grovedb-version/src/version/v1.rs +++ b/grovedb-version/src/version/v1.rs @@ -32,6 +32,7 @@ pub const GROVE_V1: GroveVersion = GroveVersion { estimated_case_operations_for_batch: 0, delete_tree_cleanup_type_source: 0, overwrite_indexed_cleanup_inspection: 0, + keyless_op_cost_dispatch: 0, }, element: GroveDBElementMethodVersions { delete: 0, @@ -190,6 +191,7 @@ pub const GROVE_V1: GroveVersion = GroveVersion { add_average_case_get_raw_cost: 0, add_average_case_get_raw_tree_cost: 0, add_average_case_get_cost: 0, + average_case_commitment_tree_insert: 0, }, worst_case: GroveDBOperationsWorstCaseVersions { add_worst_case_get_merk_at_path: 0, @@ -204,6 +206,7 @@ pub const GROVE_V1: GroveVersion = GroveVersion { add_worst_case_get_raw_tree_cost: 0, add_worst_case_get_raw_cost: 0, add_worst_case_get_cost: 0, + worst_case_commitment_tree_insert: 0, }, }, aggregate_sum_path_query_methods: GroveDBAggregateSumPathQueryMethodVersions { merge: 0 }, diff --git a/grovedb-version/src/version/v2.rs b/grovedb-version/src/version/v2.rs index 8c92dac9e..5788c54e3 100644 --- a/grovedb-version/src/version/v2.rs +++ b/grovedb-version/src/version/v2.rs @@ -32,6 +32,7 @@ pub const GROVE_V2: GroveVersion = GroveVersion { estimated_case_operations_for_batch: 0, delete_tree_cleanup_type_source: 0, overwrite_indexed_cleanup_inspection: 0, + keyless_op_cost_dispatch: 0, }, element: GroveDBElementMethodVersions { delete: 0, @@ -190,6 +191,7 @@ pub const GROVE_V2: GroveVersion = GroveVersion { add_average_case_get_raw_cost: 0, add_average_case_get_raw_tree_cost: 0, add_average_case_get_cost: 0, + average_case_commitment_tree_insert: 0, }, worst_case: GroveDBOperationsWorstCaseVersions { add_worst_case_get_merk_at_path: 0, @@ -204,6 +206,7 @@ pub const GROVE_V2: GroveVersion = GroveVersion { add_worst_case_get_raw_tree_cost: 0, add_worst_case_get_raw_cost: 0, add_worst_case_get_cost: 0, + worst_case_commitment_tree_insert: 0, }, }, aggregate_sum_path_query_methods: GroveDBAggregateSumPathQueryMethodVersions { merge: 0 }, diff --git a/grovedb-version/src/version/v3.rs b/grovedb-version/src/version/v3.rs index 220c619df..88e486f7b 100644 --- a/grovedb-version/src/version/v3.rs +++ b/grovedb-version/src/version/v3.rs @@ -32,6 +32,7 @@ pub const GROVE_V3: GroveVersion = GroveVersion { estimated_case_operations_for_batch: 0, delete_tree_cleanup_type_source: 0, overwrite_indexed_cleanup_inspection: 0, + keyless_op_cost_dispatch: 0, }, element: GroveDBElementMethodVersions { delete: 0, @@ -194,6 +195,7 @@ pub const GROVE_V3: GroveVersion = GroveVersion { add_average_case_get_raw_cost: 0, add_average_case_get_raw_tree_cost: 0, add_average_case_get_cost: 0, + average_case_commitment_tree_insert: 0, }, worst_case: GroveDBOperationsWorstCaseVersions { add_worst_case_get_merk_at_path: 0, @@ -208,6 +210,7 @@ pub const GROVE_V3: GroveVersion = GroveVersion { add_worst_case_get_raw_tree_cost: 0, add_worst_case_get_raw_cost: 0, add_worst_case_get_cost: 0, + worst_case_commitment_tree_insert: 0, }, }, aggregate_sum_path_query_methods: GroveDBAggregateSumPathQueryMethodVersions { merge: 0 }, diff --git a/grovedb-version/src/version/v4.rs b/grovedb-version/src/version/v4.rs index 45caf8a22..352737ad4 100644 --- a/grovedb-version/src/version/v4.rs +++ b/grovedb-version/src/version/v4.rs @@ -62,6 +62,27 @@ //! the version-2 `Query` wire encoding outright, so the slot's `0` value //! is the in-process mirror of that fail-closed decode. //! +//! - `apply_batch.keyless_op_cost_dispatch: 1` — keyless append-only ops +//! (`CommitmentTreeInsert`, `MmrTreeAppend`, `BulkAppend`, +//! `DenseTreeInsert`) reach the cost dispatch in the estimated-cost batch +//! structure, filed under unique synthetic keys so every append is +//! charged. V1..V3 silently skip them — the append estimates as free, +//! the under-estimate behind issue #812's admission-control bypass — +//! preserved so historical admission decisions replay identically. The +//! apply path is unaffected on every version (preprocessing rewrites +//! keyless ops before the batch structure is built). +//! +//! - `operations.average_case.average_case_commitment_tree_insert: 1` and +//! `operations.worst_case.worst_case_commitment_tree_insert: 1` — the +//! `CommitmentTreeInsert` estimation arms charge the depth-derived +//! upper-bound model (full ommer cascade, dense-buffer recompute, epoch +//! compaction, flags-bounded element load). V1..V3 keep the legacy +//! constants (average-case 33 Sinsemilla / 554-byte frontier; worst-case +//! 64 / 1066 but no compaction), which are NOT upper bounds — preserved +//! for replay only. Gated because downstream the estimate is the +//! admission bound: raising it ungated would make already-committed +//! shield transitions re-validate as under-funded and brick sync. +//! //! Note that `GroveVersion::latest()` resolves to this version, so anything //! defaulting to "latest" — tests, benchmarks, tools — exercises every gate //! listed above rather than V3 behaviour. @@ -112,6 +133,7 @@ pub const GROVE_V4: GroveVersion = GroveVersion { estimated_case_operations_for_batch: 0, delete_tree_cleanup_type_source: 1, overwrite_indexed_cleanup_inspection: 1, + keyless_op_cost_dispatch: 1, }, element: GroveDBElementMethodVersions { delete: 0, @@ -274,6 +296,7 @@ pub const GROVE_V4: GroveVersion = GroveVersion { add_average_case_get_raw_cost: 0, add_average_case_get_raw_tree_cost: 0, add_average_case_get_cost: 0, + average_case_commitment_tree_insert: 1, }, worst_case: GroveDBOperationsWorstCaseVersions { add_worst_case_get_merk_at_path: 0, @@ -288,6 +311,7 @@ pub const GROVE_V4: GroveVersion = GroveVersion { add_worst_case_get_raw_tree_cost: 0, add_worst_case_get_raw_cost: 0, add_worst_case_get_cost: 0, + worst_case_commitment_tree_insert: 1, }, }, aggregate_sum_path_query_methods: GroveDBAggregateSumPathQueryMethodVersions { merge: 0 }, diff --git a/grovedb/src/batch/batch_structure.rs b/grovedb/src/batch/batch_structure.rs index ba0441fd0..765f7e64f 100644 --- a/grovedb/src/batch/batch_structure.rs +++ b/grovedb/src/batch/batch_structure.rs @@ -13,6 +13,8 @@ use grovedb_merk::element::tree_type::ElementTreeTypeExtensions; #[cfg(feature = "minimal")] use grovedb_storage::worst_case_costs::WorstKeyLength; #[cfg(feature = "minimal")] +use grovedb_version::version::GroveVersion; +#[cfg(feature = "minimal")] use grovedb_visualize::{DebugByteVectors, DebugBytes}; #[cfg(feature = "minimal")] use intmap::IntMap; @@ -121,6 +123,7 @@ where update_element_flags_function: F, split_remove_bytes_function: SR, merk_tree_cache: C, + grove_version: &GroveVersion, ) -> CostResult, Error> { Self::continue_from_ops( None, @@ -128,6 +131,7 @@ where update_element_flags_function, split_remove_bytes_function, merk_tree_cache, + grove_version, ) } @@ -138,7 +142,13 @@ where update_element_flags_function: F, split_remove_bytes_function: SR, mut merk_tree_cache: C, + grove_version: &GroveVersion, ) -> CostResult, Error> { + let keyless_ops_reach_cost_dispatch = grove_version + .grovedb_versions + .apply_batch + .keyless_op_cost_dispatch + >= 1; let mut cost = OperationCost::default(); let mut ops_by_level_paths: OpsByLevelPath = previous_ops.unwrap_or_default(); @@ -161,8 +171,11 @@ where // rewritten into keyed ops by preprocessing before reaching here; // in the estimated-cost paths there is no preprocessing, so split // the tree key off the path and let the op flow to the cost - // dispatch. Silently dropping them here (as this code used to do) - // made every append estimate as free — see issue #812. + // dispatch. Silently dropping them (as V1..V3 do below) makes + // every append estimate as free — see issue #812. The old skip + // is version-gated, not deleted: downstream the estimate is an + // admission bound, and historical blocks admitted under the old + // under-estimate must re-validate identically on replay. // // The synthetic key (see `keyless_op_synthetic_key`) sizes // estimates with the real tree-key length while keeping one map @@ -172,6 +185,7 @@ where // silent drop. let (op_path, key, is_keyless_append) = match op_key { Some(k) => (op_path, k, false), + None if !keyless_ops_reach_cost_dispatch => continue, None => { let mut path = op_path; let Some(tree_key) = path.0.pop() else { diff --git a/grovedb/src/batch/estimated_costs/average_case_costs.rs b/grovedb/src/batch/estimated_costs/average_case_costs.rs index 13bd8d887..0bb41ca01 100644 --- a/grovedb/src/batch/estimated_costs/average_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/average_case_costs.rs @@ -215,9 +215,53 @@ impl GroveOp { grove_version, ), GroveOp::CommitmentTreeInsert { payload, .. } => { - // In the apply path, preprocessing rewrites this op into - // ReplaceNonMerkTreeRoot. The base cost is a tree root key - // replacement in the parent Merk; the append work itself + // Version-gated cost model — downstream the estimate is an + // admission bound, so historical blocks admitted under the + // legacy numbers must re-validate identically on replay. + if grove_version + .grovedb_versions + .operations + .average_case + .average_case_commitment_tree_insert + == 0 + { + // Legacy (V1..V3) model: averages, NOT upper bounds. + // Kept byte-for-byte for replay of historical admission + // decisions; unreachable from the batch estimation path + // on those versions (keyless ops are skipped there) but + // reachable through direct dispatch. + let item_cost = GroveDb::average_case_merk_replace_tree( + key, + layer_element_estimates, + TreeType::CommitmentTree(0), + propagate, + grove_version, + ); + use grovedb_costs::storage_cost::{removal::StorageRemovedBytes, StorageCost}; + // Average frontier size with ~16 ommers: + // 1 (flag) + 8 (position) + 32 (leaf) + 1 (count) + 16*32 + const AVG_FRONTIER_SIZE: u32 = 554; + // Buffer entry: cmx (32) + rho (32) + cv_net (32) + payload + let buffer_entry_size = 96 + payload.len() as u32; + // 32 (root computation) + 1 (avg ommer updates) = 33 + const AVG_SINSEMILLA_HASHES: u32 = 33; + // 1 blake3 for the running buffer hash + const AVG_BLAKE3_HASHES: u32 = 1; + return item_cost.add_cost(OperationCost { + seek_count: 3, // frontier load + frontier save + buffer write + storage_cost: StorageCost { + added_bytes: buffer_entry_size, + replaced_bytes: AVG_FRONTIER_SIZE, + removed_bytes: StorageRemovedBytes::NoStorageRemoval, + }, + storage_loaded_bytes: AVG_FRONTIER_SIZE as u64, + hash_node_calls: AVG_BLAKE3_HASHES, + sinsemilla_hash_calls: AVG_SINSEMILLA_HASHES, + }); + } + // V4+: in the apply path, preprocessing rewrites this op + // into ReplaceNonMerkTreeRoot. The base cost is a tree root + // key replacement in the parent Merk; the append work itself // (frontier I/O, Sinsemilla hashing, note write, epoch // compaction) is charged by the shared upper-bound model — // deliberately NOT an average, since the append cost is @@ -2427,4 +2471,69 @@ mod tests { actual.storage_loaded_bytes ); } + + /// Replay guarantee: the V1..V3 average-case CommitmentTreeInsert arm + /// must keep producing the LEGACY numbers byte-for-byte — 33 Sinsemilla + /// hashes, a 554-byte frontier charged as replaced and loaded bytes, + /// 1 blake3, 3 seeks on top of the parent-node replace — because + /// historical admission bounds were computed with them. The upper-bound + /// model is gated to V4+ (`average_case_commitment_tree_insert`). + #[test] + fn test_commitment_tree_insert_average_case_cost_pinned_before_v4() { + use grovedb_version::version::v3::GROVE_V3; + let grove_version = &GROVE_V3; + + let payload_len: u32 = 216; + let op = GroveOp::CommitmentTreeInsert { + cmx: [1u8; 32], + rho: [2u8; 32], + cv_net: [3u8; 32], + payload: vec![0u8; payload_len as usize], + }; + let key = KeyInfo::KnownKey(b"pool".to_vec()); + let layer_info = EstimatedLayerInformation { + tree_type: TreeType::NormalTree, + estimated_layer_count: EstimatedLevel(1, false), + estimated_layer_sizes: AllSubtrees(4, NoSumTrees, None), + }; + + let arm_cost = op + .average_case_cost(&key, &layer_info, None, false, grove_version) + .cost_as_result() + .expect("expected V3 average case cost"); + // A declared chunk power must not change the V3 output — the + // declared-layer machinery is part of the V4+ model only. + let arm_cost_with_declared_chunk_power = op + .average_case_cost(&key, &layer_info, Some(4), false, grove_version) + .cost_as_result() + .expect("expected V3 average case cost with declared chunk power"); + assert_eq!(arm_cost, arm_cost_with_declared_chunk_power); + + let replace_part = GroveDb::average_case_merk_replace_tree( + &key, + &layer_info, + grovedb_merk::tree_type::TreeType::CommitmentTree(0), + false, + grove_version, + ) + .cost_as_result() + .expect("expected replace-tree part"); + let legacy_flat = OperationCost { + seek_count: 3, + storage_cost: StorageCost { + added_bytes: 96 + payload_len, + replaced_bytes: 554, + removed_bytes: NoStorageRemoval, + }, + storage_loaded_bytes: 554, + hash_node_calls: 1, + sinsemilla_hash_calls: 33, + }; + assert_eq!( + arm_cost, + replace_part + legacy_flat, + "V3 average-case CommitmentTreeInsert output changed — this breaks replay of \ + historical admission bounds", + ); + } } diff --git a/grovedb/src/batch/estimated_costs/worst_case_costs.rs b/grovedb/src/batch/estimated_costs/worst_case_costs.rs index 17dd99231..289f43bed 100644 --- a/grovedb/src/batch/estimated_costs/worst_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/worst_case_costs.rs @@ -193,9 +193,54 @@ impl GroveOp { grove_version, ), GroveOp::CommitmentTreeInsert { payload, .. } => { - // In the apply path, preprocessing rewrites this op into - // ReplaceNonMerkTreeRoot. The base cost is a tree root key - // replacement in the parent Merk; the append work itself + // Version-gated cost model — downstream the estimate is an + // admission bound, so historical blocks admitted under the + // legacy numbers must re-validate identically on replay. + if grove_version + .grovedb_versions + .operations + .worst_case + .worst_case_commitment_tree_insert + == 0 + { + // Legacy (V1..V3) model: depth-correct Sinsemilla and + // frontier bounds but no dense-buffer recompute or epoch + // compaction. Kept byte-for-byte for replay of historical + // admission decisions; unreachable from the batch + // estimation path on those versions (keyless ops are + // skipped there) but reachable through direct dispatch. + let item_cost = GroveDb::worst_case_merk_replace_tree( + key, + TreeType::CommitmentTree(0), + in_parent_tree_type, + worst_case_layer_element_estimates, + propagate, + grove_version, + ); + use grovedb_costs::storage_cost::{removal::StorageRemovedBytes, StorageCost}; + // 1 (flag) + 8 (position) + 32 (leaf) + 1 (count) + 32*32 + const MAX_FRONTIER_SIZE: u32 = 1066; + // Buffer entry: cmx (32) + rho (32) + cv_net (32) + payload + let buffer_entry_size = 96 + payload.len() as u32; + // 32 (root computation) + 32 (all ommers cascade) = 64 + const MAX_SINSEMILLA_HASHES: u32 = 64; + // 1 blake3 for the running buffer hash + const MAX_BLAKE3_HASHES: u32 = 1; + return item_cost.add_cost(OperationCost { + seek_count: 3, // frontier load + frontier save + buffer write + storage_cost: StorageCost { + added_bytes: buffer_entry_size, + replaced_bytes: MAX_FRONTIER_SIZE, + removed_bytes: StorageRemovedBytes::NoStorageRemoval, + }, + storage_loaded_bytes: MAX_FRONTIER_SIZE as u64, + hash_node_calls: MAX_BLAKE3_HASHES, + sinsemilla_hash_calls: MAX_SINSEMILLA_HASHES, + }); + } + // V4+: in the apply path, preprocessing rewrites this op + // into ReplaceNonMerkTreeRoot. The base cost is a tree root + // key replacement in the parent Merk; the append work itself // (frontier I/O, Sinsemilla hashing, note write, epoch // compaction) is charged by the shared upper-bound model with // constants derived from the frontier depth. The epoch scale @@ -1556,4 +1601,70 @@ mod tests { cost_count, ); } + + /// Replay guarantee: the V1..V3 worst-case CommitmentTreeInsert arm + /// must keep producing the LEGACY numbers byte-for-byte — 64 Sinsemilla + /// hashes, a 1066-byte frontier charged as replaced and loaded bytes, + /// 1 blake3, 3 seeks on top of the parent-node replace, no epoch + /// compaction — because historical admission bounds were computed with + /// them. The upper-bound model is gated to V4+ + /// (`worst_case_commitment_tree_insert`). + #[test] + fn test_commitment_tree_insert_worst_case_cost_pinned_before_v4() { + use grovedb_costs::{ + storage_cost::{removal::StorageRemovedBytes::NoStorageRemoval, StorageCost}, + OperationCost, + }; + use grovedb_version::version::v3::GROVE_V3; + let grove_version = &GROVE_V3; + + let payload_len: u32 = 216; + let op = GroveOp::CommitmentTreeInsert { + cmx: [1u8; 32], + rho: [2u8; 32], + cv_net: [3u8; 32], + payload: vec![0u8; payload_len as usize], + }; + let key = KeyInfo::KnownKey(b"pool".to_vec()); + let layer_info = MaxElementsNumber(100); + + let arm_cost = op + .worst_case_cost( + &key, + TreeType::NormalTree, + &layer_info, + false, + grove_version, + ) + .cost_as_result() + .expect("expected V3 worst case cost"); + + let replace_part = GroveDb::worst_case_merk_replace_tree( + &key, + grovedb_merk::tree_type::TreeType::CommitmentTree(0), + TreeType::NormalTree, + &layer_info, + false, + grove_version, + ) + .cost_as_result() + .expect("expected replace-tree part"); + let legacy_flat = OperationCost { + seek_count: 3, + storage_cost: StorageCost { + added_bytes: 96 + payload_len, + replaced_bytes: 1066, + removed_bytes: NoStorageRemoval, + }, + storage_loaded_bytes: 1066, + hash_node_calls: 1, + sinsemilla_hash_calls: 64, + }; + assert_eq!( + arm_cost, + replace_part + legacy_flat, + "V3 worst-case CommitmentTreeInsert output changed — this breaks replay of \ + historical admission bounds", + ); + } } diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index 12a75936d..e01228bd3 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -4461,7 +4461,8 @@ impl GroveDb { indexed_secondary_after_apply: Default::default(), cidx_overwrite_cleanup_paths: Default::default(), deleted_tree_actual_types: Default::default(), - } + }, + grove_version ) ); Self::apply_batch_structure(batch_structure, batch_apply_options, grove_version) @@ -4521,7 +4522,8 @@ impl GroveDb { indexed_secondary_after_apply: Default::default(), cidx_overwrite_cleanup_paths: Default::default(), deleted_tree_actual_types: Default::default(), - } + }, + grove_version ) ); Self::apply_batch_structure(batch_structure, batch_apply_options, grove_version) @@ -6266,7 +6268,8 @@ impl GroveDb { split_removal_bytes_function, AverageCaseTreeCacheKnownPaths::new_with_estimated_layer_information( estimated_layer_information - ) + ), + grove_version ) ); cost_return_on_error!( @@ -6288,7 +6291,8 @@ impl GroveDb { split_removal_bytes_function, WorstCaseTreeCacheKnownPaths::new_with_worst_case_layer_information( worst_case_layer_information - ) + ), + grove_version ) ); cost_return_on_error!( diff --git a/grovedb/src/tests/commitment_tree_cost_bound_tests.rs b/grovedb/src/tests/commitment_tree_cost_bound_tests.rs index eb6b0afe6..9eae19dbd 100644 --- a/grovedb/src/tests/commitment_tree_cost_bound_tests.rs +++ b/grovedb/src/tests/commitment_tree_cost_bound_tests.rs @@ -453,6 +453,37 @@ fn test_other_keyless_append_ops_reach_estimation() { } } +/// Replay guarantee: on grove versions at or below V3 the estimated-cost +/// batch structure must keep SKIPPING keyless append ops — the append +/// contributes zero, exactly the (under-counting) estimate historical +/// blocks were admitted under. The new cost dispatch is V4-gated +/// (`apply_batch.keyless_op_cost_dispatch`); flipping it for old versions +/// would change historical admission bounds and brick chain sync replay. +#[test] +fn test_keyless_append_ops_still_estimate_as_free_before_v4() { + use grovedb_version::version::v3::GROVE_V3; + + for op in [ + ct_op(0), + QualifiedGroveDbOp::mmr_tree_append_op(vec![b"mmr".to_vec()], vec![1u8; 64]), + QualifiedGroveDbOp::bulk_append_op(vec![b"bulk".to_vec()], vec![2u8; 64]), + QualifiedGroveDbOp::dense_tree_insert_op(vec![b"dense".to_vec()], vec![3u8; 64]), + ] { + let average = average_case_estimate(vec![op.clone()], &GROVE_V3); + assert_eq!( + average, + OperationCost::default(), + "V3 average-case estimate for a keyless append op must stay zero (op {op:?})", + ); + let worst = worst_case_estimate(vec![op.clone()], &GROVE_V3); + assert_eq!( + worst, + OperationCost::default(), + "V3 worst-case estimate for a keyless append op must stay zero (op {op:?})", + ); + } +} + /// A commitment tree with large caller-supplied flags: the preprocessing /// read loads the flags too, so the estimate's element-load bound must /// cover them. The average-case estimator derives the bound from the From 582d8a76c09ebc44e4be89ebb70f0ca5027e10da Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Tue, 18 Aug 2026 02:54:03 +0700 Subject: [PATCH 6/7] refactor: declaration-required chunk_power and versioned cost dispatch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two changes from maintainer feedback on #813: 1. Drop the MAX_COMMITMENT_TREE_CHUNK_POWER creation cap. The validated constructors go back to accepting chunk_power <= 31, and the estimator no longer carries a policy constant. Instead, the V4 average-case model REQUIRES the commitment tree's own layer declared with TreeType::CommitmentTree(chunk_power) in the estimation paths — the same declare-your-layers contract every other estimated op follows, and the shape Dash Platform already registers — erroring loudly when it is missing. The worst-case model, which has no declaration channel, charges the physical ceiling (2^16, the dense buffer's u16 count limit — a structural invariant, not policy). 2. Move both version-gated CommitmentTreeInsert arms out of inline if-version blocks into the standard versioned-function pattern: {average,worst}_case_commitment_tree_insert dispatchers matching on the version slot with _v0 (legacy, byte-for-byte) and _v1 (upper-bound model) variants and UnknownVersionMismatch on unrecognized versions, mirroring average_case_merk_replace_tree. Tests updated to the declaration contract (all dominance sweeps declare the tree's layer as Platform does), plus a new test pinning that an undeclared CommitmentTreeInsert estimation fails loudly. The V3 replay pinning tests are unchanged and still pass. Co-Authored-By: Claude Fable 5 --- grovedb-element/src/element/constructor.rs | 22 +- grovedb-element/src/element/mod.rs | 13 - .../estimated_costs/average_case_costs.rs | 234 ++++++++++++------ grovedb/src/batch/estimated_costs/mod.rs | 36 +-- .../batch/estimated_costs/worst_case_costs.rs | 195 ++++++++++----- .../tests/commitment_tree_cost_bound_tests.rs | 123 ++++----- 6 files changed, 371 insertions(+), 252 deletions(-) diff --git a/grovedb-element/src/element/constructor.rs b/grovedb-element/src/element/constructor.rs index 71f975dd7..8949ca1aa 100644 --- a/grovedb-element/src/element/constructor.rs +++ b/grovedb-element/src/element/constructor.rs @@ -2,10 +2,7 @@ //! Functions for setting an element's type use crate::{ - element::{ - BigSumValue, CountValue, Element, ElementFlags, MaxReferenceHop, SumValue, - MAX_COMMITMENT_TREE_CHUNK_POWER, - }, + element::{BigSumValue, CountValue, Element, ElementFlags, MaxReferenceHop, SumValue}, error::ElementError, reference_path::ReferencePathType, }; @@ -411,28 +408,23 @@ impl Element { /// Set element to an empty commitment tree. /// - /// Returns `InvalidInput` if `chunk_power > - /// MAX_COMMITMENT_TREE_CHUNK_POWER` (11) — the estimated-cost model - /// only covers epochs up to that size, and an estimate that is not - /// an upper bound is an admission-control bypass for consumers. + /// Returns `InvalidInput` if `chunk_power > 31`. pub fn empty_commitment_tree(chunk_power: u8) -> Result { - if chunk_power > MAX_COMMITMENT_TREE_CHUNK_POWER { - return Err(ElementError::InvalidInput("chunk_power must be <= 11")); + if chunk_power > 31 { + return Err(ElementError::InvalidInput("chunk_power must be <= 31")); } Ok(Element::CommitmentTree(0, chunk_power, None)) } /// Set element to an empty commitment tree with flags. /// - /// Returns `InvalidInput` if `chunk_power > - /// MAX_COMMITMENT_TREE_CHUNK_POWER` (11) — see - /// [`empty_commitment_tree`](Self::empty_commitment_tree). + /// Returns `InvalidInput` if `chunk_power > 31`. pub fn empty_commitment_tree_with_flags( chunk_power: u8, flags: Option, ) -> Result { - if chunk_power > MAX_COMMITMENT_TREE_CHUNK_POWER { - return Err(ElementError::InvalidInput("chunk_power must be <= 11")); + if chunk_power > 31 { + return Err(ElementError::InvalidInput("chunk_power must be <= 31")); } Ok(Element::CommitmentTree(0, chunk_power, flags)) } diff --git a/grovedb-element/src/element/mod.rs b/grovedb-element/src/element/mod.rs index 129388a66..3fc5e9022 100644 --- a/grovedb-element/src/element/mod.rs +++ b/grovedb-element/src/element/mod.rs @@ -17,19 +17,6 @@ use bincode::{Decode, Encode}; use crate::{element_type::ElementType, reference_path::ReferencePathType}; -/// Largest `chunk_power` accepted when creating a commitment tree -/// (2^11 = 2048-entry epochs, matching the largest deployed value — Dash -/// Platform's shielded notes pool). -/// -/// This is the authoritative cap that GroveDB's estimated-cost model for -/// `CommitmentTreeInsert` covers when the actual chunk power is not -/// declared in the estimation layer information: the per-append dense -/// buffer recompute and the epoch-compaction blob both scale with -/// `2^chunk_power`, so a creatable tree must never exceed what the -/// estimator charges (issue #812). Raising this constant loosens that -/// fallback estimate proportionally. -pub const MAX_COMMITMENT_TREE_CHUNK_POWER: u8 = 11; - /// Optional meta-data to be stored per element pub type ElementFlags = Vec; diff --git a/grovedb/src/batch/estimated_costs/average_case_costs.rs b/grovedb/src/batch/estimated_costs/average_case_costs.rs index 0bb41ca01..d5f1b7c88 100644 --- a/grovedb/src/batch/estimated_costs/average_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/average_case_costs.rs @@ -22,7 +22,7 @@ use grovedb_merk::{ use grovedb_storage::rocksdb_storage::RocksDbStorage; #[cfg(feature = "minimal")] use grovedb_storage::worst_case_costs::WorstKeyLength; -use grovedb_version::version::GroveVersion; +use grovedb_version::{error::GroveVersionError, version::GroveVersion}; #[cfg(feature = "minimal")] use integer_encoding::VarInt; #[cfg(feature = "minimal")] @@ -215,87 +215,14 @@ impl GroveOp { grove_version, ), GroveOp::CommitmentTreeInsert { payload, .. } => { - // Version-gated cost model — downstream the estimate is an - // admission bound, so historical blocks admitted under the - // legacy numbers must re-validate identically on replay. - if grove_version - .grovedb_versions - .operations - .average_case - .average_case_commitment_tree_insert - == 0 - { - // Legacy (V1..V3) model: averages, NOT upper bounds. - // Kept byte-for-byte for replay of historical admission - // decisions; unreachable from the batch estimation path - // on those versions (keyless ops are skipped there) but - // reachable through direct dispatch. - let item_cost = GroveDb::average_case_merk_replace_tree( - key, - layer_element_estimates, - TreeType::CommitmentTree(0), - propagate, - grove_version, - ); - use grovedb_costs::storage_cost::{removal::StorageRemovedBytes, StorageCost}; - // Average frontier size with ~16 ommers: - // 1 (flag) + 8 (position) + 32 (leaf) + 1 (count) + 16*32 - const AVG_FRONTIER_SIZE: u32 = 554; - // Buffer entry: cmx (32) + rho (32) + cv_net (32) + payload - let buffer_entry_size = 96 + payload.len() as u32; - // 32 (root computation) + 1 (avg ommer updates) = 33 - const AVG_SINSEMILLA_HASHES: u32 = 33; - // 1 blake3 for the running buffer hash - const AVG_BLAKE3_HASHES: u32 = 1; - return item_cost.add_cost(OperationCost { - seek_count: 3, // frontier load + frontier save + buffer write - storage_cost: StorageCost { - added_bytes: buffer_entry_size, - replaced_bytes: AVG_FRONTIER_SIZE, - removed_bytes: StorageRemovedBytes::NoStorageRemoval, - }, - storage_loaded_bytes: AVG_FRONTIER_SIZE as u64, - hash_node_calls: AVG_BLAKE3_HASHES, - sinsemilla_hash_calls: AVG_SINSEMILLA_HASHES, - }); - } - // V4+: in the apply path, preprocessing rewrites this op - // into ReplaceNonMerkTreeRoot. The base cost is a tree root - // key replacement in the parent Merk; the append work itself - // (frontier I/O, Sinsemilla hashing, note write, epoch - // compaction) is charged by the shared upper-bound model — - // deliberately NOT an average, since the append cost is - // position-dependent and the position is adversary-chosen. - // See `commitment_tree_insert_op_cost`. - // - // The preprocessing read of the stored element loads its - // caller-supplied flags too; bound them with the parent - // layer's declared flags size — the same metadata the - // parent-node replace below uses, so an undeclared flag - // size undercounts both consistently. - let element_flags_load_bound = match layer_element_estimates - .estimated_layer_sizes - .layered_flags_size() - { - Ok(flags_size) => flags_size - .map(|f| f + f.required_space() as u32) - .unwrap_or_default(), - Err(e) => { - return Err(Error::MerkError(e)).wrap_with_cost(OperationCost::default()) - } - }; - GroveDb::average_case_merk_replace_tree( + Self::average_case_commitment_tree_insert( + payload, key, layer_element_estimates, - TreeType::CommitmentTree(ct_chunk_power.unwrap_or(0)), + ct_chunk_power, propagate, grove_version, ) - .add_cost(super::commitment_tree_insert_op_cost( - payload.len() as u32, - ct_chunk_power, - element_flags_load_bound, - )) } GroveOp::MmrTreeAppend { value } => { // Cost of updating parent element in the Merk @@ -461,6 +388,151 @@ impl GroveOp { } } } + + /// Versioned cost of a `CommitmentTreeInsert` op in the average-case + /// estimator. Downstream the estimate is an admission bound, so + /// historical blocks admitted under the legacy numbers must re-validate + /// identically on replay — the model is dispatched on + /// `average_case_commitment_tree_insert`. + fn average_case_commitment_tree_insert( + payload: &[u8], + key: &KeyInfo, + layer_element_estimates: &EstimatedLayerInformation, + ct_chunk_power: Option, + propagate: bool, + grove_version: &GroveVersion, + ) -> CostResult<(), Error> { + match grove_version + .grovedb_versions + .operations + .average_case + .average_case_commitment_tree_insert + { + 0 => Self::average_case_commitment_tree_insert_v0( + payload, + key, + layer_element_estimates, + propagate, + grove_version, + ), + 1 => Self::average_case_commitment_tree_insert_v1( + payload, + key, + layer_element_estimates, + ct_chunk_power, + propagate, + grove_version, + ), + version => Err(Error::VersionError( + GroveVersionError::UnknownVersionMismatch { + method: "average_case_commitment_tree_insert".to_string(), + known_versions: vec![0, 1], + received: version, + }, + )) + .wrap_with_cost(OperationCost::default()), + } + } + + /// Legacy (V1..V3) model: averages, NOT upper bounds. Kept byte-for-byte + /// for replay of historical admission decisions; unreachable from the + /// batch estimation path on those versions (keyless ops are skipped + /// there) but reachable through direct dispatch. + fn average_case_commitment_tree_insert_v0( + payload: &[u8], + key: &KeyInfo, + layer_element_estimates: &EstimatedLayerInformation, + propagate: bool, + grove_version: &GroveVersion, + ) -> CostResult<(), Error> { + let item_cost = GroveDb::average_case_merk_replace_tree( + key, + layer_element_estimates, + TreeType::CommitmentTree(0), + propagate, + grove_version, + ); + use grovedb_costs::storage_cost::{removal::StorageRemovedBytes, StorageCost}; + // Average frontier size with ~16 ommers: + // 1 (flag) + 8 (position) + 32 (leaf) + 1 (count) + 16*32 + const AVG_FRONTIER_SIZE: u32 = 554; + // Buffer entry: cmx (32) + rho (32) + cv_net (32) + payload + let buffer_entry_size = 96 + payload.len() as u32; + // 32 (root computation) + 1 (avg ommer updates) = 33 + const AVG_SINSEMILLA_HASHES: u32 = 33; + // 1 blake3 for the running buffer hash + const AVG_BLAKE3_HASHES: u32 = 1; + item_cost.add_cost(OperationCost { + seek_count: 3, // frontier load + frontier save + buffer write + storage_cost: StorageCost { + added_bytes: buffer_entry_size, + replaced_bytes: AVG_FRONTIER_SIZE, + removed_bytes: StorageRemovedBytes::NoStorageRemoval, + }, + storage_loaded_bytes: AVG_FRONTIER_SIZE as u64, + hash_node_calls: AVG_BLAKE3_HASHES, + sinsemilla_hash_calls: AVG_SINSEMILLA_HASHES, + }) + } + + /// V4+ model: in the apply path, preprocessing rewrites the op into + /// ReplaceNonMerkTreeRoot. The base cost is a tree root key replacement + /// in the parent Merk; the append work itself (frontier I/O, Sinsemilla + /// hashing, note write, epoch compaction) is charged by the shared + /// upper-bound model — deliberately NOT an average, since the append + /// cost is position-dependent and the position is adversary-chosen. See + /// `commitment_tree_insert_op_cost`. + fn average_case_commitment_tree_insert_v1( + payload: &[u8], + key: &KeyInfo, + layer_element_estimates: &EstimatedLayerInformation, + ct_chunk_power: Option, + propagate: bool, + grove_version: &GroveVersion, + ) -> CostResult<(), Error> { + // The dense-recompute and compaction terms scale with 2^chunk_power, + // which the op does not carry, so the tree's own layer MUST be + // declared with `TreeType::CommitmentTree(chunk_power)` in the + // estimation paths — the same declare-your-layers contract every + // other estimated op follows. A silent fallback here would either + // under-bound (too small) or grotesquely over-reserve (the physical + // ceiling), both worse than a loud error at integration time. + let Some(chunk_power) = ct_chunk_power else { + return Err(Error::PathNotFoundInCacheForEstimatedCosts( + "CommitmentTreeInsert estimation requires the commitment tree's own layer \ + declared with TreeType::CommitmentTree(chunk_power) in the estimated layer \ + information" + .to_string(), + )) + .wrap_with_cost(OperationCost::default()); + }; + // The preprocessing read of the stored element loads its + // caller-supplied flags too; bound them with the parent layer's + // declared flags size — the same metadata the parent-node replace + // below uses, so an undeclared flag size undercounts both + // consistently. + let element_flags_load_bound = match layer_element_estimates + .estimated_layer_sizes + .layered_flags_size() + { + Ok(flags_size) => flags_size + .map(|f| f + f.required_space() as u32) + .unwrap_or_default(), + Err(e) => return Err(Error::MerkError(e)).wrap_with_cost(OperationCost::default()), + }; + GroveDb::average_case_merk_replace_tree( + key, + layer_element_estimates, + TreeType::CommitmentTree(chunk_power), + propagate, + grove_version, + ) + .add_cost(super::commitment_tree_insert_op_cost( + payload.len() as u32, + chunk_power, + element_flags_load_bound, + )) + } } #[cfg(feature = "minimal")] @@ -1720,9 +1792,15 @@ mod tests { estimated_layer_count: ApproximateElements(10), estimated_layer_sizes: AllSubtrees(4, NoSumTrees, None), }; - let cost = op + // The V4+ model requires the tree's chunk power (normally read from + // the tree's own declared layer); an undeclared dispatch errors. + assert!(op .average_case_cost(&key, &layer_info, None, false, grove_version) .cost_as_result() + .is_err()); + let cost = op + .average_case_cost(&key, &layer_info, Some(10), false, grove_version) + .cost_as_result() .expect("expected cost for commitment tree insert"); // CommitmentTreeInsert includes frontier I/O and buffer writes plus // Sinsemilla hashing for the commitment tree anchor. diff --git a/grovedb/src/batch/estimated_costs/mod.rs b/grovedb/src/batch/estimated_costs/mod.rs index b473dd1ed..586b7f2e7 100644 --- a/grovedb/src/batch/estimated_costs/mod.rs +++ b/grovedb/src/batch/estimated_costs/mod.rs @@ -73,23 +73,15 @@ pub const MAX_SINSEMILLA_HASHES_PER_APPEND: u32 = FRONTIER_DEPTH + FRONTIER_DEPT #[cfg(feature = "minimal")] pub const MAX_FRONTIER_SIZE: u32 = 1 + 8 + 32 + 1 + FRONTIER_DEPTH * 32; -/// Largest `chunk_power` the CommitmentTreeInsert estimate charges when -/// the actual value is not declared: the cap enforced by the validated -/// element constructors ([`grovedb_element::MAX_COMMITMENT_TREE_CHUNK_POWER`], -/// 2^11 = 2048-entry epochs), so no creatable tree exceeds the fallback -/// estimate. The average-case estimator uses the ACTUAL chunk power -/// instead when the caller declares the tree's own layer with -/// `TreeType::CommitmentTree(chunk_power)` in the estimation paths. -#[cfg(feature = "minimal")] -pub const MAX_ESTIMATED_CHUNK_POWER: u8 = grovedb_element::MAX_COMMITMENT_TREE_CHUNK_POWER; - /// Physical ceiling on `chunk_power`: the dense buffer's `u16` count /// limits the underlying tree height to 16, and `BulkAppendTree` /// construction rejects anything larger, so no tree beyond this can -/// exist on disk. Declared chunk powers are clamped here to keep the -/// `1 << chunk_power` epoch arithmetic in range. +/// function on disk. The worst-case estimator (which has no channel for +/// the tree's declared shape) charges this ceiling, and declared chunk +/// powers are clamped here to keep the `1 << chunk_power` epoch +/// arithmetic in range. #[cfg(feature = "minimal")] -const PHYSICAL_MAX_CHUNK_POWER: u8 = 16; +pub const PHYSICAL_MAX_CHUNK_POWER: u8 = 16; /// Per-put storage overhead charged on data-storage writes: the 32-byte /// blake3 path prefix, the logical key (dense positions, MMR indices, @@ -112,11 +104,12 @@ const CT_ELEMENT_LOAD_BASE: u32 = 256; /// root recompute), and a full epoch compaction (chunk-blob write plus /// MMR merge cascade). /// -/// `chunk_power` is the tree's declared epoch scale: the average-case -/// estimator reads it from the tree's own layer in the estimation paths -/// (`TreeType::CommitmentTree(chunk_power)`); pass `None` when it is -/// unknown — the [`MAX_ESTIMATED_CHUNK_POWER`] cap, which the validated -/// element constructors enforce at creation, is charged instead. +/// `chunk_power` is the tree's epoch scale: the average-case estimator +/// requires it declared in the tree's own layer in the estimation paths +/// (`TreeType::CommitmentTree(chunk_power)`) and errors when it is +/// missing; the worst-case estimator, which has no declaration channel, +/// passes [`PHYSICAL_MAX_CHUNK_POWER`]. Values above the physical +/// ceiling are clamped to it. /// /// `element_flags_load_bound` bounds the caller-supplied flags on the /// stored `CommitmentTree` element, which the preprocessing read loads: @@ -133,7 +126,7 @@ const CT_ELEMENT_LOAD_BASE: u32 = 256; #[cfg(feature = "minimal")] pub(in crate::batch) fn commitment_tree_insert_op_cost( payload_len: u32, - chunk_power: Option, + chunk_power: u8, element_flags_load_bound: u32, ) -> OperationCost { // A stored note entry: cmx (32) || rho (32) || cv_net (32) || payload. @@ -142,10 +135,7 @@ pub(in crate::batch) fn commitment_tree_insert_op_cost( // Epoch size for the compaction and dense-recompute bounds. Clamped // to the physical ceiling so hand-built layer information cannot // overflow the shift. - let epoch_size: u32 = 1u32 - << chunk_power - .unwrap_or(MAX_ESTIMATED_CHUNK_POWER) - .min(PHYSICAL_MAX_CHUNK_POWER); + let epoch_size: u32 = 1u32 << chunk_power.min(PHYSICAL_MAX_CHUNK_POWER); // Chunk-blob serialization overhead per entry (length prefix) and // per blob (entry count, MMR leaf node framing). diff --git a/grovedb/src/batch/estimated_costs/worst_case_costs.rs b/grovedb/src/batch/estimated_costs/worst_case_costs.rs index 289f43bed..169227c2d 100644 --- a/grovedb/src/batch/estimated_costs/worst_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/worst_case_costs.rs @@ -23,7 +23,7 @@ use grovedb_merk::{ use grovedb_storage::rocksdb_storage::RocksDbStorage; #[cfg(feature = "minimal")] use grovedb_storage::worst_case_costs::WorstKeyLength; -use grovedb_version::version::GroveVersion; +use grovedb_version::{error::GroveVersionError, version::GroveVersion}; #[cfg(feature = "minimal")] use itertools::Itertools; @@ -193,78 +193,14 @@ impl GroveOp { grove_version, ), GroveOp::CommitmentTreeInsert { payload, .. } => { - // Version-gated cost model — downstream the estimate is an - // admission bound, so historical blocks admitted under the - // legacy numbers must re-validate identically on replay. - if grove_version - .grovedb_versions - .operations - .worst_case - .worst_case_commitment_tree_insert - == 0 - { - // Legacy (V1..V3) model: depth-correct Sinsemilla and - // frontier bounds but no dense-buffer recompute or epoch - // compaction. Kept byte-for-byte for replay of historical - // admission decisions; unreachable from the batch - // estimation path on those versions (keyless ops are - // skipped there) but reachable through direct dispatch. - let item_cost = GroveDb::worst_case_merk_replace_tree( - key, - TreeType::CommitmentTree(0), - in_parent_tree_type, - worst_case_layer_element_estimates, - propagate, - grove_version, - ); - use grovedb_costs::storage_cost::{removal::StorageRemovedBytes, StorageCost}; - // 1 (flag) + 8 (position) + 32 (leaf) + 1 (count) + 32*32 - const MAX_FRONTIER_SIZE: u32 = 1066; - // Buffer entry: cmx (32) + rho (32) + cv_net (32) + payload - let buffer_entry_size = 96 + payload.len() as u32; - // 32 (root computation) + 32 (all ommers cascade) = 64 - const MAX_SINSEMILLA_HASHES: u32 = 64; - // 1 blake3 for the running buffer hash - const MAX_BLAKE3_HASHES: u32 = 1; - return item_cost.add_cost(OperationCost { - seek_count: 3, // frontier load + frontier save + buffer write - storage_cost: StorageCost { - added_bytes: buffer_entry_size, - replaced_bytes: MAX_FRONTIER_SIZE, - removed_bytes: StorageRemovedBytes::NoStorageRemoval, - }, - storage_loaded_bytes: MAX_FRONTIER_SIZE as u64, - hash_node_calls: MAX_BLAKE3_HASHES, - sinsemilla_hash_calls: MAX_SINSEMILLA_HASHES, - }); - } - // V4+: in the apply path, preprocessing rewrites this op - // into ReplaceNonMerkTreeRoot. The base cost is a tree root - // key replacement in the parent Merk; the append work itself - // (frontier I/O, Sinsemilla hashing, note write, epoch - // compaction) is charged by the shared upper-bound model with - // constants derived from the frontier depth. The epoch scale - // is the constructor-enforced cap: unlike the average-case - // paths, `WorstCaseLayerInformation` carries no tree type, so - // the tree's actual chunk power cannot be declared here. See - // `commitment_tree_insert_op_cost`. - GroveDb::worst_case_merk_replace_tree( + Self::worst_case_commitment_tree_insert( + payload, key, - TreeType::CommitmentTree(0), in_parent_tree_type, worst_case_layer_element_estimates, propagate, grove_version, ) - .add_cost(super::commitment_tree_insert_op_cost( - payload.len() as u32, - None, - // Caller-supplied element flags have no declared bound - // in the worst-case paths — charge the largest value a - // Merk node can store, consistent with the rest of the - // worst-case machinery. - MERK_BIGGEST_VALUE_SIZE, - )) } GroveOp::MmrTreeAppend { value } => { // Cost of updating parent element in the Merk @@ -446,6 +382,131 @@ impl GroveOp { ), } } + + /// Versioned cost of a `CommitmentTreeInsert` op in the worst-case + /// estimator. Downstream the estimate is an admission bound, so + /// historical blocks admitted under the legacy numbers must re-validate + /// identically on replay — the model is dispatched on + /// `worst_case_commitment_tree_insert`. + fn worst_case_commitment_tree_insert( + payload: &[u8], + key: &KeyInfo, + in_parent_tree_type: TreeType, + worst_case_layer_element_estimates: &WorstCaseLayerInformation, + propagate: bool, + grove_version: &GroveVersion, + ) -> CostResult<(), Error> { + match grove_version + .grovedb_versions + .operations + .worst_case + .worst_case_commitment_tree_insert + { + 0 => Self::worst_case_commitment_tree_insert_v0( + payload, + key, + in_parent_tree_type, + worst_case_layer_element_estimates, + propagate, + grove_version, + ), + 1 => Self::worst_case_commitment_tree_insert_v1( + payload, + key, + in_parent_tree_type, + worst_case_layer_element_estimates, + propagate, + grove_version, + ), + version => Err(Error::VersionError( + GroveVersionError::UnknownVersionMismatch { + method: "worst_case_commitment_tree_insert".to_string(), + known_versions: vec![0, 1], + received: version, + }, + )) + .wrap_with_cost(OperationCost::default()), + } + } + + /// Legacy (V1..V3) model: depth-correct Sinsemilla and frontier bounds + /// but no dense-buffer recompute or epoch compaction. Kept byte-for-byte + /// for replay of historical admission decisions; unreachable from the + /// batch estimation path on those versions (keyless ops are skipped + /// there) but reachable through direct dispatch. + fn worst_case_commitment_tree_insert_v0( + payload: &[u8], + key: &KeyInfo, + in_parent_tree_type: TreeType, + worst_case_layer_element_estimates: &WorstCaseLayerInformation, + propagate: bool, + grove_version: &GroveVersion, + ) -> CostResult<(), Error> { + let item_cost = GroveDb::worst_case_merk_replace_tree( + key, + TreeType::CommitmentTree(0), + in_parent_tree_type, + worst_case_layer_element_estimates, + propagate, + grove_version, + ); + use grovedb_costs::storage_cost::{removal::StorageRemovedBytes, StorageCost}; + // 1 (flag) + 8 (position) + 32 (leaf) + 1 (count) + 32*32 + const MAX_FRONTIER_SIZE: u32 = 1066; + // Buffer entry: cmx (32) + rho (32) + cv_net (32) + payload + let buffer_entry_size = 96 + payload.len() as u32; + // 32 (root computation) + 32 (all ommers cascade) = 64 + const MAX_SINSEMILLA_HASHES: u32 = 64; + // 1 blake3 for the running buffer hash + const MAX_BLAKE3_HASHES: u32 = 1; + item_cost.add_cost(OperationCost { + seek_count: 3, // frontier load + frontier save + buffer write + storage_cost: StorageCost { + added_bytes: buffer_entry_size, + replaced_bytes: MAX_FRONTIER_SIZE, + removed_bytes: StorageRemovedBytes::NoStorageRemoval, + }, + storage_loaded_bytes: MAX_FRONTIER_SIZE as u64, + hash_node_calls: MAX_BLAKE3_HASHES, + sinsemilla_hash_calls: MAX_SINSEMILLA_HASHES, + }) + } + + /// V4+ model: in the apply path, preprocessing rewrites the op into + /// ReplaceNonMerkTreeRoot. The base cost is a tree root key replacement + /// in the parent Merk; the append work itself (frontier I/O, Sinsemilla + /// hashing, note write, epoch compaction) is charged by the shared + /// upper-bound model with constants derived from the frontier depth. The + /// epoch scale is the PHYSICAL ceiling (2^16, the dense buffer's u16 + /// count limit — no tree beyond it can function): unlike the + /// average-case paths, `WorstCaseLayerInformation` carries no tree type, + /// so the tree's actual chunk power cannot be declared here. See + /// `commitment_tree_insert_op_cost`. + fn worst_case_commitment_tree_insert_v1( + payload: &[u8], + key: &KeyInfo, + in_parent_tree_type: TreeType, + worst_case_layer_element_estimates: &WorstCaseLayerInformation, + propagate: bool, + grove_version: &GroveVersion, + ) -> CostResult<(), Error> { + GroveDb::worst_case_merk_replace_tree( + key, + TreeType::CommitmentTree(0), + in_parent_tree_type, + worst_case_layer_element_estimates, + propagate, + grove_version, + ) + .add_cost(super::commitment_tree_insert_op_cost( + payload.len() as u32, + super::PHYSICAL_MAX_CHUNK_POWER, + // Caller-supplied element flags have no declared bound in the + // worst-case paths — charge the largest value a Merk node can + // store, consistent with the rest of the worst-case machinery. + MERK_BIGGEST_VALUE_SIZE, + )) + } } #[cfg(feature = "minimal")] diff --git a/grovedb/src/tests/commitment_tree_cost_bound_tests.rs b/grovedb/src/tests/commitment_tree_cost_bound_tests.rs index 9eae19dbd..d95ec51b3 100644 --- a/grovedb/src/tests/commitment_tree_cost_bound_tests.rs +++ b/grovedb/src/tests/commitment_tree_cost_bound_tests.rs @@ -85,16 +85,17 @@ fn ct_op(index: u32) -> QualifiedGroveDbOp { ) } -/// Average-case estimate for a batch of `ops` against a root layer holding a -/// handful of subtrees. When `declared_chunk_power` is set, the commitment -/// tree's own layer is declared with `TreeType::CommitmentTree(chunk_power)` -/// — the shape Dash Platform registers — so the estimator charges the tree's -/// actual epoch scale instead of the constructor-enforced cap. -fn average_case_estimate_with_layers( +/// Average-case estimation result for a batch of `ops` against a root layer +/// holding a handful of subtrees. When `declared_chunk_power` is set, the +/// commitment tree's own layer is declared with +/// `TreeType::CommitmentTree(chunk_power)` — the shape Dash Platform +/// registers, and the declaration the estimator REQUIRES for +/// CommitmentTreeInsert ops. +fn try_average_case_estimate( ops: Vec, declared_chunk_power: Option, grove_version: &GroveVersion, -) -> OperationCost { +) -> Result { let mut paths = HashMap::new(); paths.insert( KeyInfoPath(vec![]), @@ -123,11 +124,21 @@ fn average_case_estimate_with_layers( grove_version, ) .cost_as_result() - .expect("expected to compute average case costs for CommitmentTreeInsert") } -/// Average-case estimate without declaring the tree's own layer, so the -/// estimator falls back to the constructor-enforced chunk-power cap. +/// Average-case estimate with the commitment tree's layer declared. +fn average_case_estimate_with_layers( + ops: Vec, + declared_chunk_power: Option, + grove_version: &GroveVersion, +) -> OperationCost { + try_average_case_estimate(ops, declared_chunk_power, grove_version) + .expect("expected to compute average case costs for CommitmentTreeInsert") +} + +/// Average-case estimate without the commitment tree's own layer declared — +/// valid for non-commitment-tree ops and for grove versions that skip +/// keyless ops. fn average_case_estimate( ops: Vec, grove_version: &GroveVersion, @@ -222,7 +233,8 @@ fn test_commitment_tree_insert_estimated_covers_actual_positions_chunk_power_4() } let op = ct_op(next_index); - let average = average_case_estimate(vec![op.clone()], grove_version); + let average = + average_case_estimate_with_layers(vec![op.clone()], Some(CHUNK_POWER), grove_version); let worst = worst_case_estimate(vec![op.clone()], grove_version); let CostContext { value, @@ -235,17 +247,16 @@ fn test_commitment_tree_insert_estimated_covers_actual_positions_chunk_power_4() } } -/// Cross the epoch boundary at the estimator's chunk-power cap -/// (`MAX_COMMITMENT_TREE_CHUNK_POWER` = 11, the value Dash Platform's -/// shielded notes pool uses): position 2046 maximizes the dense buffer's +/// Cross the epoch boundary at chunk_power 11 — the value Dash Platform's +/// shielded notes pool uses: position 2046 maximizes the dense buffer's /// per-append root recompute, and position 2047 triggers compaction of a -/// full 2048-entry epoch — the single most expensive append a creatable -/// tree can produce. +/// full 2048-entry epoch — the single most expensive append such a tree +/// can produce. #[test] -fn test_commitment_tree_insert_estimated_covers_actual_epoch_boundary_at_cap() { +fn test_commitment_tree_insert_estimated_covers_actual_epoch_boundary_chunk_power_11() { let grove_version = GroveVersion::latest(); let db = make_empty_grovedb(); - const CHUNK_POWER: u8 = grovedb_element::MAX_COMMITMENT_TREE_CHUNK_POWER; + const CHUNK_POWER: u8 = 11; const EPOCH: u32 = 1 << CHUNK_POWER as u32; db.insert( @@ -267,8 +278,7 @@ fn test_commitment_tree_insert_estimated_covers_actual_epoch_boundary_at_cap() { for index in [EPOCH - 2, EPOCH - 1, EPOCH] { let op = ct_op(index); - let average = average_case_estimate(vec![op.clone()], grove_version); - let declared = + let average = average_case_estimate_with_layers(vec![op.clone()], Some(CHUNK_POWER), grove_version); let worst = worst_case_estimate(vec![op.clone()], grove_version); let CostContext { @@ -278,13 +288,6 @@ fn test_commitment_tree_insert_estimated_covers_actual_epoch_boundary_at_cap() { value.expect("append should succeed"); assert_estimates_dominate(index as u64, CHUNK_POWER, &average, &worst, &actual); - // The declared-layer estimate (the shape Platform registers) must - // also dominate at the tree's own epoch scale. - assert!( - declared.worse_or_eq_than(&actual), - "declared-chunk-power estimate must dominate actual at position {index};\nestimated \ - {declared:?}\nactual {actual:?}", - ); } } @@ -319,18 +322,19 @@ fn test_commitment_tree_insert_declared_chunk_power_tightens_estimate() { let op = ct_op(15); let declared = average_case_estimate_with_layers(vec![op.clone()], Some(CHUNK_POWER), grove_version); - let fallback = average_case_estimate(vec![op.clone()], grove_version); + let worst = worst_case_estimate(vec![op.clone()], grove_version); let CostContext { value, cost: actual, } = db.apply_batch(vec![op], None, None, grove_version); value.expect("compaction append should succeed"); - // Tighter than the cap-based fallback (2^4 vs 2^11 epoch)... + // Far tighter than the worst-case physical-ceiling assumption + // (2^4 vs 2^16 epoch)... assert!( - declared.storage_cost.added_bytes < fallback.storage_cost.added_bytes / 8, - "declared estimate should be far tighter than the fallback; declared {declared:?}\ - \nfallback {fallback:?}", + declared.storage_cost.added_bytes < worst.storage_cost.added_bytes / 100, + "declared estimate should be far tighter than the physical-ceiling worst case; declared \ + {declared:?}\nworst {worst:?}", ); // ...while still an upper bound of the compaction append. assert!( @@ -340,6 +344,20 @@ fn test_commitment_tree_insert_declared_chunk_power_tightens_estimate() { ); } +/// The average-case estimator REQUIRES the commitment tree's own layer to be +/// declared: an undeclared CommitmentTreeInsert estimation fails loudly +/// instead of silently guessing an epoch scale that could under-bound (too +/// small) or grotesquely over-reserve (the physical ceiling). +#[test] +fn test_commitment_tree_insert_estimation_requires_declared_layer() { + let grove_version = GroveVersion::latest(); + let result = try_average_case_estimate(vec![ct_op(0)], None, grove_version); + assert!( + result.is_err(), + "undeclared CommitmentTreeInsert estimation must error, got {result:?}", + ); +} + /// A batch with several appends to the SAME tree must charge every append — /// the ops share (path, key), and before the fix for issue #812 the batch /// structure either dropped them entirely (keyless skip) or would have @@ -350,8 +368,12 @@ fn test_commitment_tree_insert_declared_chunk_power_tightens_estimate() { fn test_commitment_tree_insert_estimate_charges_every_append_in_batch() { let grove_version = GroveVersion::latest(); - let one = average_case_estimate(vec![ct_op(0)], grove_version); - let three = average_case_estimate(vec![ct_op(0), ct_op(1), ct_op(2)], grove_version); + let one = average_case_estimate_with_layers(vec![ct_op(0)], Some(11), grove_version); + let three = average_case_estimate_with_layers( + vec![ct_op(0), ct_op(1), ct_op(2)], + Some(11), + grove_version, + ); // Each additional op must contribute at least the flat append cost's // Sinsemilla component (the parent-node replacement may be shared). @@ -406,7 +428,7 @@ fn test_commitment_tree_insert_estimated_covers_actual_multi_op_batch() { // A 4-op batch covering positions 14..=17 (compaction at 15). let ops: Vec<_> = (14..18).map(ct_op).collect(); - let average = average_case_estimate(ops.clone(), grove_version); + let average = average_case_estimate_with_layers(ops.clone(), Some(CHUNK_POWER), grove_version); let worst = worst_case_estimate(ops.clone(), grove_version); let CostContext { value, @@ -512,7 +534,8 @@ fn test_commitment_tree_insert_estimated_covers_actual_with_large_flags() { let op = ct_op(0); - // Average case with the flags size declared in the parent layer. + // Average case with the flags size declared in the parent layer and the + // commitment tree's own layer declared with its chunk power. let mut paths = HashMap::new(); paths.insert( KeyInfoPath(vec![]), @@ -522,6 +545,14 @@ fn test_commitment_tree_insert_estimated_covers_actual_with_large_flags() { estimated_layer_sizes: AllSubtrees(4, NoSumTrees, Some(FLAGS_LEN as u32)), }, ); + paths.insert( + KeyInfoPath::from_known_owned_path(vec![b"pool".to_vec()]), + EstimatedLayerInformation { + tree_type: TreeType::CommitmentTree(CHUNK_POWER), + estimated_layer_count: EstimatedLevel(16, false), + estimated_layer_sizes: AllItems(8, 312, None), + }, + ); let average = GroveDb::estimated_case_operations_for_batch( AverageCaseCostsType(paths), vec![op.clone()], @@ -542,23 +573,3 @@ fn test_commitment_tree_insert_estimated_covers_actual_with_large_flags() { assert_estimates_dominate(0, CHUNK_POWER, &average, &worst, &actual); } - -/// The validated constructors enforce the chunk-power cap the estimator -/// charges as its fallback, so no creatable tree can exceed the estimate. -/// A revert to the old `<= 31` bound must fail here. -#[test] -fn test_commitment_tree_creation_rejects_chunk_power_above_estimator_cap() { - const CAP: u8 = grovedb_element::MAX_COMMITMENT_TREE_CHUNK_POWER; - - assert!(Element::empty_commitment_tree(CAP).is_ok()); - assert!(Element::empty_commitment_tree_with_flags(CAP, Some(vec![1])).is_ok()); - assert!( - Element::empty_commitment_tree(CAP + 1).is_err(), - "chunk_power above MAX_COMMITMENT_TREE_CHUNK_POWER must be rejected", - ); - assert!( - Element::empty_commitment_tree_with_flags(CAP + 1, Some(vec![1])).is_err(), - "chunk_power above MAX_COMMITMENT_TREE_CHUNK_POWER must be rejected", - ); - assert!(Element::empty_commitment_tree(31).is_err()); -} From 2d5eacb3acd23c88372cb7ded96df54b3a0a6617 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Tue, 18 Aug 2026 03:02:20 +0700 Subject: [PATCH 7/7] fix: saturate the epoch term in the CommitmentTreeInsert estimate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses CodeRabbit's overflow finding on #813: the epoch term multiplies the entry size by up to 2^16 (the physical ceiling), which overflows u32 for hand-built ops with oversized payloads — the op type is public and estimation runs before the apply path rejects wrong-sized payloads. A wrapped added_bytes would silently UNDER-estimate, the exact failure the model exists to prevent. The byte sum is now computed in u64 and saturated at u32::MAX, with a boundary test pinning the saturation (a debug build would previously panic there). Co-Authored-By: Claude Fable 5 --- grovedb/src/batch/estimated_costs/mod.rs | 29 ++++++++++------ .../batch/estimated_costs/worst_case_costs.rs | 34 +++++++++++++++++++ 2 files changed, 53 insertions(+), 10 deletions(-) diff --git a/grovedb/src/batch/estimated_costs/mod.rs b/grovedb/src/batch/estimated_costs/mod.rs index 586b7f2e7..c38f68419 100644 --- a/grovedb/src/batch/estimated_costs/mod.rs +++ b/grovedb/src/batch/estimated_costs/mod.rs @@ -130,7 +130,7 @@ pub(in crate::batch) fn commitment_tree_insert_op_cost( element_flags_load_bound: u32, ) -> OperationCost { // A stored note entry: cmx (32) || rho (32) || cv_net (32) || payload. - let entry_size = 96 + payload_len; + let entry_size = 96u64 + payload_len as u64; // Epoch size for the compaction and dense-recompute bounds. Clamped // to the physical ceiling so hand-built layer information cannot @@ -139,10 +139,24 @@ pub(in crate::batch) fn commitment_tree_insert_op_cost( // Chunk-blob serialization overhead per entry (length prefix) and // per blob (entry count, MMR leaf node framing). - const CHUNK_ENTRY_OVERHEAD: u32 = 16; - const CHUNK_BLOB_OVERHEAD: u32 = 64; + const CHUNK_ENTRY_OVERHEAD: u64 = 16; + const CHUNK_BLOB_OVERHEAD: u64 = 64; // An MMR internal node: 1 (flag) + 32 (hash). - const MMR_INTERNAL_NODE_SIZE: u32 = 33; + const MMR_INTERNAL_NODE_SIZE: u64 = 33; + + // The epoch term multiplies the entry size by up to 2^16, which + // overflows u32 for hand-built ops with oversized payloads (the op + // is public; the apply path only rejects wrong-sized payloads + // later). Sum in u64 and saturate at u32::MAX — a wrapped + // added_bytes would silently UNDER-estimate, the exact failure this + // model exists to prevent, while a saturated one merely + // over-reserves for an op the apply would reject anyway. + let added_bytes_u64: u64 = (entry_size + PER_PUT_OVERHEAD as u64) + + (MAX_FRONTIER_SIZE as u64 + PER_PUT_OVERHEAD as u64) + + (epoch_size as u64 * (entry_size + CHUNK_ENTRY_OVERHEAD) + + CHUNK_BLOB_OVERHEAD + + PER_PUT_OVERHEAD as u64) + + FRONTIER_DEPTH as u64 * (MMR_INTERNAL_NODE_SIZE + PER_PUT_OVERHEAD as u64); OperationCost { // 2 reads (CommitmentTree element + frontier) and up to @@ -160,12 +174,7 @@ pub(in crate::batch) fn commitment_tree_insert_op_cost( // - on compaction: the epoch's chunk blob (every entry is // re-written once into the blob) and the MMR merge // cascade's internal nodes. - added_bytes: (entry_size + PER_PUT_OVERHEAD) - + (MAX_FRONTIER_SIZE + PER_PUT_OVERHEAD) - + (epoch_size * (entry_size + CHUNK_ENTRY_OVERHEAD) - + CHUNK_BLOB_OVERHEAD - + PER_PUT_OVERHEAD) - + FRONTIER_DEPTH * (MMR_INTERNAL_NODE_SIZE + PER_PUT_OVERHEAD), + added_bytes: u32::try_from(added_bytes_u64).unwrap_or(u32::MAX), // The parent-Merk node replacement is charged by the // replace_tree part; the append itself replaces nothing. replaced_bytes: 0, diff --git a/grovedb/src/batch/estimated_costs/worst_case_costs.rs b/grovedb/src/batch/estimated_costs/worst_case_costs.rs index 169227c2d..18cd586ba 100644 --- a/grovedb/src/batch/estimated_costs/worst_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/worst_case_costs.rs @@ -1728,4 +1728,38 @@ mod tests { historical admission bounds", ); } + + /// Boundary test for the saturating epoch arithmetic: a hand-built op + /// with an oversized payload (the op type is public; the apply path only + /// rejects wrong-sized payloads later) drives the physical-ceiling epoch + /// term past u32. The estimate must saturate at u32::MAX — never panic + /// in debug builds nor wrap in release builds, since a wrapped + /// added_bytes would silently UNDER-estimate. + #[test] + fn test_commitment_tree_insert_worst_case_cost_oversized_payload_saturates() { + let grove_version = GroveVersion::latest(); + let op = GroveOp::CommitmentTreeInsert { + cmx: [1u8; 32], + rho: [2u8; 32], + cv_net: [3u8; 32], + // 2^16 epoch x (96 + 70_000 + 16) bytes ≈ 4.6e9 > u32::MAX. + payload: vec![0u8; 70_000], + }; + let key = KeyInfo::KnownKey(b"tree_key".to_vec()); + let cost = op + .worst_case_cost( + &key, + TreeType::NormalTree, + &MaxElementsNumber(100), + false, + grove_version, + ) + .cost_as_result() + .expect("expected worst case cost for oversized payload"); + assert_eq!( + cost.storage_cost.added_bytes, + u32::MAX, + "oversized-payload estimate must saturate, not wrap", + ); + } }