fix(drive-abci): rebuild a ranked envelope that loses its race with a commit

A ranked page is committed to by an envelope built from several
independent storage reads, and those are not isolated from a concurrent
block commit. A commit landing inside that window leaves the ancestor
chain unreconcilable, grovedb rejects it, and the request fails. Before
this surface read through the prover, only `prove = true` was exposed;
now the default read path is too.

`query::service` already re-runs a query whose committed height moved,
which absorbs almost all of it, but `finalize_block` makes a commit
visible before publishing that height, and a request finishing inside
that gap is not retried. Rebuild the envelope instead of returning: the
condition is transient and a fresh envelope over settled state resolves
it.

Deliberately narrow. Only the chain mismatch is retried, bounded at two
extra attempts (~400 us total), and genuine corruption produces the same
rejection every time and so still surfaces — one envelope later, with a
`warn` that says a burst under load is the race while a persistent or
unloaded occurrence is not.

Also covers three defensive paths that had no tests: the shared entry
decoder's axis-shape and `k` guards, unreachable through grovedb today
and therefore worth pinning precisely because nothing else can reach
them, and the chain-mismatch detector's narrowness.

Reviewed-by: CodeRabbit (retry suggestion adopted; its exact-match
suggestion for the empty-tree marker was rejected — grovedb wraps merk's
constant in its own prefix, so exact matching would stop the mapper from
ever firing)

Author: shumkov

packages/rs-drive-abci/src/query/document_query/v1/mod.rs

@@ -1317,26 +1317,45 @@ impl<C> Platform<C> {
                 document_type_name, contract_id
             ))));
 
-        let drive_request = DocumentRankedRequest {
-            contract: contract_ref,
-            document_type,
-            group_by: &group_by,
-            select,
-            having: &having,
-            order_by: &order_clauses,
-            where_clauses: &where_clauses,
-            limit,
-            offset,
-            has_start_at: start.is_some(),
-            prove,
-        };
+        // A ranked page is committed to by an envelope built from several
+        // independent storage reads, which are not isolated from a
+        // concurrent block commit. A commit landing inside that window
+        // leaves the envelope's ancestor chain unreconcilable and grovedb
+        // rejects it — a transient condition that a fresh envelope over
+        // settled state resolves, so it is retried here rather than
+        // returned. `query::service` already re-runs a query whose
+        // committed height moved, but there is a window between a commit
+        // becoming visible and that height being published where it does
+        // not, and this closes it.
+        //
+        // Deliberately narrow: only the chain mismatch is retried, and
+        // only a bounded number of times. Genuine corruption produces the
+        // same rejection on every attempt and so still surfaces, one
+        // envelope later.
+        let mut attempts_left = RANKED_CHAIN_MISMATCH_RETRIES;
+        let drive_response = loop {
+            // Rebuilt per attempt rather than cloned: every field is a
+            // borrow of state that outlives the loop, plus the request's
+            // one owned value.
+            let drive_request = DocumentRankedRequest {
+                contract: contract_ref,
+                document_type,
+                group_by: &group_by,
+                select: select.clone(),
+                having: &having,
+                order_by: &order_clauses,
+                where_clauses: &where_clauses,
+                limit,
+                offset,
+                has_start_at: start.is_some(),
+                prove,
+            };
 
-        let drive_response =
             match self
                 .drive
                 .execute_document_ranked_request(drive_request, None, platform_version)
             {
-                Ok(r) => r,
+                Ok(r) => break r,
                 Err(drive::error::Error::Query(qe)) => {
                     return Ok(QueryValidationResult::new_with_error(QueryError::Query(qe)));
                 }
@@ -1345,11 +1364,16 @@ impl<C> Platform<C> {
                         return Ok(QueryValidationResult::new_with_error(rejection));
                     }
                     None => {
+                        if is_ranked_chain_mismatch(&e) && attempts_left > 0 {
+                            attempts_left -= 1;
+                            continue;
+                        }
                         annotate_ranked_chain_mismatch(&e);
                         return Err(e.into());
                     }
                 },
-            };
+            }
+        };
 
         let response = match drive_response {
             DocumentRankedResponse::Entries(page) => GetDocumentsResponseV1 {
@@ -1471,6 +1495,35 @@ fn into_v1_ranked_entry(e: DriveRankedEntry) -> RankedEntry {
 /// narrow: any other `CorruptedData` still propagates as an internal
 /// error, because for every other cause that classification is
 /// correct.
+/// How many times a ranked request is rebuilt when its envelope loses the
+/// race against a block commit.
+///
+/// The condition needs a commit to land inside one envelope's read window,
+/// which takes microseconds, so a single retry over settled state is
+/// already overwhelmingly likely to succeed; two bounds the cost at three
+/// envelopes (~400 µs) for a request that would otherwise have failed.
+/// Raising it would trade real work against a vanishing tail.
+const RANKED_CHAIN_MISMATCH_RETRIES: u8 = 2;
+
+/// Whether a drive error is the ancestor-chain reconciliation failure that
+/// a concurrent commit produces.
+///
+/// Detected by variant plus marker substring rather than by a typed error,
+/// for the same reason [`empty_ranking_proof_rejection`] is: grovedb
+/// flattens the failure into a `CorruptedData(String)` at the indexed-axis
+/// proof boundary. "chain mismatch" is the substring both of grovedb's
+/// reconciliation errors carry (the deepest layer's and the intermediate
+/// ancestors').
+fn is_ranked_chain_mismatch(error: &drive::error::Error) -> bool {
+    let drive::error::Error::GroveDB(grove_error) = error else {
+        return false;
+    };
+    let drive::query::GroveError::CorruptedData(message) = grove_error.as_ref() else {
+        return false;
+    };
+    message.contains("chain mismatch")
+}
+
 /// Name the benign cause of a ranked chain-mismatch error in the log,
 /// without reclassifying it.
 ///
@@ -1493,20 +1546,14 @@ fn into_v1_ranked_entry(e: DriveRankedEntry) -> RankedEntry {
 /// exposure; unproved reads share it now that they are served from the
 /// same envelope.
 fn annotate_ranked_chain_mismatch(error: &drive::error::Error) {
-    let drive::error::Error::GroveDB(grove_error) = error else {
-        return;
-    };
-    let drive::query::GroveError::CorruptedData(message) = grove_error.as_ref() else {
-        return;
-    };
-    if !message.contains("chain mismatch") {
+    if !is_ranked_chain_mismatch(error) {
         return;
     }
     tracing::warn!(
-        error = message.as_str(),
-        "ranked query failed to reconcile its proof's ancestor chain; an isolated \
-         occurrence under load is a query racing a block commit rather than corrupted \
-         state, but repeated occurrences — especially without load — are not"
+        error = %error,
+        "ranked query failed to reconcile its proof's ancestor chain on every attempt; \
+         a burst under load is queries racing block commits, but a persistent or \
+         unloaded occurrence is not and should be treated as suspected corruption"
     );
 }
 

packages/rs-drive-abci/src/query/document_query/v1/tests.rs

@@ -3089,4 +3089,48 @@ mod ranked_tests {
             "the marker string alone must not reclassify a non-grovedb error"
         );
     }
+
+    /// The chain-mismatch detector decides whether a ranked request is
+    /// rebuilt or surfaced, so its narrowness is the whole safety property:
+    /// retrying the wrong error class would spend three envelopes on a
+    /// failure that cannot improve, and — worse — retrying *everything*
+    /// would turn a genuine fault into three of them.
+    ///
+    /// The message it matches is grovedb's, wrapped: the reconciliation
+    /// failure arrives as `CorruptedData` with the marker embedded in
+    /// grovedb's own prefix, never as the bare marker, which is why this
+    /// matches a substring rather than the whole string.
+    #[test]
+    fn only_a_grovedb_chain_mismatch_is_retried() {
+        use drive::error::Error as DriveError;
+        use drive::query::GroveError;
+
+        let wrapped = "indexed-axis paginated proof: intermediate layer at depth 2 chain \
+                       mismatch — parent recorded value_hash ab, computed cd";
+        assert!(
+            is_ranked_chain_mismatch(&DriveError::GroveDB(Box::new(GroveError::CorruptedData(
+                wrapped.to_string()
+            )))),
+            "the reconciliation failure must be recognised inside grovedb's wrapper text"
+        );
+
+        // Everything else is surfaced on the first attempt.
+        for other in [
+            GroveError::CorruptedData("Cannot create proof for empty tree".to_string()),
+            GroveError::CorruptedData("some unrelated corruption".to_string()),
+            GroveError::PathNotFound("no such subtree".to_string()),
+        ] {
+            let label = format!("{other:?}");
+            assert!(
+                !is_ranked_chain_mismatch(&DriveError::GroveDB(Box::new(other))),
+                "must not be retried: {label}"
+            );
+        }
+        assert!(
+            !is_ranked_chain_mismatch(&DriveError::Drive(
+                drive::error::drive::DriveError::CorruptedDriveState("chain mismatch".to_string())
+            )),
+            "the marker alone must not make a non-grovedb error retriable"
+        );
+    }
 }

packages/rs-drive/src/query/drive_document_ranked_query/tests.rs

@@ -35,6 +35,7 @@ use super::index_picker::find_ranked_index_for_axis;
 use super::mode_detection::{detect_ranked_mode, detect_ranked_mode_v0};
 use super::*;
 use crate::drive::Drive;
+use crate::error::drive::DriveError;
 use crate::error::query::QuerySyntaxError;
 use crate::error::Error;
 use crate::query::having::{
@@ -57,6 +58,7 @@ use dpp::prelude::DataContract;
 use dpp::tests::json_document::json_document_to_contract;
 use dpp::version::PlatformVersion;
 use grovedb::element::indexed::compute_avg_fixed_point;
+use grovedb::operations::proof::indexed_axis::AxisEntries;
 use grovedb_costs::{CostContext, OperationCost};
 use std::collections::BTreeMap;
 
@@ -1685,11 +1687,17 @@ fn a_deep_offset_does_not_cost_a_walk_of_the_skipped_region() {
             .expect("the proof must build")
             .len()
     };
+    // Its own allowance, derived from what an envelope actually carries per
+    // level of the counted descent — a handful of collapsed ops, each a key
+    // plus a 32-byte hash and a count. Deliberately not `byte_slack`, which
+    // bounds *storage reads*: the two are unrelated quantities, and sharing
+    // one constant would let a change to either silently move the other.
+    let proof_size_slack = 128 * log2_population as usize;
     let (shallow, mid) = (proof_bytes_at(0), proof_bytes_at(population / 2));
     assert!(
-        shallow.abs_diff(mid) <= byte_slack as usize,
+        shallow.abs_diff(mid) <= proof_size_slack,
         "the envelope's size must not grow with the offset it attests: {shallow} bytes at \
-         offset 0 against {mid} at offset {}",
+         offset 0 against {mid} at offset {}, allowing {proof_size_slack}",
         population / 2
     );
     let (at_end, absurd) = (proof_bytes_at(population), proof_bytes_at(4_000_000_000));
@@ -1702,6 +1710,64 @@ fn a_deep_offset_does_not_cost_a_walk_of_the_skipped_region() {
     );
 }
 
+/// **The shared decoder's two defensive arms, exercised directly.**
+///
+/// [`DriveDocumentRankedQuery::ranked_entries_from`] guards against an
+/// envelope whose entries do not match the request: a different axis
+/// shape, or more entries than `k` authorized. Neither is reachable
+/// through grovedb today — the verifier builds the `AxisEntries` variant
+/// from the caller's own `expected_axis`, and it enforces `k` itself — so
+/// the arms exist for the day a decoder change decouples the two, and are
+/// worth pinning precisely *because* nothing else can reach them. Calling
+/// the decoder directly is the only way to prove they fire rather than
+/// silently pass a mis-typed number to the caller.
+#[test]
+fn the_shared_entry_decoder_rejects_a_mismatched_axis_and_an_over_long_page() {
+    let (drive, contract) = setup_restaurants();
+    insert_docs(&drive, &contract, "tip", "amount", 1, &[("alpha", 10)]);
+
+    // A Count request handed Sum entries: the tag check upstream would
+    // normally have caught this, so reaching the caller would mean a
+    // `Sum` scalar being read as a document count.
+    let count_query = client_side_query(&contract, &RankedCase::count(false, Some(4)));
+    let error = count_query
+        .ranked_entries_from(AxisEntries::Sum(vec![(7, b"alpha".to_vec())]))
+        .expect_err("a Count query must not accept Sum entries");
+    assert!(
+        matches!(error, Error::Drive(DriveError::CorruptedDriveState(ref m))
+            if m.contains("different axis shape")),
+        "expected an axis-shape rejection, got {error}"
+    );
+
+    // More entries than `k`: the request authorized one page, and a
+    // longer list would mean the envelope committed a longer walk.
+    let one_entry_query = client_side_query(&contract, &RankedCase::sum(false, Some(1)));
+    let error = one_entry_query
+        .ranked_entries_from(AxisEntries::Sum(vec![
+            (10, b"alpha".to_vec()),
+            (9, b"beta".to_vec()),
+        ]))
+        .expect_err("k = 1 must not accept two entries");
+    assert!(
+        matches!(error, Error::Drive(DriveError::CorruptedDriveState(ref m))
+            if m.contains("for k = 1")),
+        "expected a k rejection naming the limit, got {error}"
+    );
+
+    // And the happy path still decodes, so the guards are not simply
+    // rejecting everything.
+    let entries = one_entry_query
+        .ranked_entries_from(AxisEntries::Sum(vec![(10, b"alpha".to_vec())]))
+        .expect("a well-shaped single-entry page decodes");
+    assert_eq!(
+        entries,
+        vec![RankedEntry {
+            key: b"alpha".to_vec(),
+            value: RankedEntryValue::Sum(10),
+        }]
+    );
+}
+
 /// A proof of one page must not verify as another page of the same
 /// ranking. `offset` is echoed in the envelope and re-checked, which is
 /// what stops a server from answering "the 5th best" with a proof of