[DK] WIP inc

Author: coderofstuff

consensus/src/consensus/services.rs

@@ -13,10 +13,10 @@ use crate::{
     },
     processes::{
         block_depth::BlockDepthManager, coinbase::CoinbaseManager, dagknight::DagknightCounters,
-        dagknight::protocol::DagknightExecutor, ghostdag::protocol::GhostdagManager,
-        parents_builder::ParentsManager, pruning::PruningPointManager, pruning_proof::PruningProofManager,
-        sync::SyncManager, transaction_validator::TransactionValidator, traversal_manager::DagTraversalManager,
-        window::SampledWindowManager,
+        dagknight::protocol::DagknightExecutor, dagknight::umc_cascade_persistence::DbUmcCascadeStore,
+        ghostdag::protocol::GhostdagManager, parents_builder::ParentsManager, pruning::PruningPointManager,
+        pruning_proof::PruningProofManager, sync::SyncManager, transaction_validator::TransactionValidator,
+        traversal_manager::DagTraversalManager, window::SampledWindowManager,
     },
 };
 use kaspa_consensus_core::mass::MassCalculator;
@@ -52,7 +52,7 @@ pub type DbPruningPointManager = PruningPointManager<
 pub type DbBlockDepthManager = BlockDepthManager<DbDepthStore, DbReachabilityStore, DbGhostdagStore, DbHeadersStore>;
 pub type DbParentsManager = ParentsManager<DbHeadersStore, DbReachabilityStore, MTRelationsService<DbRelationsStore>>;
 pub type DbDagknightExecutor =
-    DagknightExecutor<DbDagknightStore, DbHeadersStore, MTRelationsService<DbRelationsStore>, DbReachabilityStore>;
+    DagknightExecutor<DbDagknightStore, DbHeadersStore, MTRelationsService<DbRelationsStore>, DbUmcCascadeStore, DbReachabilityStore>;
 
 pub struct ConsensusServices {
     // Underlying storage
@@ -149,6 +149,7 @@ impl ConsensusServices {
             relations_store: Arc::new(RwLock::new(relations_service.clone())),
             reachability_service: reachability_service.clone(),
             counters: dagknight_counters.clone(),
+            umc_persistence_store: storage.umc_persistence_store.clone(),
         });
 
         let coinbase_manager = CoinbaseManager::new(

consensus/src/consensus/storage.rs

@@ -24,12 +24,15 @@ use crate::{
         utxo_multisets::DbUtxoMultisetsStore,
         virtual_state::{LkgVirtualState, VirtualStores},
     },
-    processes::{ghostdag::ordering::SortableBlock, reachability::inquirer as reachability, relations},
+    processes::{
+        dagknight::umc_cascade_persistence::DbUmcCascadeStore, ghostdag::ordering::SortableBlock,
+        reachability::inquirer as reachability, relations,
+    },
 };
 
 use super::cache_policy_builder::CachePolicyBuilder as PolicyBuilder;
 use kaspa_consensus_core::{BlockHashSet, blockstatus::BlockStatus};
-use kaspa_database::registry::DatabaseStorePrefixes;
+use kaspa_database::{prelude::CachePolicy, registry::DatabaseStorePrefixes};
 use kaspa_hashes::Hash;
 use parking_lot::RwLock;
 use std::{ops::DerefMut, sync::Arc};
@@ -50,6 +53,7 @@ pub struct ConsensusStorage {
     pub virtual_stores: Arc<RwLock<VirtualStores>>,
     pub selected_chain_store: Arc<RwLock<DbSelectedChainStore>>,
     pub dagknight_store: Option<Arc<DbDagknightStore>>,
+    pub umc_persistence_store: Arc<DbUmcCascadeStore>,
 
     // Append-only stores
     pub topology_ghostdag_store: Arc<DbGhostdagStore>,
@@ -249,6 +253,7 @@ impl ConsensusStorage {
         let dagknight_builder = PolicyBuilder::new().bytes_budget(scaled(5_000_000)).tracked_bytes();
         // TODO[DK]: Use a config or ForkActivation to gate this
         let dagknight_store = Some(Arc::new(DbDagknightStore::new(db.clone(), dagknight_builder.build())));
+        let umc_persistence_store = Arc::new(DbUmcCascadeStore::new(db.clone(), CachePolicy::Count(256)));
 
         Arc::new(Self {
             _db: db,
@@ -277,6 +282,7 @@ impl ConsensusStorage {
             block_window_cache_for_past_median_time,
             lkg_virtual_state,
             dagknight_store,
+            umc_persistence_store,
         })
     }
 }

consensus/src/processes/dagknight/appendable_segment_tree_api.rs

@@ -62,4 +62,7 @@ where
     fn flip_to_positive(&mut self, leaf: T);
 
     fn score(&mut self, leaf: T) -> S;
+
+    /// Returns all leaves in left-to-right (position) order.
+    fn leaves(&self) -> Vec<T>;
 }

consensus/src/processes/dagknight/appendable_segment_tree_impl.rs

@@ -227,6 +227,20 @@ where
         self.point_score(ROOT_NODE, self.full_leaf_range(), target_position)
     }
 
+    /// Returns all leaves in left-to-right (position) order.
+    pub fn leaves(&self) -> Vec<T> {
+        (0..self.len)
+            .map(|pos| {
+                let leaf = &self.nodes[self.leaf_node(pos)];
+                if let Some(candidate) = leaf.min_positive {
+                    candidate.leaf
+                } else {
+                    leaf.max_negative.expect("occupied leaf must have a candidate").leaf
+                }
+            })
+            .collect()
+    }
+
     // ---------------------------------------------------------------------
     // Internal queries and bucket transitions
     // ---------------------------------------------------------------------
@@ -479,6 +493,10 @@ where
     fn score(&mut self, leaf: T) -> S {
         AppendableSegmentTree::score(self, leaf)
     }
+
+    fn leaves(&self) -> Vec<T> {
+        AppendableSegmentTree::leaves(self)
+    }
 }
 
 #[cfg(test)]
@@ -648,4 +666,42 @@ mod tests {
         assert_eq!(tree.score(1), -5);
         assert_eq!(tree.score(2), 0);
     }
+
+    #[test]
+    fn test_leaves_returns_in_position_order() {
+        let mut tree = AppendableSegmentTree::<u64>::new();
+        tree.append_leaf(10, 1);
+        tree.append_leaf(20, 2);
+        tree.append_leaf(30, 3);
+        tree.append_leaf(40, 4);
+        tree.append_leaf(50, 5);
+
+        let leaves = tree.leaves();
+        assert_eq!(leaves, vec![10, 20, 30, 40, 50]);
+    }
+
+    #[test]
+    fn test_leaves_after_prefix_add() {
+        let mut tree = AppendableSegmentTree::<u64>::new();
+        tree.append_leaf(1, 10);
+        tree.append_leaf(2, 20);
+        tree.append_leaf(3, 30);
+
+        tree.prefix_add(2, -5);
+
+        // Leaves should still be in position order regardless of score changes
+        let leaves = tree.leaves();
+        assert_eq!(leaves, vec![1, 2, 3]);
+
+        // Verify scores are correct after prefix add
+        assert_eq!(tree.score(1), 5);
+        assert_eq!(tree.score(2), 15);
+        assert_eq!(tree.score(3), 30);
+    }
+
+    #[test]
+    fn test_leaves_empty_tree() {
+        let tree = AppendableSegmentTree::<u64>::new();
+        assert!(tree.leaves().is_empty());
+    }
 }

consensus/src/processes/dagknight/mod.rs

@@ -16,6 +16,7 @@ pub mod protocol;
 pub mod rank_search;
 pub mod tie_breaking;
 pub mod umc_cascade;
+pub mod umc_cascade_persistence;
 
 pub struct GroupMetadata {
     conflict_genesis: Hash,
@@ -34,6 +35,16 @@ pub struct DagknightCounters {
     total_cascade_flips: AtomicU64,
     /// Maximum flips in a single cascade run
     max_cascade_flips: AtomicU64,
+    /// Number of UMC cascade calls that started from a persisted checkpoint
+    checkpoint_from_checkpoint: AtomicU64,
+    /// Number of UMC cascade calls that started from scratch
+    checkpoint_from_scratch: AtomicU64,
+    /// Total estimated effort saved across all checkpoint loads,
+    /// representing estimated blue blocks skipped by checkpointing
+    checkpoint_estimated_effort_saved: AtomicU64,
+    /// Total estimated effort across all UMC calls (sum of virtual_gd.blue_score).
+    /// Used as denominator for effort_saved percentage.
+    checkpoint_estimated_effort_total: AtomicU64,
     /// Baseline true, cascade false
     #[cfg(feature = "baseline-debugging")]
     baseline_true_cascade_false: AtomicU64,
@@ -55,6 +66,10 @@ impl DagknightCounters {
             total_voting_blocks: AtomicU64::new(0),
             total_cascade_flips: AtomicU64::new(0),
             max_cascade_flips: AtomicU64::new(0),
+            checkpoint_from_checkpoint: AtomicU64::new(0),
+            checkpoint_from_scratch: AtomicU64::new(0),
+            checkpoint_estimated_effort_saved: AtomicU64::new(0),
+            checkpoint_estimated_effort_total: AtomicU64::new(0),
             #[cfg(feature = "baseline-debugging")]
             baseline_true_cascade_false: AtomicU64::new(0),
             #[cfg(feature = "baseline-debugging")]
@@ -90,13 +105,28 @@ impl DagknightCounters {
         }
     }
 
+    /// Record checkpoint persistence statistics from a single run_cascade call.
+    pub fn record_checkpoint_stats(&self, from_checkpoint: bool, estimated_effort_saved: u64, estimated_effort_total: u64) {
+        if from_checkpoint {
+            self.checkpoint_from_checkpoint.fetch_add(1, Ordering::Relaxed);
+        } else {
+            self.checkpoint_from_scratch.fetch_add(1, Ordering::Relaxed);
+        }
+        self.checkpoint_estimated_effort_saved.fetch_add(estimated_effort_saved, Ordering::Relaxed);
+        self.checkpoint_estimated_effort_total.fetch_add(estimated_effort_total, Ordering::Relaxed);
+    }
+
     /// Take a point-in-time snapshot of all counters.
     pub fn snapshot(&self) -> DagknightCountersSnapshot {
         DagknightCountersSnapshot {
             total_calls: self.total_calls.load(Ordering::Relaxed),
             total_voting_blocks: self.total_voting_blocks.load(Ordering::Relaxed),
             total_cascade_flips: self.total_cascade_flips.load(Ordering::Relaxed),
             max_cascade_flips: self.max_cascade_flips.load(Ordering::Relaxed),
+            checkpoint_from_checkpoint: self.checkpoint_from_checkpoint.load(Ordering::Relaxed),
+            checkpoint_from_scratch: self.checkpoint_from_scratch.load(Ordering::Relaxed),
+            checkpoint_estimated_effort_saved: self.checkpoint_estimated_effort_saved.load(Ordering::Relaxed),
+            checkpoint_estimated_effort_total: self.checkpoint_estimated_effort_total.load(Ordering::Relaxed),
             #[cfg(feature = "baseline-debugging")]
             baseline_true_cascade_false: self.baseline_true_cascade_false.load(Ordering::Relaxed),
             #[cfg(feature = "baseline-debugging")]
@@ -112,6 +142,14 @@ pub struct DagknightCountersSnapshot {
     pub total_voting_blocks: u64,
     pub total_cascade_flips: u64,
     pub max_cascade_flips: u64,
+    /// Number of UMC cascade calls that started from a persisted checkpoint
+    pub checkpoint_from_checkpoint: u64,
+    /// Number of UMC cascade calls that started from scratch
+    pub checkpoint_from_scratch: u64,
+    /// Total estimated effort saved across all checkpoint loads
+    pub checkpoint_estimated_effort_saved: u64,
+    /// Total estimated effort across all UMC calls (sum of virtual_gd.blue_score)
+    pub checkpoint_estimated_effort_total: u64,
     #[cfg(feature = "baseline-debugging")]
     pub baseline_true_cascade_false: u64,
     #[cfg(feature = "baseline-debugging")]
@@ -129,6 +167,21 @@ impl DagknightCountersSnapshot {
         if self.total_calls == 0 { 0.0 } else { self.total_cascade_flips as f64 / self.total_calls as f64 }
     }
 
+    /// Percentage of UMC calls that used a checkpoint.
+    pub fn checkpoint_hit_rate(&self) -> f64 {
+        let total = self.checkpoint_from_checkpoint + self.checkpoint_from_scratch;
+        if total == 0 { 0.0 } else { self.checkpoint_from_checkpoint as f64 / total as f64 * 100.0 }
+    }
+
+    /// Percentage of blue work saved by checkpointing relative to total blue work in conflict zones.
+    pub fn checkpoint_effort_saved(&self) -> f64 {
+        if self.checkpoint_estimated_effort_total == 0 {
+            0.0
+        } else {
+            self.checkpoint_estimated_effort_saved as f64 / self.checkpoint_estimated_effort_total as f64 * 100.0
+        }
+    }
+
     /// Percentage of calls where baseline accepted and cascade rejected.
     #[cfg(feature = "baseline-debugging")]
     pub fn baseline_true_cascade_false_percentage(&self) -> f64 {
@@ -151,6 +204,14 @@ impl std::ops::Sub for &DagknightCountersSnapshot {
             total_voting_blocks: self.total_voting_blocks.saturating_sub(rhs.total_voting_blocks),
             total_cascade_flips: self.total_cascade_flips.saturating_sub(rhs.total_cascade_flips),
             max_cascade_flips: 0, // max doesn't subtract meaningfully
+            checkpoint_from_checkpoint: self.checkpoint_from_checkpoint.saturating_sub(rhs.checkpoint_from_checkpoint),
+            checkpoint_from_scratch: self.checkpoint_from_scratch.saturating_sub(rhs.checkpoint_from_scratch),
+            checkpoint_estimated_effort_saved: self
+                .checkpoint_estimated_effort_saved
+                .saturating_sub(rhs.checkpoint_estimated_effort_saved),
+            checkpoint_estimated_effort_total: self
+                .checkpoint_estimated_effort_total
+                .saturating_sub(rhs.checkpoint_estimated_effort_total),
             #[cfg(feature = "baseline-debugging")]
             baseline_true_cascade_false: self.baseline_true_cascade_false.saturating_sub(rhs.baseline_true_cascade_false),
             #[cfg(feature = "baseline-debugging")]