Add dedicated IBD rules for Nakamoto

stackslib/src/net/download/nakamoto/download_state_machine.rs

@@ -109,6 +109,8 @@ pub struct NakamotoDownloadStateMachine {
     tenure_block_ids: HashMap<NeighborAddress, AvailableTenures>,
     /// Who can serve a given tenure
     pub(crate) available_tenures: HashMap<ConsensusHash, Vec<NeighborAddress>>,
+    /// What is the highest available tenure, if known?
+    pub(crate) highest_available_tenure: Option<ConsensusHash>,
     /// Confirmed tenure download schedule
     pub(crate) tenure_download_schedule: VecDeque<ConsensusHash>,
     /// Unconfirmed tenure download schedule
@@ -140,6 +142,7 @@ impl NakamotoDownloadStateMachine {
             state: NakamotoDownloadState::Confirmed,
             tenure_block_ids: HashMap::new(),
             available_tenures: HashMap::new(),
+            highest_available_tenure: None,
             tenure_download_schedule: VecDeque::new(),
             unconfirmed_tenure_download_schedule: VecDeque::new(),
             tenure_downloads: NakamotoTenureDownloaderSet::new(),
@@ -862,6 +865,14 @@ impl NakamotoDownloadStateMachine {
         self.tenure_download_schedule = schedule;
         self.tenure_block_ids = tenure_block_ids;
         self.available_tenures = available;
+
+        let highest_available_tenure = self.find_highest_available_tenure();
+        self.highest_available_tenure = highest_available_tenure;
+
+        test_debug!(
+            "new highest_available_tenure: {:?}",
+            &self.highest_available_tenure
+        );
     }
 
     /// Update our tenure download state machines, given our download schedule, our peers' tenure
@@ -958,14 +969,14 @@ impl NakamotoDownloadStateMachine {
             return false;
         }
 
-        let (unconfirmed_tenure_opt, confirmed_tenure_opt) = Self::find_unconfirmed_tenure_ids(
+        let (confirmed_tenure_opt, unconfirmed_tenure_opt) = Self::find_unconfirmed_tenure_ids(
             wanted_tenures,
             prev_wanted_tenures,
             available_tenures,
         );
         debug!(
             "Check unconfirmed tenures: highest two available tenures are {:?}, {:?}",
-            &unconfirmed_tenure_opt, &confirmed_tenure_opt
+            &confirmed_tenure_opt, &unconfirmed_tenure_opt
         );
 
         // see if we need any tenures still
@@ -980,11 +991,11 @@ impl NakamotoDownloadStateMachine {
             });
 
             if !is_available_and_processed {
-                let is_unconfirmed = unconfirmed_tenure_opt
+                let is_unconfirmed = confirmed_tenure_opt
                     .as_ref()
                     .map(|ch| *ch == wt.tenure_id_consensus_hash)
                     .unwrap_or(false)
-                    || confirmed_tenure_opt
+                    || unconfirmed_tenure_opt
                         .as_ref()
                         .map(|ch| *ch == wt.tenure_id_consensus_hash)
                         .unwrap_or(false);
@@ -1549,6 +1560,24 @@ impl NakamotoDownloadStateMachine {
         }
     }
 
+    /// Find the highest available tenure ID.
+    /// Returns Some(consensus_hash) for the highest tenure available from at least one node.
+    /// Returns None if no tenures are available from any peer.
+    fn find_highest_available_tenure(&self) -> Option<ConsensusHash> {
+        let (t1, t2) = Self::find_unconfirmed_tenure_ids(
+            &self.wanted_tenures,
+            self.prev_wanted_tenures.as_ref().unwrap_or(&vec![]),
+            &self.available_tenures,
+        );
+        if let Some(ch) = t2 {
+            return Some(ch);
+        } else if let Some(ch) = t1 {
+            return Some(ch);
+        } else {
+            return None;
+        }
+    }
+
     /// Go and get tenures. Returns list of blocks per tenure, identified by consensus hash.
     /// The blocks will be sorted by height, but may not be contiguous.
     pub fn run(

stackslib/src/net/download/nakamoto/tenure_downloader_set.rs

@@ -113,9 +113,11 @@ pub struct NakamotoTenureDownloaderSet {
     /// The set of tenures that have been successfully downloaded (but possibly not yet stored or
     /// processed)
     pub(crate) completed_tenures: HashSet<CompletedTenure>,
-    /// Number of times a tenure download was attempted
+    /// Number of times a tenure download was attempted.  This counter is incremented before the
+    /// downloader starts
     pub(crate) attempted_tenures: HashMap<ConsensusHash, u64>,
-    /// Number of times a tenure download failed
+    /// Number of times a tenure download failed.  This counter is incremented after the downloader
+    /// finishes in an error state.
     pub(crate) attempt_failed_tenures: HashMap<ConsensusHash, u64>,
     /// Peers that should be deprioritized because they're dead (maps to when they can be used
     /// again)
@@ -451,7 +453,13 @@ impl NakamotoTenureDownloaderSet {
                 continue;
             };
             if tenure_info.processed {
-                // we already have this tenure
+                // we already have tried to download this tenure,
+                // but do remove it from `self.completed_tenures` in order to (1) avoid a memory
+                // leak, and (2) account for the chance that the end-block has changed due to a
+                // Bitcoin reorg.  This way, a subsequent call with the same tenure in `schedule`
+                // will succeed in starting a downloader.  Since `schedule` is derived from on-disk
+                // state, the only way a "completed" tenure will show up in `schedule` again is if
+                // it is later determined that the tenure we stored is incomplete or not canonical.
                 debug!("Already have processed tenure {ch}");
                 self.completed_tenures
                     .remove(&CompletedTenure::from(tenure_info));

stackslib/src/net/mod.rs

@@ -1469,6 +1469,8 @@ pub const DENY_MIN_BAN_DURATION: u64 = 2;
 pub struct NetworkResult {
     /// Stacks chain tip when we began this pass
     pub stacks_tip: StacksBlockId,
+    /// Stacks chain tip's tenure ID when we began this pass
+    pub stacks_tip_tenure_id: ConsensusHash,
     /// PoX ID as it was when we begin downloading blocks (set if we have downloaded new blocks)
     pub download_pox_id: Option<PoxId>,
     /// Network messages we received but did not handle
@@ -1519,15 +1521,22 @@ pub struct NetworkResult {
     pub coinbase_height: u64,
     /// The observed stacks tip height (different in Nakamoto from coinbase height)
     pub stacks_tip_height: u64,
-    /// The consensus hash of the stacks tip (prefixed `rc_` for historical reasons)
+    /// The consensus hash of the highest complete Stacks tenure at the time the canonical
+    /// sortition tip was processed.  Not guaranteed to be the same across all nodes for the same
+    /// given sortition tip.
+    ///
+    /// TODO: remove this and use canonical Stacks tenure ID instead.
     pub rc_consensus_hash: ConsensusHash,
     /// The current StackerDB configs
     pub stacker_db_configs: HashMap<QualifiedContractIdentifier, StackerDBConfig>,
+    /// Highest available tenure, if known
+    pub highest_available_tenure: Option<ConsensusHash>,
 }
 
 impl NetworkResult {
     pub fn new(
         stacks_tip: StacksBlockId,
+        stacks_tip_tenure_id: ConsensusHash,
         num_state_machine_passes: u64,
         num_inv_sync_passes: u64,
         num_download_passes: u64,
@@ -1537,9 +1546,11 @@ impl NetworkResult {
         stacks_tip_height: u64,
         rc_consensus_hash: ConsensusHash,
         stacker_db_configs: HashMap<QualifiedContractIdentifier, StackerDBConfig>,
+        highest_available_tenure: Option<ConsensusHash>,
     ) -> NetworkResult {
         NetworkResult {
             stacks_tip,
+            stacks_tip_tenure_id,
             unhandled_messages: HashMap::new(),
             download_pox_id: None,
             blocks: vec![],
@@ -1567,6 +1578,7 @@ impl NetworkResult {
             stacks_tip_height,
             rc_consensus_hash,
             stacker_db_configs,
+            highest_available_tenure,
         }
     }
 

stackslib/src/net/p2p.rs

@@ -5187,7 +5187,7 @@ impl PeerNetwork {
         poll_timeout: u64,
         handler_args: &RPCHandlerArgs,
     ) -> Result<NetworkResult, net_error> {
-        debug!(">>>>>>>>>>>>>>>>>>>>>>> Begin Network Dispatch (poll for {}) >>>>>>>>>>>>>>>>>>>>>>>>>>>>", poll_timeout);
+        debug!(">>>>>>>>>>>>>>>>>>>>>>> Begin Network Dispatch (poll for {}, ibd={}) >>>>>>>>>>>>>>>>>>>>>>>>>>>>", poll_timeout, ibd);
         let mut poll_states = match self.network {
             None => {
                 debug!("{:?}: network not connected", &self.local_peer);
@@ -5226,6 +5226,7 @@ impl PeerNetwork {
         );
         let mut network_result = NetworkResult::new(
             self.stacks_tip.block_id(),
+            self.stacks_tip.consensus_hash.clone(),
             self.num_state_machine_passes,
             self.num_inv_sync_passes,
             self.num_downloader_passes,
@@ -5235,6 +5236,10 @@ impl PeerNetwork {
             self.stacks_tip.height,
             self.chain_view.rc_consensus_hash.clone(),
             self.get_stacker_db_configs_owned(),
+            self.block_downloader_nakamoto
+                .as_ref()
+                .map(|dler| dler.highest_available_tenure.clone())
+                .flatten(),
         );
 
         network_result.consume_unsolicited(unsolicited_buffered_messages);

stackslib/src/net/tests/mod.rs

@@ -1155,6 +1155,7 @@ fn test_boot_nakamoto_peer() {
 fn test_network_result_update() {
     let mut network_result_1 = NetworkResult::new(
         StacksBlockId([0x11; 32]),
+        ConsensusHash([0x01; 20]),
         1,
         1,
         1,
@@ -1164,10 +1165,12 @@ fn test_network_result_update() {
         1,
         ConsensusHash([0x11; 20]),
         HashMap::new(),
+        None,
     );
 
     let mut network_result_2 = NetworkResult::new(
         StacksBlockId([0x22; 32]),
+        ConsensusHash([0x01; 20]),
         2,
         2,
         2,
@@ -1177,6 +1180,7 @@ fn test_network_result_update() {
         2,
         ConsensusHash([0x22; 20]),
         HashMap::new(),
+        None,
     );
 
     let nk1 = NeighborKey {
@@ -1589,6 +1593,7 @@ fn test_network_result_update() {
     // stackerdb uploaded chunks get consolidated correctly
     let mut old = NetworkResult::new(
         StacksBlockId([0xaa; 32]),
+        ConsensusHash([0x01; 20]),
         10,
         10,
         10,
@@ -1598,6 +1603,7 @@ fn test_network_result_update() {
         10,
         ConsensusHash([0xaa; 20]),
         HashMap::new(),
+        None,
     );
     let mut new = old.clone();
 
@@ -1648,6 +1654,7 @@ fn test_network_result_update() {
     // stackerdb pushed chunks get consolidated correctly
     let mut old = NetworkResult::new(
         StacksBlockId([0xaa; 32]),
+        ConsensusHash([0x01; 20]),
         10,
         10,
         10,
@@ -1657,6 +1664,7 @@ fn test_network_result_update() {
         10,
         ConsensusHash([0xaa; 20]),
         HashMap::new(),
+        None,
     );
     let mut new = old.clone();
 
@@ -1707,6 +1715,7 @@ fn test_network_result_update() {
     // nakamoto blocks obtained via download, upload, or pushed get consoldated
     let mut old = NetworkResult::new(
         StacksBlockId([0xbb; 32]),
+        ConsensusHash([0x01; 20]),
         11,
         11,
         11,
@@ -1716,6 +1725,7 @@ fn test_network_result_update() {
         11,
         ConsensusHash([0xbb; 20]),
         HashMap::new(),
+        None,
     );
     old.nakamoto_blocks.insert(nblk1.block_id(), nblk1.clone());
     old.pushed_nakamoto_blocks.insert(
@@ -1731,6 +1741,7 @@ fn test_network_result_update() {
 
     let new = NetworkResult::new(
         StacksBlockId([0xbb; 32]),
+        ConsensusHash([0x01; 20]),
         11,
         11,
         11,
@@ -1740,6 +1751,7 @@ fn test_network_result_update() {
         11,
         ConsensusHash([0xbb; 20]),
         HashMap::new(),
+        None,
     );
 
     let mut new_pushed = new.clone();

stackslib/src/net/tests/relay/epoch2x.rs

@@ -3076,6 +3076,7 @@ fn process_new_blocks_rejects_problematic_asts() {
 
     let mut network_result = NetworkResult::new(
         peer.network.stacks_tip.block_id(),
+        ConsensusHash([0x01; 20]),
         0,
         0,
         0,
@@ -3085,6 +3086,7 @@ fn process_new_blocks_rejects_problematic_asts() {
         0,
         ConsensusHash([0x01; 20]),
         HashMap::new(),
+        None,
     );
     network_result.consume_unsolicited(unsolicited);
 

testnet/stacks-node/src/burnchains/bitcoin_regtest_controller.rs

@@ -2179,6 +2179,7 @@ impl BurnchainController for BitcoinRegtestController {
         }
     }
 
+    /// NOTE: this is 1-indexed. If there are 10 headers, then this returns 11
     fn get_headers_height(&self) -> u64 {
         let (_, network_id) = self.config.burnchain.get_bitcoin_network();
         let spv_client = SpvClient::new(

testnet/stacks-node/src/globals.rs

@@ -129,12 +129,17 @@ impl<T> Globals<T> {
         }
     }
 
-    /// Does the inventory sync watcher think we still need to
-    /// catch up to the chain tip?
+    /// Are we still in the initial block download period?  As in, are there more sortitions to
+    /// process and/or more tenure-start blocks to process?
     pub fn in_initial_block_download(&self) -> bool {
         self.sync_comms.get_ibd()
     }
 
+    /// Flag whether or not the node is in IBD
+    pub fn set_initial_block_download(&mut self, ibd: bool) {
+        self.sync_comms.set_ibd(ibd);
+    }
+
     /// Get the last sortition processed by the relayer thread
     pub fn get_last_sortition(&self) -> Option<BlockSnapshot> {
         self.last_sortition

testnet/stacks-node/src/nakamoto_node.rs

@@ -313,14 +313,12 @@ impl StacksNode {
 
     /// Process a state coming from the burnchain, by extracting the validated KeyRegisterOp
     /// and inspecting if a sortition was won.
-    /// `ibd`: boolean indicating whether or not we are in the initial block download
     /// Called from the main thread.
     pub fn process_burnchain_state(
         &mut self,
         config: &Config,
         sortdb: &SortitionDB,
         sort_id: &SortitionId,
-        ibd: bool,
     ) -> Result<(), Error> {
         let ic = sortdb.index_conn();
 
@@ -374,7 +372,6 @@ impl StacksNode {
             "burn_height" => block_height,
             "leader_keys_count" => num_key_registers,
             "block_commits_count" => num_block_commits,
-            "in_initial_block_download?" => ibd,
         );
 
         self.globals.set_last_sortition(block_snapshot.clone());

testnet/stacks-node/src/nakamoto_node/miner.rs

@@ -64,11 +64,12 @@ use crate::run_loop::RegisteredKey;
 /// Test flag to stall the miner thread
 pub static TEST_MINE_STALL: LazyLock<TestFlag<bool>> = LazyLock::new(TestFlag::default);
 #[cfg(test)]
+/// Test flag to stall the miner from announcing a block
+pub static TEST_BLOCK_ANNOUNCE_STALL: LazyLock<TestFlag<bool>> = LazyLock::new(TestFlag::default);
+#[cfg(test)]
 /// Test flag to stall block proposal broadcasting
 pub static TEST_BROADCAST_STALL: LazyLock<TestFlag<bool>> = LazyLock::new(TestFlag::default);
 #[cfg(test)]
-pub static TEST_BLOCK_ANNOUNCE_STALL: LazyLock<TestFlag<bool>> = LazyLock::new(TestFlag::default);
-#[cfg(test)]
 pub static TEST_SKIP_P2P_BROADCAST: LazyLock<TestFlag<bool>> = LazyLock::new(TestFlag::default);
 
 /// If the miner was interrupted while mining a block, how long should the

testnet/stacks-node/src/nakamoto_node/peer.rs

@@ -235,7 +235,7 @@ impl PeerThread {
         fee_estimator: Option<&Box<dyn FeeEstimator>>,
     ) -> bool {
         // initial block download?
-        let ibd = self.globals.sync_comms.get_ibd();
+        let ibd = self.globals.in_initial_block_download();
         let download_backpressure = self
             .results_with_data
             .as_ref()