From 06b2d04e671e4b9f913ffd73e1411066d1c0ab74 Mon Sep 17 00:00:00 2001 From: perfogic Date: Sat, 8 Aug 2026 20:46:44 +0700 Subject: [PATCH 01/17] fix(rpc): resolve the head and genesis block ids MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both were rejected with a 404, so `/eth/v1/beacon/headers/head` — the ordinary way to ask a node where it is — returned "This ID type is currently not supported" and any consumer polling it concluded the node was offline. Head resolves through fork choice, which reads entirely out of the database, and genesis through the slot index. --- crates/rpc/beacon/src/handlers/block.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/rpc/beacon/src/handlers/block.rs b/crates/rpc/beacon/src/handlers/block.rs index b92ddbc3b..e1d77445e 100644 --- a/crates/rpc/beacon/src/handlers/block.rs +++ b/crates/rpc/beacon/src/handlers/block.rs @@ -39,6 +39,7 @@ use ream_consensus_misc::constants::beacon::{ }; use ream_fork_choice_beacon::store::Store; use ream_network_manager::p2p_sender::P2PSender; +use ream_operation_pool::OperationPool; use ream_network_spec::networks::beacon_network_spec; use ream_operation_pool::OperationPool; use ream_p2p::{ From 702d5276f121a884b45f3449c08526a5d9ae1b32 Mon Sep 17 00:00:00 2001 From: perfogic Date: Wed, 19 Aug 2026 00:17:11 +0700 Subject: [PATCH 02/17] style: format kurtosis integration changes --- crates/rpc/beacon/src/handlers/block.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/rpc/beacon/src/handlers/block.rs b/crates/rpc/beacon/src/handlers/block.rs index e1d77445e..b92ddbc3b 100644 --- a/crates/rpc/beacon/src/handlers/block.rs +++ b/crates/rpc/beacon/src/handlers/block.rs @@ -39,7 +39,6 @@ use ream_consensus_misc::constants::beacon::{ }; use ream_fork_choice_beacon::store::Store; use ream_network_manager::p2p_sender::P2PSender; -use ream_operation_pool::OperationPool; use ream_network_spec::networks::beacon_network_spec; use ream_operation_pool::OperationPool; use ream_p2p::{ From 39172ce14991a1d726504f5afb22abbfff76df0c Mon Sep 17 00:00:00 2001 From: vuonghuuhung Date: Sat, 15 Aug 2026 23:30:28 +0700 Subject: [PATCH 03/17] fix(syncer): sync past finalized to head, and stop crashing on post-Fulu blobs --- crates/networking/manager/src/service.rs | 2 +- .../syncer/src/block_range/block_cache.rs | 16 ++++---- .../networking/syncer/src/block_range/mod.rs | 38 ++++++++++++++----- .../syncer/src/block_range/peer_manager.rs | 8 ++++ 4 files changed, 46 insertions(+), 18 deletions(-) diff --git a/crates/networking/manager/src/service.rs b/crates/networking/manager/src/service.rs index 6de6eccf4..d22008b90 100644 --- a/crates/networking/manager/src/service.rs +++ b/crates/networking/manager/src/service.rs @@ -551,7 +551,7 @@ impl NetworkManagerService { } }; - if !block_range_syncer.is_synced_to_finalized_slot().await { + if !block_range_syncer.is_synced_to_head_slot().await { syncer_handle = block_range_syncer.start(); syncer_active = true; } diff --git a/crates/networking/syncer/src/block_range/block_cache.rs b/crates/networking/syncer/src/block_range/block_cache.rs index e36ec7c2b..90b1b1f86 100644 --- a/crates/networking/syncer/src/block_range/block_cache.rs +++ b/crates/networking/syncer/src/block_range/block_cache.rs @@ -151,14 +151,14 @@ impl BlockCache { self.block_ranges_to_retry.push(range); } - pub fn data_to_fetch(&mut self, finalized_slot: u64, current_epoch: u64) -> DataToFetch { + pub fn data_to_fetch(&mut self, target_slot: u64, current_epoch: u64) -> DataToFetch { match self.block_ranges_to_retry.pop() { Some(range) => return DataToFetch::BlockRange(range), None => { let estimated_blocks_to_fetch = self.estimated_blocks_to_fetch(); - if estimated_blocks_to_fetch > 0 && self.next_start_slot < finalized_slot { + if estimated_blocks_to_fetch > 0 && self.next_start_slot < target_slot { let blocks_to_fill = estimated_blocks_to_fetch - .min(MAX_BLOCKS_PER_REQUEST.min(finalized_slot - self.next_start_slot)); + .min(MAX_BLOCKS_PER_REQUEST.min(target_slot - self.next_start_slot)); let start_slot = self.next_start_slot + 1; self.next_start_slot += blocks_to_fill; return DataToFetch::BlockRange(Range::new(start_slot, blocks_to_fill)); @@ -293,7 +293,7 @@ mod tests { use super::*; #[test] - fn data_to_fetch_finishes_at_finalized_slot() { + fn data_to_fetch_finishes_at_target_slot() { initialize_test_network_spec(); let mut cache = BlockCache::new(B256::ZERO, 10); @@ -317,7 +317,7 @@ mod tests { } #[test] - fn blob_fetching_uses_the_data_availability_retention_boundary() { + fn post_fulu_blob_commitments_never_request_legacy_blob_identifiers() { initialize_test_network_spec(); let network_spec = beacon_network_spec(); let current_epoch = network_spec.fulu_fork_epoch @@ -350,10 +350,10 @@ mod tests { retained .add_blocks(vec![block_with_blob(boundary_slot)], false) .expect("boundary block should enter cache"); - assert!(matches!( + assert_eq!( retained.data_to_fetch(boundary_slot, current_epoch), - DataToFetch::MissingBlobIdentifiers(identifiers) if identifiers.len() == 1 - )); + DataToFetch::Finished + ); let expired_slot = compute_start_slot_at_epoch(boundary_epoch - 1); let mut expired = BlockCache::new(parent_root, expired_slot); diff --git a/crates/networking/syncer/src/block_range/mod.rs b/crates/networking/syncer/src/block_range/mod.rs index 9c5609496..121c229e7 100644 --- a/crates/networking/syncer/src/block_range/mod.rs +++ b/crates/networking/syncer/src/block_range/mod.rs @@ -43,12 +43,14 @@ const MAX_BLOCKS_PER_REQUEST: u64 = 10; const SLEEP_DURATION: Duration = Duration::from_secs(5); /// Validates downloaded blob sidecars and derives the columns needed for data availability. +/// An empty `blob_sidecars` map defers to the block-lookup coordinator instead of erroring, +/// since range sync never fetches legacy blob sidecars for post-Fulu blocks. fn build_data_columns_from_blob_sidecars( block: &SignedBeaconBlock, blob_sidecars: &std::collections::HashMap, verify_data_availability: bool, ) -> anyhow::Result> { - if !verify_data_availability { + if !verify_data_availability || blob_sidecars.is_empty() { return Ok(Vec::new()); } @@ -118,8 +120,11 @@ impl BlockRangeSyncer { } } - pub async fn is_synced_to_finalized_slot(&self) -> bool { - let finalized_slot = self.peer_manager.finalized_slot(); + pub async fn is_synced_to_head_slot(&self) -> bool { + let target_slot = self + .peer_manager + .head_slot() + .or_else(|| self.peer_manager.finalized_slot()); let latest_synced_slot = self .beacon_chain .store @@ -131,7 +136,7 @@ impl BlockRangeSyncer { .unwrap_or_default() .unwrap_or(0); - finalized_slot <= Some(latest_synced_slot) + target_slot <= Some(latest_synced_slot) } pub fn start(mut self) -> JoinHandle>> { @@ -170,10 +175,14 @@ impl BlockRangeSyncer { loop { poll_ready_tasks(&mut task_handles, &mut block_cache, &mut self.peer_manager)?; - let finalized_slot = match self.peer_manager.finalized_slot() { - Some(finalized_slot) => finalized_slot, + let target_slot = match self + .peer_manager + .head_slot() + .or_else(|| self.peer_manager.finalized_slot()) + { + Some(target_slot) => target_slot, None => { - warn!("No peers available to determine finalized slot, retrying..."); + warn!("No peers available to determine sync target, retrying..."); sleep(SLEEP_DURATION).await; self.peer_manager.update_peer_set(); continue; @@ -186,7 +195,7 @@ impl BlockRangeSyncer { .lock() .await .get_current_store_epoch()?; - let data_to_fetch = block_cache.data_to_fetch(finalized_slot, current_epoch); + let data_to_fetch = block_cache.data_to_fetch(target_slot, current_epoch); info!( "Forward sync status: Downloaded Blocks {}, Downloaded Blobs {}/{}, Stage {data_to_fetch}", block_cache.block_count(), @@ -630,7 +639,11 @@ mod tests { .expect("expired blocks should not require blob sidecars") .is_empty() ); - assert!(build_data_columns_from_blob_sidecars(&block, &HashMap::new(), true).is_err()); + assert!( + build_data_columns_from_blob_sidecars(&block, &HashMap::new(), true) + .expect("blocks awaiting column fetch should not error") + .is_empty() + ); let columns = build_data_columns_from_blob_sidecars(&block, &sidecars, true) .expect("valid blobs should produce data columns"); @@ -641,5 +654,12 @@ mod tests { .expect("test sidecar should exist") .kzg_proof[0] ^= 1; assert!(build_data_columns_from_blob_sidecars(&block, &sidecars, true).is_err()); + + let wrong_identifier = BlobIdentifier::new(B256::repeat_byte(0xAB), 0); + let mismatched = HashMap::from([( + wrong_identifier, + sidecars.remove(&identifier).expect("sidecar should exist"), + )]); + assert!(build_data_columns_from_blob_sidecars(&block, &mismatched, true).is_err()); } } diff --git a/crates/networking/syncer/src/block_range/peer_manager.rs b/crates/networking/syncer/src/block_range/peer_manager.rs index f2f0a209a..876864157 100644 --- a/crates/networking/syncer/src/block_range/peer_manager.rs +++ b/crates/networking/syncer/src/block_range/peer_manager.rs @@ -125,4 +125,12 @@ impl PeerManager { .max_by_key(|&(_, count)| count) .map(|(slot, _)| slot) } + + pub fn head_slot(&self) -> Option { + self.peers + .values() + .filter_map(|peer| peer.peer.status.as_ref()) + .map(|status| status.head_slot) + .max() + } } From 80df15d583f27ba56f20197d5fc3a62e319f43e7 Mon Sep 17 00:00:00 2001 From: vuonghuuhung Date: Mon, 24 Aug 2026 00:34:00 +0700 Subject: [PATCH 04/17] feat: harden range sync against thin peer votes, permanent stalls, and post-Fulu DA gaps --- Cargo.lock | 6 + crates/networking/manager/src/service.rs | 70 +- .../p2p/src/network/beacon/channel.rs | 7 + .../networking/p2p/src/network/beacon/mod.rs | 18 +- crates/networking/syncer/Cargo.toml | 6 + .../syncer/src/block_range/block_cache.rs | 311 ++++++- .../networking/syncer/src/block_range/mod.rs | 838 ++++++++++++++---- .../syncer/src/block_range/peer_manager.rs | 285 +++++- .../src/block_range/peer_range_downloader.rs | 312 ++++--- 9 files changed, 1509 insertions(+), 344 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 080764023..c68c0d06f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7170,12 +7170,14 @@ version = "0.1.0" dependencies = [ "alloy-primitives", "anyhow", + "discv5", "ethereum_ssz", "futures", "kzg", "libp2p", "libp2p-identity", "libp2p-mplex", + "parking_lot", "ream-bls", "ream-chain-beacon", "ream-consensus-beacon", @@ -7183,10 +7185,14 @@ dependencies = [ "ream-execution-rpc-types", "ream-executor", "ream-network-spec", + "ream-operation-pool", "ream-p2p", + "ream-peer", "ream-polynomial-commitments", "ream-req-resp", "ream-storage", + "ream-sync-committee-pool", + "tempfile", "tokio", "tracing", "tree_hash", diff --git a/crates/networking/manager/src/service.rs b/crates/networking/manager/src/service.rs index d22008b90..eba22e66a 100644 --- a/crates/networking/manager/src/service.rs +++ b/crates/networking/manager/src/service.rs @@ -8,7 +8,8 @@ use alloy_primitives::B256; use libp2p::PeerId; use ream_chain_beacon::beacon_chain::BeaconChain; use ream_consensus_misc::{ - constants::beacon::NUM_CUSTODY_GROUPS, misc::compute_start_slot_at_epoch, + constants::beacon::{NUM_CUSTODY_GROUPS, SLOTS_PER_EPOCH}, + misc::compute_start_slot_at_epoch, }; use ream_discv5::{ config::DiscoveryConfig, @@ -37,7 +38,10 @@ use ream_syncer::{ block_range::BlockRangeSyncer, unknown_parent_lookups::{MAX_LOOKUPS, UnknownBlockMeta, UnknownParentLookupCoordinator}, }; -use tokio::{sync::mpsc, time::interval}; +use tokio::{ + sync::mpsc, + time::{interval, sleep}, +}; use tracing::{error, info, warn}; use tree_hash::TreeHash; @@ -107,6 +111,7 @@ pub struct NetworkManagerService { pub ream_db: BeaconDB, pub cached_db: Arc, pub sync_committee_pool: Arc, + pub executor: ReamExecutor, } struct ReconciledBlockLookupState { @@ -274,6 +279,7 @@ impl NetworkManagerService { ream_db, cached_db, sync_committee_pool, + executor, }) } @@ -290,9 +296,13 @@ impl NetworkManagerService { cached_db, network_state, block_range_syncer, + executor, .. } = self; + let mut resync_check_interval = interval(Duration::from_secs( + SLOTS_PER_EPOCH * beacon_network_spec().seconds_per_slot(), + )); let mut interval = interval(Duration::from_secs( beacon_network_spec().seconds_per_slot(), )); @@ -314,6 +324,8 @@ impl NetworkManagerService { let mut syncer_handle = block_range_syncer.start(); // Avoid polling a completed JoinHandle after the syncer has caught up. let mut syncer_active = true; + // Kept alive (not dropped) once synced, so the periodic re-check can restart it later. + let mut idle_syncer: Option = None; loop { tokio::select! { // Drive unknown-parent lookup actions and results. @@ -527,31 +539,43 @@ impl NetworkManagerService { // Restart range sync until the finalized target is reached. result = &mut syncer_handle, if syncer_active => { syncer_active = false; - let joined_result = match result { - Ok(joined_result) => joined_result, - Err(err) => { - error!("Block range syncer failed to join task: {err}"); - continue; + match result { + Ok(Ok((block_range_syncer, sync_result))) => { + if let Err(err) = sync_result { + warn!("Block range sync segment failed: {err:?}"); + } + if block_range_syncer.is_synced_to_head_slot().await { + idle_syncer = Some(block_range_syncer); + } else { + syncer_handle = block_range_syncer.start(); + syncer_active = true; + } } - }; - - let thread_result = match joined_result { - Ok(result) => result, - Err(err) => { - error!("Block range syncer thread failed: {err}"); - continue; + Ok(Err(err)) => { + // Executor shutdown cancelled the task; do not re-arm. + error!("Block range syncer task cancelled: {err}"); } - }; - - let block_range_syncer = match thread_result { - Ok(syncer) => syncer, Err(err) => { - error!("Block range syncer failed to start: {err}"); - continue; + error!("Block range syncer task panicked, reconstructing: {err}"); + sleep(Duration::from_secs(5)).await; + let block_range_syncer = BlockRangeSyncer::new( + beacon_chain.clone(), + p2p_sender.0.clone(), + network_state.clone(), + executor.clone(), + ); + syncer_handle = block_range_syncer.start(); + syncer_active = true; } - }; - - if !block_range_syncer.is_synced_to_head_slot().await { + } + } + _ = resync_check_interval.tick(), if !syncer_active && idle_syncer.is_some() => { + let block_range_syncer = idle_syncer + .take() + .expect("checked by the select guard above"); + if block_range_syncer.is_synced_to_head_slot().await { + idle_syncer = Some(block_range_syncer); + } else { syncer_handle = block_range_syncer.start(); syncer_active = true; } diff --git a/crates/networking/p2p/src/network/beacon/channel.rs b/crates/networking/p2p/src/network/beacon/channel.rs index d7ff2bd2e..012633205 100644 --- a/crates/networking/p2p/src/network/beacon/channel.rs +++ b/crates/networking/p2p/src/network/beacon/channel.rs @@ -56,6 +56,13 @@ pub enum P2PRequest { blob_identifiers: Vec, callback: mpsc::Sender>, }, + DataColumnRange { + peer_id: PeerId, + start: u64, + count: u64, + columns: Vec, + callback: mpsc::Sender>, + }, DataColumnIdentifiers { peer_id: PeerId, column_identifiers: Vec, diff --git a/crates/networking/p2p/src/network/beacon/mod.rs b/crates/networking/p2p/src/network/beacon/mod.rs index f3dcbfaaf..90d13b649 100644 --- a/crates/networking/p2p/src/network/beacon/mod.rs +++ b/crates/networking/p2p/src/network/beacon/mod.rs @@ -43,7 +43,9 @@ use ream_req_resp::{ BeaconRequestMessage, BeaconResponseMessage, blob_sidecars::BlobSidecarsByRootV1Request, blocks::{BeaconBlocksByRangeV2Request, BeaconBlocksByRootV2Request}, - data_column_sidecars::DataColumnSidecarsByRootV1Request, + data_column_sidecars::{ + DataColumnSidecarsByRangeV1Request, DataColumnSidecarsByRootV1Request, + }, meta_data::GetMetaDataV3, ping::Ping, status::Status, @@ -52,6 +54,7 @@ use ream_req_resp::{ handler::{ReqRespMessageError, ReqRespMessageReceived, RespMessage}, messages::{RequestMessage, ResponseMessage}, }; +use ssz_types::VariableList; use tokio::{ sync::mpsc::{self, UnboundedReceiver, UnboundedSender}, time::interval, @@ -364,6 +367,19 @@ impl Network { warn!("Failed to send error response: {err:?}"); } }, + P2PRequest::DataColumnRange { peer_id, start, count, columns, callback } => { + let request = DataColumnSidecarsByRangeV1Request { + start_slot: start, + count, + columns: VariableList::new(columns) + .expect("Too many columns were requested"), + }; + if let Some(request_id) = self.send_request(peer_id, BeaconRequestMessage::DataColumnSidecarsByRange(request)) { + self.callbacks.insert(request_id, callback); + } else if let Err(err) = callback.send(Ok(P2PCallbackResponse::Disconnected)).await { + warn!("Failed to send error response: {err:?}"); + } + }, P2PRequest::DataColumnIdentifiers { peer_id, column_identifiers, callback } => { if let Some(request_id) = self.send_request(peer_id, BeaconRequestMessage::DataColumnSidecarsByRoot(DataColumnSidecarsByRootV1Request::new(column_identifiers))) { self.callbacks.insert(request_id, callback); diff --git a/crates/networking/syncer/Cargo.toml b/crates/networking/syncer/Cargo.toml index 9ba1f3566..6e09872f3 100644 --- a/crates/networking/syncer/Cargo.toml +++ b/crates/networking/syncer/Cargo.toml @@ -38,8 +38,14 @@ ream-req-resp.workspace = true ream-storage.workspace = true [dev-dependencies] +discv5.workspace = true kzg.workspace = true +parking_lot.workspace = true ream-execution-rpc-types.workspace = true +ream-operation-pool.workspace = true +ream-peer.workspace = true +ream-sync-committee-pool.workspace = true +tempfile.workspace = true [lints] workspace = true diff --git a/crates/networking/syncer/src/block_range/block_cache.rs b/crates/networking/syncer/src/block_range/block_cache.rs index 90b1b1f86..f782cbf06 100644 --- a/crates/networking/syncer/src/block_range/block_cache.rs +++ b/crates/networking/syncer/src/block_range/block_cache.rs @@ -5,10 +5,12 @@ use anyhow::{bail, ensure}; use ream_chain_beacon::beacon_chain::is_data_availability_check_required; use ream_consensus_beacon::{ blob_sidecar::{BlobIdentifier, BlobSidecar}, + data_column_sidecar::{ColumnIdentifier, DataColumnSidecar}, electra::beacon_block::SignedBeaconBlock, }; use ream_consensus_misc::misc::compute_epoch_at_slot; use ream_network_spec::networks::beacon_network_spec; +use ream_polynomial_commitments::handlers::verify_data_column_sidecar_kzg_proofs; use ssz::Encode; use tree_hash::TreeHash; @@ -17,6 +19,7 @@ use super::{MAX_BLOCKS_PER_REQUEST, peer_range_downloader::Range}; pub struct BlockAndBlobBundle { pub block: SignedBeaconBlock, pub blobs: HashMap, + pub columns: HashMap, } impl BlockAndBlobBundle { @@ -24,6 +27,7 @@ impl BlockAndBlobBundle { Self { block, blobs: HashMap::new(), + columns: HashMap::new(), } } } @@ -35,8 +39,12 @@ pub struct BlockCache { block_ranges_to_retry: Vec, initial_slot: u64, next_start_slot: u64, + block_ranges_in_progress: HashSet, block_roots_in_progress: HashSet, blob_identifiers_in_progress: HashSet, + column_ranges_to_fetch: Vec, + column_ranges_in_progress: HashSet, + data_column_identifiers_in_progress: HashSet, } impl BlockCache { @@ -48,8 +56,12 @@ impl BlockCache { block_ranges_to_retry: vec![], initial_slot: next_start_slot, next_start_slot, + block_ranges_in_progress: HashSet::new(), block_roots_in_progress: HashSet::new(), blob_identifiers_in_progress: HashSet::new(), + column_ranges_to_fetch: vec![], + column_ranges_in_progress: HashSet::new(), + data_column_identifiers_in_progress: HashSet::new(), } } @@ -101,6 +113,40 @@ impl BlockCache { Ok(()) } + pub fn add_data_columns( + &mut self, + columns: Vec, + required_columns: &HashSet, + ) -> anyhow::Result<()> { + for column in columns { + if !required_columns.contains(&column.index) { + continue; + } + + let block_root = column.signed_block_header.message.tree_hash_root(); + let Some(bundle) = self.blocks_and_blobs.get_mut(&block_root) else { + bail!("Block root {block_root} not found in cache, this should be impossible"); + }; + + ensure!( + column.verify_inclusion_proof(), + "Invalid inclusion proof for column {} of block {block_root}", + column.index + ); + ensure!( + verify_data_column_sidecar_kzg_proofs(&column)?, + "Invalid KZG proof for column {} of block {block_root}", + column.index + ); + + bundle + .columns + .insert(ColumnIdentifier::new(block_root, column.index), column); + } + + Ok(()) + } + pub fn extend_block_roots_in_progress(&mut self, block_roots: &[B256]) { self.block_roots_in_progress.extend(block_roots); } @@ -121,6 +167,28 @@ impl BlockCache { } } + pub fn push_column_range(&mut self, range: Range) { + self.column_ranges_to_fetch.push(range); + } + + pub fn mark_column_range_in_progress(&mut self, range: Range) { + self.column_ranges_in_progress.insert(range); + } + + pub fn remove_column_range_in_progress(&mut self, range: &Range) { + self.column_ranges_in_progress.remove(range); + } + + pub fn extend_data_column_identifiers_in_progress(&mut self, identifiers: &[ColumnIdentifier]) { + self.data_column_identifiers_in_progress.extend(identifiers); + } + + pub fn remove_data_column_identifiers_in_progress(&mut self, identifiers: &[ColumnIdentifier]) { + for identifier in identifiers { + self.data_column_identifiers_in_progress.remove(identifier); + } + } + pub fn block_count(&self) -> u64 { self.blocks_and_blobs.len() as u64 } @@ -151,7 +219,20 @@ impl BlockCache { self.block_ranges_to_retry.push(range); } - pub fn data_to_fetch(&mut self, target_slot: u64, current_epoch: u64) -> DataToFetch { + pub fn mark_block_range_in_progress(&mut self, range: Range) { + self.block_ranges_in_progress.insert(range); + } + + pub fn remove_block_range_in_progress(&mut self, range: &Range) { + self.block_ranges_in_progress.remove(range); + } + + pub fn data_to_fetch( + &mut self, + target_slot: u64, + current_epoch: u64, + required_columns: &HashSet, + ) -> DataToFetch { match self.block_ranges_to_retry.pop() { Some(range) => return DataToFetch::BlockRange(range), None => { @@ -166,6 +247,10 @@ impl BlockCache { } } + if let Some(range) = self.column_ranges_to_fetch.pop() { + return DataToFetch::DataColumnRange(range); + } + let mut block_roots_left_to_fetch = self.get_missing_block_roots(); let missing_block_roots_len = block_roots_left_to_fetch.len(); block_roots_left_to_fetch.retain(|root| !self.block_roots_in_progress.contains(root)); @@ -175,6 +260,15 @@ impl BlockCache { blob_identifiers_left_to_fetch .retain(|blob_identifier| !self.blob_identifiers_in_progress.contains(blob_identifier)); + let mut data_column_identifiers_left_to_fetch = + self.get_missing_data_column_identifiers(current_epoch, required_columns); + let missing_data_column_identifiers_len = data_column_identifiers_left_to_fetch.len(); + data_column_identifiers_left_to_fetch.retain(|identifier| { + !self + .data_column_identifiers_in_progress + .contains(identifier) + }); + if !block_roots_left_to_fetch.is_empty() { return DataToFetch::MissingBlockRoots(block_roots_left_to_fetch); } @@ -183,7 +277,18 @@ impl BlockCache { return DataToFetch::MissingBlobIdentifiers(blob_identifiers_left_to_fetch); } - if missing_block_roots_len > 0 || missing_blob_identifiers_len > 0 { + if !data_column_identifiers_left_to_fetch.is_empty() { + return DataToFetch::MissingDataColumnIdentifiers( + data_column_identifiers_left_to_fetch, + ); + } + + if missing_block_roots_len > 0 + || missing_blob_identifiers_len > 0 + || missing_data_column_identifiers_len > 0 + || !self.block_ranges_in_progress.is_empty() + || !self.column_ranges_in_progress.is_empty() + { return DataToFetch::DownloadsInProgress; } @@ -255,13 +360,53 @@ impl BlockCache { } missing_roots } + + /// ByRoot fallback for columns a range fetch missed (e.g. root-fetched blocks). + fn get_missing_data_column_identifiers( + &self, + current_epoch: u64, + required_columns: &HashSet, + ) -> Vec { + let network_spec = beacon_network_spec(); + let mut missing_identifiers = Vec::new(); + for block in self.blocks_and_blobs.values() { + if block.block.message.body.blob_kzg_commitments.is_empty() { + continue; + } + + if !is_data_availability_check_required( + compute_epoch_at_slot(block.block.message.slot), + current_epoch, + network_spec.fulu_fork_epoch, + network_spec.min_epochs_for_data_column_sidecars_requests, + ) { + continue; + } + + if compute_epoch_at_slot(block.block.message.slot) < network_spec.fulu_fork_epoch { + continue; + } + + let block_root = block.block.message.tree_hash_root(); + for column_index in required_columns { + let identifier = ColumnIdentifier::new(block_root, *column_index); + if block.columns.contains_key(&identifier) { + continue; + } + missing_identifiers.push(identifier); + } + } + missing_identifiers + } } #[derive(Debug, Clone, PartialEq, Eq)] pub enum DataToFetch { BlockRange(Range), + DataColumnRange(Range), MissingBlockRoots(Vec), MissingBlobIdentifiers(Vec), + MissingDataColumnIdentifiers(Vec), DownloadsInProgress, Finished, } @@ -270,12 +415,16 @@ impl std::fmt::Display for DataToFetch { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { DataToFetch::BlockRange(range) => write!(f, "BlockRange({range:?})"), + DataToFetch::DataColumnRange(range) => write!(f, "DataColumnRange({range:?})"), DataToFetch::MissingBlockRoots(roots) => { write!(f, "MissingBlockRoots({})", roots.len()) } DataToFetch::MissingBlobIdentifiers(identifiers) => { write!(f, "MissingBlobIdentifiers({})", identifiers.len()) } + DataToFetch::MissingDataColumnIdentifiers(identifiers) => { + write!(f, "MissingDataColumnIdentifiers({})", identifiers.len()) + } DataToFetch::DownloadsInProgress => write!(f, "DownloadsInProgress"), DataToFetch::Finished => write!(f, "Finished"), } @@ -284,10 +433,14 @@ impl std::fmt::Display for DataToFetch { #[cfg(test)] mod tests { - use ream_consensus_beacon::electra::beacon_block::BeaconBlock; + use ream_consensus_beacon::{ + electra::beacon_block::BeaconBlock, + matrix_entry::{compute_cells_and_kzg_proofs, das_context}, + }; use ream_consensus_misc::{ misc::compute_start_slot_at_epoch, polynomial_commitments::kzg_commitment::KZGCommitment, }; + use ream_execution_rpc_types::get_blobs::Blob; use ream_network_spec::networks::beacon::initialize_test_network_spec; use super::*; @@ -297,7 +450,31 @@ mod tests { initialize_test_network_spec(); let mut cache = BlockCache::new(B256::ZERO, 10); - assert_eq!(cache.data_to_fetch(10, 0), DataToFetch::Finished); + assert_eq!( + cache.data_to_fetch(10, 0, &HashSet::new()), + DataToFetch::Finished + ); + } + + /// `Finished` must not fire while a `BlockRange` is still in flight, or the download (and + /// its peer) gets silently orphaned. + #[test] + fn data_to_fetch_waits_for_in_flight_block_ranges_before_finishing() { + initialize_test_network_spec(); + let mut cache = BlockCache::new(B256::ZERO, 10); + let range = Range::new(11, 10); + cache.mark_block_range_in_progress(range); + + assert_eq!( + cache.data_to_fetch(10, 0, &HashSet::new()), + DataToFetch::DownloadsInProgress + ); + + cache.remove_block_range_in_progress(&range); + assert_eq!( + cache.data_to_fetch(10, 0, &HashSet::new()), + DataToFetch::Finished + ); } #[test] @@ -306,14 +483,17 @@ mod tests { let mut cache = BlockCache::new(B256::ZERO, 10); assert_eq!( - cache.data_to_fetch(25, 0), + cache.data_to_fetch(25, 0, &HashSet::new()), DataToFetch::BlockRange(Range::new(11, 10)) ); assert_eq!( - cache.data_to_fetch(25, 0), + cache.data_to_fetch(25, 0, &HashSet::new()), DataToFetch::BlockRange(Range::new(21, 5)) ); - assert_eq!(cache.data_to_fetch(25, 0), DataToFetch::Finished); + assert_eq!( + cache.data_to_fetch(25, 0, &HashSet::new()), + DataToFetch::Finished + ); } #[test] @@ -351,7 +531,7 @@ mod tests { .add_blocks(vec![block_with_blob(boundary_slot)], false) .expect("boundary block should enter cache"); assert_eq!( - retained.data_to_fetch(boundary_slot, current_epoch), + retained.data_to_fetch(boundary_slot, current_epoch, &HashSet::new()), DataToFetch::Finished ); @@ -361,8 +541,121 @@ mod tests { .add_blocks(vec![block_with_blob(expired_slot)], false) .expect("expired block should enter cache"); assert_eq!( - expired.data_to_fetch(expired_slot, current_epoch), + expired.data_to_fetch(expired_slot, current_epoch, &HashSet::new()), DataToFetch::Finished ); } + + #[test] + fn post_fulu_blob_commitments_request_required_data_column_identifiers() { + initialize_test_network_spec(); + let network_spec = beacon_network_spec(); + let current_epoch = network_spec.fulu_fork_epoch + + network_spec.min_epochs_for_data_column_sidecars_requests + + 10; + let boundary_epoch = + current_epoch - network_spec.min_epochs_for_data_column_sidecars_requests; + let parent_root = B256::repeat_byte(1); + let boundary_slot = compute_start_slot_at_epoch(boundary_epoch); + + let mut block = SignedBeaconBlock { + message: BeaconBlock { + slot: boundary_slot, + parent_root, + ..Default::default() + }, + signature: Default::default(), + }; + block + .message + .body + .blob_kzg_commitments + .push(KZGCommitment::empty_for_testing()) + .expect("one commitment should fit"); + let block_root = block.message.tree_hash_root(); + + let mut cache = BlockCache::new(parent_root, boundary_slot); + cache + .add_blocks(vec![block], false) + .expect("boundary block should enter cache"); + + let required_columns = HashSet::from([1, 5]); + match cache.data_to_fetch(boundary_slot, current_epoch, &required_columns) { + DataToFetch::MissingDataColumnIdentifiers(identifiers) => { + let mut identifiers = identifiers; + identifiers.sort(); + assert_eq!( + identifiers, + vec![ + ColumnIdentifier::new(block_root, 1), + ColumnIdentifier::new(block_root, 5), + ] + ); + } + other => panic!("expected MissingDataColumnIdentifiers, got {other:?}"), + } + + // Unneeded columns never count as missing. + assert_eq!( + cache.data_to_fetch(boundary_slot, current_epoch, &HashSet::new()), + DataToFetch::Finished + ); + } + + #[test] + fn add_data_columns_ignores_extras_and_rejects_bad_proofs() { + use ream_consensus_beacon::data_column_sidecar::get_data_column_sidecars_from_block; + + let blob = Blob::default(); + let blob_bytes = blob.to_fixed_bytes(); + let raw_commitment = das_context() + .blob_to_kzg_commitment(&blob_bytes) + .expect("test blob should produce a commitment"); + let commitment = KZGCommitment(raw_commitment); + + let mut block = SignedBeaconBlock { + message: Default::default(), + signature: Default::default(), + }; + block + .message + .body + .blob_kzg_commitments + .push(commitment) + .expect("one commitment should fit"); + let block_root = block.message.tree_hash_root(); + + let cells_and_kzg_proofs = compute_cells_and_kzg_proofs(&blob, das_context()) + .expect("test blob should produce cells and proofs"); + let columns = get_data_column_sidecars_from_block(&block, vec![cells_and_kzg_proofs]) + .expect("test block should produce data columns"); + + let mut cache = BlockCache::new(B256::ZERO, 0); + cache + .add_blocks(vec![block], false) + .expect("block should enter cache"); + + // Only column 0 was requested; extra columns in the response are ignored, not fatal. + let required_columns = HashSet::from([0]); + cache + .add_data_columns(columns.clone(), &required_columns) + .expect("valid, requested column should be accepted"); + assert_eq!( + cache + .blocks_and_blobs + .get(&block_root) + .expect("block should be cached") + .columns + .len(), + 1 + ); + + let mut tampered = columns[0].clone(); + tampered.kzg_proofs[0][0] ^= 1; + assert!( + cache + .add_data_columns(vec![tampered], &required_columns) + .is_err() + ); + } } diff --git a/crates/networking/syncer/src/block_range/mod.rs b/crates/networking/syncer/src/block_range/mod.rs index 121c229e7..da10b5ce2 100644 --- a/crates/networking/syncer/src/block_range/mod.rs +++ b/crates/networking/syncer/src/block_range/mod.rs @@ -3,6 +3,7 @@ mod peer_manager; mod peer_range_downloader; use std::{ + collections::HashSet, pin::Pin, sync::Arc, task::{Context, Poll}, @@ -14,14 +15,19 @@ use anyhow::{anyhow, bail, ensure}; use block_cache::{BlockAndBlobBundle, BlockCache, DataToFetch}; use futures::task::noop_waker; use libp2p::PeerId; -use peer_manager::PeerManager; -use peer_range_downloader::{PeerBlobIdentifierDownloader, PeerRootsDownloader}; +use peer_manager::{BanReason, PeerManager}; +use peer_range_downloader::{ + PeerBlobIdentifierDownloader, PeerDataColumnIdentifierDownloader, + PeerDataColumnRangeDownloader, PeerRootsDownloader, +}; use ream_chain_beacon::beacon_chain::{ BeaconChain, BlockProcessingOutcome, is_data_availability_check_required, }; use ream_consensus_beacon::{ blob_sidecar::{BlobIdentifier, BlobSidecar}, - data_column_sidecar::{DataColumnSidecar, get_data_column_sidecars_from_block}, + data_column_sidecar::{ + ColumnIdentifier, DataColumnSidecar, get_data_column_sidecars_from_block, + }, electra::beacon_block::SignedBeaconBlock, matrix_entry::{compute_cells_and_kzg_proofs, das_context}, }; @@ -30,7 +36,9 @@ use ream_executor::ReamExecutor; use ream_network_spec::networks::beacon_network_spec; use ream_p2p::network::beacon::{channel::P2PMessage, network_state::NetworkState}; use ream_polynomial_commitments::handlers::verify_blob_kzg_proof_batch; -use ream_req_resp::MAX_CONCURRENT_REQUESTS; +use ream_req_resp::{ + MAX_CONCURRENT_REQUESTS, beacon::messages::data_column_sidecars::DataColumnsByRootIdentifier, +}; use ream_storage::tables::table::{CustomTable, REDBTable}; use tokio::{sync::mpsc::UnboundedSender, task::JoinHandle, time::sleep}; use tracing::{info, warn}; @@ -42,6 +50,10 @@ const MAX_BLOBS_PER_REQUEST: usize = 6; const MAX_BLOCKS_PER_REQUEST: u64 = 10; const SLEEP_DURATION: Duration = Duration::from_secs(5); +/// Max slots behind wall-clock to still count as synced (like Lighthouse's +/// `SLOT_IMPORT_TOLERANCE`). A thin/early peer sample can be fooled; the clock can't. +const SLOT_IMPORT_TOLERANCE: u64 = 32; + /// Validates downloaded blob sidecars and derives the columns needed for data availability. /// An empty `blob_sidecars` map defers to the block-lookup coordinator instead of erroring, /// since range sync never fetches legacy blob sidecars for post-Fulu blocks. @@ -121,247 +133,384 @@ impl BlockRangeSyncer { } pub async fn is_synced_to_head_slot(&self) -> bool { - let target_slot = self - .peer_manager - .head_slot() - .or_else(|| self.peer_manager.finalized_slot()); - let latest_synced_slot = self - .beacon_chain - .store - .lock() - .await + let Some(sync_target) = self.peer_manager.sync_target() else { + return false; + }; + + let store = self.beacon_chain.store.lock().await; + let latest_synced_slot = store .db .slot_index_provider() .get_highest_slot() .unwrap_or_default() .unwrap_or(0); + let Ok(current_slot) = store.get_current_slot() else { + return false; + }; + drop(store); + + let peers_report_caught_up = sync_target.slot <= latest_synced_slot; + // The clock is ground truth peers can't fake. + let clock_confirms_caught_up = + current_slot.saturating_sub(latest_synced_slot) <= SLOT_IMPORT_TOLERANCE; - target_slot <= Some(latest_synced_slot) + peers_report_caught_up && clock_confirms_caught_up } - pub fn start(mut self) -> JoinHandle>> { + pub fn start(mut self) -> JoinHandle)>> { let executor = self.executor.clone(); executor.spawn(async move { - let Some(latest_synced_root) = self + let result = self.run_segment().await; + (self, result) + }) + } + + async fn run_segment(&mut self) -> anyhow::Result<()> { + let Some(latest_synced_root) = self + .beacon_chain + .store + .lock() + .await + .db + .slot_index_provider() + .get_highest_root() + .map_err(|err| anyhow!("Failed to get highest root: {err}"))? + else { + bail!("No synced root found in the database"); + }; + + let Some(latest_synced_slot) = self + .beacon_chain + .store + .lock() + .await + .db + .slot_index_provider() + .get_highest_slot() + .map_err(|err| anyhow!("Failed to get highest slot: {err}"))? + else { + bail!("No synced slot found in the database"); + }; + + // phase 1: download majority of blocks from ranges + let mut block_cache = BlockCache::new(latest_synced_root, latest_synced_slot); + let mut task_handles = vec![]; + loop { + let required_columns = self .beacon_chain .store .lock() .await - .db - .slot_index_provider() - .get_highest_root() - .map_err(|err| anyhow!("Failed to get highest root: {err}"))? - else { - bail!("No synced root found in the database"); + .data_availability_checker + .required_columns() + .clone(); + + poll_ready_tasks( + &mut task_handles, + &mut block_cache, + &mut self.peer_manager, + &required_columns, + )?; + + let Some(sync_target) = self.peer_manager.sync_target() else { + warn!("No peers available to determine sync target, retrying..."); + sleep(SLEEP_DURATION).await; + self.peer_manager.update_peer_set(); + continue; }; - let Some(latest_synced_slot) = self + let current_epoch = self .beacon_chain .store .lock() .await - .db - .slot_index_provider() - .get_highest_slot() - .map_err(|err| anyhow!("Failed to get highest slot: {err}"))? - else { - bail!("No synced slot found in the database"); - }; + .get_current_store_epoch()?; + let data_to_fetch = + block_cache.data_to_fetch(sync_target.slot, current_epoch, &required_columns); + info!( + "Forward sync status: Downloaded Blocks {}, Downloaded Blobs {}/{}, Stage {data_to_fetch}", + block_cache.block_count(), + block_cache.downloaded_blob_count(), + block_cache.blob_count(), + ); - // phase 1: download majority of blocks from ranges - let mut block_cache = - BlockCache::new(latest_synced_root, latest_synced_slot); - let mut task_handles = vec![]; - loop { - poll_ready_tasks(&mut task_handles, &mut block_cache, &mut self.peer_manager)?; - - let target_slot = match self - .peer_manager - .head_slot() - .or_else(|| self.peer_manager.finalized_slot()) - { - Some(target_slot) => target_slot, - None => { - warn!("No peers available to determine sync target, retrying..."); - sleep(SLEEP_DURATION).await; + match data_to_fetch { + DataToFetch::BlockRange(range) => { + let Some(peer) = self + .peer_manager + .fetch_idle_peer_preferring(Some(sync_target.root)) + else { self.peer_manager.update_peer_set(); + info!("No idle peers available for block range sync."); + block_cache.push_retry_range(range); + sleep(SLEEP_DURATION).await; continue; - } - }; + }; - let current_epoch = self - .beacon_chain - .store - .lock() - .await - .get_current_store_epoch()?; - let data_to_fetch = block_cache.data_to_fetch(target_slot, current_epoch); - info!( - "Forward sync status: Downloaded Blocks {}, Downloaded Blobs {}/{}, Stage {data_to_fetch}", - block_cache.block_count(), - block_cache.downloaded_blob_count(), - block_cache.blob_count(), - ); + block_cache.mark_block_range_in_progress(range); + task_handles.push(DownloadTask::new_block_range( + PeerRangeDownloader::start( + peer.peer_id, + self.p2p_sender.clone(), + self.executor.clone(), + range, + ), + range, + peer.peer_id, + )); + } + DataToFetch::DataColumnRange(range) => { + let Some(peer) = self + .peer_manager + .fetch_idle_peer_preferring(Some(sync_target.root)) + else { + self.peer_manager.update_peer_set(); + info!("No idle peers available for data column range sync."); + block_cache.push_column_range(range); + sleep(SLEEP_DURATION).await; + continue; + }; - match data_to_fetch { - DataToFetch::BlockRange(range) => { - let Some(peer) = self.peer_manager.fetch_idle_peer() else { + block_cache.mark_column_range_in_progress(range); + task_handles.push(DownloadTask::new_data_column_range( + PeerDataColumnRangeDownloader::start( + peer.peer_id, + self.p2p_sender.clone(), + self.executor.clone(), + range, + required_columns.iter().copied().collect(), + ), + range, + peer.peer_id, + )); + } + DataToFetch::MissingBlockRoots(block_roots) => { + for block_roots_chunk in block_roots.chunks(MAX_CONCURRENT_REQUESTS) { + let Some(peer) = self + .peer_manager + .fetch_idle_peer_preferring(Some(sync_target.root)) + else { self.peer_manager.update_peer_set(); - info!("No idle peers available for block range sync."); + info!("No idle peers available for block roots sync."); sleep(SLEEP_DURATION).await; - continue; + break; }; + block_cache.extend_block_roots_in_progress(block_roots_chunk); - task_handles.push(DownloadTask::new_block_range( - PeerRangeDownloader::start( + task_handles.push(DownloadTask::new_block_roots( + PeerRootsDownloader::start( peer.peer_id, self.p2p_sender.clone(), self.executor.clone(), - range, + block_roots_chunk.to_vec(), ), - range, + block_roots_chunk.to_vec(), peer.peer_id, )); } - DataToFetch::MissingBlockRoots(block_roots) => { - for block_roots_chunk in block_roots.chunks(MAX_CONCURRENT_REQUESTS) { - let Some(peer) = self.peer_manager.fetch_idle_peer() else { - self.peer_manager.update_peer_set(); - info!("No idle peers available for block roots sync."); - sleep(SLEEP_DURATION).await; - break; - }; - block_cache.extend_block_roots_in_progress(block_roots_chunk); - - task_handles.push(DownloadTask::new_block_roots( - PeerRootsDownloader::start( - peer.peer_id, - self.p2p_sender.clone(), - self.executor.clone(), - block_roots_chunk.to_vec(), - ), - block_roots_chunk.to_vec(), + } + DataToFetch::MissingBlobIdentifiers(blob_identifiers) => { + for blob_identifiers_chunk in blob_identifiers.chunks(MAX_BLOBS_PER_REQUEST) { + let Some(peer) = self + .peer_manager + .fetch_idle_peer_preferring(Some(sync_target.root)) + else { + self.peer_manager.update_peer_set(); + info!( + "No idle peers available for blob sync. {}", + self.peer_manager.peer_counts() + ); + sleep(SLEEP_DURATION).await; + break; + }; + + block_cache.extend_blob_identifiers_in_progress(blob_identifiers_chunk); + + task_handles.push(DownloadTask::new_blob_identifiers( + PeerBlobIdentifierDownloader::start( peer.peer_id, - )); - } - } - DataToFetch::MissingBlobIdentifiers(blob_identifiers) => { - for blob_identifiers_chunk in blob_identifiers.chunks(MAX_BLOBS_PER_REQUEST) { - let Some(peer) = self.peer_manager.fetch_idle_peer() else { - self.peer_manager.update_peer_set(); - info!("No idle peers available for blob sync. {}", self.peer_manager.peer_counts()); - sleep(SLEEP_DURATION).await; - break; - }; - - block_cache.extend_blob_identifiers_in_progress(blob_identifiers_chunk); - - task_handles.push(DownloadTask::new_blob_identifiers( - PeerBlobIdentifierDownloader::start( - peer.peer_id, - self.p2p_sender.clone(), - self.executor.clone(), - blob_identifiers_chunk.to_vec(), - ), + self.p2p_sender.clone(), + self.executor.clone(), blob_identifiers_chunk.to_vec(), - peer.peer_id, - )); - } + ), + blob_identifiers_chunk.to_vec(), + peer.peer_id, + )); } - DataToFetch::DownloadsInProgress => { - info!("Waiting for ongoing downloads to complete... {}", self.peer_manager.peer_counts()); - sleep(Duration::from_secs(10)).await; + } + DataToFetch::MissingDataColumnIdentifiers(identifiers) => { + for identifiers_chunk in identifiers.chunks(MAX_CONCURRENT_REQUESTS) { + let Some(peer) = self + .peer_manager + .fetch_idle_peer_preferring(Some(sync_target.root)) + else { + self.peer_manager.update_peer_set(); + info!("No idle peers available for data column sync."); + sleep(SLEEP_DURATION).await; + break; + }; + + block_cache.extend_data_column_identifiers_in_progress(identifiers_chunk); + + task_handles.push(DownloadTask::new_data_column_identifiers( + PeerDataColumnIdentifierDownloader::start( + peer.peer_id, + self.p2p_sender.clone(), + self.executor.clone(), + group_by_block_root(identifiers_chunk), + ), + identifiers_chunk.to_vec(), + peer.peer_id, + )); } - DataToFetch::Finished => break, } + DataToFetch::DownloadsInProgress => { + info!( + "Waiting for ongoing downloads to complete... {}", + self.peer_manager.peer_counts() + ); + sleep(Duration::from_secs(10)).await; + } + DataToFetch::Finished => break, } + } - info!("Block range sync completed a segment successfully with {} blocks and {} blobs.", - block_cache.block_count(), - block_cache.downloaded_blob_count(), - ); + info!( + "Block range sync completed a segment successfully with {} blocks and {} blobs.", + block_cache.block_count(), + block_cache.downloaded_blob_count(), + ); - // execute all the blocks downloaded - for BlockAndBlobBundle { block, blobs } in block_cache.get_blocks_and_blobs()? { - info!("Processing block with slot {}", - block.message.slot, - ); + let fulu_fork_epoch = beacon_network_spec().fulu_fork_epoch; - let (block, columns) = if block.message.body.blob_kzg_commitments.is_empty() { - ensure!( - blobs.is_empty(), - "Range-sync block without blob commitments had downloaded blob sidecars" - ); - (block, Vec::new()) - } else { - let (blobs_provider, required_columns, verify_data_availability) = { - let store = self.beacon_chain.store.lock().await; - let network_spec = beacon_network_spec(); - ( - store.db.blobs_and_proofs_provider(), - store.data_availability_checker.required_columns().clone(), - is_data_availability_check_required( - compute_epoch_at_slot(block.message.slot), - store.get_current_store_epoch()?, - network_spec.fulu_fork_epoch, - network_spec.min_epochs_for_data_column_sidecars_requests, - ), - ) - }; - tokio::task::spawn_blocking(move || { - let columns = build_data_columns_from_blob_sidecars( - &block, - &blobs, - verify_data_availability, - )? - .into_iter() - .filter(|column| required_columns.contains(&column.index)) - .collect::>(); - for (identifier, sidecar) in blobs { - blobs_provider.insert(identifier, sidecar.into())?; - } - Ok::<_, anyhow::Error>((block, columns)) - }) + // execute all the blocks downloaded + for BlockAndBlobBundle { + block, + blobs, + columns, + } in block_cache.get_blocks_and_blobs()? + { + info!("Processing block with slot {}", block.message.slot,); + + let (block, columns) = if block.message.body.blob_kzg_commitments.is_empty() { + ensure!( + blobs.is_empty(), + "Range-sync block without blob commitments had downloaded blob sidecars" + ); + ensure!( + columns.is_empty(), + "Range-sync block without blob commitments had downloaded data columns" + ); + (block, Vec::new()) + } else if compute_epoch_at_slot(block.message.slot) >= fulu_fork_epoch { + let required_columns = self + .beacon_chain + .store + .lock() .await - .map_err(|err| anyhow!("Range-sync data-column task failed: {err}"))?? + .data_availability_checker + .required_columns() + .clone(); + let columns = columns + .into_values() + .filter(|column| required_columns.contains(&column.index)) + .collect::>(); + (block, columns) + } else { + let (blobs_provider, required_columns, verify_data_availability) = { + let store = self.beacon_chain.store.lock().await; + let network_spec = beacon_network_spec(); + ( + store.db.blobs_and_proofs_provider(), + store.data_availability_checker.required_columns().clone(), + is_data_availability_check_required( + compute_epoch_at_slot(block.message.slot), + store.get_current_store_epoch()?, + network_spec.fulu_fork_epoch, + network_spec.min_epochs_for_data_column_sidecars_requests, + ), + ) }; + tokio::task::spawn_blocking(move || { + let columns = build_data_columns_from_blob_sidecars( + &block, + &blobs, + verify_data_availability, + )? + .into_iter() + .filter(|column| required_columns.contains(&column.index)) + .collect::>(); + for (identifier, sidecar) in blobs { + blobs_provider.insert(identifier, sidecar.into())?; + } + Ok::<_, anyhow::Error>((block, columns)) + }) + .await + .map_err(|err| anyhow!("Range-sync data-column task failed: {err}"))?? + }; - match self.beacon_chain.process_block(block).await? { - BlockProcessingOutcome::Imported { .. } => {} - BlockProcessingOutcome::PendingAvailability { block_root } => { - for column in columns { - self.beacon_chain - .import_data_column_sidecar_if(column, |_| Ok(())) - .await?; - } - ensure!( - self.beacon_chain - .store - .lock() - .await - .db - .block_provider() - .get(block_root)? - .is_some(), - "Range-sync block {block_root} remained pending after processing its downloaded data" - ); + match self.beacon_chain.process_block(block).await? { + BlockProcessingOutcome::Imported { .. } => {} + BlockProcessingOutcome::PendingAvailability { block_root } => { + for column in columns { + self.beacon_chain + .import_data_column_sidecar_if(column, |_| Ok(())) + .await?; } + ensure!( + self.beacon_chain + .store + .lock() + .await + .db + .block_provider() + .get(block_root)? + .is_some(), + "Range-sync block {block_root} remained pending after processing its downloaded data" + ); } } + } - info!("All blocks processed successfully."); + info!("All blocks processed successfully."); - Ok(self) - }) + Ok(()) } } +fn group_by_block_root(identifiers: &[ColumnIdentifier]) -> Vec { + let mut by_root: std::collections::HashMap> = std::collections::HashMap::new(); + for identifier in identifiers { + by_root + .entry(identifier.block_root) + .or_default() + .push(identifier.index); + } + by_root + .into_iter() + .filter_map( + |(block_root, columns)| match DataColumnsByRootIdentifier::new(block_root, columns) { + Ok(identifier) => Some(identifier), + Err(err) => { + warn!("Failed to build data column identifier for {block_root}: {err}"); + None + } + }, + ) + .collect() +} + pub enum DownloadTask { BlockRange { handle: JoinHandle>>>, range: Range, peer_id: PeerId, }, + DataColumnRange { + handle: JoinHandle>>>, + range: Range, + peer_id: PeerId, + }, BlockRoots { handle: JoinHandle>>>, roots: Vec, @@ -372,6 +521,11 @@ pub enum DownloadTask { blob_identifiers: Vec, peer_id: PeerId, }, + DataColumnIdentifiers { + handle: JoinHandle>>>, + identifiers: Vec, + peer_id: PeerId, + }, } impl DownloadTask { @@ -387,6 +541,18 @@ impl DownloadTask { } } + pub fn new_data_column_range( + handle: JoinHandle>>>, + range: Range, + peer_id: PeerId, + ) -> Self { + DownloadTask::DataColumnRange { + handle, + range, + peer_id, + } + } + pub fn new_block_roots( handle: JoinHandle>>>, roots: Vec, @@ -410,16 +576,30 @@ impl DownloadTask { peer_id, } } + + pub fn new_data_column_identifiers( + handle: JoinHandle>>>, + identifiers: Vec, + peer_id: PeerId, + ) -> Self { + DownloadTask::DataColumnIdentifiers { + handle, + identifiers, + peer_id, + } + } } fn poll_ready_tasks( tasks: &mut Vec, block_cache: &mut BlockCache, peer_manager: &mut PeerManager, + required_columns: &HashSet, ) -> anyhow::Result<()> { let waker = noop_waker(); let mut context = Context::from_waker(&waker); let mut indexes_to_remove = vec![]; + let fulu_fork_epoch = beacon_network_spec().fulu_fork_epoch; for index in (0..tasks.len()).rev() { let Some(task) = tasks.get_mut(index) else { @@ -437,6 +617,7 @@ fn poll_ready_tasks( match pinned.poll(&mut context) { Poll::Ready(Ok(blocks_result)) => { indexes_to_remove.push(index); + block_cache.remove_block_range_in_progress(range); peer_manager.mark_peer_as_idle(peer_id); let blocks = match blocks_result { Ok(blocks) => blocks, @@ -451,8 +632,10 @@ fn poll_ready_tasks( Ok(blocks) => blocks, Err(err) => { block_cache.push_retry_range(*range); - peer_manager - .ban_peer(peer_id, format!("Failed to fetch blocks: {err:?}")); + peer_manager.ban_peer( + peer_id, + BanReason::ProtocolError(format!("{err:?}")), + ); continue; } }; @@ -460,18 +643,85 @@ fn poll_ready_tasks( if blocks.is_empty() { warn!("Received empty block range from peer: {peer_id}"); block_cache.push_retry_range(*range); - peer_manager - .ban_peer(peer_id, "Received empty block range".to_string()); + peer_manager.ban_peer(peer_id, BanReason::EmptyResponse); continue; } + let needs_columns = !required_columns.is_empty() + && blocks.iter().any(|block| { + !block.message.body.blob_kzg_commitments.is_empty() + && compute_epoch_at_slot(block.message.slot) >= fulu_fork_epoch + }); + if let Err(err) = block_cache.add_blocks(blocks, true) { warn!("Failed to add downloaded blocks to cache: {err:?}"); block_cache.push_retry_range(*range); + } else if needs_columns { + block_cache.push_column_range(*range); + } + } + Poll::Ready(Err(err)) => { + warn!("Forward fill task failed: {err}"); + peer_manager.mark_peer_as_idle(peer_id); + block_cache.remove_block_range_in_progress(range); + block_cache.push_retry_range(*range); + indexes_to_remove.push(index); + } + Poll::Pending => {} + } + } + DownloadTask::DataColumnRange { + handle, + range, + peer_id, + } => { + let pinned = Pin::new(handle); + + match pinned.poll(&mut context) { + Poll::Ready(Ok(columns_result)) => { + indexes_to_remove.push(index); + block_cache.remove_column_range_in_progress(range); + peer_manager.mark_peer_as_idle(peer_id); + let columns = match columns_result { + Ok(columns) => columns, + Err(err) => { + warn!("Failed to fetch data columns from peer: {err:?}"); + block_cache.push_column_range(*range); + continue; + } + }; + + let columns = match columns { + Ok(columns) => columns, + Err(err) => { + block_cache.push_column_range(*range); + peer_manager.ban_peer( + peer_id, + BanReason::ProtocolError(format!("{err:?}")), + ); + continue; + } + }; + + if columns.is_empty() { + warn!("Received empty data column range from peer: {peer_id}"); + block_cache.push_column_range(*range); + peer_manager.ban_peer(peer_id, BanReason::EmptyResponse); + continue; + } + + if let Err(err) = block_cache.add_data_columns(columns, required_columns) { + warn!("Failed to add downloaded data columns to cache: {err:?}"); + block_cache.push_column_range(*range); + peer_manager + .ban_peer(peer_id, BanReason::InvalidProof(format!("{err:?}"))); } } Poll::Ready(Err(err)) => { warn!("Forward fill task failed: {err}"); + peer_manager.mark_peer_as_idle(peer_id); + block_cache.remove_column_range_in_progress(range); + block_cache.push_column_range(*range); indexes_to_remove.push(index); } Poll::Pending => {} @@ -503,7 +753,7 @@ fn poll_ready_tasks( warn!("Failed to fetch blocks from roots: {err:?}"); peer_manager.ban_peer( peer_id, - format!("Failed to fetch blocks from receipts: {err:?}"), + BanReason::ProtocolError(format!("{err:?}")), ); continue; } @@ -511,8 +761,7 @@ fn poll_ready_tasks( if blocks.is_empty() { warn!("Received empty block roots from peer: {peer_id}"); - peer_manager - .ban_peer(peer_id, "Received empty block roots".to_string()); + peer_manager.ban_peer(peer_id, BanReason::EmptyResponse); continue; } @@ -522,6 +771,8 @@ fn poll_ready_tasks( } Poll::Ready(Err(err)) => { warn!("Forward fill task failed: {err}"); + peer_manager.mark_peer_as_idle(peer_id); + block_cache.remove_block_roots_in_progress(roots); indexes_to_remove.push(index); } Poll::Pending => {} @@ -553,7 +804,7 @@ fn poll_ready_tasks( warn!("Failed to fetch blobs from identifiers: {err:?}"); peer_manager.ban_peer( peer_id, - format!("Failed to fetch blobs from identifiers: {err:?}"), + BanReason::ProtocolError(format!("{err:?}")), ); continue; } @@ -561,8 +812,7 @@ fn poll_ready_tasks( if blob_sidecars.is_empty() { warn!("Received empty blob identifiers from peer: {peer_id}"); - peer_manager - .ban_peer(peer_id, "Received empty blob identifiers".to_string()); + peer_manager.ban_peer(peer_id, BanReason::EmptyResponse); continue; } @@ -572,6 +822,61 @@ fn poll_ready_tasks( } Poll::Ready(Err(err)) => { warn!("Forward fill task failed: {err}"); + peer_manager.mark_peer_as_idle(peer_id); + block_cache.remove_blob_identifiers_in_progress(blob_identifiers); + indexes_to_remove.push(index); + } + Poll::Pending => {} + } + } + DownloadTask::DataColumnIdentifiers { + handle, + identifiers, + peer_id, + } => { + let pinned = Pin::new(handle); + + match pinned.poll(&mut context) { + Poll::Ready(Ok(columns_result)) => { + indexes_to_remove.push(index); + block_cache.remove_data_column_identifiers_in_progress(identifiers); + peer_manager.mark_peer_as_idle(peer_id); + let columns = match columns_result { + Ok(columns) => columns, + Err(err) => { + warn!("Failed to fetch data columns from peer: {err:?}"); + continue; + } + }; + + let columns = match columns { + Ok(columns) => columns, + Err(err) => { + warn!("Failed to fetch data columns from identifiers: {err:?}"); + peer_manager.ban_peer( + peer_id, + BanReason::ProtocolError(format!("{err:?}")), + ); + continue; + } + }; + + if columns.is_empty() { + warn!("Received empty data column identifiers from peer: {peer_id}"); + peer_manager.ban_peer(peer_id, BanReason::EmptyResponse); + continue; + } + + if let Err(err) = block_cache.add_data_columns(columns, required_columns) { + warn!("Failed to add downloaded data columns to cache: {err:?}"); + peer_manager + .ban_peer(peer_id, BanReason::InvalidProof(format!("{err:?}"))); + } + } + Poll::Ready(Err(err)) => { + warn!("Forward fill task failed: {err}"); + peer_manager.mark_peer_as_idle(peer_id); + block_cache.remove_data_column_identifiers_in_progress(identifiers); indexes_to_remove.push(index); } Poll::Pending => {} @@ -589,14 +894,24 @@ fn poll_ready_tasks( #[cfg(test)] mod tests { - use std::collections::HashMap; + use std::{collections::HashMap, path::PathBuf}; + use discv5::{Enr, enr::CombinedKey}; use kzg::{G1, eip_4844::compute_blob_kzg_proof_raw}; + use parking_lot::RwLock; use ream_consensus_beacon::data_column_sidecar::NUMBER_OF_COLUMNS; use ream_consensus_misc::polynomial_commitments::{ kzg_commitment::KZGCommitment, kzg_proof::KZGProof, }; use ream_execution_rpc_types::get_blobs::{Blob, BlobAndProofV1}; + use ream_network_spec::networks::beacon::initialize_test_network_spec; + use ream_operation_pool::OperationPool; + use ream_p2p::network::beacon::peer::CachedPeer; + use ream_peer::{ConnectionState, Direction}; + use ream_req_resp::beacon::messages::{meta_data::GetMetaDataV3, status::Status}; + use ream_storage::{db::ReamDB, tables::field::REDBField}; + use ream_sync_committee_pool::SyncCommitteePool; + use tempfile::TempDir; use super::*; @@ -662,4 +977,139 @@ mod tests { )]); assert!(build_data_columns_from_blob_sidecars(&block, &mismatched, true).is_err()); } + + /// Kept alongside the chain so the dir isn't dropped (and cleaned up) until the test ends. + fn test_beacon_chain() -> (TempDir, BeaconChain) { + let data_dir = tempfile::tempdir().expect("tempdir should be created"); + let beacon_db = ReamDB::new(data_dir.path().to_path_buf()) + .expect("ReamDB should init") + .init_beacon_db() + .expect("beacon DB tables should init"); + let beacon_chain = BeaconChain::new( + beacon_db, + Arc::new(OperationPool::default()), + Arc::new(SyncCommitteePool::default()), + None, + None, + ); + (data_dir, beacon_chain) + } + + fn test_network_state() -> Arc { + let enr_key = CombinedKey::generate_secp256k1(); + Arc::new(NetworkState { + local_enr: RwLock::new(Enr::builder().build(&enr_key).expect("valid enr")), + peer_table: RwLock::new(HashMap::new()), + meta_data: RwLock::new(GetMetaDataV3::default()), + status: RwLock::new(Status::default()), + data_dir: PathBuf::new(), + }) + } + + /// `start()` must hand `self` back out even on an ordinary segment error, not just success, + /// or the caller loses the syncer (and its warmed-up peer table) and can never retry. + #[test] + fn start_returns_self_after_a_failed_segment() { + initialize_test_network_spec(); + // Empty DB has no highest synced slot, so run_segment fails immediately. + let (_data_dir, beacon_chain) = test_beacon_chain(); + let executor = ReamExecutor::new().expect("executor should start"); + let (p2p_sender, _p2p_receiver) = tokio::sync::mpsc::unbounded_channel(); + + let syncer = BlockRangeSyncer::new( + Arc::new(beacon_chain), + p2p_sender, + test_network_state(), + executor, + ); + + // ReamExecutor owns its own runtime; #[tokio::test] would make dropping it panic. + let (_syncer, sync_result) = futures::executor::block_on(syncer.start()) + .expect("task should not panic") + .expect("task should not be cancelled by shutdown"); + + assert!( + sync_result.is_err(), + "expected the empty-DB segment to fail" + ); + } + + /// A lone/early peer must not be enough to call the node synced. The wall clock has to + /// agree too, or one thin sample could "vote" the node caught up while it's still far behind. + #[test] + fn is_synced_to_head_slot_requires_wall_clock_agreement() { + // ReamExecutor owns its own runtime; #[tokio::test] would make dropping it panic. + futures::executor::block_on(is_synced_to_head_slot_requires_wall_clock_agreement_inner()); + } + + async fn is_synced_to_head_slot_requires_wall_clock_agreement_inner() { + initialize_test_network_spec(); + let (_data_dir, beacon_chain) = test_beacon_chain(); + let seconds_per_slot = beacon_network_spec().seconds_per_slot(); + let highest_slot = 100u64; + let highest_root = B256::repeat_byte(0x42); + + { + let store = beacon_chain.store.lock().await; + store + .db + .genesis_time_provider() + .insert(0) + .expect("insert genesis time"); + store + .db + .slot_index_provider() + .insert(highest_slot, highest_root) + .expect("insert highest synced slot"); + } + + let network_state = test_network_state(); + let peer_id = PeerId::random(); + let mut peer = CachedPeer::new( + peer_id, + None, + ConnectionState::Connected, + Direction::Outbound, + None, + ); + peer.status = Some(Status { + head_slot: highest_slot, + head_root: highest_root, + ..Default::default() + }); + network_state.peer_table.write().insert(peer_id, peer); + + let (p2p_sender, _p2p_receiver) = tokio::sync::mpsc::unbounded_channel(); + let mut syncer = BlockRangeSyncer::new( + Arc::new(beacon_chain), + p2p_sender, + network_state, + ReamExecutor::new().expect("executor should start"), + ); + syncer.peer_manager.update_peer_set(); + + // Clock agrees with the peer: genuinely caught up. + syncer + .beacon_chain + .store + .lock() + .await + .db + .time_provider() + .insert(highest_slot * seconds_per_slot) + .expect("insert time"); + assert!(syncer.is_synced_to_head_slot().await); + + // Clock disagrees: must not report synced, even though the peer still does. + syncer + .beacon_chain + .store + .lock() + .await + .db + .time_provider() + .insert((highest_slot + 10_000) * seconds_per_slot) + .expect("insert time"); + assert!(!syncer.is_synced_to_head_slot().await); + } } diff --git a/crates/networking/syncer/src/block_range/peer_manager.rs b/crates/networking/syncer/src/block_range/peer_manager.rs index 876864157..b73d38208 100644 --- a/crates/networking/syncer/src/block_range/peer_manager.rs +++ b/crates/networking/syncer/src/block_range/peer_manager.rs @@ -1,14 +1,46 @@ use std::{ collections::{HashMap, hash_map::Entry}, sync::Arc, - time::Instant, + time::{Duration, Instant}, }; +use alloy_primitives::B256; use libp2p::PeerId; use ream_consensus_misc::constants::beacon::SLOTS_PER_EPOCH; use ream_p2p::network::beacon::{network_state::NetworkState, peer::CachedPeer}; use tracing::warn; +/// How long a peer stays banned before it becomes eligible to rejoin the peer set. +const BAN_DURATION: Duration = Duration::from_secs(300); + +/// Majority-agreed (slot, root), tagged with root so peer selection can target this chain. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SyncTarget { + pub slot: u64, + pub root: B256, +} + +/// Why a peer was banned. Still just bans outright either way, but structured (instead of a +/// free-text string) so a future scoring system can weigh severities without touching call sites. +#[derive(Debug, Clone)] +pub enum BanReason { + EmptyResponse, + /// Disconnect, timeout, decode error, or a response that didn't fit the cache. + ProtocolError(String), + /// Failed KZG or inclusion proof verification. + InvalidProof(String), +} + +impl std::fmt::Display for BanReason { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + BanReason::EmptyResponse => write!(f, "empty response"), + BanReason::ProtocolError(detail) => write!(f, "protocol error: {detail}"), + BanReason::InvalidProof(detail) => write!(f, "invalid proof: {detail}"), + } + } +} + #[derive(Debug, Clone)] pub enum PeerStatus { Idle, @@ -25,7 +57,7 @@ pub struct PeerManager { network_state: Arc, peers: HashMap, banned_peers: HashMap, - ban_reasons: HashMap, + ban_reasons: HashMap, } impl PeerManager { @@ -39,6 +71,12 @@ impl PeerManager { } pub fn update_peer_set(&mut self) { + let now = Instant::now(); + self.banned_peers + .retain(|_, banned_at| now.duration_since(*banned_at) < BAN_DURATION); + self.ban_reasons + .retain(|peer_id, _| self.banned_peers.contains_key(peer_id)); + let connected_peers = self.network_state.connected_peers(); for peer in &connected_peers { if self.banned_peers.contains_key(&peer.peer_id) { @@ -63,8 +101,7 @@ impl PeerManager { .retain(|peer_id, _| connected_peers.iter().any(|peer| peer.peer_id == *peer_id)); } - /// Bans a peer - pub fn ban_peer(&mut self, peer_id: &PeerId, reason: String) { + pub fn ban_peer(&mut self, peer_id: &PeerId, reason: BanReason) { self.ban_reasons.insert(*peer_id, reason); if let Some(peer_info) = self.peers.remove(peer_id) { self.banned_peers @@ -87,6 +124,22 @@ impl PeerManager { None } + pub fn fetch_idle_peer_preferring(&mut self, target_root: Option) -> Option { + if let Some(target_root) = target_root { + for peer_info in self.peers.values_mut() { + let matches_target = peer_info.peer.status.as_ref().is_some_and(|status| { + status.head_root == target_root || status.finalized_root == target_root + }); + if matches_target && matches!(peer_info.peer_status, PeerStatus::Idle) { + peer_info.peer_status = PeerStatus::Downloading; + return Some(peer_info.peer.clone()); + } + } + } + + self.fetch_idle_peer() + } + pub fn peer_counts(&self) -> String { let total_peers = self.peers.len(); let idle_peers = self @@ -109,13 +162,17 @@ impl PeerManager { } } - pub fn finalized_slot(&self) -> Option { - let mut frequencies = HashMap::new(); + /// Majority vote on (epoch, root), not just epoch, so two forks can't be merged as "agreed". + pub fn finalized_target(&self) -> Option { + let mut frequencies: HashMap<(u64, B256), usize> = HashMap::new(); for peer in self.peers.values() { if let Some(status) = &peer.peer.status { *frequencies - .entry(status.finalized_epoch * SLOTS_PER_EPOCH) + .entry(( + status.finalized_epoch * SLOTS_PER_EPOCH, + status.finalized_root, + )) .or_insert(0) += 1; } } @@ -123,14 +180,214 @@ impl PeerManager { frequencies .into_iter() .max_by_key(|&(_, count)| count) - .map(|(slot, _)| slot) + .map(|((slot, root), _)| SyncTarget { slot, root }) } - pub fn head_slot(&self) -> Option { - self.peers - .values() - .filter_map(|peer| peer.peer.status.as_ref()) - .map(|status| status.head_slot) - .max() + /// Majority vote on (slot, root); replaces a bare max() so one outlier can't skew the target. + pub fn head_target(&self) -> Option { + let mut frequencies: HashMap<(u64, B256), usize> = HashMap::new(); + + for peer in self.peers.values() { + if let Some(status) = &peer.peer.status { + *frequencies + .entry((status.head_slot, status.head_root)) + .or_insert(0) += 1; + } + } + + frequencies + .into_iter() + .max_by_key(|&(_, count)| count) + .map(|((slot, root), _)| SyncTarget { slot, root }) + } + + pub fn sync_target(&self) -> Option { + self.head_target().or_else(|| self.finalized_target()) + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use discv5::{Enr, enr::CombinedKey}; + use parking_lot::RwLock; + use ream_peer::{ConnectionState, Direction}; + use ream_req_resp::beacon::messages::{meta_data::GetMetaDataV3, status::Status}; + + use super::*; + + fn test_network_state() -> Arc { + let enr_key = CombinedKey::generate_secp256k1(); + Arc::new(NetworkState { + local_enr: RwLock::new(Enr::builder().build(&enr_key).expect("valid enr")), + peer_table: RwLock::new(HashMap::new()), + meta_data: RwLock::new(GetMetaDataV3::default()), + status: RwLock::new(Status::default()), + data_dir: PathBuf::new(), + }) + } + + fn test_peer(status: Status) -> CachedPeer { + let mut peer = CachedPeer::new( + PeerId::random(), + None, + ConnectionState::Connected, + Direction::Outbound, + None, + ); + peer.status = Some(status); + peer + } + + fn insert_idle(peer_manager: &mut PeerManager, peer: CachedPeer) { + peer_manager.peers.insert( + peer.peer_id, + PeerInfo { + peer, + peer_status: PeerStatus::Idle, + }, + ); + } + + #[test] + fn finalized_target_does_not_merge_different_forks_with_same_epoch() { + let mut peer_manager = PeerManager::new(test_network_state()); + let root_a = B256::repeat_byte(0xAA); + let root_b = B256::repeat_byte(0xBB); + + insert_idle( + &mut peer_manager, + test_peer(Status { + finalized_epoch: 10, + finalized_root: root_a, + ..Default::default() + }), + ); + insert_idle( + &mut peer_manager, + test_peer(Status { + finalized_epoch: 10, + finalized_root: root_b, + ..Default::default() + }), + ); + + // 1 vote each: must not merge into a false "2 votes" just because the epoch matches. + let target = peer_manager + .finalized_target() + .expect("a target should be picked"); + assert!(target.root == root_a || target.root == root_b); + } + + #[test] + fn head_target_ignores_outlier_without_plurality() { + let mut peer_manager = PeerManager::new(test_network_state()); + let honest_root = B256::repeat_byte(0x11); + let lying_root = B256::repeat_byte(0x22); + + for _ in 0..3 { + insert_idle( + &mut peer_manager, + test_peer(Status { + head_slot: 100, + head_root: honest_root, + ..Default::default() + }), + ); + } + insert_idle( + &mut peer_manager, + test_peer(Status { + head_slot: 999_999, + head_root: lying_root, + ..Default::default() + }), + ); + + let target = peer_manager + .head_target() + .expect("a target should be picked"); + assert_eq!(target.slot, 100); + assert_eq!(target.root, honest_root); + } + + #[test] + fn fetch_idle_peer_preferring_prefers_matching_root_then_falls_back() { + let mut peer_manager = PeerManager::new(test_network_state()); + let target_root = B256::repeat_byte(0x33); + let other_root = B256::repeat_byte(0x44); + + let matching = test_peer(Status { + head_root: target_root, + ..Default::default() + }); + let matching_id = matching.peer_id; + insert_idle( + &mut peer_manager, + test_peer(Status { + head_root: other_root, + ..Default::default() + }), + ); + insert_idle(&mut peer_manager, matching); + + let peer = peer_manager + .fetch_idle_peer_preferring(Some(target_root)) + .expect("a peer should be found"); + assert_eq!(peer.peer_id, matching_id); + + // No peer matches this root: falls back to any idle peer instead of returning None. + let unrelated_root = B256::repeat_byte(0x55); + assert!( + peer_manager + .fetch_idle_peer_preferring(Some(unrelated_root)) + .is_some() + ); + } + + #[test] + fn bans_expire_after_ban_duration() { + let mut peer_manager = PeerManager::new(test_network_state()); + let peer = test_peer(Status::default()); + let peer_id = peer.peer_id; + insert_idle(&mut peer_manager, peer); + + peer_manager.ban_peer(&peer_id, BanReason::EmptyResponse); + assert!(peer_manager.banned_peers.contains_key(&peer_id)); + + // Simulate the ban having happened BAN_DURATION ago. + peer_manager.banned_peers.insert( + peer_id, + Instant::now() - BAN_DURATION - Duration::from_secs(1), + ); + + peer_manager.update_peer_set(); + assert!(!peer_manager.banned_peers.contains_key(&peer_id)); + } + + #[test] + fn mark_peer_as_idle_recovers_a_stuck_downloading_peer() { + let mut peer_manager = PeerManager::new(test_network_state()); + let peer = test_peer(Status::default()); + let peer_id = peer.peer_id; + peer_manager.peers.insert( + peer_id, + PeerInfo { + peer, + peer_status: PeerStatus::Downloading, + }, + ); + + peer_manager.mark_peer_as_idle(&peer_id); + + assert!(matches!( + peer_manager + .peers + .get(&peer_id) + .expect("peer exists") + .peer_status, + PeerStatus::Idle + )); } } diff --git a/crates/networking/syncer/src/block_range/peer_range_downloader.rs b/crates/networking/syncer/src/block_range/peer_range_downloader.rs index 9ea0976c2..6f2ce5171 100644 --- a/crates/networking/syncer/src/block_range/peer_range_downloader.rs +++ b/crates/networking/syncer/src/block_range/peer_range_downloader.rs @@ -3,19 +3,21 @@ use anyhow::bail; use libp2p::PeerId; use ream_consensus_beacon::{ blob_sidecar::{BlobIdentifier, BlobSidecar}, + data_column_sidecar::DataColumnSidecar, electra::beacon_block::SignedBeaconBlock, }; use ream_executor::ReamExecutor; use ream_p2p::network::beacon::channel::{P2PCallbackResponse, P2PMessage, P2PRequest}; -use ream_req_resp::beacon::messages::BeaconResponseMessage; -use ssz::Encode; +use ream_req_resp::beacon::messages::{ + BeaconResponseMessage, data_column_sidecars::DataColumnsByRootIdentifier, +}; use tokio::{ sync::mpsc::{self, UnboundedSender}, task::JoinHandle, }; use tracing::info; -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct Range { pub start_slot: u64, pub count: u64, @@ -27,6 +29,38 @@ impl Range { } } +async fn drain_responses( + mut rx: mpsc::Receiver>, + mut extract: impl FnMut(BeaconResponseMessage) -> Option, +) -> anyhow::Result> { + let mut items = vec![]; + + while let Some(response) = rx.recv().await { + match response { + Ok(P2PCallbackResponse::ResponseMessage(message)) => { + if let Some(item) = extract(message.as_ref().clone()) { + items.push(item); + } + } + Ok(P2PCallbackResponse::EndOfStream) => { + info!("End of request stream received."); + break; + } + Ok(P2PCallbackResponse::Disconnected) => { + bail!("Peer disconnected while receiving response."); + } + Ok(P2PCallbackResponse::Timeout) => { + bail!("Request timed out."); + } + Err(err) => { + info!("Error receiving response: {err:?}"); + } + } + } + + Ok(items) +} + pub struct PeerRangeDownloader; impl PeerRangeDownloader { @@ -37,8 +71,7 @@ impl PeerRangeDownloader { range: Range, ) -> JoinHandle>>> { executor.spawn(async move { - let mut beacon_blocks = vec![]; - let (callback, mut rx) = mpsc::channel(100); + let (callback, rx) = mpsc::channel(100); p2p_sender .send(P2PMessage::Request(P2PRequest::BlockRange { peer_id, @@ -48,37 +81,11 @@ impl PeerRangeDownloader { })) .expect("Failed to send block range request"); - while let Some(response) = rx.recv().await { - match response { - Ok(P2PCallbackResponse::ResponseMessage(message)) => { - if let BeaconResponseMessage::BeaconBlocksByRange(blocks) = - message.as_ref().clone() - { - info!( - "Received block response with slot {} length {}", - blocks.message.slot, - blocks.as_ssz_bytes().len() - ); - beacon_blocks.push(blocks); - } - } - Ok(P2PCallbackResponse::EndOfStream) => { - info!("End of block range request stream received."); - break; - } - Ok(P2PCallbackResponse::Disconnected) => { - bail!("Peer disconnected while receiving block range."); - } - Ok(P2PCallbackResponse::Timeout) => { - bail!("Block range request timed out."); - } - Err(err) => { - info!("Error receiving BeaconBlocks from block range request: {err:?}"); - } - } - } - - Ok(beacon_blocks) + drain_responses(rx, |message| match message { + BeaconResponseMessage::BeaconBlocksByRange(block) => Some(block), + _ => None, + }) + .await }) } } @@ -93,47 +100,20 @@ impl PeerRootsDownloader { roots: Vec, ) -> JoinHandle>>> { executor.spawn(async move { - let mut beacon_blocks = vec![]; - let (callback, mut rx) = mpsc::channel(100); + let (callback, rx) = mpsc::channel(100); p2p_sender .send(P2PMessage::Request(P2PRequest::BlockRoots { peer_id, - roots: roots.to_vec(), + roots, callback, })) .expect("Failed to send block roots request"); - while let Some(response) = rx.recv().await { - match response { - Ok(P2PCallbackResponse::ResponseMessage(message)) => { - if let BeaconResponseMessage::BeaconBlocksByRoot(blocks) = - message.as_ref().clone() - { - info!( - "Received block response with slot {} length {}", - blocks.message.slot, - blocks.as_ssz_bytes().len() - ); - beacon_blocks.push(blocks); - } - } - Ok(P2PCallbackResponse::EndOfStream) => { - info!("End of block roots request stream received."); - break; - } - Ok(P2PCallbackResponse::Disconnected) => { - bail!("Peer disconnected while receiving block roots."); - } - Ok(P2PCallbackResponse::Timeout) => { - bail!("Block roots request timed out."); - } - Err(err) => { - info!("Error receiving BeaconBlocks from block roots request: {err:?}"); - } - } - } - - Ok(beacon_blocks) + drain_responses(rx, |message| match message { + BeaconResponseMessage::BeaconBlocksByRoot(block) => Some(block), + _ => None, + }) + .await }) } } @@ -148,47 +128,173 @@ impl PeerBlobIdentifierDownloader { blob_identifiers: Vec, ) -> JoinHandle>>> { executor.spawn(async move { - let mut blob_sidecars = vec![]; - let (callback, mut rx) = mpsc::channel(100); + let (callback, rx) = mpsc::channel(100); p2p_sender .send(P2PMessage::Request(P2PRequest::BlobIdentifiers { peer_id, - blob_identifiers: blob_identifiers.to_vec(), + blob_identifiers, callback, })) .expect("Failed to send blob identifiers request"); - while let Some(response) = rx.recv().await { - match response { - Ok(P2PCallbackResponse::ResponseMessage(message)) => { - if let BeaconResponseMessage::BlobSidecarsByRoot(blob_sidecar) = - message.as_ref().clone() - { - info!( - "Received blob sidecar response with index {} length {}", - blob_sidecar.index, - blob_sidecar.as_ssz_bytes().len() - ); - blob_sidecars.push(blob_sidecar); - } - } - Ok(P2PCallbackResponse::EndOfStream) => { - info!("End of blob roots request stream received."); - break; - } - Ok(P2PCallbackResponse::Disconnected) => { - bail!("Peer disconnected while receiving blob sidecars."); - } - Ok(P2PCallbackResponse::Timeout) => { - bail!("Blob identifiers request timed out."); - } - Err(err) => { - info!("Error receiving blobs from blob roots request: {err:?}"); - } - } - } + drain_responses(rx, |message| match message { + BeaconResponseMessage::BlobSidecarsByRoot(blob_sidecar) => Some(blob_sidecar), + _ => None, + }) + .await + }) + } +} - Ok(blob_sidecars) +pub struct PeerDataColumnRangeDownloader; + +impl PeerDataColumnRangeDownloader { + pub fn start( + peer_id: PeerId, + p2p_sender: UnboundedSender, + executor: ReamExecutor, + range: Range, + columns: Vec, + ) -> JoinHandle>>> { + executor.spawn(async move { + let (callback, rx) = mpsc::channel(100); + p2p_sender + .send(P2PMessage::Request(P2PRequest::DataColumnRange { + peer_id, + start: range.start_slot, + count: range.count, + columns, + callback, + })) + .expect("Failed to send data column range request"); + + drain_responses(rx, |message| match message { + BeaconResponseMessage::DataColumnSidecarsByRange(column) => Some(column), + _ => None, + }) + .await }) } } + +pub struct PeerDataColumnIdentifierDownloader; + +impl PeerDataColumnIdentifierDownloader { + pub fn start( + peer_id: PeerId, + p2p_sender: UnboundedSender, + executor: ReamExecutor, + identifiers: Vec, + ) -> JoinHandle>>> { + executor.spawn(async move { + let (callback, rx) = mpsc::channel(100); + p2p_sender + .send(P2PMessage::Request(P2PRequest::DataColumnIdentifiers { + peer_id, + column_identifiers: identifiers, + callback, + })) + .expect("Failed to send data column identifiers request"); + + drain_responses(rx, |message| match message { + BeaconResponseMessage::DataColumnSidecarsByRoot(column) => Some(column), + _ => None, + }) + .await + }) + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use super::*; + + fn test_block() -> SignedBeaconBlock { + SignedBeaconBlock { + message: Default::default(), + signature: Default::default(), + } + } + + #[tokio::test] + async fn drain_responses_extracts_matching_and_ignores_other_variants() { + let (tx, rx) = mpsc::channel(10); + tx.send(Ok(P2PCallbackResponse::ResponseMessage(Arc::new( + BeaconResponseMessage::BeaconBlocksByRange(test_block()), + )))) + .await + .expect("send should succeed"); + // A response for a different request kind must be ignored, not collected or errored on. + tx.send(Ok(P2PCallbackResponse::ResponseMessage(Arc::new( + BeaconResponseMessage::BeaconBlocksByRoot(test_block()), + )))) + .await + .expect("send should succeed"); + tx.send(Ok(P2PCallbackResponse::EndOfStream)) + .await + .expect("send should succeed"); + + let items = drain_responses(rx, |message| match message { + BeaconResponseMessage::BeaconBlocksByRange(block) => Some(block), + _ => None, + }) + .await + .expect("drain should succeed"); + + assert_eq!(items.len(), 1); + } + + #[tokio::test] + async fn drain_responses_returns_collected_items_when_channel_closes() { + let (tx, rx) = mpsc::channel(10); + tx.send(Ok(P2PCallbackResponse::ResponseMessage(Arc::new( + BeaconResponseMessage::BeaconBlocksByRange(test_block()), + )))) + .await + .expect("send should succeed"); + drop(tx); + + let items = drain_responses(rx, |message| match message { + BeaconResponseMessage::BeaconBlocksByRange(block) => Some(block), + _ => None, + }) + .await + .expect("drain should succeed"); + + assert_eq!(items.len(), 1); + } + + #[tokio::test] + async fn drain_responses_fails_on_disconnect() { + let (tx, rx) = mpsc::channel(10); + tx.send(Ok(P2PCallbackResponse::Disconnected)) + .await + .expect("send should succeed"); + + let result = drain_responses(rx, |message| match message { + BeaconResponseMessage::BeaconBlocksByRange(block) => Some(block), + _ => None, + }) + .await; + + assert!(result.is_err()); + } + + #[tokio::test] + async fn drain_responses_fails_on_timeout() { + let (tx, rx) = mpsc::channel(10); + tx.send(Ok(P2PCallbackResponse::Timeout)) + .await + .expect("send should succeed"); + + let result = drain_responses(rx, |message| match message { + BeaconResponseMessage::BeaconBlocksByRange(block) => Some(block), + _ => None, + }) + .await; + + assert!(result.is_err()); + } +} From b1b2032865120ea29604169233e12e7778ce0094 Mon Sep 17 00:00:00 2001 From: vuonghuuhung Date: Sat, 29 Aug 2026 02:20:48 +0700 Subject: [PATCH 05/17] fix: tighten range-sync quorum, candidate pool, and response validation --- crates/networking/manager/src/service.rs | 4 +- .../p2p/src/network/beacon/channel.rs | 19 +- .../networking/p2p/src/network/beacon/mod.rs | 10 +- crates/networking/req_resp/src/error.rs | 12 +- crates/networking/req_resp/src/handler.rs | 1 + .../req_resp/src/outbound_protocol.rs | 16 +- .../syncer/src/block_range/block_cache.rs | 557 +++++++- .../networking/syncer/src/block_range/mod.rs | 1204 ++++++++++++++--- .../syncer/src/block_range/peer_manager.rs | 361 +++-- .../src/block_range/peer_range_downloader.rs | 217 ++- 10 files changed, 1973 insertions(+), 428 deletions(-) diff --git a/crates/networking/manager/src/service.rs b/crates/networking/manager/src/service.rs index eba22e66a..2e4a3d162 100644 --- a/crates/networking/manager/src/service.rs +++ b/crates/networking/manager/src/service.rs @@ -540,7 +540,7 @@ impl NetworkManagerService { result = &mut syncer_handle, if syncer_active => { syncer_active = false; match result { - Ok(Ok((block_range_syncer, sync_result))) => { + Ok(Ok((mut block_range_syncer, sync_result))) => { if let Err(err) = sync_result { warn!("Block range sync segment failed: {err:?}"); } @@ -570,7 +570,7 @@ impl NetworkManagerService { } } _ = resync_check_interval.tick(), if !syncer_active && idle_syncer.is_some() => { - let block_range_syncer = idle_syncer + let mut block_range_syncer = idle_syncer .take() .expect("checked by the select guard above"); if block_range_syncer.is_synced_to_head_slot().await { diff --git a/crates/networking/p2p/src/network/beacon/channel.rs b/crates/networking/p2p/src/network/beacon/channel.rs index 012633205..abb6c3dcd 100644 --- a/crates/networking/p2p/src/network/beacon/channel.rs +++ b/crates/networking/p2p/src/network/beacon/channel.rs @@ -11,6 +11,7 @@ use ream_req_resp::{ beacon::messages::{ BeaconResponseMessage, data_column_sidecars::DataColumnsByRootIdentifier, status::Status, }, + error::ReqRespError, handler::RespMessage, }; use tokio::sync::mpsc; @@ -24,6 +25,14 @@ pub enum P2PCallbackResponse { EndOfStream, } +#[derive(thiserror::Error, Debug)] +pub enum P2PCallbackError { + #[error(transparent)] + ReqResp(#[from] ReqRespError), + #[error(transparent)] + Other(#[from] anyhow::Error), +} + pub enum P2PMessage { Request(P2PRequest), Response(P2PResponse), @@ -44,29 +53,29 @@ pub enum P2PRequest { peer_id: PeerId, start: u64, count: u64, - callback: mpsc::Sender>, + callback: mpsc::Sender>, }, BlockRoots { peer_id: PeerId, roots: Vec, - callback: mpsc::Sender>, + callback: mpsc::Sender>, }, BlobIdentifiers { peer_id: PeerId, blob_identifiers: Vec, - callback: mpsc::Sender>, + callback: mpsc::Sender>, }, DataColumnRange { peer_id: PeerId, start: u64, count: u64, columns: Vec, - callback: mpsc::Sender>, + callback: mpsc::Sender>, }, DataColumnIdentifiers { peer_id: PeerId, column_identifiers: Vec, - callback: mpsc::Sender>, + callback: mpsc::Sender>, }, } diff --git a/crates/networking/p2p/src/network/beacon/mod.rs b/crates/networking/p2p/src/network/beacon/mod.rs index 90d13b649..8ef759538 100644 --- a/crates/networking/p2p/src/network/beacon/mod.rs +++ b/crates/networking/p2p/src/network/beacon/mod.rs @@ -12,7 +12,7 @@ use std::{ }; use anyhow::anyhow; -use channel::{P2PCallbackResponse, P2PMessage, P2PRequest, P2PResponse}; +use channel::{P2PCallbackError, P2PCallbackResponse, P2PMessage, P2PRequest, P2PResponse}; use delay_map::{HashMapDelay, HashSetDelay}; use discv5::Enr; use libp2p::{ @@ -109,7 +109,7 @@ pub struct Network { peer_id: PeerId, swarm: Swarm, subscribed_topics: Arc>>, - callbacks: HashMapDelay>>, + callbacks: HashMapDelay>>, request_id: u64, network_state: Arc, peers_to_ping: HashSetDelay, @@ -687,9 +687,9 @@ impl Network { let message = match message { Ok(message) => message, Err(err) => { - if let ReqRespMessageError::Outbound { request_id, .. } = &err - && let Some(callback) = self.callbacks.get(request_id) - && let Err(err) = callback.send(Err(anyhow!("{err:?}"))).await + if let ReqRespMessageError::Outbound { request_id, err } = err + && let Some(callback) = self.callbacks.remove(&request_id) + && let Err(err) = callback.send(Err(P2PCallbackError::ReqResp(err))).await { warn!("Failed to send error response: {err:?}"); } diff --git a/crates/networking/req_resp/src/error.rs b/crates/networking/req_resp/src/error.rs index c41a20944..6213e0387 100644 --- a/crates/networking/req_resp/src/error.rs +++ b/crates/networking/req_resp/src/error.rs @@ -1,6 +1,6 @@ use std::io::{self}; -use ssz_types::{VariableList, typenum::U256}; +use crate::inbound_protocol::ResponseCode; #[derive(thiserror::Error, Debug)] pub enum ReqRespError { @@ -13,6 +13,9 @@ pub enum ReqRespError { #[error("Invalid data {0}")] InvalidData(String), + #[error("Remote error [{code:?}]: {message}")] + RemoteError { code: ResponseCode, message: String }, + #[error("Incomplete stream")] IncompleteStream, @@ -34,10 +37,3 @@ impl From for ReqRespError { ReqRespError::InvalidData(format!("Failed to decode ssz: {err:?}")) } } - -impl From> for ReqRespError { - fn from(err: VariableList) -> Self { - let err = String::from_utf8(Vec::from(err)).unwrap_or("Invalid UTF-8".to_string()); - ReqRespError::InvalidData(format!("ReqResp error message from peer: {err:?}")) - } -} diff --git a/crates/networking/req_resp/src/handler.rs b/crates/networking/req_resp/src/handler.rs index 8b3dc9cfe..77a901a03 100644 --- a/crates/networking/req_resp/src/handler.rs +++ b/crates/networking/req_resp/src/handler.rs @@ -66,6 +66,7 @@ impl RespMessage { | ReqRespError::Anyhow(_) | ReqRespError::IoError(_) => Some(ResponseCode::ServerError), ReqRespError::InvalidData(_) => Some(ResponseCode::InvalidRequest), + ReqRespError::RemoteError { code, .. } => Some(*code), ReqRespError::Disconnected | ReqRespError::StreamTimedOut(_) | ReqRespError::TokioTimedOut(_) => Some(ResponseCode::ResourceUnavailable), diff --git a/crates/networking/req_resp/src/outbound_protocol.rs b/crates/networking/req_resp/src/outbound_protocol.rs index 75365c8a9..ed9ffeac4 100644 --- a/crates/networking/req_resp/src/outbound_protocol.rs +++ b/crates/networking/req_resp/src/outbound_protocol.rs @@ -285,9 +285,19 @@ impl Decoder for OutboundSSZSnappyCodec { } } } else { - Ok(Some(RespMessage::Error( - VariableList::::from_ssz_bytes(&buf).map(ReqRespError::from).map_err(|err| anyhow!("OutboundSSZSnappyCodec::decode: protocol: {:?}, response_code: {response_code:?}, err: {err:?}", self.protocol.protocol))?, - ))) + let message = match VariableList::::from_ssz_bytes(&buf) { + Ok(bytes) => String::from_utf8(Vec::from(bytes)) + .unwrap_or_else(|_| "".to_string()), + Err(err) => { + return Ok(Some(RespMessage::Error(ReqRespError::InvalidData( + format!("Failed to decode error body: {err:?}"), + )))); + } + }; + Ok(Some(RespMessage::Error(ReqRespError::RemoteError { + code: response_code, + message, + }))) } } Err(err) => match err.kind() { diff --git a/crates/networking/syncer/src/block_range/block_cache.rs b/crates/networking/syncer/src/block_range/block_cache.rs index f782cbf06..fca93c42f 100644 --- a/crates/networking/syncer/src/block_range/block_cache.rs +++ b/crates/networking/syncer/src/block_range/block_cache.rs @@ -1,7 +1,11 @@ -use std::collections::{HashMap, HashSet}; +use std::{ + collections::{HashMap, HashSet}, + time::{Duration, Instant}, +}; use alloy_primitives::B256; use anyhow::{bail, ensure}; +use libp2p::PeerId; use ream_chain_beacon::beacon_chain::is_data_availability_check_required; use ream_consensus_beacon::{ blob_sidecar::{BlobIdentifier, BlobSidecar}, @@ -10,12 +14,31 @@ use ream_consensus_beacon::{ }; use ream_consensus_misc::misc::compute_epoch_at_slot; use ream_network_spec::networks::beacon_network_spec; -use ream_polynomial_commitments::handlers::verify_data_column_sidecar_kzg_proofs; +use ream_polynomial_commitments::handlers::{ + verify_blob_kzg_proof_batch, verify_data_column_sidecar_kzg_proofs, +}; use ssz::Encode; use tree_hash::TreeHash; use super::{MAX_BLOCKS_PER_REQUEST, peer_range_downloader::Range}; +const ATTEMPT_COOLDOWN: Duration = Duration::from_secs(30); + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum RequestKey { + BlockRange(Range), + ColumnRange(Range), + BlockRoot(B256), + Blob(BlobIdentifier), + Column(ColumnIdentifier), +} + +#[derive(Default)] +struct AttemptState { + attempted_peers: HashSet, + cooldown_until: Option, +} + pub struct BlockAndBlobBundle { pub block: SignedBeaconBlock, pub blobs: HashMap, @@ -45,6 +68,7 @@ pub struct BlockCache { column_ranges_to_fetch: Vec, column_ranges_in_progress: HashSet, data_column_identifiers_in_progress: HashSet, + attempts: HashMap, } impl BlockCache { @@ -62,9 +86,63 @@ impl BlockCache { column_ranges_to_fetch: vec![], column_ranges_in_progress: HashSet::new(), data_column_identifiers_in_progress: HashSet::new(), + attempts: HashMap::new(), + } + } + + fn refresh_attempt_round(&mut self, key: &RequestKey, now: Instant) { + if let Some(state) = self.attempts.get_mut(key) + && let Some(cooldown_until) = state.cooldown_until + && now >= cooldown_until + { + state.attempted_peers.clear(); + state.cooldown_until = None; + } + } + + fn is_schedulable( + &mut self, + key: RequestKey, + candidate_peers: &[PeerId], + now: Instant, + ) -> bool { + self.refresh_attempt_round(&key, now); + match self.attempts.get(&key) { + None => true, + Some(state) => candidate_peers + .iter() + .any(|peer_id| !state.attempted_peers.contains(peer_id)), } } + pub fn attempted_peers_for(&self, key: RequestKey) -> HashSet { + self.attempts + .get(&key) + .map(|state| state.attempted_peers.clone()) + .unwrap_or_default() + } + + pub fn mark_attempted( + &mut self, + key: RequestKey, + peer_id: PeerId, + candidate_peers: &[PeerId], + now: Instant, + ) { + let state = self.attempts.entry(key).or_default(); + state.attempted_peers.insert(peer_id); + if candidate_peers + .iter() + .all(|peer_id| state.attempted_peers.contains(peer_id)) + { + state.cooldown_until = Some(now + ATTEMPT_COOLDOWN); + } + } + + pub fn clear_attempted(&mut self, key: RequestKey) { + self.attempts.remove(&key); + } + pub fn add_blocks( &mut self, blocks: Vec, @@ -78,6 +156,10 @@ impl BlockCache { block.message.parent_root == blocks[index - 1].message.tree_hash_root(), "Block at index {index} has a parent root that does not match the previous block's tree hash root", ); + ensure!( + block.message.slot > blocks[index - 1].message.slot, + "Block at index {index} does not have a strictly increasing slot", + ); } } } @@ -94,20 +176,60 @@ impl BlockCache { } pub fn add_blobs(&mut self, blobs: Vec) -> anyhow::Result<()> { + let mut validated = Vec::with_capacity(blobs.len()); for blob_sidecar in blobs { let block_root = blob_sidecar.signed_block_header.message.tree_hash_root(); - - if let Some(bundle) = self.blocks_and_blobs.get_mut(&block_root) { - bundle.blobs.insert( - BlobIdentifier { - block_root, - index: blob_sidecar.index, - }, - blob_sidecar, - ); - } else { + let Some(bundle) = self.blocks_and_blobs.get(&block_root) else { bail!("Block root {block_root} not found in cache, this should be impossible"); - } + }; + + ensure!( + blob_sidecar.signed_block_header == bundle.block.signed_header(), + "Blob sidecar {} does not belong to block {block_root}", + blob_sidecar.index + ); + let commitments = &bundle.block.message.body.blob_kzg_commitments; + let index = blob_sidecar.index as usize; + ensure!( + index < commitments.len(), + "Blob sidecar index {} out of range for block {block_root}", + blob_sidecar.index + ); + ensure!( + blob_sidecar.kzg_commitment == commitments[index], + "Blob sidecar {} commitment does not match block {block_root}", + blob_sidecar.index + ); + ensure!( + blob_sidecar.verify_blob_sidecar_inclusion_proof(), + "Invalid inclusion proof for blob {} of block {block_root}", + blob_sidecar.index + ); + ensure!( + verify_blob_kzg_proof_batch( + std::slice::from_ref(&blob_sidecar.blob), + std::slice::from_ref(&blob_sidecar.kzg_commitment), + std::slice::from_ref(&blob_sidecar.kzg_proof), + )?, + "Invalid KZG proof for blob {} of block {block_root}", + blob_sidecar.index + ); + + validated.push((block_root, blob_sidecar)); + } + + for (block_root, blob_sidecar) in validated { + let bundle = self + .blocks_and_blobs + .get_mut(&block_root) + .expect("presence already checked in the validation pass above"); + bundle.blobs.insert( + BlobIdentifier { + block_root, + index: blob_sidecar.index, + }, + blob_sidecar, + ); } Ok(()) @@ -118,16 +240,26 @@ impl BlockCache { columns: Vec, required_columns: &HashSet, ) -> anyhow::Result<()> { + let mut validated = Vec::with_capacity(columns.len()); for column in columns { if !required_columns.contains(&column.index) { continue; } let block_root = column.signed_block_header.message.tree_hash_root(); - let Some(bundle) = self.blocks_and_blobs.get_mut(&block_root) else { - bail!("Block root {block_root} not found in cache, this should be impossible"); - }; + if !self.blocks_and_blobs.contains_key(&block_root) { + continue; + } + let bundle = self + .blocks_and_blobs + .get(&block_root) + .expect("presence just checked above"); + ensure!( + column.signed_block_header == bundle.block.signed_header(), + "Data column sidecar {} does not belong to block {block_root}", + column.index + ); ensure!( column.verify_inclusion_proof(), "Invalid inclusion proof for column {} of block {block_root}", @@ -139,6 +271,14 @@ impl BlockCache { column.index ); + validated.push((block_root, column)); + } + + for (block_root, column) in validated { + let bundle = self + .blocks_and_blobs + .get_mut(&block_root) + .expect("presence already checked in the validation pass above"); bundle .columns .insert(ColumnIdentifier::new(block_root, column.index), column); @@ -207,6 +347,10 @@ impl BlockCache { .sum() } + pub fn next_start_slot(&self) -> u64 { + self.next_start_slot + } + pub fn estimated_blocks_to_fetch(&self) -> u64 { if self.next_start_slot.saturating_sub(self.initial_slot) > 30 { return 0; @@ -227,38 +371,72 @@ impl BlockCache { self.block_ranges_in_progress.remove(range); } + fn take_schedulable_retry_range( + &mut self, + candidate_peers: &[PeerId], + now: Instant, + ) -> Option { + let ranges = self.block_ranges_to_retry.clone(); + for (index, range) in ranges.iter().enumerate().rev() { + if self.is_schedulable(RequestKey::BlockRange(*range), candidate_peers, now) { + return Some(self.block_ranges_to_retry.remove(index)); + } + } + None + } + + fn take_schedulable_column_range( + &mut self, + candidate_peers: &[PeerId], + now: Instant, + ) -> Option { + let ranges = self.column_ranges_to_fetch.clone(); + for (index, range) in ranges.iter().enumerate().rev() { + if self.is_schedulable(RequestKey::ColumnRange(*range), candidate_peers, now) { + return Some(self.column_ranges_to_fetch.remove(index)); + } + } + None + } + pub fn data_to_fetch( &mut self, target_slot: u64, current_epoch: u64, required_columns: &HashSet, + candidate_peers: &[PeerId], + now: Instant, ) -> DataToFetch { - match self.block_ranges_to_retry.pop() { - Some(range) => return DataToFetch::BlockRange(range), - None => { - let estimated_blocks_to_fetch = self.estimated_blocks_to_fetch(); - if estimated_blocks_to_fetch > 0 && self.next_start_slot < target_slot { - let blocks_to_fill = estimated_blocks_to_fetch - .min(MAX_BLOCKS_PER_REQUEST.min(target_slot - self.next_start_slot)); - let start_slot = self.next_start_slot + 1; - self.next_start_slot += blocks_to_fill; - return DataToFetch::BlockRange(Range::new(start_slot, blocks_to_fill)); - } - } + if let Some(range) = self.take_schedulable_retry_range(candidate_peers, now) { + return DataToFetch::BlockRange(range); } - if let Some(range) = self.column_ranges_to_fetch.pop() { + let estimated_blocks_to_fetch = self.estimated_blocks_to_fetch(); + if estimated_blocks_to_fetch > 0 && self.next_start_slot < target_slot { + let blocks_to_fill = estimated_blocks_to_fetch + .min(MAX_BLOCKS_PER_REQUEST.min(target_slot - self.next_start_slot)); + let start_slot = self.next_start_slot + 1; + self.next_start_slot += blocks_to_fill; + return DataToFetch::BlockRange(Range::new(start_slot, blocks_to_fill)); + } + + if let Some(range) = self.take_schedulable_column_range(candidate_peers, now) { return DataToFetch::DataColumnRange(range); } let mut block_roots_left_to_fetch = self.get_missing_block_roots(); let missing_block_roots_len = block_roots_left_to_fetch.len(); - block_roots_left_to_fetch.retain(|root| !self.block_roots_in_progress.contains(root)); + block_roots_left_to_fetch.retain(|root| { + !self.block_roots_in_progress.contains(root) + && self.is_schedulable(RequestKey::BlockRoot(*root), candidate_peers, now) + }); let mut blob_identifiers_left_to_fetch = self.get_missing_blob_identifiers(current_epoch); let missing_blob_identifiers_len = blob_identifiers_left_to_fetch.len(); - blob_identifiers_left_to_fetch - .retain(|blob_identifier| !self.blob_identifiers_in_progress.contains(blob_identifier)); + blob_identifiers_left_to_fetch.retain(|blob_identifier| { + !self.blob_identifiers_in_progress.contains(blob_identifier) + && self.is_schedulable(RequestKey::Blob(*blob_identifier), candidate_peers, now) + }); let mut data_column_identifiers_left_to_fetch = self.get_missing_data_column_identifiers(current_epoch, required_columns); @@ -267,6 +445,7 @@ impl BlockCache { !self .data_column_identifiers_in_progress .contains(identifier) + && self.is_schedulable(RequestKey::Column(*identifier), candidate_peers, now) }); if !block_roots_left_to_fetch.is_empty() { @@ -288,6 +467,8 @@ impl BlockCache { || missing_data_column_identifiers_len > 0 || !self.block_ranges_in_progress.is_empty() || !self.column_ranges_in_progress.is_empty() + || !self.block_ranges_to_retry.is_empty() + || !self.column_ranges_to_fetch.is_empty() { return DataToFetch::DownloadsInProgress; } @@ -398,6 +579,29 @@ impl BlockCache { } missing_identifiers } + + pub fn expected_column_identifiers_in_range( + &self, + range: Range, + required_columns: &HashSet, + ) -> Vec { + let range_end = range.start_slot + range.count; + let mut expected = Vec::new(); + for block in self.blocks_and_blobs.values() { + if block.block.message.slot < range.start_slot || block.block.message.slot >= range_end + { + continue; + } + let block_root = block.block.message.tree_hash_root(); + for column_index in required_columns { + let identifier = ColumnIdentifier::new(block_root, *column_index); + if !block.columns.contains_key(&identifier) { + expected.push(identifier); + } + } + } + expected + } } #[derive(Debug, Clone, PartialEq, Eq)] @@ -451,7 +655,7 @@ mod tests { let mut cache = BlockCache::new(B256::ZERO, 10); assert_eq!( - cache.data_to_fetch(10, 0, &HashSet::new()), + cache.data_to_fetch(10, 0, &HashSet::new(), &[], Instant::now()), DataToFetch::Finished ); } @@ -466,13 +670,13 @@ mod tests { cache.mark_block_range_in_progress(range); assert_eq!( - cache.data_to_fetch(10, 0, &HashSet::new()), + cache.data_to_fetch(10, 0, &HashSet::new(), &[], Instant::now()), DataToFetch::DownloadsInProgress ); cache.remove_block_range_in_progress(&range); assert_eq!( - cache.data_to_fetch(10, 0, &HashSet::new()), + cache.data_to_fetch(10, 0, &HashSet::new(), &[], Instant::now()), DataToFetch::Finished ); } @@ -483,15 +687,15 @@ mod tests { let mut cache = BlockCache::new(B256::ZERO, 10); assert_eq!( - cache.data_to_fetch(25, 0, &HashSet::new()), + cache.data_to_fetch(25, 0, &HashSet::new(), &[], Instant::now()), DataToFetch::BlockRange(Range::new(11, 10)) ); assert_eq!( - cache.data_to_fetch(25, 0, &HashSet::new()), + cache.data_to_fetch(25, 0, &HashSet::new(), &[], Instant::now()), DataToFetch::BlockRange(Range::new(21, 5)) ); assert_eq!( - cache.data_to_fetch(25, 0, &HashSet::new()), + cache.data_to_fetch(25, 0, &HashSet::new(), &[], Instant::now()), DataToFetch::Finished ); } @@ -531,7 +735,13 @@ mod tests { .add_blocks(vec![block_with_blob(boundary_slot)], false) .expect("boundary block should enter cache"); assert_eq!( - retained.data_to_fetch(boundary_slot, current_epoch, &HashSet::new()), + retained.data_to_fetch( + boundary_slot, + current_epoch, + &HashSet::new(), + &[], + Instant::now() + ), DataToFetch::Finished ); @@ -541,7 +751,13 @@ mod tests { .add_blocks(vec![block_with_blob(expired_slot)], false) .expect("expired block should enter cache"); assert_eq!( - expired.data_to_fetch(expired_slot, current_epoch, &HashSet::new()), + expired.data_to_fetch( + expired_slot, + current_epoch, + &HashSet::new(), + &[], + Instant::now() + ), DataToFetch::Finished ); } @@ -580,7 +796,13 @@ mod tests { .expect("boundary block should enter cache"); let required_columns = HashSet::from([1, 5]); - match cache.data_to_fetch(boundary_slot, current_epoch, &required_columns) { + match cache.data_to_fetch( + boundary_slot, + current_epoch, + &required_columns, + &[], + Instant::now(), + ) { DataToFetch::MissingDataColumnIdentifiers(identifiers) => { let mut identifiers = identifiers; identifiers.sort(); @@ -597,7 +819,13 @@ mod tests { // Unneeded columns never count as missing. assert_eq!( - cache.data_to_fetch(boundary_slot, current_epoch, &HashSet::new()), + cache.data_to_fetch( + boundary_slot, + current_epoch, + &HashSet::new(), + &[], + Instant::now() + ), DataToFetch::Finished ); } @@ -658,4 +886,249 @@ mod tests { .is_err() ); } + + #[test] + fn add_data_columns_is_atomic_a_bad_item_does_not_leave_earlier_items_mutated() { + use ream_consensus_beacon::data_column_sidecar::get_data_column_sidecars_from_block; + + let blob = Blob::default(); + let blob_bytes = blob.to_fixed_bytes(); + let raw_commitment = das_context() + .blob_to_kzg_commitment(&blob_bytes) + .expect("test blob should produce a commitment"); + let commitment = KZGCommitment(raw_commitment); + + let mut block = SignedBeaconBlock { + message: Default::default(), + signature: Default::default(), + }; + block + .message + .body + .blob_kzg_commitments + .push(commitment) + .expect("one commitment should fit"); + + let cells_and_kzg_proofs = compute_cells_and_kzg_proofs(&blob, das_context()) + .expect("test blob should produce cells and proofs"); + let columns = get_data_column_sidecars_from_block(&block, vec![cells_and_kzg_proofs]) + .expect("test block should produce data columns"); + let block_root = block.message.tree_hash_root(); + + let mut cache = BlockCache::new(B256::ZERO, 0); + cache + .add_blocks(vec![block], false) + .expect("block should enter cache"); + + let mut tampered = columns[1].clone(); + tampered.kzg_proofs[0][0] ^= 1; + let required_columns = HashSet::from([0, 1]); + + let result = cache.add_data_columns(vec![columns[0].clone(), tampered], &required_columns); + assert!(result.is_err(), "the batch as a whole must fail"); + + assert!( + cache + .blocks_and_blobs + .get(&block_root) + .expect("block should still be cached") + .columns + .is_empty(), + "a failed batch must not partially mutate the cache" + ); + } + + #[test] + fn is_schedulable_false_only_when_every_candidate_is_attempted() { + let mut cache = BlockCache::new(B256::ZERO, 0); + let key = RequestKey::BlockRoot(B256::repeat_byte(1)); + let peer_a = PeerId::random(); + let peer_b = PeerId::random(); + let candidates = [peer_a, peer_b]; + let now = Instant::now(); + + assert!(cache.is_schedulable(key, &candidates, now)); + + cache.mark_attempted(key, peer_a, &candidates, now); + assert!(cache.is_schedulable(key, &candidates, now)); + + cache.mark_attempted(key, peer_b, &candidates, now); + assert!(!cache.is_schedulable(key, &candidates, now)); + } + + #[test] + fn cooldown_expiring_resets_the_attempted_set_for_a_fresh_round() { + let mut cache = BlockCache::new(B256::ZERO, 0); + let key = RequestKey::BlockRoot(B256::repeat_byte(1)); + let peer_a = PeerId::random(); + let candidates = [peer_a]; + let now = Instant::now(); + + cache.mark_attempted(key, peer_a, &candidates, now); + assert!(!cache.is_schedulable(key, &candidates, now)); + + assert!(!cache.is_schedulable(key, &candidates, now + Duration::from_secs(1))); + + assert!(cache.is_schedulable( + key, + &candidates, + now + ATTEMPT_COOLDOWN + Duration::from_secs(1) + )); + } + + #[test] + fn a_new_peer_makes_an_exhausted_key_schedulable_without_waiting_for_cooldown() { + let mut cache = BlockCache::new(B256::ZERO, 0); + let key = RequestKey::BlockRoot(B256::repeat_byte(1)); + let peer_a = PeerId::random(); + let peer_b = PeerId::random(); + let now = Instant::now(); + + cache.mark_attempted(key, peer_a, &[peer_a], now); + assert!(!cache.is_schedulable(key, &[peer_a], now)); + + assert!(cache.is_schedulable(key, &[peer_a, peer_b], now)); + } + + #[test] + fn clear_attempted_starts_a_clean_round() { + let mut cache = BlockCache::new(B256::ZERO, 0); + let key = RequestKey::BlockRoot(B256::repeat_byte(1)); + let peer_a = PeerId::random(); + let now = Instant::now(); + + cache.mark_attempted(key, peer_a, &[peer_a], now); + assert!(!cache.is_schedulable(key, &[peer_a], now)); + + cache.clear_attempted(key); + assert!(cache.is_schedulable(key, &[peer_a], now)); + assert!(cache.attempted_peers_for(key).is_empty()); + } + + #[test] + fn take_schedulable_retry_range_skips_backed_off_ranges_but_keeps_them_queued() { + let mut cache = BlockCache::new(B256::ZERO, 0); + let backed_off = Range::new(1, 10); + let schedulable = Range::new(11, 10); + let peer_a = PeerId::random(); + let now = Instant::now(); + + cache.push_retry_range(backed_off); + cache.push_retry_range(schedulable); + cache.mark_attempted(RequestKey::BlockRange(backed_off), peer_a, &[peer_a], now); + + assert_eq!( + cache.take_schedulable_retry_range(&[peer_a], now), + Some(schedulable) + ); + assert_eq!(cache.block_ranges_to_retry, vec![backed_off]); + assert_eq!(cache.take_schedulable_retry_range(&[peer_a], now), None); + } + + #[test] + fn an_exhausted_backed_off_root_does_not_starve_other_tiers_or_cause_false_finished() { + initialize_test_network_spec(); + let parent_root = B256::repeat_byte(1); + let mut block = SignedBeaconBlock { + message: BeaconBlock { + slot: 1, + parent_root, + ..Default::default() + }, + signature: Default::default(), + }; + block.message.parent_root = B256::repeat_byte(2); + let mut cache = BlockCache::new(parent_root, 1); + cache + .add_blocks(vec![block], false) + .expect("block should enter cache"); + + let peer_a = PeerId::random(); + let now = Instant::now(); + let missing_root = B256::repeat_byte(2); + + cache.mark_attempted(RequestKey::BlockRoot(missing_root), peer_a, &[peer_a], now); + + assert_eq!( + cache.data_to_fetch(1, 0, &HashSet::new(), &[peer_a], now), + DataToFetch::DownloadsInProgress + ); + } + + #[test] + fn data_to_fetch_filters_out_only_the_exhausted_root_not_the_whole_tier() { + initialize_test_network_spec(); + let parent_root = B256::repeat_byte(1); + let root_a = B256::repeat_byte(2); + let root_b = B256::repeat_byte(3); + let block1 = SignedBeaconBlock { + message: BeaconBlock { + slot: 1, + parent_root: root_a, + ..Default::default() + }, + signature: Default::default(), + }; + let block2 = SignedBeaconBlock { + message: BeaconBlock { + slot: 2, + parent_root: root_b, + ..Default::default() + }, + signature: Default::default(), + }; + + let mut cache = BlockCache::new(parent_root, 2); + cache + .add_blocks(vec![block1, block2], false) + .expect("blocks should enter cache"); + + let peer_a = PeerId::random(); + let now = Instant::now(); + cache.mark_attempted(RequestKey::BlockRoot(root_a), peer_a, &[peer_a], now); + + match cache.data_to_fetch(2, 0, &HashSet::new(), &[peer_a], now) { + DataToFetch::MissingBlockRoots(roots) => { + assert_eq!(roots, vec![root_b]); + } + other => panic!("expected MissingBlockRoots, got {other:?}"), + } + } + + #[test] + fn add_data_columns_drops_a_column_for_an_unknown_block_without_erroring_or_banning() { + use ream_consensus_beacon::data_column_sidecar::get_data_column_sidecars_from_block; + + let blob = Blob::default(); + let blob_bytes = blob.to_fixed_bytes(); + let raw_commitment = das_context() + .blob_to_kzg_commitment(&blob_bytes) + .expect("test blob should produce a commitment"); + let commitment = KZGCommitment(raw_commitment); + + let mut block = SignedBeaconBlock { + message: Default::default(), + signature: Default::default(), + }; + block + .message + .body + .blob_kzg_commitments + .push(commitment) + .expect("one commitment should fit"); + + let cells_and_kzg_proofs = compute_cells_and_kzg_proofs(&blob, das_context()) + .expect("test blob should produce cells and proofs"); + let columns = get_data_column_sidecars_from_block(&block, vec![cells_and_kzg_proofs]) + .expect("test block should produce data columns"); + + let mut cache = BlockCache::new(B256::ZERO, 0); + let required_columns = HashSet::from([0]); + let result = cache.add_data_columns(vec![columns[0].clone()], &required_columns); + + assert!( + result.is_ok(), + "a column for a block not yet in the cache must be dropped, not erroring the batch" + ); + } } diff --git a/crates/networking/syncer/src/block_range/mod.rs b/crates/networking/syncer/src/block_range/mod.rs index da10b5ce2..67c10afe3 100644 --- a/crates/networking/syncer/src/block_range/mod.rs +++ b/crates/networking/syncer/src/block_range/mod.rs @@ -7,18 +7,18 @@ use std::{ pin::Pin, sync::Arc, task::{Context, Poll}, - time::Duration, + time::{Duration, Instant}, }; use alloy_primitives::B256; use anyhow::{anyhow, bail, ensure}; -use block_cache::{BlockAndBlobBundle, BlockCache, DataToFetch}; +use block_cache::{BlockAndBlobBundle, BlockCache, DataToFetch, RequestKey}; use futures::task::noop_waker; use libp2p::PeerId; -use peer_manager::{BanReason, PeerManager}; +use peer_manager::{BanReason, MIN_SYNC_PEERS, PeerManager, TargetQualification, TargetSelection}; use peer_range_downloader::{ - PeerBlobIdentifierDownloader, PeerDataColumnIdentifierDownloader, - PeerDataColumnRangeDownloader, PeerRootsDownloader, + DownloadFailure, PeerBlobIdentifierDownloader, PeerDataColumnIdentifierDownloader, + PeerDataColumnRangeDownloader, PeerRootsDownloader, StreamOutcome, }; use ream_chain_beacon::beacon_chain::{ BeaconChain, BlockProcessingOutcome, is_data_availability_check_required, @@ -31,15 +31,19 @@ use ream_consensus_beacon::{ electra::beacon_block::SignedBeaconBlock, matrix_entry::{compute_cells_and_kzg_proofs, das_context}, }; -use ream_consensus_misc::misc::compute_epoch_at_slot; +use ream_consensus_misc::{constants::beacon::SLOTS_PER_EPOCH, misc::compute_epoch_at_slot}; use ream_executor::ReamExecutor; use ream_network_spec::networks::beacon_network_spec; use ream_p2p::network::beacon::{channel::P2PMessage, network_state::NetworkState}; use ream_polynomial_commitments::handlers::verify_blob_kzg_proof_batch; use ream_req_resp::{ MAX_CONCURRENT_REQUESTS, beacon::messages::data_column_sidecars::DataColumnsByRootIdentifier, + inbound_protocol::ResponseCode, +}; +use ream_storage::tables::{ + field::REDBField, + table::{CustomTable, REDBTable}, }; -use ream_storage::tables::table::{CustomTable, REDBTable}; use tokio::{sync::mpsc::UnboundedSender, task::JoinHandle, time::sleep}; use tracing::{info, warn}; use tree_hash::TreeHash; @@ -54,6 +58,10 @@ const SLEEP_DURATION: Duration = Duration::from_secs(5); /// `SLOT_IMPORT_TOLERANCE`). A thin/early peer sample can be fooled; the clock can't. const SLOT_IMPORT_TOLERANCE: u64 = 32; +const ZERO_PROGRESS_BACKOFF: Duration = Duration::from_secs(30); + +const CANDIDATE_EXHAUSTION_TIMEOUT: Duration = Duration::from_secs(60); + /// Validates downloaded blob sidecars and derives the columns needed for data availability. /// An empty `blob_sidecars` map defers to the block-lookup coordinator instead of erroring, /// since range sync never fetches legacy blob sidecars for post-Fulu blocks. @@ -92,6 +100,10 @@ fn build_data_columns_from_blob_sidecars( sidecar.kzg_commitment == *expected_commitment, "Blob sidecar {index} commitment does not match block {block_root}" ); + ensure!( + sidecar.verify_blob_sidecar_inclusion_proof(), + "Invalid inclusion proof for blob sidecar {index} of block {block_root}" + ); blobs.push(sidecar.blob.clone()); proofs.push(sidecar.kzg_proof); @@ -110,11 +122,24 @@ fn build_data_columns_from_blob_sidecars( .map_err(|err| anyhow!("Failed to build data columns for block {block_root}: {err}")) } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SyncPhase { + Finalized, + Head, +} + +#[derive(Debug, Clone)] +struct ActivePhaseTarget { + slot: u64, + qualification: TargetQualification, +} + pub struct BlockRangeSyncer { pub beacon_chain: Arc, pub peer_manager: PeerManager, pub p2p_sender: UnboundedSender, pub executor: ReamExecutor, + next_segment_not_before: Option, } impl BlockRangeSyncer { @@ -129,13 +154,12 @@ impl BlockRangeSyncer { p2p_sender, peer_manager: PeerManager::new(network_state), executor, + next_segment_not_before: None, } } - pub async fn is_synced_to_head_slot(&self) -> bool { - let Some(sync_target) = self.peer_manager.sync_target() else { - return false; - }; + pub async fn is_synced_to_head_slot(&mut self) -> bool { + self.peer_manager.update_peer_set(); let store = self.beacon_chain.store.lock().await; let latest_synced_slot = store @@ -144,16 +168,38 @@ impl BlockRangeSyncer { .get_highest_slot() .unwrap_or_default() .unwrap_or(0); + let Ok(finalized_epoch) = store.db.finalized_checkpoint_provider().get() else { + return false; + }; + let finalized_epoch = finalized_epoch.epoch; let Ok(current_slot) = store.get_current_slot() else { return false; }; drop(store); - let peers_report_caught_up = sync_target.slot <= latest_synced_slot; + let finalized_selection = self.peer_manager.best_finalized(finalized_epoch); + let still_behind_finalized = matches!( + &finalized_selection, + TargetSelection::Ready { target_slot, .. } if latest_synced_slot < *target_slot + ); + if still_behind_finalized { + return false; + } + + let our_head_epoch = latest_synced_slot / SLOTS_PER_EPOCH; + let head_selection = self + .peer_manager + .best_non_finalized(MIN_SYNC_PEERS, our_head_epoch); // The clock is ground truth peers can't fake. let clock_confirms_caught_up = current_slot.saturating_sub(latest_synced_slot) <= SLOT_IMPORT_TOLERANCE; + let TargetSelection::Ready { target_slot, .. } = head_selection else { + return clock_confirms_caught_up; + }; + + let peers_report_caught_up = target_slot <= latest_synced_slot; + peers_report_caught_up && clock_confirms_caught_up } @@ -166,36 +212,49 @@ impl BlockRangeSyncer { } async fn run_segment(&mut self) -> anyhow::Result<()> { - let Some(latest_synced_root) = self - .beacon_chain - .store - .lock() - .await - .db - .slot_index_provider() - .get_highest_root() - .map_err(|err| anyhow!("Failed to get highest root: {err}"))? - else { - bail!("No synced root found in the database"); - }; + if let Some(not_before) = self.next_segment_not_before.take() + && let Some(remaining) = not_before.checked_duration_since(Instant::now()) + { + info!("Backing off {remaining:?} after the previous segment made no progress..."); + sleep(remaining).await; + } - let Some(latest_synced_slot) = self - .beacon_chain - .store - .lock() - .await + let store = self.beacon_chain.store.lock().await; + let head_root = store + .get_head() + .map_err(|err| anyhow!("Failed to get canonical head: {err}"))?; + let head_slot = store .db - .slot_index_provider() - .get_highest_slot() - .map_err(|err| anyhow!("Failed to get highest slot: {err}"))? - else { - bail!("No synced slot found in the database"); - }; + .block_provider() + .get(head_root) + .map_err(|err| anyhow!("Failed to load head block: {err}"))? + .ok_or_else(|| anyhow!("Head block {head_root} not found"))? + .message + .slot; + let finalized_epoch = store + .db + .finalized_checkpoint_provider() + .get() + .map_err(|err| anyhow!("Failed to get finalized checkpoint: {err}"))? + .epoch; + drop(store); + let our_head_epoch = head_slot / SLOTS_PER_EPOCH; // phase 1: download majority of blocks from ranges - let mut block_cache = BlockCache::new(latest_synced_root, latest_synced_slot); + let mut block_cache = BlockCache::new(head_root, head_slot); let mut task_handles = vec![]; + + let mut phase = SyncPhase::Finalized; + let mut active_target: Option = None; + let mut saw_empty_range = false; + let mut finalized_phase_settled = false; + let mut finalized_target_was_ahead = false; + let mut candidate_exhausted_since: Option = None; + let mut ended_due_to_candidate_exhaustion = false; + loop { + self.peer_manager.update_peer_set(); + let required_columns = self .beacon_chain .store @@ -205,28 +264,123 @@ impl BlockRangeSyncer { .required_columns() .clone(); + let selection = match phase { + SyncPhase::Finalized => self.peer_manager.best_finalized(finalized_epoch), + SyncPhase::Head => self + .peer_manager + .best_non_finalized(MIN_SYNC_PEERS, our_head_epoch), + }; + + if active_target.is_none() { + if phase == SyncPhase::Finalized { + match &selection { + TargetSelection::Ready { + target_slot, + eligible_peers, + } => { + if block_cache.next_start_slot() >= *target_slot { + phase = SyncPhase::Head; + continue; + } + if eligible_peers.len() < MIN_SYNC_PEERS { + info!( + "Finalized target not yet confirmed by enough peers ({} < {MIN_SYNC_PEERS}), waiting...", + eligible_peers.len() + ); + sleep(SLEEP_DURATION).await; + continue; + } + } + TargetSelection::NoQuorum => { + phase = SyncPhase::Head; + continue; + } + } + } else if let TargetSelection::NoQuorum = &selection { + if finalized_phase_settled || block_cache.block_count() > 0 { + info!( + "No head-phase sync target after the finalized phase settled; ending this segment." + ); + break; + } + info!("No sync target yet, waiting for peers..."); + sleep(SLEEP_DURATION).await; + continue; + } + } + + if let TargetSelection::Ready { + target_slot, + eligible_peers, + } = &selection + { + let sufficient = match phase { + SyncPhase::Finalized => eligible_peers.len() >= MIN_SYNC_PEERS, + SyncPhase::Head => true, + }; + if sufficient { + let qualification = match phase { + SyncPhase::Finalized => { + TargetQualification::FinalizedEpoch(target_slot / SLOTS_PER_EPOCH) + } + SyncPhase::Head => { + TargetQualification::HeadEpoch(target_slot / SLOTS_PER_EPOCH) + } + }; + active_target = Some(match active_target { + Some(existing) if existing.slot > *target_slot => existing, + _ => ActivePhaseTarget { + slot: *target_slot, + qualification, + }, + }); + } + } + + let Some(target) = active_target.clone() else { + sleep(SLEEP_DURATION).await; + continue; + }; + + let candidate_peers = self.peer_manager.peers_satisfying(target.qualification); + + if candidate_peers.is_empty() { + let exhausted_since = *candidate_exhausted_since.get_or_insert_with(Instant::now); + if exhausted_since.elapsed() >= CANDIDATE_EXHAUSTION_TIMEOUT { + info!( + "No peer has qualified for the active sync target for over {CANDIDATE_EXHAUSTION_TIMEOUT:?}; ending this segment." + ); + ended_due_to_candidate_exhaustion = true; + break; + } + } else { + candidate_exhausted_since = None; + } + + let now = Instant::now(); poll_ready_tasks( &mut task_handles, &mut block_cache, &mut self.peer_manager, &required_columns, + &mut saw_empty_range, + &candidate_peers, + now, )?; - let Some(sync_target) = self.peer_manager.sync_target() else { - warn!("No peers available to determine sync target, retrying..."); - sleep(SLEEP_DURATION).await; - self.peer_manager.update_peer_set(); - continue; - }; - let current_epoch = self .beacon_chain .store .lock() .await .get_current_store_epoch()?; - let data_to_fetch = - block_cache.data_to_fetch(sync_target.slot, current_epoch, &required_columns); + let data_to_fetch = block_cache.data_to_fetch( + target.slot, + current_epoch, + &required_columns, + &candidate_peers, + now, + ); info!( "Forward sync status: Downloaded Blocks {}, Downloaded Blobs {}/{}, Stage {data_to_fetch}", block_cache.block_count(), @@ -236,9 +390,11 @@ impl BlockRangeSyncer { match data_to_fetch { DataToFetch::BlockRange(range) => { + let key = RequestKey::BlockRange(range); + let excluded = block_cache.attempted_peers_for(key); let Some(peer) = self .peer_manager - .fetch_idle_peer_preferring(Some(sync_target.root)) + .fetch_idle_peer_from_excluding(&candidate_peers, &excluded) else { self.peer_manager.update_peer_set(); info!("No idle peers available for block range sync."); @@ -260,9 +416,11 @@ impl BlockRangeSyncer { )); } DataToFetch::DataColumnRange(range) => { + let key = RequestKey::ColumnRange(range); + let excluded = block_cache.attempted_peers_for(key); let Some(peer) = self .peer_manager - .fetch_idle_peer_preferring(Some(sync_target.root)) + .fetch_idle_peer_from_excluding(&candidate_peers, &excluded) else { self.peer_manager.update_peer_set(); info!("No idle peers available for data column range sync."); @@ -271,6 +429,9 @@ impl BlockRangeSyncer { continue; }; + let expected_known_identifiers = + block_cache.expected_column_identifiers_in_range(range, &required_columns); + block_cache.mark_column_range_in_progress(range); task_handles.push(DownloadTask::new_data_column_range( PeerDataColumnRangeDownloader::start( @@ -282,38 +443,57 @@ impl BlockRangeSyncer { ), range, peer.peer_id, + expected_known_identifiers, )); } - DataToFetch::MissingBlockRoots(block_roots) => { - for block_roots_chunk in block_roots.chunks(MAX_CONCURRENT_REQUESTS) { + DataToFetch::MissingBlockRoots(mut block_roots) => { + let mut exhausted_this_tick = HashSet::new(); + while !block_roots.is_empty() { let Some(peer) = self .peer_manager - .fetch_idle_peer_preferring(Some(sync_target.root)) + .fetch_idle_peer_from_excluding(&candidate_peers, &exhausted_this_tick) else { self.peer_manager.update_peer_set(); info!("No idle peers available for block roots sync."); sleep(SLEEP_DURATION).await; break; }; - block_cache.extend_block_roots_in_progress(block_roots_chunk); + + let (assigned, remaining): (Vec, Vec) = + block_roots.into_iter().partition(|root| { + !block_cache + .attempted_peers_for(RequestKey::BlockRoot(*root)) + .contains(&peer.peer_id) + }); + block_roots = remaining; + if assigned.is_empty() { + self.peer_manager.mark_peer_as_idle(&peer.peer_id); + exhausted_this_tick.insert(peer.peer_id); + continue; + } + let chunk: Vec = + assigned.into_iter().take(MAX_CONCURRENT_REQUESTS).collect(); + + block_cache.extend_block_roots_in_progress(&chunk); task_handles.push(DownloadTask::new_block_roots( PeerRootsDownloader::start( peer.peer_id, self.p2p_sender.clone(), self.executor.clone(), - block_roots_chunk.to_vec(), + chunk.clone(), ), - block_roots_chunk.to_vec(), + chunk, peer.peer_id, )); } } - DataToFetch::MissingBlobIdentifiers(blob_identifiers) => { - for blob_identifiers_chunk in blob_identifiers.chunks(MAX_BLOBS_PER_REQUEST) { + DataToFetch::MissingBlobIdentifiers(mut blob_identifiers) => { + let mut exhausted_this_tick = HashSet::new(); + while !blob_identifiers.is_empty() { let Some(peer) = self .peer_manager - .fetch_idle_peer_preferring(Some(sync_target.root)) + .fetch_idle_peer_from_excluding(&candidate_peers, &exhausted_this_tick) else { self.peer_manager.update_peer_set(); info!( @@ -324,25 +504,41 @@ impl BlockRangeSyncer { break; }; - block_cache.extend_blob_identifiers_in_progress(blob_identifiers_chunk); + let (assigned, remaining): (Vec, Vec) = + blob_identifiers.into_iter().partition(|identifier| { + !block_cache + .attempted_peers_for(RequestKey::Blob(*identifier)) + .contains(&peer.peer_id) + }); + blob_identifiers = remaining; + if assigned.is_empty() { + self.peer_manager.mark_peer_as_idle(&peer.peer_id); + exhausted_this_tick.insert(peer.peer_id); + continue; + } + let chunk: Vec = + assigned.into_iter().take(MAX_BLOBS_PER_REQUEST).collect(); + + block_cache.extend_blob_identifiers_in_progress(&chunk); task_handles.push(DownloadTask::new_blob_identifiers( PeerBlobIdentifierDownloader::start( peer.peer_id, self.p2p_sender.clone(), self.executor.clone(), - blob_identifiers_chunk.to_vec(), + chunk.clone(), ), - blob_identifiers_chunk.to_vec(), + chunk, peer.peer_id, )); } } - DataToFetch::MissingDataColumnIdentifiers(identifiers) => { - for identifiers_chunk in identifiers.chunks(MAX_CONCURRENT_REQUESTS) { + DataToFetch::MissingDataColumnIdentifiers(mut identifiers) => { + let mut exhausted_this_tick = HashSet::new(); + while !identifiers.is_empty() { let Some(peer) = self .peer_manager - .fetch_idle_peer_preferring(Some(sync_target.root)) + .fetch_idle_peer_from_excluding(&candidate_peers, &exhausted_this_tick) else { self.peer_manager.update_peer_set(); info!("No idle peers available for data column sync."); @@ -350,16 +546,31 @@ impl BlockRangeSyncer { break; }; - block_cache.extend_data_column_identifiers_in_progress(identifiers_chunk); + let (assigned, remaining): (Vec, Vec) = + identifiers.into_iter().partition(|identifier| { + !block_cache + .attempted_peers_for(RequestKey::Column(*identifier)) + .contains(&peer.peer_id) + }); + identifiers = remaining; + if assigned.is_empty() { + self.peer_manager.mark_peer_as_idle(&peer.peer_id); + exhausted_this_tick.insert(peer.peer_id); + continue; + } + let chunk: Vec = + assigned.into_iter().take(MAX_CONCURRENT_REQUESTS).collect(); + + block_cache.extend_data_column_identifiers_in_progress(&chunk); task_handles.push(DownloadTask::new_data_column_identifiers( PeerDataColumnIdentifierDownloader::start( peer.peer_id, self.p2p_sender.clone(), self.executor.clone(), - group_by_block_root(identifiers_chunk), + group_by_block_root(&chunk), ), - identifiers_chunk.to_vec(), + chunk, peer.peer_id, )); } @@ -371,7 +582,17 @@ impl BlockRangeSyncer { ); sleep(Duration::from_secs(10)).await; } - DataToFetch::Finished => break, + DataToFetch::Finished => { + if phase == SyncPhase::Finalized && block_cache.next_start_slot() >= target.slot + { + finalized_target_was_ahead = target.slot > head_slot; + phase = SyncPhase::Head; + active_target = None; + finalized_phase_settled = true; + continue; + } + break; + } } } @@ -382,6 +603,7 @@ impl BlockRangeSyncer { ); let fulu_fork_epoch = beacon_network_spec().fulu_fork_epoch; + let mut imported_count: u64 = 0; // execute all the blocks downloaded for BlockAndBlobBundle { @@ -470,10 +692,22 @@ impl BlockRangeSyncer { ); } } + imported_count += 1; } info!("All blocks processed successfully."); + let target_was_ahead = finalized_target_was_ahead + || active_target + .as_ref() + .is_some_and(|target| target.slot > head_slot); + if target_was_ahead + && (saw_empty_range || ended_due_to_candidate_exhaustion) + && imported_count == 0 + { + self.next_segment_not_before = Some(Instant::now() + ZERO_PROGRESS_BACKOFF); + } + Ok(()) } } @@ -502,27 +736,28 @@ fn group_by_block_root(identifiers: &[ColumnIdentifier]) -> Vec>>>, + handle: JoinHandle>>, range: Range, peer_id: PeerId, }, DataColumnRange { - handle: JoinHandle>>>, + handle: JoinHandle>>, range: Range, peer_id: PeerId, + expected_known_identifiers: Vec, }, BlockRoots { - handle: JoinHandle>>>, + handle: JoinHandle>>, roots: Vec, peer_id: PeerId, }, BlobIdentifiers { - handle: JoinHandle>>>, + handle: JoinHandle>>, blob_identifiers: Vec, peer_id: PeerId, }, DataColumnIdentifiers { - handle: JoinHandle>>>, + handle: JoinHandle>>, identifiers: Vec, peer_id: PeerId, }, @@ -530,7 +765,7 @@ pub enum DownloadTask { impl DownloadTask { pub fn new_block_range( - handle: JoinHandle>>>, + handle: JoinHandle>>, range: Range, peer_id: PeerId, ) -> Self { @@ -542,19 +777,21 @@ impl DownloadTask { } pub fn new_data_column_range( - handle: JoinHandle>>>, + handle: JoinHandle>>, range: Range, peer_id: PeerId, + expected_known_identifiers: Vec, ) -> Self { DownloadTask::DataColumnRange { handle, range, peer_id, + expected_known_identifiers, } } pub fn new_block_roots( - handle: JoinHandle>>>, + handle: JoinHandle>>, roots: Vec, peer_id: PeerId, ) -> Self { @@ -566,7 +803,7 @@ impl DownloadTask { } pub fn new_blob_identifiers( - handle: JoinHandle>>>, + handle: JoinHandle>>, blob_identifiers: Vec, peer_id: PeerId, ) -> Self { @@ -578,7 +815,7 @@ impl DownloadTask { } pub fn new_data_column_identifiers( - handle: JoinHandle>>>, + handle: JoinHandle>>, identifiers: Vec, peer_id: PeerId, ) -> Self { @@ -590,11 +827,53 @@ impl DownloadTask { } } +fn handle_stream_outcome( + peer_manager: &mut PeerManager, + peer_id: &PeerId, + result: Result>, tokio::task::JoinError>, +) -> Option> { + let outcome = match result { + Ok(Ok(outcome)) => outcome, + Ok(Err(err)) => { + warn!("Forward fill task failed: {err}"); + return None; + } + Err(err) => { + warn!("Forward fill task panicked: {err}"); + return None; + } + }; + + match outcome { + StreamOutcome::Complete(items) => Some(items), + StreamOutcome::Failed(DownloadFailure::Transport(detail)) => { + info!("Transport failure from peer {peer_id}, will retry with someone else: {detail}"); + None + } + StreamOutcome::Failed(DownloadFailure::InvalidData(detail)) => { + warn!("Invalid data from peer {peer_id}: {detail}"); + peer_manager.ban_peer(peer_id, BanReason::ProtocolError(detail)); + None + } + StreamOutcome::Failed(DownloadFailure::RemoteError { code, message }) => { + if code == ResponseCode::InvalidRequest { + warn!("Peer {peer_id} reported InvalidRequest (possible local bug): {message}"); + } else { + info!("Peer {peer_id} declined the request ({code:?}): {message}"); + } + None + } + } +} + fn poll_ready_tasks( tasks: &mut Vec, block_cache: &mut BlockCache, peer_manager: &mut PeerManager, required_columns: &HashSet, + saw_empty_range: &mut bool, + candidate_peers: &[PeerId], + now: Instant, ) -> anyhow::Result<()> { let waker = noop_waker(); let mut context = Context::from_waker(&waker); @@ -615,35 +894,42 @@ fn poll_ready_tasks( let pinned = Pin::new(handle); match pinned.poll(&mut context) { - Poll::Ready(Ok(blocks_result)) => { + Poll::Ready(result) => { indexes_to_remove.push(index); block_cache.remove_block_range_in_progress(range); peer_manager.mark_peer_as_idle(peer_id); - let blocks = match blocks_result { - Ok(blocks) => blocks, - Err(err) => { - warn!("Failed to fetch blocks from peer: {err:?}"); - block_cache.push_retry_range(*range); - continue; - } - }; + let key = RequestKey::BlockRange(*range); - let blocks = match blocks { - Ok(blocks) => blocks, - Err(err) => { - block_cache.push_retry_range(*range); - peer_manager.ban_peer( - peer_id, - BanReason::ProtocolError(format!("{err:?}")), - ); - continue; - } + let Some(blocks) = handle_stream_outcome(peer_manager, peer_id, result) + else { + block_cache.mark_attempted(key, *peer_id, candidate_peers, now); + block_cache.push_retry_range(*range); + continue; }; if blocks.is_empty() { - warn!("Received empty block range from peer: {peer_id}"); + info!("Received empty block range from peer: {peer_id}"); + block_cache.clear_attempted(key); + *saw_empty_range = true; + continue; + } + + let range_end = range.start_slot + range.count; + let out_of_range = blocks.iter().any(|block| { + block.message.slot < range.start_slot || block.message.slot >= range_end + }); + if out_of_range { + warn!( + "Peer {peer_id} returned block(s) outside the requested range {range:?}" + ); + block_cache.mark_attempted(key, *peer_id, candidate_peers, now); block_cache.push_retry_range(*range); - peer_manager.ban_peer(peer_id, BanReason::EmptyResponse); + peer_manager.ban_peer( + peer_id, + BanReason::ProtocolError( + "block(s) outside the requested range".to_string(), + ), + ); continue; } @@ -655,18 +941,19 @@ fn poll_ready_tasks( if let Err(err) = block_cache.add_blocks(blocks, true) { warn!("Failed to add downloaded blocks to cache: {err:?}"); + block_cache.mark_attempted(key, *peer_id, candidate_peers, now); block_cache.push_retry_range(*range); - } else if needs_columns { - block_cache.push_column_range(*range); + peer_manager.ban_peer( + peer_id, + BanReason::ProtocolError(format!("invalid block range: {err:?}")), + ); + } else { + block_cache.clear_attempted(key); + if needs_columns { + block_cache.push_column_range(*range); + } } } - Poll::Ready(Err(err)) => { - warn!("Forward fill task failed: {err}"); - peer_manager.mark_peer_as_idle(peer_id); - block_cache.remove_block_range_in_progress(range); - block_cache.push_retry_range(*range); - indexes_to_remove.push(index); - } Poll::Pending => {} } } @@ -674,56 +961,121 @@ fn poll_ready_tasks( handle, range, peer_id, + expected_known_identifiers, } => { let pinned = Pin::new(handle); match pinned.poll(&mut context) { - Poll::Ready(Ok(columns_result)) => { + Poll::Ready(result) => { indexes_to_remove.push(index); block_cache.remove_column_range_in_progress(range); peer_manager.mark_peer_as_idle(peer_id); - let columns = match columns_result { - Ok(columns) => columns, - Err(err) => { - warn!("Failed to fetch data columns from peer: {err:?}"); - block_cache.push_column_range(*range); - continue; - } - }; + let key = RequestKey::ColumnRange(*range); - let columns = match columns { - Ok(columns) => columns, - Err(err) => { - block_cache.push_column_range(*range); - peer_manager.ban_peer( - peer_id, - BanReason::ProtocolError(format!("{err:?}")), + let Some(columns) = handle_stream_outcome(peer_manager, peer_id, result) + else { + block_cache.mark_attempted(key, *peer_id, candidate_peers, now); + for identifier in expected_known_identifiers.iter() { + block_cache.mark_attempted( + RequestKey::Column(*identifier), + *peer_id, + candidate_peers, + now, ); - continue; } + continue; }; if columns.is_empty() { - warn!("Received empty data column range from peer: {peer_id}"); - block_cache.push_column_range(*range); - peer_manager.ban_peer(peer_id, BanReason::EmptyResponse); + info!("Received empty data column range from peer: {peer_id}"); + block_cache.clear_attempted(key); + for identifier in expected_known_identifiers.iter() { + block_cache.mark_attempted( + RequestKey::Column(*identifier), + *peer_id, + candidate_peers, + now, + ); + } + *saw_empty_range = true; continue; } + let range_end = range.start_slot + range.count; + let mut seen_identifiers = HashSet::new(); + let out_of_shape = columns.iter().any(|column| { + let slot = column.signed_block_header.message.slot; + let identifier = ColumnIdentifier::new( + column.signed_block_header.message.tree_hash_root(), + column.index, + ); + slot < range.start_slot + || slot >= range_end + || !required_columns.contains(&column.index) + || !seen_identifiers.insert(identifier) + }); + if out_of_shape { + warn!( + "Peer {peer_id} returned data column(s) outside the requested range/columns, or a duplicate, for range {range:?}" + ); + block_cache.clear_attempted(key); + for identifier in expected_known_identifiers.iter() { + block_cache.mark_attempted( + RequestKey::Column(*identifier), + *peer_id, + candidate_peers, + now, + ); + } + peer_manager.ban_peer( + peer_id, + BanReason::ProtocolError( + "data column(s) outside the requested range/columns, or a duplicate" + .to_string(), + ), + ); + continue; + } + + let returned: HashSet = columns + .iter() + .map(|column| { + ColumnIdentifier::new( + column.signed_block_header.message.tree_hash_root(), + column.index, + ) + }) + .collect(); + if let Err(err) = block_cache.add_data_columns(columns, required_columns) { warn!("Failed to add downloaded data columns to cache: {err:?}"); - block_cache.push_column_range(*range); + block_cache.clear_attempted(key); + for identifier in expected_known_identifiers.iter() { + block_cache.mark_attempted( + RequestKey::Column(*identifier), + *peer_id, + candidate_peers, + now, + ); + } peer_manager .ban_peer(peer_id, BanReason::InvalidProof(format!("{err:?}"))); + } else { + block_cache.clear_attempted(key); + for identifier in expected_known_identifiers.iter() { + if returned.contains(identifier) { + block_cache.clear_attempted(RequestKey::Column(*identifier)); + } else { + block_cache.mark_attempted( + RequestKey::Column(*identifier), + *peer_id, + candidate_peers, + now, + ); + } + } } } - Poll::Ready(Err(err)) => { - warn!("Forward fill task failed: {err}"); - peer_manager.mark_peer_as_idle(peer_id); - block_cache.remove_column_range_in_progress(range); - block_cache.push_column_range(*range); - indexes_to_remove.push(index); - } Poll::Pending => {} } } @@ -735,33 +1087,68 @@ fn poll_ready_tasks( let pinned = Pin::new(handle); match pinned.poll(&mut context) { - Poll::Ready(Ok(blocks_result)) => { + Poll::Ready(result) => { indexes_to_remove.push(index); block_cache.remove_block_roots_in_progress(roots); peer_manager.mark_peer_as_idle(peer_id); - let blocks = match blocks_result { - Ok(blocks) => blocks, - Err(err) => { - warn!("Failed to fetch blocks from peer: {err:?}"); - continue; + + let Some(blocks) = handle_stream_outcome(peer_manager, peer_id, result) + else { + for root in roots.iter() { + block_cache.mark_attempted( + RequestKey::BlockRoot(*root), + *peer_id, + candidate_peers, + now, + ); } + continue; }; - let blocks = match blocks { - Ok(blocks) => blocks, - Err(err) => { - warn!("Failed to fetch blocks from roots: {err:?}"); - peer_manager.ban_peer( - peer_id, - BanReason::ProtocolError(format!("{err:?}")), + let requested: HashSet = roots.iter().copied().collect(); + let has_unexpected = blocks + .iter() + .any(|block| !requested.contains(&block.message.tree_hash_root())); + if has_unexpected { + warn!( + "Peer {peer_id} returned block(s) with an unrequested root for a BlockRoots request" + ); + for root in roots.iter() { + block_cache.mark_attempted( + RequestKey::BlockRoot(*root), + *peer_id, + candidate_peers, + now, ); - continue; } - }; + peer_manager.ban_peer( + peer_id, + BanReason::ProtocolError( + "returned block(s) with an unrequested root".to_string(), + ), + ); + continue; + } + + let returned: HashSet = blocks + .iter() + .map(|block| block.message.tree_hash_root()) + .collect(); + for root in roots.iter() { + if returned.contains(root) { + block_cache.clear_attempted(RequestKey::BlockRoot(*root)); + } else { + block_cache.mark_attempted( + RequestKey::BlockRoot(*root), + *peer_id, + candidate_peers, + now, + ); + } + } if blocks.is_empty() { warn!("Received empty block roots from peer: {peer_id}"); - peer_manager.ban_peer(peer_id, BanReason::EmptyResponse); continue; } @@ -769,12 +1156,6 @@ fn poll_ready_tasks( warn!("Failed to add downloaded blocks to cache: {err:?}"); } } - Poll::Ready(Err(err)) => { - warn!("Forward fill task failed: {err}"); - peer_manager.mark_peer_as_idle(peer_id); - block_cache.remove_block_roots_in_progress(roots); - indexes_to_remove.push(index); - } Poll::Pending => {} } } @@ -786,46 +1167,100 @@ fn poll_ready_tasks( let pinned = Pin::new(handle); match pinned.poll(&mut context) { - Poll::Ready(Ok(blob_sidecars_result)) => { + Poll::Ready(result) => { indexes_to_remove.push(index); block_cache.remove_blob_identifiers_in_progress(blob_identifiers); peer_manager.mark_peer_as_idle(peer_id); - let blob_sidecars = match blob_sidecars_result { - Ok(blob_sidecars) => blob_sidecars, - Err(err) => { - warn!("Failed to fetch blobs from peer: {err:?}"); - continue; - } - }; - let blob_sidecars = match blob_sidecars { - Ok(blob_sidecars) => blob_sidecars, - Err(err) => { - warn!("Failed to fetch blobs from identifiers: {err:?}"); - peer_manager.ban_peer( - peer_id, - BanReason::ProtocolError(format!("{err:?}")), + let Some(blob_sidecars) = + handle_stream_outcome(peer_manager, peer_id, result) + else { + for identifier in blob_identifiers.iter() { + block_cache.mark_attempted( + RequestKey::Blob(*identifier), + *peer_id, + candidate_peers, + now, ); - continue; } + continue; }; if blob_sidecars.is_empty() { warn!("Received empty blob identifiers from peer: {peer_id}"); - peer_manager.ban_peer(peer_id, BanReason::EmptyResponse); + for identifier in blob_identifiers.iter() { + block_cache.mark_attempted( + RequestKey::Blob(*identifier), + *peer_id, + candidate_peers, + now, + ); + } continue; } + let requested: HashSet = + blob_identifiers.iter().copied().collect(); + let has_unexpected = blob_sidecars.iter().any(|sidecar| { + !requested.contains(&BlobIdentifier { + block_root: sidecar.signed_block_header.message.tree_hash_root(), + index: sidecar.index, + }) + }); + if has_unexpected { + warn!("Peer {peer_id} returned unrequested blob sidecar(s)"); + for identifier in blob_identifiers.iter() { + block_cache.mark_attempted( + RequestKey::Blob(*identifier), + *peer_id, + candidate_peers, + now, + ); + } + peer_manager.ban_peer( + peer_id, + BanReason::ProtocolError( + "returned unrequested blob sidecar(s)".to_string(), + ), + ); + continue; + } + + let returned: HashSet = blob_sidecars + .iter() + .map(|sidecar| BlobIdentifier { + block_root: sidecar.signed_block_header.message.tree_hash_root(), + index: sidecar.index, + }) + .collect(); + if let Err(err) = block_cache.add_blobs(blob_sidecars) { warn!("Failed to add downloaded blobs to cache: {err:?}"); + for identifier in blob_identifiers.iter() { + block_cache.mark_attempted( + RequestKey::Blob(*identifier), + *peer_id, + candidate_peers, + now, + ); + } + peer_manager + .ban_peer(peer_id, BanReason::InvalidProof(format!("{err:?}"))); + } else { + for identifier in blob_identifiers.iter() { + if returned.contains(identifier) { + block_cache.clear_attempted(RequestKey::Blob(*identifier)); + } else { + block_cache.mark_attempted( + RequestKey::Blob(*identifier), + *peer_id, + candidate_peers, + now, + ); + } + } } } - Poll::Ready(Err(err)) => { - warn!("Forward fill task failed: {err}"); - peer_manager.mark_peer_as_idle(peer_id); - block_cache.remove_blob_identifiers_in_progress(blob_identifiers); - indexes_to_remove.push(index); - } Poll::Pending => {} } } @@ -837,48 +1272,101 @@ fn poll_ready_tasks( let pinned = Pin::new(handle); match pinned.poll(&mut context) { - Poll::Ready(Ok(columns_result)) => { + Poll::Ready(result) => { indexes_to_remove.push(index); block_cache.remove_data_column_identifiers_in_progress(identifiers); peer_manager.mark_peer_as_idle(peer_id); - let columns = match columns_result { - Ok(columns) => columns, - Err(err) => { - warn!("Failed to fetch data columns from peer: {err:?}"); - continue; - } - }; - let columns = match columns { - Ok(columns) => columns, - Err(err) => { - warn!("Failed to fetch data columns from identifiers: {err:?}"); - peer_manager.ban_peer( - peer_id, - BanReason::ProtocolError(format!("{err:?}")), + let Some(columns) = handle_stream_outcome(peer_manager, peer_id, result) + else { + for identifier in identifiers.iter() { + block_cache.mark_attempted( + RequestKey::Column(*identifier), + *peer_id, + candidate_peers, + now, ); - continue; } + continue; }; if columns.is_empty() { warn!("Received empty data column identifiers from peer: {peer_id}"); - peer_manager.ban_peer(peer_id, BanReason::EmptyResponse); + for identifier in identifiers.iter() { + block_cache.mark_attempted( + RequestKey::Column(*identifier), + *peer_id, + candidate_peers, + now, + ); + } continue; } + let requested: HashSet = + identifiers.iter().copied().collect(); + let has_unexpected = columns.iter().any(|column| { + !requested.contains(&ColumnIdentifier::new( + column.signed_block_header.message.tree_hash_root(), + column.index, + )) + }); + if has_unexpected { + warn!("Peer {peer_id} returned unrequested data column(s)"); + for identifier in identifiers.iter() { + block_cache.mark_attempted( + RequestKey::Column(*identifier), + *peer_id, + candidate_peers, + now, + ); + } + peer_manager.ban_peer( + peer_id, + BanReason::ProtocolError( + "returned unrequested data column(s)".to_string(), + ), + ); + continue; + } + + let returned: HashSet = columns + .iter() + .map(|column| { + ColumnIdentifier::new( + column.signed_block_header.message.tree_hash_root(), + column.index, + ) + }) + .collect(); + if let Err(err) = block_cache.add_data_columns(columns, required_columns) { warn!("Failed to add downloaded data columns to cache: {err:?}"); + for identifier in identifiers.iter() { + block_cache.mark_attempted( + RequestKey::Column(*identifier), + *peer_id, + candidate_peers, + now, + ); + } peer_manager .ban_peer(peer_id, BanReason::InvalidProof(format!("{err:?}"))); + } else { + for identifier in identifiers.iter() { + if returned.contains(identifier) { + block_cache.clear_attempted(RequestKey::Column(*identifier)); + } else { + block_cache.mark_attempted( + RequestKey::Column(*identifier), + *peer_id, + candidate_peers, + now, + ); + } + } } } - Poll::Ready(Err(err)) => { - warn!("Forward fill task failed: {err}"); - peer_manager.mark_peer_as_idle(peer_id); - block_cache.remove_data_column_identifiers_in_progress(identifiers); - indexes_to_remove.push(index); - } Poll::Pending => {} } } @@ -899,9 +1387,12 @@ mod tests { use discv5::{Enr, enr::CombinedKey}; use kzg::{G1, eip_4844::compute_blob_kzg_proof_raw}; use parking_lot::RwLock; - use ream_consensus_beacon::data_column_sidecar::NUMBER_OF_COLUMNS; - use ream_consensus_misc::polynomial_commitments::{ - kzg_commitment::KZGCommitment, kzg_proof::KZGProof, + use ream_consensus_beacon::{ + data_column_sidecar::NUMBER_OF_COLUMNS, electra::beacon_block::BeaconBlock, + }; + use ream_consensus_misc::{ + checkpoint::Checkpoint, + polynomial_commitments::{kzg_commitment::KZGCommitment, kzg_proof::KZGProof}, }; use ream_execution_rpc_types::get_blobs::{Blob, BlobAndProofV1}; use ream_network_spec::networks::beacon::initialize_test_network_spec; @@ -1006,6 +1497,51 @@ mod tests { }) } + fn test_peer_manager_with_one_peer() -> (PeerManager, PeerId) { + let network_state = test_network_state(); + let peer_id = PeerId::random(); + let mut peer = CachedPeer::new( + peer_id, + None, + ConnectionState::Connected, + Direction::Outbound, + None, + ); + peer.status = Some(Status::default()); + network_state.peer_table.write().insert(peer_id, peer); + + let mut peer_manager = PeerManager::new(network_state); + peer_manager.update_peer_set(); + (peer_manager, peer_id) + } + + fn poll_until_done( + tasks: &mut Vec, + block_cache: &mut BlockCache, + peer_manager: &mut PeerManager, + required_columns: &HashSet, + saw_empty_range: &mut bool, + candidate_peers: &[PeerId], + ) { + for _ in 0..500 { + poll_ready_tasks( + tasks, + block_cache, + peer_manager, + required_columns, + saw_empty_range, + candidate_peers, + Instant::now(), + ) + .expect("poll_ready_tasks should not error"); + if tasks.is_empty() { + return; + } + std::thread::sleep(Duration::from_millis(2)); + } + panic!("task did not complete within the test timeout"); + } + /// `start()` must hand `self` back out even on an ordinary segment error, not just success, /// or the caller loses the syncer (and its warmed-up peer table) and can never retry. #[test] @@ -1061,23 +1597,33 @@ mod tests { .slot_index_provider() .insert(highest_slot, highest_root) .expect("insert highest synced slot"); + store + .db + .finalized_checkpoint_provider() + .insert(Checkpoint { + epoch: 0, + root: B256::ZERO, + }) + .expect("insert finalized checkpoint"); } let network_state = test_network_state(); - let peer_id = PeerId::random(); - let mut peer = CachedPeer::new( - peer_id, - None, - ConnectionState::Connected, - Direction::Outbound, - None, - ); - peer.status = Some(Status { - head_slot: highest_slot, - head_root: highest_root, - ..Default::default() - }); - network_state.peer_table.write().insert(peer_id, peer); + for _ in 0..MIN_SYNC_PEERS { + let peer_id = PeerId::random(); + let mut peer = CachedPeer::new( + peer_id, + None, + ConnectionState::Connected, + Direction::Outbound, + None, + ); + peer.status = Some(Status { + head_slot: highest_slot, + head_root: highest_root, + ..Default::default() + }); + network_state.peer_table.write().insert(peer_id, peer); + } let (p2p_sender, _p2p_receiver) = tokio::sync::mpsc::unbounded_channel(); let mut syncer = BlockRangeSyncer::new( @@ -1112,4 +1658,224 @@ mod tests { .expect("insert time"); assert!(!syncer.is_synced_to_head_slot().await); } + + #[test] + fn poll_ready_tasks_partial_block_roots_marks_only_the_omitted_root_attempted() { + initialize_test_network_spec(); + let executor = ReamExecutor::new().expect("executor should start"); + let (mut peer_manager, peer_id) = test_peer_manager_with_one_peer(); + + let delivered_block = SignedBeaconBlock { + message: BeaconBlock { + slot: 1, + ..Default::default() + }, + signature: Default::default(), + }; + let delivered_root = delivered_block.message.tree_hash_root(); + let omitted_root = B256::repeat_byte(9); + + let mut block_cache = BlockCache::new(B256::ZERO, 0); + let handle = executor.spawn(async move { StreamOutcome::Complete(vec![delivered_block]) }); + let mut tasks = vec![DownloadTask::new_block_roots( + handle, + vec![delivered_root, omitted_root], + peer_id, + )]; + + poll_until_done( + &mut tasks, + &mut block_cache, + &mut peer_manager, + &HashSet::new(), + &mut false, + &[peer_id], + ); + + assert!( + block_cache + .attempted_peers_for(RequestKey::BlockRoot(delivered_root)) + .is_empty(), + "a delivered root must not be marked as a failed attempt" + ); + assert!( + block_cache + .attempted_peers_for(RequestKey::BlockRoot(omitted_root)) + .contains(&peer_id), + "the omitted root must be attributed to this peer, so a retry excludes it" + ); + } + + #[test] + fn poll_ready_tasks_unrequested_block_root_bans_peer_and_discards_whole_batch() { + initialize_test_network_spec(); + let executor = ReamExecutor::new().expect("executor should start"); + let (mut peer_manager, peer_id) = test_peer_manager_with_one_peer(); + + let requested_block = SignedBeaconBlock { + message: BeaconBlock { + slot: 1, + ..Default::default() + }, + signature: Default::default(), + }; + let requested_root = requested_block.message.tree_hash_root(); + let unrequested_block = SignedBeaconBlock { + message: BeaconBlock { + slot: 2, + ..Default::default() + }, + signature: Default::default(), + }; + + let mut block_cache = BlockCache::new(B256::ZERO, 0); + let handle = executor.spawn(async move { + StreamOutcome::Complete(vec![requested_block, unrequested_block]) + }); + let mut tasks = vec![DownloadTask::new_block_roots( + handle, + vec![requested_root], + peer_id, + )]; + + poll_until_done( + &mut tasks, + &mut block_cache, + &mut peer_manager, + &HashSet::new(), + &mut false, + &[peer_id], + ); + + assert_eq!( + block_cache.block_count(), + 0, + "a response with any unrequested item is atomically discarded -- even the \ + requested, otherwise-valid item in the same batch must not be committed" + ); + assert!( + block_cache + .attempted_peers_for(RequestKey::BlockRoot(requested_root)) + .contains(&peer_id), + "the requested root must still be attributed to this peer's failed attempt" + ); + assert!( + peer_manager.fetch_idle_peer_from(&[peer_id]).is_none(), + "a peer that smuggled in an unrequested item must be banned, not just marked idle" + ); + } + + #[test] + fn poll_ready_tasks_empty_data_column_range_falls_through_to_by_root_and_excludes_peer() { + initialize_test_network_spec(); + let executor = ReamExecutor::new().expect("executor should start"); + let (mut peer_manager, peer_id) = test_peer_manager_with_one_peer(); + + let range = Range::new(1, 10); + let identifier = ColumnIdentifier::new(B256::repeat_byte(3), 0); + let mut block_cache = BlockCache::new(B256::ZERO, 0); + let handle = + executor.spawn(async move { StreamOutcome::Complete(Vec::::new()) }); + let mut tasks = vec![DownloadTask::new_data_column_range( + handle, + range, + peer_id, + vec![identifier], + )]; + + let mut saw_empty_range = false; + poll_until_done( + &mut tasks, + &mut block_cache, + &mut peer_manager, + &HashSet::new(), + &mut saw_empty_range, + &[peer_id], + ); + + assert!(saw_empty_range); + assert!( + block_cache + .attempted_peers_for(RequestKey::Column(identifier)) + .contains(&peer_id), + "the expected identifier must fall through to the by-root fallback, excluding this peer" + ); + assert!( + block_cache + .attempted_peers_for(RequestKey::ColumnRange(range)) + .is_empty(), + "an empty range response is accepted (settled), not held as a pending attempt" + ); + } + + #[test] + fn poll_ready_tasks_invalid_blob_proof_bans_peer_and_marks_it_attempted() { + initialize_test_network_spec(); + let executor = ReamExecutor::new().expect("executor should start"); + let (mut peer_manager, peer_id) = test_peer_manager_with_one_peer(); + + let blob = Blob::default(); + let blob_bytes = blob.to_fixed_bytes(); + let raw_commitment = das_context() + .blob_to_kzg_commitment(&blob_bytes) + .expect("test blob should produce a commitment"); + let commitment = KZGCommitment(raw_commitment); + let proof = KZGProof::from( + compute_blob_kzg_proof_raw( + blob_bytes, + raw_commitment, + ream_polynomial_commitments::trusted_setup::blst_settings(), + ) + .expect("test blob should produce a proof") + .to_bytes(), + ); + let mut block = SignedBeaconBlock { + message: Default::default(), + signature: Default::default(), + }; + block + .message + .body + .blob_kzg_commitments + .push(commitment) + .expect("one commitment should fit"); + let block_root = block.message.tree_hash_root(); + let identifier = BlobIdentifier::new(block_root, 0); + let mut sidecar = block + .blob_sidecar(BlobAndProofV1 { blob, proof }, 0) + .expect("test sidecar should be constructed"); + sidecar.kzg_proof[0] ^= 1; + + let mut block_cache = BlockCache::new(B256::ZERO, 0); + block_cache + .add_blocks(vec![block], false) + .expect("block should enter cache"); + + let handle = executor.spawn(async move { StreamOutcome::Complete(vec![sidecar]) }); + let mut tasks = vec![DownloadTask::new_blob_identifiers( + handle, + vec![identifier], + peer_id, + )]; + + poll_until_done( + &mut tasks, + &mut block_cache, + &mut peer_manager, + &HashSet::new(), + &mut false, + &[peer_id], + ); + + assert!( + block_cache + .attempted_peers_for(RequestKey::Blob(identifier)) + .contains(&peer_id), + "the dependency must be attributed to this peer so a retry picks someone else" + ); + assert!( + peer_manager.fetch_idle_peer_from(&[peer_id]).is_none(), + "a peer that served an invalid blob proof must be banned, not just marked idle" + ); + } } diff --git a/crates/networking/syncer/src/block_range/peer_manager.rs b/crates/networking/syncer/src/block_range/peer_manager.rs index b73d38208..a01ce8406 100644 --- a/crates/networking/syncer/src/block_range/peer_manager.rs +++ b/crates/networking/syncer/src/block_range/peer_manager.rs @@ -1,10 +1,9 @@ use std::{ - collections::{HashMap, hash_map::Entry}, + collections::{HashMap, HashSet, hash_map::Entry}, sync::Arc, time::{Duration, Instant}, }; -use alloy_primitives::B256; use libp2p::PeerId; use ream_consensus_misc::constants::beacon::SLOTS_PER_EPOCH; use ream_p2p::network::beacon::{network_state::NetworkState, peer::CachedPeer}; @@ -13,18 +12,27 @@ use tracing::warn; /// How long a peer stays banned before it becomes eligible to rejoin the peer set. const BAN_DURATION: Duration = Duration::from_secs(300); -/// Majority-agreed (slot, root), tagged with root so peer selection can target this chain. +pub const MIN_SYNC_PEERS: usize = 3; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TargetSelection { + Ready { + target_slot: u64, + eligible_peers: Vec, + }, + NoQuorum, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct SyncTarget { - pub slot: u64, - pub root: B256, +pub enum TargetQualification { + FinalizedEpoch(u64), + HeadEpoch(u64), } /// Why a peer was banned. Still just bans outright either way, but structured (instead of a /// free-text string) so a future scoring system can weigh severities without touching call sites. #[derive(Debug, Clone)] pub enum BanReason { - EmptyResponse, /// Disconnect, timeout, decode error, or a response that didn't fit the cache. ProtocolError(String), /// Failed KZG or inclusion proof verification. @@ -34,7 +42,6 @@ pub enum BanReason { impl std::fmt::Display for BanReason { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - BanReason::EmptyResponse => write!(f, "empty response"), BanReason::ProtocolError(detail) => write!(f, "protocol error: {detail}"), BanReason::InvalidProof(detail) => write!(f, "invalid proof: {detail}"), } @@ -124,20 +131,35 @@ impl PeerManager { None } - pub fn fetch_idle_peer_preferring(&mut self, target_root: Option) -> Option { - if let Some(target_root) = target_root { - for peer_info in self.peers.values_mut() { - let matches_target = peer_info.peer.status.as_ref().is_some_and(|status| { - status.head_root == target_root || status.finalized_root == target_root - }); - if matches_target && matches!(peer_info.peer_status, PeerStatus::Idle) { - peer_info.peer_status = PeerStatus::Downloading; - return Some(peer_info.peer.clone()); - } + pub fn fetch_idle_peer_from(&mut self, eligible: &[PeerId]) -> Option { + for peer_id in eligible { + if let Some(peer_info) = self.peers.get_mut(peer_id) + && matches!(peer_info.peer_status, PeerStatus::Idle) + { + peer_info.peer_status = PeerStatus::Downloading; + return Some(peer_info.peer.clone()); } } + None + } - self.fetch_idle_peer() + pub fn fetch_idle_peer_from_excluding( + &mut self, + eligible: &[PeerId], + excluded: &HashSet, + ) -> Option { + for peer_id in eligible { + if excluded.contains(peer_id) { + continue; + } + if let Some(peer_info) = self.peers.get_mut(peer_id) + && matches!(peer_info.peer_status, PeerStatus::Idle) + { + peer_info.peer_status = PeerStatus::Downloading; + return Some(peer_info.peer.clone()); + } + } + None } pub fn peer_counts(&self) -> String { @@ -162,47 +184,107 @@ impl PeerManager { } } - /// Majority vote on (epoch, root), not just epoch, so two forks can't be merged as "agreed". - pub fn finalized_target(&self) -> Option { - let mut frequencies: HashMap<(u64, B256), usize> = HashMap::new(); - - for peer in self.peers.values() { - if let Some(status) = &peer.peer.status { - *frequencies - .entry(( - status.finalized_epoch * SLOTS_PER_EPOCH, - status.finalized_root, - )) - .or_insert(0) += 1; + pub fn best_finalized(&self, our_finalized_epoch: u64) -> TargetSelection { + let mut votes: HashMap = HashMap::new(); + let mut candidates: Vec<(PeerId, u64, u64)> = Vec::new(); + + for (peer_id, peer_info) in &self.peers { + let Some(status) = &peer_info.peer.status else { + continue; + }; + if status.finalized_epoch < our_finalized_epoch { + continue; } + *votes.entry(status.finalized_epoch).or_insert(0) += 1; + candidates.push((*peer_id, status.finalized_epoch, status.head_slot)); } - frequencies + let Some(winner_epoch) = votes + .iter() + .max_by(|(epoch_a, votes_a), (epoch_b, votes_b)| { + votes_a.cmp(votes_b).then(epoch_a.cmp(epoch_b)) + }) + .map(|(&epoch, _)| epoch) + else { + return TargetSelection::NoQuorum; + }; + + let mut eligible: Vec<(PeerId, u64, u64)> = candidates .into_iter() - .max_by_key(|&(_, count)| count) - .map(|((slot, root), _)| SyncTarget { slot, root }) + .filter(|(_, epoch, _)| *epoch >= winner_epoch) + .collect(); + eligible.sort_by(|a, b| b.1.cmp(&a.1).then(b.2.cmp(&a.2))); + + TargetSelection::Ready { + target_slot: winner_epoch * SLOTS_PER_EPOCH, + eligible_peers: eligible.into_iter().map(|(id, ..)| id).collect(), + } } - /// Majority vote on (slot, root); replaces a bare max() so one outlier can't skew the target. - pub fn head_target(&self) -> Option { - let mut frequencies: HashMap<(u64, B256), usize> = HashMap::new(); + pub fn best_non_finalized(&self, min_peers: usize, our_head_epoch: u64) -> TargetSelection { + let our_head_slot = our_head_epoch * SLOTS_PER_EPOCH; + let mut epoch_votes: HashMap = HashMap::new(); + let mut candidates: Vec<(PeerId, u64, u64)> = Vec::new(); - for peer in self.peers.values() { - if let Some(status) = &peer.peer.status { - *frequencies - .entry((status.head_slot, status.head_root)) - .or_insert(0) += 1; + for (peer_id, peer_info) in &self.peers { + let Some(status) = &peer_info.peer.status else { + continue; + }; + if status.head_slot <= our_head_slot { + continue; } + let epoch = status.head_slot / SLOTS_PER_EPOCH; + *epoch_votes.entry(epoch).or_insert(0) += 1; + candidates.push((*peer_id, epoch, status.head_slot)); } - frequencies + let Some(target_epoch) = epoch_votes + .iter() + .filter(|&(_, &votes)| votes >= min_peers) + .map(|(&epoch, _)| epoch) + .max() + else { + return TargetSelection::NoQuorum; + }; + + let eligible: Vec<(PeerId, u64)> = candidates .into_iter() - .max_by_key(|&(_, count)| count) - .map(|((slot, root), _)| SyncTarget { slot, root }) + .filter(|(_, epoch, _)| *epoch >= target_epoch) + .map(|(id, _, head_slot)| (id, head_slot)) + .collect(); + + let target_slot = eligible + .iter() + .filter(|(_, head_slot)| head_slot / SLOTS_PER_EPOCH == target_epoch) + .map(|(_, head_slot)| *head_slot) + .max() + .unwrap_or(target_epoch * SLOTS_PER_EPOCH); + + let mut eligible = eligible; + eligible.sort_by_key(|&(_, head_slot)| std::cmp::Reverse(head_slot)); + + TargetSelection::Ready { + target_slot, + eligible_peers: eligible.into_iter().map(|(id, _)| id).collect(), + } } - pub fn sync_target(&self) -> Option { - self.head_target().or_else(|| self.finalized_target()) + pub fn peers_satisfying(&self, qualification: TargetQualification) -> Vec { + self.peers + .iter() + .filter(|(_, info)| { + let Some(status) = &info.peer.status else { + return false; + }; + match qualification { + TargetQualification::FinalizedEpoch(epoch) => status.finalized_epoch >= epoch, + TargetQualification::HeadEpoch(epoch) => { + status.head_slot / SLOTS_PER_EPOCH >= epoch + } + } + }) + .map(|(peer_id, _)| *peer_id) + .collect() } } @@ -251,47 +333,62 @@ mod tests { } #[test] - fn finalized_target_does_not_merge_different_forks_with_same_epoch() { + fn best_finalized_excludes_peers_behind_our_epoch() { let mut peer_manager = PeerManager::new(test_network_state()); - let root_a = B256::repeat_byte(0xAA); - let root_b = B256::repeat_byte(0xBB); - insert_idle( &mut peer_manager, test_peer(Status { - finalized_epoch: 10, - finalized_root: root_a, + finalized_epoch: 5, ..Default::default() }), ); + + assert_eq!( + peer_manager.best_finalized(10), + TargetSelection::NoQuorum, + "the only peer is behind our own finalized epoch, so there's no target to chase" + ); + } + + #[test] + fn best_finalized_is_plurality_not_majority_with_tie_break_by_higher_epoch() { + let mut peer_manager = PeerManager::new(test_network_state()); + for _ in 0..2 { + insert_idle( + &mut peer_manager, + test_peer(Status { + finalized_epoch: 10, + ..Default::default() + }), + ); + } insert_idle( &mut peer_manager, test_peer(Status { - finalized_epoch: 10, - finalized_root: root_b, + finalized_epoch: 12, ..Default::default() }), ); - // 1 vote each: must not merge into a false "2 votes" just because the epoch matches. - let target = peer_manager - .finalized_target() - .expect("a target should be picked"); - assert!(target.root == root_a || target.root == root_b); + let TargetSelection::Ready { + target_slot, + eligible_peers, + } = peer_manager.best_finalized(0) + else { + panic!("expected a target"); + }; + assert_eq!(target_slot, 10 * SLOTS_PER_EPOCH); + assert_eq!(eligible_peers.len(), 3); } #[test] - fn head_target_ignores_outlier_without_plurality() { + fn best_non_finalized_uses_a_threshold_not_a_plurality_vote() { let mut peer_manager = PeerManager::new(test_network_state()); - let honest_root = B256::repeat_byte(0x11); - let lying_root = B256::repeat_byte(0x22); - for _ in 0..3 { insert_idle( &mut peer_manager, test_peer(Status { - head_slot: 100, - head_root: honest_root, + head_slot: 20 * SLOTS_PER_EPOCH, ..Default::default() }), ); @@ -299,50 +396,118 @@ mod tests { insert_idle( &mut peer_manager, test_peer(Status { - head_slot: 999_999, - head_root: lying_root, + head_slot: 21 * SLOTS_PER_EPOCH, ..Default::default() }), ); - let target = peer_manager - .head_target() - .expect("a target should be picked"); - assert_eq!(target.slot, 100); - assert_eq!(target.root, honest_root); + let TargetSelection::Ready { target_slot, .. } = peer_manager.best_non_finalized(1, 0) + else { + panic!("expected a target"); + }; + assert_eq!(target_slot, 21 * SLOTS_PER_EPOCH); } #[test] - fn fetch_idle_peer_preferring_prefers_matching_root_then_falls_back() { + fn best_non_finalized_returns_no_quorum_below_min_peers_even_with_peers_ahead() { let mut peer_manager = PeerManager::new(test_network_state()); - let target_root = B256::repeat_byte(0x33); - let other_root = B256::repeat_byte(0x44); + insert_idle( + &mut peer_manager, + test_peer(Status { + head_slot: 20 * SLOTS_PER_EPOCH, + ..Default::default() + }), + ); - let matching = test_peer(Status { - head_root: target_root, - ..Default::default() - }); - let matching_id = matching.peer_id; + assert_eq!( + peer_manager.best_non_finalized(3, 0), + TargetSelection::NoQuorum, + "one peer is ahead of us, but not enough to clear min_peers=3" + ); + } + + #[test] + fn best_non_finalized_refines_target_to_highest_actual_head_slot_in_winning_epoch() { + let mut peer_manager = PeerManager::new(test_network_state()); + insert_idle( + &mut peer_manager, + test_peer(Status { + head_slot: 20 * SLOTS_PER_EPOCH + 3, + ..Default::default() + }), + ); insert_idle( &mut peer_manager, test_peer(Status { - head_root: other_root, + head_slot: 20 * SLOTS_PER_EPOCH + 9, ..Default::default() }), ); - insert_idle(&mut peer_manager, matching); + + let TargetSelection::Ready { target_slot, .. } = peer_manager.best_non_finalized(1, 0) + else { + panic!("expected a target"); + }; + assert_eq!(target_slot, 20 * SLOTS_PER_EPOCH + 9); + } + + #[test] + fn fetch_idle_peer_from_returns_first_idle_in_given_order() { + let mut peer_manager = PeerManager::new(test_network_state()); + let first = test_peer(Status::default()); + let second = test_peer(Status::default()); + let first_id = first.peer_id; + let second_id = second.peer_id; + insert_idle(&mut peer_manager, first); + insert_idle(&mut peer_manager, second); let peer = peer_manager - .fetch_idle_peer_preferring(Some(target_root)) + .fetch_idle_peer_from(&[second_id, first_id]) .expect("a peer should be found"); - assert_eq!(peer.peer_id, matching_id); + assert_eq!(peer.peer_id, second_id); - // No peer matches this root: falls back to any idle peer instead of returning None. - let unrelated_root = B256::repeat_byte(0x55); assert!( peer_manager - .fetch_idle_peer_preferring(Some(unrelated_root)) - .is_some() + .fetch_idle_peer_from(&[PeerId::random()]) + .is_none() + ); + } + + #[test] + fn fetch_idle_peer_from_excluding_skips_excluded_peers() { + let mut peer_manager = PeerManager::new(test_network_state()); + let peer = test_peer(Status::default()); + let peer_id = peer.peer_id; + insert_idle(&mut peer_manager, peer); + + let mut excluded = HashSet::new(); + excluded.insert(peer_id); + assert!( + peer_manager + .fetch_idle_peer_from_excluding(&[peer_id], &excluded) + .is_none() + ); + } + + #[test] + fn peers_satisfying_distinguishes_finalized_and_head_qualification() { + let mut peer_manager = PeerManager::new(test_network_state()); + let high_head_low_finalized = test_peer(Status { + head_slot: 1_000, + finalized_epoch: 1, + ..Default::default() + }); + let id = high_head_low_finalized.peer_id; + insert_idle(&mut peer_manager, high_head_low_finalized); + + assert!( + peer_manager + .peers_satisfying(TargetQualification::FinalizedEpoch(10)) + .is_empty() + ); + assert_eq!( + peer_manager.peers_satisfying(TargetQualification::HeadEpoch(0)), + vec![id] ); } @@ -353,7 +518,7 @@ mod tests { let peer_id = peer.peer_id; insert_idle(&mut peer_manager, peer); - peer_manager.ban_peer(&peer_id, BanReason::EmptyResponse); + peer_manager.ban_peer(&peer_id, BanReason::ProtocolError("test".to_string())); assert!(peer_manager.banned_peers.contains_key(&peer_id)); // Simulate the ban having happened BAN_DURATION ago. @@ -390,4 +555,20 @@ mod tests { PeerStatus::Idle )); } + + #[test] + fn peers_satisfying_head_epoch_includes_a_peer_below_the_refined_max_slot() { + let mut peer_manager = PeerManager::new(test_network_state()); + let below_refined_max = test_peer(Status { + head_slot: 20 * SLOTS_PER_EPOCH + 3, + ..Default::default() + }); + let id = below_refined_max.peer_id; + insert_idle(&mut peer_manager, below_refined_max); + + assert_eq!( + peer_manager.peers_satisfying(TargetQualification::HeadEpoch(20)), + vec![id] + ); + } } diff --git a/crates/networking/syncer/src/block_range/peer_range_downloader.rs b/crates/networking/syncer/src/block_range/peer_range_downloader.rs index 6f2ce5171..c8c773de0 100644 --- a/crates/networking/syncer/src/block_range/peer_range_downloader.rs +++ b/crates/networking/syncer/src/block_range/peer_range_downloader.rs @@ -1,5 +1,4 @@ use alloy_primitives::B256; -use anyhow::bail; use libp2p::PeerId; use ream_consensus_beacon::{ blob_sidecar::{BlobIdentifier, BlobSidecar}, @@ -7,9 +6,13 @@ use ream_consensus_beacon::{ electra::beacon_block::SignedBeaconBlock, }; use ream_executor::ReamExecutor; -use ream_p2p::network::beacon::channel::{P2PCallbackResponse, P2PMessage, P2PRequest}; -use ream_req_resp::beacon::messages::{ - BeaconResponseMessage, data_column_sidecars::DataColumnsByRootIdentifier, +use ream_p2p::network::beacon::channel::{ + P2PCallbackError, P2PCallbackResponse, P2PMessage, P2PRequest, +}; +use ream_req_resp::{ + beacon::messages::{BeaconResponseMessage, data_column_sidecars::DataColumnsByRootIdentifier}, + error::ReqRespError, + inbound_protocol::ResponseCode, }; use tokio::{ sync::mpsc::{self, UnboundedSender}, @@ -29,36 +32,68 @@ impl Range { } } +#[derive(Debug)] +pub enum DownloadFailure { + Transport(String), + InvalidData(String), + RemoteError { code: ResponseCode, message: String }, +} + +pub enum StreamOutcome { + Complete(Vec), + Failed(DownloadFailure), +} + +fn classify_req_resp_error(err: ReqRespError) -> DownloadFailure { + match err { + ReqRespError::RemoteError { code, message } => { + DownloadFailure::RemoteError { code, message } + } + ReqRespError::InvalidData(message) => DownloadFailure::InvalidData(message), + other => DownloadFailure::Transport(format!("{other:?}")), + } +} + async fn drain_responses( - mut rx: mpsc::Receiver>, - mut extract: impl FnMut(BeaconResponseMessage) -> Option, -) -> anyhow::Result> { + mut rx: mpsc::Receiver>, + mut extract: impl FnMut(BeaconResponseMessage) -> Result, +) -> StreamOutcome { let mut items = vec![]; while let Some(response) = rx.recv().await { match response { Ok(P2PCallbackResponse::ResponseMessage(message)) => { - if let Some(item) = extract(message.as_ref().clone()) { - items.push(item); + match extract(message.as_ref().clone()) { + Ok(item) => items.push(item), + Err(err) => return StreamOutcome::Failed(err), } } Ok(P2PCallbackResponse::EndOfStream) => { info!("End of request stream received."); - break; + return StreamOutcome::Complete(items); } Ok(P2PCallbackResponse::Disconnected) => { - bail!("Peer disconnected while receiving response."); + return StreamOutcome::Failed(DownloadFailure::Transport( + "peer disconnected while receiving response".to_string(), + )); } Ok(P2PCallbackResponse::Timeout) => { - bail!("Request timed out."); + return StreamOutcome::Failed(DownloadFailure::Transport( + "request timed out".to_string(), + )); + } + Err(P2PCallbackError::ReqResp(err)) => { + return StreamOutcome::Failed(classify_req_resp_error(err)); } - Err(err) => { - info!("Error receiving response: {err:?}"); + Err(P2PCallbackError::Other(err)) => { + return StreamOutcome::Failed(DownloadFailure::Transport(format!("{err:?}"))); } } } - Ok(items) + StreamOutcome::Failed(DownloadFailure::Transport( + "channel closed before EndOfStream".to_string(), + )) } pub struct PeerRangeDownloader; @@ -69,7 +104,7 @@ impl PeerRangeDownloader { p2p_sender: UnboundedSender, executor: ReamExecutor, range: Range, - ) -> JoinHandle>>> { + ) -> JoinHandle>> { executor.spawn(async move { let (callback, rx) = mpsc::channel(100); p2p_sender @@ -82,8 +117,10 @@ impl PeerRangeDownloader { .expect("Failed to send block range request"); drain_responses(rx, |message| match message { - BeaconResponseMessage::BeaconBlocksByRange(block) => Some(block), - _ => None, + BeaconResponseMessage::BeaconBlocksByRange(block) => Ok(block), + other => Err(DownloadFailure::InvalidData(format!( + "unexpected response variant for a BlockRange request: {other:?}" + ))), }) .await }) @@ -98,7 +135,7 @@ impl PeerRootsDownloader { p2p_sender: UnboundedSender, executor: ReamExecutor, roots: Vec, - ) -> JoinHandle>>> { + ) -> JoinHandle>> { executor.spawn(async move { let (callback, rx) = mpsc::channel(100); p2p_sender @@ -110,8 +147,10 @@ impl PeerRootsDownloader { .expect("Failed to send block roots request"); drain_responses(rx, |message| match message { - BeaconResponseMessage::BeaconBlocksByRoot(block) => Some(block), - _ => None, + BeaconResponseMessage::BeaconBlocksByRoot(block) => Ok(block), + other => Err(DownloadFailure::InvalidData(format!( + "unexpected response variant for a BlockRoots request: {other:?}" + ))), }) .await }) @@ -126,7 +165,7 @@ impl PeerBlobIdentifierDownloader { p2p_sender: UnboundedSender, executor: ReamExecutor, blob_identifiers: Vec, - ) -> JoinHandle>>> { + ) -> JoinHandle>> { executor.spawn(async move { let (callback, rx) = mpsc::channel(100); p2p_sender @@ -138,8 +177,10 @@ impl PeerBlobIdentifierDownloader { .expect("Failed to send blob identifiers request"); drain_responses(rx, |message| match message { - BeaconResponseMessage::BlobSidecarsByRoot(blob_sidecar) => Some(blob_sidecar), - _ => None, + BeaconResponseMessage::BlobSidecarsByRoot(blob_sidecar) => Ok(blob_sidecar), + other => Err(DownloadFailure::InvalidData(format!( + "unexpected response variant for a BlobIdentifiers request: {other:?}" + ))), }) .await }) @@ -155,7 +196,7 @@ impl PeerDataColumnRangeDownloader { executor: ReamExecutor, range: Range, columns: Vec, - ) -> JoinHandle>>> { + ) -> JoinHandle>> { executor.spawn(async move { let (callback, rx) = mpsc::channel(100); p2p_sender @@ -169,8 +210,10 @@ impl PeerDataColumnRangeDownloader { .expect("Failed to send data column range request"); drain_responses(rx, |message| match message { - BeaconResponseMessage::DataColumnSidecarsByRange(column) => Some(column), - _ => None, + BeaconResponseMessage::DataColumnSidecarsByRange(column) => Ok(column), + other => Err(DownloadFailure::InvalidData(format!( + "unexpected response variant for a DataColumnRange request: {other:?}" + ))), }) .await }) @@ -185,7 +228,7 @@ impl PeerDataColumnIdentifierDownloader { p2p_sender: UnboundedSender, executor: ReamExecutor, identifiers: Vec, - ) -> JoinHandle>>> { + ) -> JoinHandle>> { executor.spawn(async move { let (callback, rx) = mpsc::channel(100); p2p_sender @@ -197,8 +240,10 @@ impl PeerDataColumnIdentifierDownloader { .expect("Failed to send data column identifiers request"); drain_responses(rx, |message| match message { - BeaconResponseMessage::DataColumnSidecarsByRoot(column) => Some(column), - _ => None, + BeaconResponseMessage::DataColumnSidecarsByRoot(column) => Ok(column), + other => Err(DownloadFailure::InvalidData(format!( + "unexpected response variant for a DataColumnIdentifiers request: {other:?}" + ))), }) .await }) @@ -219,35 +264,61 @@ mod tests { } #[tokio::test] - async fn drain_responses_extracts_matching_and_ignores_other_variants() { + async fn drain_responses_rejects_an_unexpected_variant_as_invalid_data() { let (tx, rx) = mpsc::channel(10); tx.send(Ok(P2PCallbackResponse::ResponseMessage(Arc::new( BeaconResponseMessage::BeaconBlocksByRange(test_block()), )))) .await .expect("send should succeed"); - // A response for a different request kind must be ignored, not collected or errored on. tx.send(Ok(P2PCallbackResponse::ResponseMessage(Arc::new( BeaconResponseMessage::BeaconBlocksByRoot(test_block()), )))) .await .expect("send should succeed"); + + let outcome = drain_responses(rx, |message| match message { + BeaconResponseMessage::BeaconBlocksByRange(block) => Ok(block), + other => Err(DownloadFailure::InvalidData(format!( + "unexpected: {other:?}" + ))), + }) + .await; + + assert!(matches!( + outcome, + StreamOutcome::Failed(DownloadFailure::InvalidData(_)) + )); + } + + #[tokio::test] + async fn drain_responses_completes_on_clean_end_of_stream() { + let (tx, rx) = mpsc::channel(10); + tx.send(Ok(P2PCallbackResponse::ResponseMessage(Arc::new( + BeaconResponseMessage::BeaconBlocksByRange(test_block()), + )))) + .await + .expect("send should succeed"); tx.send(Ok(P2PCallbackResponse::EndOfStream)) .await .expect("send should succeed"); - let items = drain_responses(rx, |message| match message { - BeaconResponseMessage::BeaconBlocksByRange(block) => Some(block), - _ => None, + let outcome = drain_responses(rx, |message| match message { + BeaconResponseMessage::BeaconBlocksByRange(block) => Ok(block), + other => Err(DownloadFailure::InvalidData(format!( + "unexpected: {other:?}" + ))), }) - .await - .expect("drain should succeed"); + .await; + let StreamOutcome::Complete(items) = outcome else { + panic!("expected a complete outcome"); + }; assert_eq!(items.len(), 1); } #[tokio::test] - async fn drain_responses_returns_collected_items_when_channel_closes() { + async fn drain_responses_treats_channel_close_without_end_of_stream_as_transport_failure() { let (tx, rx) = mpsc::channel(10); tx.send(Ok(P2PCallbackResponse::ResponseMessage(Arc::new( BeaconResponseMessage::BeaconBlocksByRange(test_block()), @@ -256,45 +327,83 @@ mod tests { .expect("send should succeed"); drop(tx); - let items = drain_responses(rx, |message| match message { - BeaconResponseMessage::BeaconBlocksByRange(block) => Some(block), - _ => None, + let outcome = drain_responses(rx, |message| match message { + BeaconResponseMessage::BeaconBlocksByRange(block) => Ok(block), + other => Err(DownloadFailure::InvalidData(format!( + "unexpected: {other:?}" + ))), }) - .await - .expect("drain should succeed"); + .await; - assert_eq!(items.len(), 1); + assert!(matches!( + outcome, + StreamOutcome::Failed(DownloadFailure::Transport(_)) + )); } #[tokio::test] - async fn drain_responses_fails_on_disconnect() { + async fn drain_responses_fails_on_disconnect_without_banning() { let (tx, rx) = mpsc::channel(10); tx.send(Ok(P2PCallbackResponse::Disconnected)) .await .expect("send should succeed"); - let result = drain_responses(rx, |message| match message { - BeaconResponseMessage::BeaconBlocksByRange(block) => Some(block), - _ => None, + let outcome = drain_responses(rx, |message| match message { + BeaconResponseMessage::BeaconBlocksByRange(block) => Ok(block), + other => Err(DownloadFailure::InvalidData(format!( + "unexpected: {other:?}" + ))), }) .await; - assert!(result.is_err()); + assert!(matches!( + outcome, + StreamOutcome::Failed(DownloadFailure::Transport(_)) + )); } #[tokio::test] - async fn drain_responses_fails_on_timeout() { + async fn drain_responses_fails_on_timeout_without_banning() { let (tx, rx) = mpsc::channel(10); tx.send(Ok(P2PCallbackResponse::Timeout)) .await .expect("send should succeed"); - let result = drain_responses(rx, |message| match message { - BeaconResponseMessage::BeaconBlocksByRange(block) => Some(block), - _ => None, + let outcome = drain_responses(rx, |message| match message { + BeaconResponseMessage::BeaconBlocksByRange(block) => Ok(block), + other => Err(DownloadFailure::InvalidData(format!( + "unexpected: {other:?}" + ))), + }) + .await; + + assert!(matches!( + outcome, + StreamOutcome::Failed(DownloadFailure::Transport(_)) + )); + } + + #[tokio::test] + async fn drain_responses_classifies_remote_error_separately_from_invalid_data() { + let (tx, rx) = mpsc::channel(10); + tx.send(Err(P2PCallbackError::ReqResp(ReqRespError::RemoteError { + code: ResponseCode::ResourceUnavailable, + message: "no data for this range".to_string(), + }))) + .await + .expect("send should succeed"); + + let outcome = drain_responses(rx, |message| match message { + BeaconResponseMessage::BeaconBlocksByRange(block) => Ok(block), + other => Err(DownloadFailure::InvalidData(format!( + "unexpected: {other:?}" + ))), }) .await; - assert!(result.is_err()); + assert!(matches!( + outcome, + StreamOutcome::Failed(DownloadFailure::RemoteError { .. }) + )); } } From d5db4ab9057e90c6fd5b641984d8b33c0156aefc Mon Sep 17 00:00:00 2001 From: vuonghuuhung Date: Sat, 29 Aug 2026 23:13:33 +0700 Subject: [PATCH 06/17] fix: fix peer-stuck and duplicate-response bugs in range sync --- .../networking/syncer/src/block_range/mod.rs | 296 ++++++++++++++++-- 1 file changed, 268 insertions(+), 28 deletions(-) diff --git a/crates/networking/syncer/src/block_range/mod.rs b/crates/networking/syncer/src/block_range/mod.rs index 67c10afe3..fe9202f7a 100644 --- a/crates/networking/syncer/src/block_range/mod.rs +++ b/crates/networking/syncer/src/block_range/mod.rs @@ -62,6 +62,14 @@ const ZERO_PROGRESS_BACKOFF: Duration = Duration::from_secs(30); const CANDIDATE_EXHAUSTION_TIMEOUT: Duration = Duration::from_secs(60); +fn should_abort_for_candidate_exhaustion(elapsed: Duration, tasks_in_flight: bool) -> bool { + elapsed >= CANDIDATE_EXHAUSTION_TIMEOUT && !tasks_in_flight +} + +fn candidate_exhaustion_deserves_backoff(target_slot: u64, head_slot: u64) -> bool { + target_slot > head_slot +} + /// Validates downloaded blob sidecars and derives the columns needed for data availability. /// An empty `blob_sidecars` map defers to the block-lookup coordinator instead of erroring, /// since range sync never fetches legacy blob sidecars for post-Fulu blocks. @@ -250,7 +258,6 @@ impl BlockRangeSyncer { let mut finalized_phase_settled = false; let mut finalized_target_was_ahead = false; let mut candidate_exhausted_since: Option = None; - let mut ended_due_to_candidate_exhaustion = false; loop { self.peer_manager.update_peer_set(); @@ -344,19 +351,6 @@ impl BlockRangeSyncer { let candidate_peers = self.peer_manager.peers_satisfying(target.qualification); - if candidate_peers.is_empty() { - let exhausted_since = *candidate_exhausted_since.get_or_insert_with(Instant::now); - if exhausted_since.elapsed() >= CANDIDATE_EXHAUSTION_TIMEOUT { - info!( - "No peer has qualified for the active sync target for over {CANDIDATE_EXHAUSTION_TIMEOUT:?}; ending this segment." - ); - ended_due_to_candidate_exhaustion = true; - break; - } - } else { - candidate_exhausted_since = None; - } - let now = Instant::now(); poll_ready_tasks( &mut task_handles, @@ -368,6 +362,24 @@ impl BlockRangeSyncer { now, )?; + if candidate_peers.is_empty() { + let exhausted_since = *candidate_exhausted_since.get_or_insert_with(Instant::now); + if should_abort_for_candidate_exhaustion( + exhausted_since.elapsed(), + !task_handles.is_empty(), + ) { + info!( + "No peer has qualified for the active sync target for over {CANDIDATE_EXHAUSTION_TIMEOUT:?}; ending this segment." + ); + if candidate_exhaustion_deserves_backoff(target.slot, head_slot) { + self.next_segment_not_before = Some(Instant::now() + ZERO_PROGRESS_BACKOFF); + } + break; + } + } else { + candidate_exhausted_since = None; + } + let current_epoch = self .beacon_chain .store @@ -701,10 +713,7 @@ impl BlockRangeSyncer { || active_target .as_ref() .is_some_and(|target| target.slot > head_slot); - if target_was_ahead - && (saw_empty_range || ended_due_to_candidate_exhaustion) - && imported_count == 0 - { + if target_was_ahead && saw_empty_range && imported_count == 0 { self.next_segment_not_before = Some(Instant::now() + ZERO_PROGRESS_BACKOFF); } @@ -1106,12 +1115,14 @@ fn poll_ready_tasks( }; let requested: HashSet = roots.iter().copied().collect(); - let has_unexpected = blocks - .iter() - .any(|block| !requested.contains(&block.message.tree_hash_root())); + let mut seen_roots = HashSet::new(); + let has_unexpected = blocks.iter().any(|block| { + let root = block.message.tree_hash_root(); + !requested.contains(&root) || !seen_roots.insert(root) + }); if has_unexpected { warn!( - "Peer {peer_id} returned block(s) with an unrequested root for a BlockRoots request" + "Peer {peer_id} returned an unrequested or duplicate root for a BlockRoots request" ); for root in roots.iter() { block_cache.mark_attempted( @@ -1201,14 +1212,18 @@ fn poll_ready_tasks( let requested: HashSet = blob_identifiers.iter().copied().collect(); + let mut seen_identifiers = HashSet::new(); let has_unexpected = blob_sidecars.iter().any(|sidecar| { - !requested.contains(&BlobIdentifier { + let identifier = BlobIdentifier { block_root: sidecar.signed_block_header.message.tree_hash_root(), index: sidecar.index, - }) + }; + !requested.contains(&identifier) || !seen_identifiers.insert(identifier) }); if has_unexpected { - warn!("Peer {peer_id} returned unrequested blob sidecar(s)"); + warn!( + "Peer {peer_id} returned an unrequested or duplicate blob sidecar" + ); for identifier in blob_identifiers.iter() { block_cache.mark_attempted( RequestKey::Blob(*identifier), @@ -1305,14 +1320,18 @@ fn poll_ready_tasks( let requested: HashSet = identifiers.iter().copied().collect(); + let mut seen_identifiers = HashSet::new(); let has_unexpected = columns.iter().any(|column| { - !requested.contains(&ColumnIdentifier::new( + let identifier = ColumnIdentifier::new( column.signed_block_header.message.tree_hash_root(), column.index, - )) + ); + !requested.contains(&identifier) || !seen_identifiers.insert(identifier) }); if has_unexpected { - warn!("Peer {peer_id} returned unrequested data column(s)"); + warn!( + "Peer {peer_id} returned an unrequested or duplicate data column" + ); for identifier in identifiers.iter() { block_cache.mark_attempted( RequestKey::Column(*identifier), @@ -1406,6 +1425,33 @@ mod tests { use super::*; + #[test] + fn should_abort_for_candidate_exhaustion_waits_for_the_timeout() { + assert!(!should_abort_for_candidate_exhaustion( + CANDIDATE_EXHAUSTION_TIMEOUT - Duration::from_secs(1), + false, + )); + assert!(should_abort_for_candidate_exhaustion( + CANDIDATE_EXHAUSTION_TIMEOUT, + false, + )); + } + + #[test] + fn should_abort_for_candidate_exhaustion_never_fires_with_a_task_still_in_flight() { + assert!(!should_abort_for_candidate_exhaustion( + CANDIDATE_EXHAUSTION_TIMEOUT + Duration::from_secs(60), + true, + )); + } + + #[test] + fn candidate_exhaustion_deserves_backoff_only_when_the_target_was_ahead() { + assert!(candidate_exhaustion_deserves_backoff(100, 50)); + assert!(!candidate_exhaustion_deserves_backoff(50, 50)); + assert!(!candidate_exhaustion_deserves_backoff(50, 100)); + } + #[test] fn range_blob_sidecars_build_valid_columns_and_reject_bad_proofs() { let blob = Blob::default(); @@ -1765,6 +1811,62 @@ mod tests { ); } + #[test] + fn poll_ready_tasks_duplicate_block_root_bans_peer_and_discards_whole_batch() { + initialize_test_network_spec(); + let executor = ReamExecutor::new().expect("executor should start"); + let (mut peer_manager, peer_id) = test_peer_manager_with_one_peer(); + + let block_a = SignedBeaconBlock { + message: BeaconBlock { + slot: 1, + ..Default::default() + }, + signature: Default::default(), + }; + let requested_root = block_a.message.tree_hash_root(); + let block_b = SignedBeaconBlock { + message: BeaconBlock { + slot: 1, + ..Default::default() + }, + signature: Default::default(), + }; + + let mut block_cache = BlockCache::new(B256::ZERO, 0); + let handle = executor.spawn(async move { StreamOutcome::Complete(vec![block_a, block_b]) }); + let mut tasks = vec![DownloadTask::new_block_roots( + handle, + vec![requested_root], + peer_id, + )]; + + poll_until_done( + &mut tasks, + &mut block_cache, + &mut peer_manager, + &HashSet::new(), + &mut false, + &[peer_id], + ); + + assert_eq!( + block_cache.block_count(), + 0, + "a response returning the same requested root twice is atomically discarded" + ); + assert!( + block_cache + .attempted_peers_for(RequestKey::BlockRoot(requested_root)) + .contains(&peer_id), + "the requested root must still be attributed to this peer's failed attempt" + ); + assert!( + peer_manager.fetch_idle_peer_from(&[peer_id]).is_none(), + "a peer that returned a duplicate item must be banned, not just marked idle" + ); + } + #[test] fn poll_ready_tasks_empty_data_column_range_falls_through_to_by_root_and_excludes_peer() { initialize_test_network_spec(); @@ -1878,4 +1980,142 @@ mod tests { "a peer that served an invalid blob proof must be banned, not just marked idle" ); } + + #[test] + fn poll_ready_tasks_duplicate_blob_identifier_bans_peer_and_discards_whole_batch() { + initialize_test_network_spec(); + let executor = ReamExecutor::new().expect("executor should start"); + let (mut peer_manager, peer_id) = test_peer_manager_with_one_peer(); + + let blob = Blob::default(); + let blob_bytes = blob.to_fixed_bytes(); + let raw_commitment = das_context() + .blob_to_kzg_commitment(&blob_bytes) + .expect("test blob should produce a commitment"); + let commitment = KZGCommitment(raw_commitment); + let proof = KZGProof::from( + compute_blob_kzg_proof_raw( + blob_bytes, + raw_commitment, + ream_polynomial_commitments::trusted_setup::blst_settings(), + ) + .expect("test blob should produce a proof") + .to_bytes(), + ); + let mut block = SignedBeaconBlock { + message: Default::default(), + signature: Default::default(), + }; + block + .message + .body + .blob_kzg_commitments + .push(commitment) + .expect("one commitment should fit"); + let block_root = block.message.tree_hash_root(); + let identifier = BlobIdentifier::new(block_root, 0); + let sidecar = block + .blob_sidecar(BlobAndProofV1 { blob, proof }, 0) + .expect("test sidecar should be constructed"); + + let mut block_cache = BlockCache::new(B256::ZERO, 0); + block_cache + .add_blocks(vec![block], false) + .expect("block should enter cache"); + + let handle = + executor.spawn(async move { StreamOutcome::Complete(vec![sidecar.clone(), sidecar]) }); + let mut tasks = vec![DownloadTask::new_blob_identifiers( + handle, + vec![identifier], + peer_id, + )]; + + poll_until_done( + &mut tasks, + &mut block_cache, + &mut peer_manager, + &HashSet::new(), + &mut false, + &[peer_id], + ); + + assert!( + block_cache + .attempted_peers_for(RequestKey::Blob(identifier)) + .contains(&peer_id), + "the requested identifier must still be attributed to this peer's failed attempt" + ); + assert!( + peer_manager.fetch_idle_peer_from(&[peer_id]).is_none(), + "a peer that returned a duplicate blob sidecar must be banned, not just marked idle" + ); + } + + #[test] + fn poll_ready_tasks_duplicate_data_column_bans_peer_and_discards_whole_batch() { + use ream_consensus_beacon::data_column_sidecar::get_data_column_sidecars_from_block; + + initialize_test_network_spec(); + let executor = ReamExecutor::new().expect("executor should start"); + let (mut peer_manager, peer_id) = test_peer_manager_with_one_peer(); + + let blob = Blob::default(); + let blob_bytes = blob.to_fixed_bytes(); + let raw_commitment = das_context() + .blob_to_kzg_commitment(&blob_bytes) + .expect("test blob should produce a commitment"); + let commitment = KZGCommitment(raw_commitment); + let mut block = SignedBeaconBlock { + message: Default::default(), + signature: Default::default(), + }; + block + .message + .body + .blob_kzg_commitments + .push(commitment) + .expect("one commitment should fit"); + let block_root = block.message.tree_hash_root(); + + let cells_and_kzg_proofs = compute_cells_and_kzg_proofs(&blob, das_context()) + .expect("test blob should produce cells and proofs"); + let columns = get_data_column_sidecars_from_block(&block, vec![cells_and_kzg_proofs]) + .expect("test block should produce data columns"); + let column = columns[0].clone(); + let identifier = ColumnIdentifier::new(block_root, column.index); + + let mut block_cache = BlockCache::new(B256::ZERO, 0); + block_cache + .add_blocks(vec![block], false) + .expect("block should enter cache"); + + let handle = + executor.spawn(async move { StreamOutcome::Complete(vec![column.clone(), column]) }); + let mut tasks = vec![DownloadTask::new_data_column_identifiers( + handle, + vec![identifier], + peer_id, + )]; + + poll_until_done( + &mut tasks, + &mut block_cache, + &mut peer_manager, + &HashSet::from([identifier.index]), + &mut false, + &[peer_id], + ); + + assert!( + block_cache + .attempted_peers_for(RequestKey::Column(identifier)) + .contains(&peer_id), + "the requested identifier must still be attributed to this peer's failed attempt" + ); + assert!( + peer_manager.fetch_idle_peer_from(&[peer_id]).is_none(), + "a peer that returned a duplicate data column must be banned, not just marked idle" + ); + } } From ac36815ec60a2f33e22e386e396373f14db9cff8 Mon Sep 17 00:00:00 2001 From: vuonghuuhung Date: Fri, 4 Sep 2026 00:32:35 +0700 Subject: [PATCH 07/17] feat: add stuck/fork recovery and peer-quality tracking to range sync --- Cargo.lock | 1 + crates/networking/manager/src/p2p_sender.rs | 19 + crates/networking/manager/src/req_resp.rs | 148 +- .../req_resp/src/beacon/messages/mod.rs | 13 +- crates/networking/syncer/Cargo.toml | 1 + .../syncer/src/block_range/block_cache.rs | 380 ++++- .../networking/syncer/src/block_range/mod.rs | 1487 +++++++++++++++-- .../syncer/src/block_range/peer_manager.rs | 135 +- .../syncer/src/block_range/recovery.rs | 844 ++++++++++ 9 files changed, 2815 insertions(+), 213 deletions(-) create mode 100644 crates/networking/syncer/src/block_range/recovery.rs diff --git a/Cargo.lock b/Cargo.lock index c68c0d06f..8bcbc1f3e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7184,6 +7184,7 @@ dependencies = [ "ream-consensus-misc", "ream-execution-rpc-types", "ream-executor", + "ream-fork-choice-beacon", "ream-network-spec", "ream-operation-pool", "ream-p2p", diff --git a/crates/networking/manager/src/p2p_sender.rs b/crates/networking/manager/src/p2p_sender.rs index 7f559be22..b1386b8ae 100644 --- a/crates/networking/manager/src/p2p_sender.rs +++ b/crates/networking/manager/src/p2p_sender.rs @@ -90,4 +90,23 @@ impl P2PSender { warn!("Failed to send error response: {err}"); } } + + pub fn send_invalid_request( + &self, + peer_id: PeerId, + connection_id: ConnectionId, + stream_id: u64, + error: &str, + ) { + if let Err(err) = self.0.send(P2PMessage::Response(P2PResponse { + peer_id, + connection_id, + stream_id, + message: Box::new(RespMessage::Error(ReqRespError::InvalidData( + error.to_string(), + ))), + })) { + warn!("Failed to send invalid-request response: {err}"); + } + } } diff --git a/crates/networking/manager/src/req_resp.rs b/crates/networking/manager/src/req_resp.rs index e5ef0eb6e..a6c9c18fa 100644 --- a/crates/networking/manager/src/req_resp.rs +++ b/crates/networking/manager/src/req_resp.rs @@ -3,11 +3,18 @@ use std::sync::Arc; use libp2p::{PeerId, swarm::ConnectionId}; use ream_consensus_beacon::{blob_sidecar::BlobIdentifier, data_column_sidecar::ColumnIdentifier}; use ream_p2p::network::beacon::network_state::NetworkState; -use ream_req_resp::beacon::messages::{ - BeaconRequestMessage, BeaconResponseMessage, - blob_sidecars::{BlobSidecarsByRangeV1Request, BlobSidecarsByRootV1Request}, - blocks::{BeaconBlocksByRangeV2Request, BeaconBlocksByRootV2Request}, - data_column_sidecars::{DataColumnSidecarsByRangeV1Request, DataColumnSidecarsByRootV1Request}, +use ream_req_resp::{ + beacon::messages::{ + BeaconRequestMessage, BeaconResponseMessage, + blob_sidecars::{BlobSidecarsByRangeV1Request, BlobSidecarsByRootV1Request}, + blocks::{BeaconBlocksByRangeV2Request, BeaconBlocksByRootV2Request}, + data_column_sidecars::{ + DataColumnSidecarsByRangeV1Request, DataColumnSidecarsByRootV1Request, + }, + }, + constants::{ + MAX_REQUEST_BLOCKS, MAX_REQUEST_BLOCKS_DENEB, MAX_REQUEST_DATA_COLUMN_SIDECARS_PER_COLUMN, + }, }; use ream_storage::{ db::beacon::BeaconDB, @@ -50,16 +57,41 @@ pub async fn handle_req_resp_message( count, .. }) => { - for slot in start_slot..start_slot + count { - let Ok(Some(block_root)) = ream_db.slot_index_provider().get(slot) else { - trace!("No block root found for slot {slot}"); - p2p_sender.send_error_response( - peer_id, - connection_id, - stream_id, - &format!("No block root found for slot {slot}"), - ); - return; + if count > MAX_REQUEST_BLOCKS { + p2p_sender.send_invalid_request( + peer_id, + connection_id, + stream_id, + &format!("Requested count {count} exceeds MAX_REQUEST_BLOCKS"), + ); + return; + } + let Some(end_slot_exclusive) = start_slot.checked_add(count) else { + p2p_sender.send_invalid_request( + peer_id, + connection_id, + stream_id, + &format!("start_slot {start_slot} + count {count} overflows"), + ); + return; + }; + + for slot in start_slot..end_slot_exclusive { + let block_root = match ream_db.slot_index_provider().get(slot) { + Ok(Some(block_root)) => block_root, + Ok(None) => { + trace!("No block root found for slot {slot}"); + continue; + } + Err(err) => { + p2p_sender.send_error_response( + peer_id, + connection_id, + stream_id, + &format!("Failed to read slot index for slot {slot}: {err:?}"), + ); + return; + } }; let Ok(Some(block)) = ream_db.block_provider().get(block_root) else { trace!("No block found for root {block_root}"); @@ -109,16 +141,41 @@ pub async fn handle_req_resp_message( start_slot, count, }) => { - for slot in start_slot..start_slot + count { - let Ok(Some(block_root)) = ream_db.slot_index_provider().get(slot) else { - trace!("No block root found for slot {slot}"); - p2p_sender.send_error_response( - peer_id, - connection_id, - stream_id, - &format!("No block root found for slot {slot}"), - ); - return; + if count > MAX_REQUEST_BLOCKS_DENEB { + p2p_sender.send_invalid_request( + peer_id, + connection_id, + stream_id, + &format!("Requested count {count} exceeds MAX_REQUEST_BLOCKS_DENEB"), + ); + return; + } + let Some(end_slot_exclusive) = start_slot.checked_add(count) else { + p2p_sender.send_invalid_request( + peer_id, + connection_id, + stream_id, + &format!("start_slot {start_slot} + count {count} overflows"), + ); + return; + }; + + for slot in start_slot..end_slot_exclusive { + let block_root = match ream_db.slot_index_provider().get(slot) { + Ok(Some(block_root)) => block_root, + Ok(None) => { + trace!("No block root found for slot {slot}"); + continue; + } + Err(err) => { + p2p_sender.send_error_response( + peer_id, + connection_id, + stream_id, + &format!("Failed to read slot index for slot {slot}: {err:?}"), + ); + return; + } }; let Ok(Some(block)) = ream_db.block_provider().get(block_root) else { trace!("No block found for root {block_root}"); @@ -232,10 +289,43 @@ pub async fn handle_req_resp_message( count, columns, }) => { - for slot in start_slot..start_slot + count { - let Ok(Some(block_root)) = ream_db.slot_index_provider().get(slot) else { - trace!("No block root found for slot {slot}"); - continue; + if count > MAX_REQUEST_DATA_COLUMN_SIDECARS_PER_COLUMN { + p2p_sender.send_invalid_request( + peer_id, + connection_id, + stream_id, + &format!( + "Requested count {count} exceeds MAX_REQUEST_DATA_COLUMN_SIDECARS_PER_COLUMN" + ), + ); + return; + } + let Some(end_slot_exclusive) = start_slot.checked_add(count) else { + p2p_sender.send_invalid_request( + peer_id, + connection_id, + stream_id, + &format!("start_slot {start_slot} + count {count} overflows"), + ); + return; + }; + + for slot in start_slot..end_slot_exclusive { + let block_root = match ream_db.slot_index_provider().get(slot) { + Ok(Some(block_root)) => block_root, + Ok(None) => { + trace!("No block root found for slot {slot}"); + continue; + } + Err(err) => { + p2p_sender.send_error_response( + peer_id, + connection_id, + stream_id, + &format!("Failed to read slot index for slot {slot}: {err:?}"), + ); + return; + } }; for &column_index in &columns { diff --git a/crates/networking/req_resp/src/beacon/messages/mod.rs b/crates/networking/req_resp/src/beacon/messages/mod.rs index 515c81c81..11d960e03 100644 --- a/crates/networking/req_resp/src/beacon/messages/mod.rs +++ b/crates/networking/req_resp/src/beacon/messages/mod.rs @@ -110,14 +110,17 @@ impl BeaconRequestMessage { request.count.min(MAX_REQUEST_BLOCKS) } BeaconRequestMessage::BeaconBlocksByRoot(request) => request.inner.len() as u64, - BeaconRequestMessage::BlobSidecarsByRange(request) => { - (request.count * MAX_BLOBS_PER_BLOCK).min(MAX_REQUEST_BLOB_SIDECARS) - } + BeaconRequestMessage::BlobSidecarsByRange(request) => request + .count + .saturating_mul(MAX_BLOBS_PER_BLOCK) + .min(MAX_REQUEST_BLOB_SIDECARS), BeaconRequestMessage::BlobSidecarsByRoot(request) => request.inner.len() as u64, BeaconRequestMessage::DataColumnSidecarsByRange(request) => { let num_columns = request.columns.len() as u64; - (request.count * num_columns) - .min(MAX_REQUEST_DATA_COLUMN_SIDECARS_PER_COLUMN * num_columns) + request + .count + .saturating_mul(num_columns) + .min(MAX_REQUEST_DATA_COLUMN_SIDECARS_PER_COLUMN.saturating_mul(num_columns)) } BeaconRequestMessage::DataColumnSidecarsByRoot(request) => { request.inner.iter().map(|id| id.columns.len() as u64).sum() diff --git a/crates/networking/syncer/Cargo.toml b/crates/networking/syncer/Cargo.toml index 6e09872f3..6819d3c97 100644 --- a/crates/networking/syncer/Cargo.toml +++ b/crates/networking/syncer/Cargo.toml @@ -31,6 +31,7 @@ ream-chain-beacon.workspace = true ream-consensus-beacon.workspace = true ream-consensus-misc.workspace = true ream-executor.workspace = true +ream-fork-choice-beacon.workspace = true ream-network-spec.workspace = true ream-p2p.workspace = true ream-polynomial-commitments.workspace = true diff --git a/crates/networking/syncer/src/block_range/block_cache.rs b/crates/networking/syncer/src/block_range/block_cache.rs index fca93c42f..35e56cdf9 100644 --- a/crates/networking/syncer/src/block_range/block_cache.rs +++ b/crates/networking/syncer/src/block_range/block_cache.rs @@ -20,10 +20,67 @@ use ream_polynomial_commitments::handlers::{ use ssz::Encode; use tree_hash::TreeHash; -use super::{MAX_BLOCKS_PER_REQUEST, peer_range_downloader::Range}; +use super::{FrontierObservation, MAX_BLOCKS_PER_REQUEST, peer_range_downloader::Range}; const ATTEMPT_COOLDOWN: Duration = Duration::from_secs(30); +#[derive(Debug, Clone)] +pub struct CoverageAnchor { + pub frontier: FrontierObservation, + pub original_parent_root: B256, + pub original_parent_slot: u64, + pub covered_through_slot: u64, + pub confirming_peers: HashSet, +} + +#[derive(Debug)] +pub enum AddBlocksError { + InvalidBatch(anyhow::Error), + CoverageDivergence { + expected_parent: B256, + actual_parent: B256, + anchor: Box, + }, +} + +impl std::fmt::Display for AddBlocksError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + AddBlocksError::InvalidBatch(err) => write!(f, "invalid batch: {err:?}"), + AddBlocksError::CoverageDivergence { + expected_parent, + actual_parent, + .. + } => write!( + f, + "coverage divergence: expected parent {expected_parent}, got {actual_parent}" + ), + } + } +} + +impl From for AddBlocksError { + fn from(err: anyhow::Error) -> Self { + AddBlocksError::InvalidBatch(err) + } +} + +pub(super) fn validate_range_chain(blocks: &[SignedBeaconBlock]) -> anyhow::Result<()> { + for (index, block) in blocks.iter().enumerate().rev() { + if index > 0 { + ensure!( + block.message.parent_root == blocks[index - 1].message.tree_hash_root(), + "Block at index {index} has a parent root that does not match the previous block's tree hash root", + ); + ensure!( + block.message.slot > blocks[index - 1].message.slot, + "Block at index {index} does not have a strictly increasing slot", + ); + } + } + Ok(()) +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum RequestKey { BlockRange(Range), @@ -43,14 +100,16 @@ pub struct BlockAndBlobBundle { pub block: SignedBeaconBlock, pub blobs: HashMap, pub columns: HashMap, + pub source_peer: PeerId, } impl BlockAndBlobBundle { - pub fn new(block: SignedBeaconBlock) -> Self { + pub fn new(block: SignedBeaconBlock, source_peer: PeerId) -> Self { Self { block, blobs: HashMap::new(), columns: HashMap::new(), + source_peer, } } } @@ -69,6 +128,7 @@ pub struct BlockCache { column_ranges_in_progress: HashSet, data_column_identifiers_in_progress: HashSet, attempts: HashMap, + coverage_anchor: Option, } impl BlockCache { @@ -87,9 +147,110 @@ impl BlockCache { column_ranges_in_progress: HashSet::new(), data_column_identifiers_in_progress: HashSet::new(), attempts: HashMap::new(), + coverage_anchor: None, } } + pub fn from_recovery_seed(seed: super::recovery::RecoverySeed) -> anyhow::Result { + let super::recovery::RecoverySeed { + ancestor_root, + ancestor_slot, + forward_blocks, + target_slot, + source_peer, + } = seed; + + ensure!( + !forward_blocks.is_empty(), + "a recovery seed must contain at least one new block" + ); + validate_range_chain(&forward_blocks)?; + + let first = &forward_blocks[0]; + ensure!( + first.message.parent_root == ancestor_root, + "recovery seed's first block does not connect to the ancestor root" + ); + ensure!( + first.message.slot > ancestor_slot, + "recovery seed's first block is not after the ancestor slot" + ); + + let mut seen_roots = HashSet::new(); + for block in &forward_blocks { + ensure!( + block.message.slot <= target_slot, + "recovery seed block at slot {} exceeds target_slot {target_slot}", + block.message.slot + ); + ensure!( + seen_roots.insert(block.message.tree_hash_root()), + "duplicate root in recovery seed" + ); + } + + let tip_slot = forward_blocks + .last() + .expect("checked non-empty above") + .message + .slot; + + let mut cache = BlockCache::new(ancestor_root, ancestor_slot); + cache + .add_blocks(forward_blocks, true, source_peer) + .map_err(|err| anyhow::anyhow!("{err}"))?; + cache.next_start_slot = tip_slot; + Ok(cache) + } + + fn is_pristine_for_advance(&self) -> bool { + self.blocks_and_blobs.is_empty() + && self.block_ranges_to_retry.is_empty() + && self.block_ranges_in_progress.is_empty() + && self.block_roots_in_progress.is_empty() + && self.blob_identifiers_in_progress.is_empty() + && self.column_ranges_to_fetch.is_empty() + && self.column_ranges_in_progress.is_empty() + && self.data_column_identifiers_in_progress.is_empty() + } + + pub fn is_pristine_for_restore(&self) -> bool { + self.coverage_anchor.is_none() && self.is_pristine_for_advance() + } + + pub fn advance_empty_coverage( + &mut self, + frontier: FrontierObservation, + parent_root: B256, + covered_through_slot: u64, + confirming_peers: HashSet, + ) -> anyhow::Result<()> { + match &mut self.coverage_anchor { + Some(anchor) => { + anchor.covered_through_slot = covered_through_slot; + anchor.confirming_peers.extend(confirming_peers); + } + None => { + ensure!( + self.is_pristine_for_advance(), + "advance_empty_coverage requires a pristine cache to start a new generation" + ); + self.coverage_anchor = Some(CoverageAnchor { + frontier, + original_parent_root: self.initial_parent_root, + original_parent_slot: self.initial_slot, + covered_through_slot, + confirming_peers, + }); + } + } + + self.initial_parent_root = parent_root; + self.next_start_slot = covered_through_slot; + self.initial_slot = covered_through_slot; + Ok(()) + } + fn refresh_attempt_round(&mut self, key: &RequestKey, now: Instant) { if let Some(state) = self.attempts.get_mut(key) && let Some(cooldown_until) = state.cooldown_until @@ -147,20 +308,24 @@ impl BlockCache { &mut self, blocks: Vec, is_range: bool, - ) -> anyhow::Result<()> { + source_peer: PeerId, + ) -> Result<(), AddBlocksError> { // Ensure that all blocks form a chain if is_range { - for (index, block) in blocks.iter().enumerate().rev() { - if index > 0 { - ensure!( - block.message.parent_root == blocks[index - 1].message.tree_hash_root(), - "Block at index {index} has a parent root that does not match the previous block's tree hash root", - ); - ensure!( - block.message.slot > blocks[index - 1].message.slot, - "Block at index {index} does not have a strictly increasing slot", - ); + validate_range_chain(&blocks)?; + + if self.blocks_and_blobs.is_empty() + && let Some(anchor) = &self.coverage_anchor + && let Some(first) = blocks.first() + { + if first.message.parent_root != self.initial_parent_root { + return Err(AddBlocksError::CoverageDivergence { + expected_parent: self.initial_parent_root, + actual_parent: first.message.parent_root, + anchor: Box::new(anchor.clone()), + }); } + self.coverage_anchor = None; } } @@ -168,7 +333,7 @@ impl BlockCache { self.current_cache_size += block.as_ssz_bytes().len() as u64; self.blocks_and_blobs.insert( block.message.tree_hash_root(), - BlockAndBlobBundle::new(block), + BlockAndBlobBundle::new(block, source_peer), ); } @@ -351,6 +516,10 @@ impl BlockCache { self.next_start_slot } + pub fn initial_parent_root(&self) -> B256 { + self.initial_parent_root + } + pub fn estimated_blocks_to_fetch(&self) -> u64 { if self.next_start_slot.saturating_sub(self.initial_slot) > 30 { return 0; @@ -406,7 +575,13 @@ impl BlockCache { required_columns: &HashSet, candidate_peers: &[PeerId], now: Instant, + frontier_tracked: bool, ) -> DataToFetch { + let single_flight = frontier_tracked || self.coverage_anchor.is_some(); + if single_flight && !self.block_ranges_in_progress.is_empty() { + return DataToFetch::DownloadsInProgress; + } + if let Some(range) = self.take_schedulable_retry_range(candidate_peers, now) { return DataToFetch::BlockRange(range); } @@ -655,7 +830,7 @@ mod tests { let mut cache = BlockCache::new(B256::ZERO, 10); assert_eq!( - cache.data_to_fetch(10, 0, &HashSet::new(), &[], Instant::now()), + cache.data_to_fetch(10, 0, &HashSet::new(), &[], Instant::now(), false), DataToFetch::Finished ); } @@ -670,13 +845,13 @@ mod tests { cache.mark_block_range_in_progress(range); assert_eq!( - cache.data_to_fetch(10, 0, &HashSet::new(), &[], Instant::now()), + cache.data_to_fetch(10, 0, &HashSet::new(), &[], Instant::now(), false), DataToFetch::DownloadsInProgress ); cache.remove_block_range_in_progress(&range); assert_eq!( - cache.data_to_fetch(10, 0, &HashSet::new(), &[], Instant::now()), + cache.data_to_fetch(10, 0, &HashSet::new(), &[], Instant::now(), false), DataToFetch::Finished ); } @@ -687,15 +862,15 @@ mod tests { let mut cache = BlockCache::new(B256::ZERO, 10); assert_eq!( - cache.data_to_fetch(25, 0, &HashSet::new(), &[], Instant::now()), + cache.data_to_fetch(25, 0, &HashSet::new(), &[], Instant::now(), false), DataToFetch::BlockRange(Range::new(11, 10)) ); assert_eq!( - cache.data_to_fetch(25, 0, &HashSet::new(), &[], Instant::now()), + cache.data_to_fetch(25, 0, &HashSet::new(), &[], Instant::now(), false), DataToFetch::BlockRange(Range::new(21, 5)) ); assert_eq!( - cache.data_to_fetch(25, 0, &HashSet::new(), &[], Instant::now()), + cache.data_to_fetch(25, 0, &HashSet::new(), &[], Instant::now(), false), DataToFetch::Finished ); } @@ -732,7 +907,11 @@ mod tests { let boundary_slot = compute_start_slot_at_epoch(boundary_epoch); let mut retained = BlockCache::new(parent_root, boundary_slot); retained - .add_blocks(vec![block_with_blob(boundary_slot)], false) + .add_blocks( + vec![block_with_blob(boundary_slot)], + false, + PeerId::random(), + ) .expect("boundary block should enter cache"); assert_eq!( retained.data_to_fetch( @@ -740,7 +919,8 @@ mod tests { current_epoch, &HashSet::new(), &[], - Instant::now() + Instant::now(), + false ), DataToFetch::Finished ); @@ -748,7 +928,7 @@ mod tests { let expired_slot = compute_start_slot_at_epoch(boundary_epoch - 1); let mut expired = BlockCache::new(parent_root, expired_slot); expired - .add_blocks(vec![block_with_blob(expired_slot)], false) + .add_blocks(vec![block_with_blob(expired_slot)], false, PeerId::random()) .expect("expired block should enter cache"); assert_eq!( expired.data_to_fetch( @@ -756,7 +936,8 @@ mod tests { current_epoch, &HashSet::new(), &[], - Instant::now() + Instant::now(), + false ), DataToFetch::Finished ); @@ -792,7 +973,7 @@ mod tests { let mut cache = BlockCache::new(parent_root, boundary_slot); cache - .add_blocks(vec![block], false) + .add_blocks(vec![block], false, PeerId::random()) .expect("boundary block should enter cache"); let required_columns = HashSet::from([1, 5]); @@ -802,6 +983,7 @@ mod tests { &required_columns, &[], Instant::now(), + false, ) { DataToFetch::MissingDataColumnIdentifiers(identifiers) => { let mut identifiers = identifiers; @@ -824,7 +1006,8 @@ mod tests { current_epoch, &HashSet::new(), &[], - Instant::now() + Instant::now(), + false ), DataToFetch::Finished ); @@ -860,7 +1043,7 @@ mod tests { let mut cache = BlockCache::new(B256::ZERO, 0); cache - .add_blocks(vec![block], false) + .add_blocks(vec![block], false, PeerId::random()) .expect("block should enter cache"); // Only column 0 was requested; extra columns in the response are ignored, not fatal. @@ -917,7 +1100,7 @@ mod tests { let mut cache = BlockCache::new(B256::ZERO, 0); cache - .add_blocks(vec![block], false) + .add_blocks(vec![block], false, PeerId::random()) .expect("block should enter cache"); let mut tampered = columns[1].clone(); @@ -1040,7 +1223,7 @@ mod tests { block.message.parent_root = B256::repeat_byte(2); let mut cache = BlockCache::new(parent_root, 1); cache - .add_blocks(vec![block], false) + .add_blocks(vec![block], false, PeerId::random()) .expect("block should enter cache"); let peer_a = PeerId::random(); @@ -1050,7 +1233,7 @@ mod tests { cache.mark_attempted(RequestKey::BlockRoot(missing_root), peer_a, &[peer_a], now); assert_eq!( - cache.data_to_fetch(1, 0, &HashSet::new(), &[peer_a], now), + cache.data_to_fetch(1, 0, &HashSet::new(), &[peer_a], now, false), DataToFetch::DownloadsInProgress ); } @@ -1080,14 +1263,14 @@ mod tests { let mut cache = BlockCache::new(parent_root, 2); cache - .add_blocks(vec![block1, block2], false) + .add_blocks(vec![block1, block2], false, PeerId::random()) .expect("blocks should enter cache"); let peer_a = PeerId::random(); let now = Instant::now(); cache.mark_attempted(RequestKey::BlockRoot(root_a), peer_a, &[peer_a], now); - match cache.data_to_fetch(2, 0, &HashSet::new(), &[peer_a], now) { + match cache.data_to_fetch(2, 0, &HashSet::new(), &[peer_a], now, false) { DataToFetch::MissingBlockRoots(roots) => { assert_eq!(roots, vec![root_b]); } @@ -1131,4 +1314,137 @@ mod tests { "a column for a block not yet in the cache must be dropped, not erroring the batch" ); } + + fn test_observation(anchor_root: B256, anchor_slot: u64) -> FrontierObservation { + FrontierObservation { + anchor_root, + anchor_slot, + phase: super::super::SyncPhase::Finalized, + scan_start_slot: anchor_slot, + target_slot: anchor_slot + 100, + } + } + + fn child_block(parent_root: B256, slot: u64) -> SignedBeaconBlock { + SignedBeaconBlock { + message: BeaconBlock { + slot, + parent_root, + ..Default::default() + }, + signature: Default::default(), + } + } + + #[test] + fn advance_empty_coverage_opens_a_coverage_anchor_that_a_connecting_block_clears() { + let root = B256::repeat_byte(1); + let mut cache = BlockCache::new(root, 10); + assert!(cache.coverage_anchor.is_none()); + + cache + .advance_empty_coverage(test_observation(root, 10), root, 20, HashSet::new()) + .expect("advancing a pristine cache should succeed"); + assert!( + cache.coverage_anchor.is_some(), + "a coverage advance must open an anchor pending confirmation" + ); + assert_eq!(cache.next_start_slot(), 20); + assert_eq!( + cache.initial_parent_root(), + root, + "an empty span never changes the anchor root" + ); + + cache + .add_blocks(vec![child_block(root, 21)], true, PeerId::random()) + .expect("a directly-connecting block must be accepted"); + assert!( + cache.coverage_anchor.is_none(), + "a connecting first block after the advance must clear the anchor" + ); + } + + #[test] + fn a_non_connecting_block_after_a_coverage_advance_is_reported_as_divergence_not_a_missing_parent() + { + let root = B256::repeat_byte(1); + let mut cache = BlockCache::new(root, 10); + cache + .advance_empty_coverage(test_observation(root, 10), root, 20, HashSet::new()) + .expect("advancing a pristine cache should succeed"); + + let wrong_parent = B256::repeat_byte(2); + let result = cache.add_blocks(vec![child_block(wrong_parent, 21)], true, PeerId::random()); + + match result { + Err(AddBlocksError::CoverageDivergence { + expected_parent, + actual_parent, + .. + }) => { + assert_eq!(expected_parent, root); + assert_eq!(actual_parent, wrong_parent); + } + other => panic!("expected CoverageDivergence, got {other:?}"), + } + assert_eq!( + cache.block_count(), + 0, + "a divergent batch must not be committed to the cache" + ); + } + + #[test] + fn advance_empty_coverage_extends_within_the_same_generation_without_losing_the_original_anchor() + { + let root = B256::repeat_byte(1); + let mut cache = BlockCache::new(root, 10); + cache + .advance_empty_coverage( + test_observation(root, 10), + root, + 20, + HashSet::from([PeerId::random()]), + ) + .expect("first advance should succeed"); + let second_peer = PeerId::random(); + cache + .advance_empty_coverage( + test_observation(root, 10), + root, + 30, + HashSet::from([second_peer]), + ) + .expect("a second advance within the same generation should succeed"); + + assert_eq!(cache.next_start_slot(), 30); + let anchor = cache + .coverage_anchor + .as_ref() + .expect("anchor should still be open"); + assert_eq!( + anchor.original_parent_root, root, + "the original pre-generation anchor must survive a second advance" + ); + assert_eq!(anchor.covered_through_slot, 30); + assert!(anchor.confirming_peers.contains(&second_peer)); + } + + #[test] + fn single_flight_blocks_a_second_block_range_while_a_frontier_is_tracked() { + let mut untracked = BlockCache::new(B256::ZERO, 10); + untracked.mark_block_range_in_progress(Range::new(11, 10)); + assert_ne!( + untracked.data_to_fetch(100, 0, &HashSet::new(), &[], Instant::now(), false), + DataToFetch::DownloadsInProgress, + ); + + let mut tracked = BlockCache::new(B256::ZERO, 10); + tracked.mark_block_range_in_progress(Range::new(11, 10)); + assert_eq!( + tracked.data_to_fetch(100, 0, &HashSet::new(), &[], Instant::now(), true), + DataToFetch::DownloadsInProgress, + ); + } } diff --git a/crates/networking/syncer/src/block_range/mod.rs b/crates/networking/syncer/src/block_range/mod.rs index fe9202f7a..8de851ef1 100644 --- a/crates/networking/syncer/src/block_range/mod.rs +++ b/crates/networking/syncer/src/block_range/mod.rs @@ -1,6 +1,7 @@ mod block_cache; mod peer_manager; mod peer_range_downloader; +mod recovery; use std::{ collections::HashSet, @@ -12,7 +13,7 @@ use std::{ use alloy_primitives::B256; use anyhow::{anyhow, bail, ensure}; -use block_cache::{BlockAndBlobBundle, BlockCache, DataToFetch, RequestKey}; +use block_cache::{AddBlocksError, BlockAndBlobBundle, BlockCache, DataToFetch, RequestKey}; use futures::task::noop_waker; use libp2p::PeerId; use peer_manager::{BanReason, MIN_SYNC_PEERS, PeerManager, TargetQualification, TargetSelection}; @@ -44,6 +45,7 @@ use ream_storage::tables::{ field::REDBField, table::{CustomTable, REDBTable}, }; +use recovery::{CoverageAdvance, RECOVERY_ROUND_TIMEOUT, RecoveryOutcome}; use tokio::{sync::mpsc::UnboundedSender, task::JoinHandle, time::sleep}; use tracing::{info, warn}; use tree_hash::TreeHash; @@ -60,6 +62,8 @@ const SLOT_IMPORT_TOLERANCE: u64 = 32; const ZERO_PROGRESS_BACKOFF: Duration = Duration::from_secs(30); +const RECOVERY_ATTEMPT_COOLDOWN: Duration = Duration::from_secs(30); + const CANDIDATE_EXHAUSTION_TIMEOUT: Duration = Duration::from_secs(60); fn should_abort_for_candidate_exhaustion(elapsed: Duration, tasks_in_flight: bool) -> bool { @@ -130,7 +134,7 @@ fn build_data_columns_from_blob_sidecars( .map_err(|err| anyhow!("Failed to build data columns for block {block_root}: {err}")) } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] enum SyncPhase { Finalized, Head, @@ -142,12 +146,300 @@ struct ActivePhaseTarget { qualification: TargetQualification, } +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct FrontierKey { + anchor_root: B256, + phase: SyncPhase, + scan_start_slot: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct FrontierId { + key: FrontierKey, + generation: u64, +} + +#[derive(Debug, Clone)] +struct FrontierObservation { + anchor_root: B256, + anchor_slot: u64, + phase: SyncPhase, + scan_start_slot: u64, + target_slot: u64, +} + +#[derive(Debug, Clone, Copy)] +struct EmptyCoverage { + start_slot: u64, + end_slot_exclusive: u64, +} + +#[derive(Debug, Clone)] +struct ConfirmedEmptyCoverage { + frontier_id: FrontierId, + origin_anchor: (B256, u64), + start_slot: u64, + end_slot_exclusive: u64, + confirming_peers: HashSet, +} + +impl ConfirmedEmptyCoverage { + fn merge(&mut self, coverage: EmptyCoverage, confirming_peers: &HashSet) -> bool { + if coverage.start_slot <= self.end_slot_exclusive + && coverage.end_slot_exclusive >= self.start_slot + { + self.end_slot_exclusive = self.end_slot_exclusive.max(coverage.end_slot_exclusive); + self.start_slot = self.start_slot.min(coverage.start_slot); + self.confirming_peers + .extend(confirming_peers.iter().copied()); + true + } else { + false + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +struct FailedRecoveryCandidate { + peer_id: PeerId, + candidate_root: B256, + target_bucket: u64, +} + +fn target_bucket(target_slot: u64) -> u64 { + target_slot / SLOTS_PER_EPOCH +} + +#[derive(Debug, Clone)] +struct StuckFrontier { + id: FrontierId, + anchor_root: B256, + anchor_slot: u64, + phase: SyncPhase, + scan_start_slot: u64, + highest_observed_target: u64, + consecutive_no_progress: u32, + attempted_peers: HashSet, + cooldown_until: Option, + recovery_round_not_before: Option, + failed_candidates: HashSet, + confirmed_empty_through: Option, +} + +impl StuckFrontier { + fn matches(&self, observation: &FrontierObservation) -> bool { + self.anchor_root == observation.anchor_root + && self.phase == observation.phase + && self.scan_start_slot == observation.scan_start_slot + } + + fn tier(&self) -> u8 { + match self.consecutive_no_progress { + 0..=2 => 0, + 3..=5 => 2, + _ => 3, + } + } + + fn refresh_attempt_round(&mut self, now: Instant) { + if let Some(cooldown_until) = self.cooldown_until + && now >= cooldown_until + { + self.attempted_peers.clear(); + self.failed_candidates.clear(); + self.cooldown_until = None; + } + } +} + +fn qualification_for(phase: SyncPhase, target_slot: u64) -> TargetQualification { + match phase { + SyncPhase::Finalized => TargetQualification::FinalizedEpoch(target_slot / SLOTS_PER_EPOCH), + SyncPhase::Head => TargetQualification::HeadEpoch(target_slot / SLOTS_PER_EPOCH), + } +} + +fn observation_from_frontier(frontier: &StuckFrontier) -> FrontierObservation { + FrontierObservation { + anchor_root: frontier.anchor_root, + anchor_slot: frontier.anchor_slot, + phase: frontier.phase, + scan_start_slot: frontier.scan_start_slot, + target_slot: frontier.highest_observed_target, + } +} + +struct FrontierStore<'a> { + finalized: &'a mut Option, + head: &'a mut Option, +} + +impl<'a> FrontierStore<'a> { + fn for_phase(&self, phase: SyncPhase) -> &Option { + match phase { + SyncPhase::Finalized => self.finalized, + SyncPhase::Head => self.head, + } + } + + fn for_phase_mut(&mut self, phase: SyncPhase) -> &mut Option { + match phase { + SyncPhase::Finalized => self.finalized, + SyncPhase::Head => self.head, + } + } +} + +struct CoverageDivergenceOutcome { + original_parent_root: B256, + original_parent_slot: u64, + frontier: FrontierObservation, + confirming_peers: HashSet, + non_connecting_peer: PeerId, +} + +struct ImportFailure { + imported_count: u64, + error: anyhow::Error, +} + +enum PollTasksOutcome { + Settled, + StaleGeneration, + CoverageDivergence(Box), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum RemoteNoProgressReason { + CandidateExhausted, + SettledEmptyCoverage, + ProbeNotFound, + UnconnectedProbe, + AncestorNotFound, + NoNewDescendants, + RecoveryBudgetExhausted, +} + +impl RemoteNoProgressReason { + fn precedence(self) -> u8 { + match self { + RemoteNoProgressReason::CandidateExhausted => 6, + RemoteNoProgressReason::RecoveryBudgetExhausted => 5, + RemoteNoProgressReason::AncestorNotFound | RemoteNoProgressReason::UnconnectedProbe => { + 4 + } + RemoteNoProgressReason::ProbeNotFound => 3, + RemoteNoProgressReason::NoNewDescendants => 2, + RemoteNoProgressReason::SettledEmptyCoverage => 1, + } + } +} + +#[derive(Debug, Clone)] +struct RemoteObservation { + observation: FrontierObservation, + reason: RemoteNoProgressReason, + implicated_peers: HashSet, + failed_candidates: HashSet, +} + +#[derive(Default)] +struct PendingConclusions { + finalized: Option, + head: Option, +} + +impl PendingConclusions { + fn for_phase(&self, phase: SyncPhase) -> &Option { + match phase { + SyncPhase::Finalized => &self.finalized, + SyncPhase::Head => &self.head, + } + } + + fn slot_mut(&mut self, phase: SyncPhase) -> &mut Option { + match phase { + SyncPhase::Finalized => &mut self.finalized, + SyncPhase::Head => &mut self.head, + } + } + + fn observe( + &mut self, + phase: SyncPhase, + observation: FrontierObservation, + reason: RemoteNoProgressReason, + implicated_peers: HashSet, + failed_candidates: HashSet, + ) { + let slot = self.slot_mut(phase); + match slot { + None => { + *slot = Some(RemoteObservation { + observation, + reason, + implicated_peers, + failed_candidates, + }); + } + Some(existing) => { + let target_slot = existing + .observation + .target_slot + .max(observation.target_slot); + if reason.precedence() >= existing.reason.precedence() { + existing.reason = reason; + existing.observation = observation; + } + existing.observation.target_slot = target_slot; + existing.implicated_peers.extend(implicated_peers); + existing.failed_candidates.extend(failed_candidates); + } + } + } + + fn observe_divergence(&mut self, phase: SyncPhase, divergence: &CoverageDivergenceOutcome) { + self.observe( + phase, + divergence.frontier.clone(), + RemoteNoProgressReason::UnconnectedProbe, + divergence.confirming_peers.clone(), + HashSet::new(), + ); + } +} + +#[derive(Default)] +struct SegmentExclusions { + finalized: HashSet, + head: HashSet, +} + +impl SegmentExclusions { + fn for_phase(&self, phase: SyncPhase) -> &HashSet { + match phase { + SyncPhase::Finalized => &self.finalized, + SyncPhase::Head => &self.head, + } + } + + fn for_phase_mut(&mut self, phase: SyncPhase) -> &mut HashSet { + match phase { + SyncPhase::Finalized => &mut self.finalized, + SyncPhase::Head => &mut self.head, + } + } +} + pub struct BlockRangeSyncer { pub beacon_chain: Arc, pub peer_manager: PeerManager, pub p2p_sender: UnboundedSender, pub executor: ReamExecutor, next_segment_not_before: Option, + next_generation: u64, + finalized_frontier: Option, + head_frontier: Option, } impl BlockRangeSyncer { @@ -163,6 +455,118 @@ impl BlockRangeSyncer { peer_manager: PeerManager::new(network_state), executor, next_segment_not_before: None, + next_generation: 0, + finalized_frontier: None, + head_frontier: None, + } + } + + fn allocate_generation(&mut self) -> u64 { + let generation = self.next_generation; + self.next_generation += 1; + generation + } + + fn frontier_for(&self, phase: SyncPhase) -> &Option { + match phase { + SyncPhase::Finalized => &self.finalized_frontier, + SyncPhase::Head => &self.head_frontier, + } + } + + fn frontier_for_mut(&mut self, phase: SyncPhase) -> &mut Option { + match phase { + SyncPhase::Finalized => &mut self.finalized_frontier, + SyncPhase::Head => &mut self.head_frontier, + } + } + + fn new_frontier(&mut self, observation: &FrontierObservation) -> StuckFrontier { + let key = FrontierKey { + anchor_root: observation.anchor_root, + phase: observation.phase, + scan_start_slot: observation.scan_start_slot, + }; + StuckFrontier { + id: FrontierId { + key, + generation: self.allocate_generation(), + }, + anchor_root: observation.anchor_root, + anchor_slot: observation.anchor_slot, + phase: observation.phase, + scan_start_slot: observation.scan_start_slot, + highest_observed_target: observation.target_slot, + consecutive_no_progress: 1, + attempted_peers: HashSet::new(), + cooldown_until: None, + recovery_round_not_before: None, + failed_candidates: HashSet::new(), + confirmed_empty_through: None, + } + } + + fn commit_pending_conclusions( + &mut self, + pending: PendingConclusions, + exclusions: SegmentExclusions, + ) { + for phase in [SyncPhase::Finalized, SyncPhase::Head] { + if let Some(record) = pending.for_phase(phase).clone() { + self.record_remote_no_progress(phase, record, exclusions.for_phase(phase).clone()); + } + } + } + + fn record_remote_no_progress( + &mut self, + phase: SyncPhase, + record: RemoteObservation, + excluded: HashSet, + ) { + let observation = &record.observation; + let mut frontier = match self.frontier_for(phase).clone() { + Some(existing) + if existing.matches(observation) + && observation.target_slot >= existing.highest_observed_target => + { + let mut existing = existing; + existing.consecutive_no_progress = + existing.consecutive_no_progress.saturating_add(1); + existing.highest_observed_target = observation.target_slot; + existing + } + _ => self.new_frontier(observation), + }; + frontier.attempted_peers.extend(record.implicated_peers); + frontier.attempted_peers.extend(excluded); + frontier.failed_candidates.extend(record.failed_candidates); + if !frontier.attempted_peers.is_empty() && frontier.cooldown_until.is_none() { + frontier.cooldown_until = Some(Instant::now() + RECOVERY_ATTEMPT_COOLDOWN); + } + if record.reason == RemoteNoProgressReason::RecoveryBudgetExhausted { + frontier.recovery_round_not_before = Some(Instant::now() + RECOVERY_ROUND_TIMEOUT); + } + *self.frontier_for_mut(phase) = Some(frontier); + } + + fn clear_tracker(&mut self) { + self.finalized_frontier = None; + self.head_frontier = None; + self.next_segment_not_before = None; + } + + fn reconcile_not_ahead(&mut self, observation: &FrontierObservation) { + let phase = observation.phase; + let should_clear = match self.frontier_for(phase) { + Some(frontier) if frontier.anchor_root != observation.anchor_root => true, + Some(frontier) => { + frontier.matches(observation) && observation.target_slot <= frontier.anchor_slot + } + None => false, + }; + if should_clear { + *self.frontier_for_mut(phase) = None; } } @@ -259,6 +663,13 @@ impl BlockRangeSyncer { let mut finalized_target_was_ahead = false; let mut candidate_exhausted_since: Option = None; + let scan_start_slot = head_slot; + let mut restore_window_open = true; + let mut recovery_window_open = true; + let mut recovery_decision_made_this_segment = false; + let mut pending_conclusions = PendingConclusions::default(); + let mut segment_exclusions = SegmentExclusions::default(); + loop { self.peer_manager.update_peer_set(); @@ -326,14 +737,7 @@ impl BlockRangeSyncer { SyncPhase::Head => true, }; if sufficient { - let qualification = match phase { - SyncPhase::Finalized => { - TargetQualification::FinalizedEpoch(target_slot / SLOTS_PER_EPOCH) - } - SyncPhase::Head => { - TargetQualification::HeadEpoch(target_slot / SLOTS_PER_EPOCH) - } - }; + let qualification = qualification_for(phase, *target_slot); active_target = Some(match active_target { Some(existing) if existing.slot > *target_slot => existing, _ => ActivePhaseTarget { @@ -349,18 +753,61 @@ impl BlockRangeSyncer { continue; }; + let observation = FrontierObservation { + anchor_root: head_root, + anchor_slot: head_slot, + phase, + scan_start_slot, + target_slot: target.slot, + }; + self.reconcile_not_ahead(&observation); + let candidate_peers = self.peer_manager.peers_satisfying(target.qualification); let now = Instant::now(); - poll_ready_tasks( - &mut task_handles, - &mut block_cache, - &mut self.peer_manager, - &required_columns, - &mut saw_empty_range, - &candidate_peers, - now, - )?; + let poll_outcome = { + let mut frontiers = FrontierStore { + finalized: &mut self.finalized_frontier, + head: &mut self.head_frontier, + }; + poll_ready_tasks( + &mut task_handles, + &mut block_cache, + &mut self.peer_manager, + &mut frontiers, + &required_columns, + &mut saw_empty_range, + &candidate_peers, + )? + }; + + if let PollTasksOutcome::CoverageDivergence(divergence) = poll_outcome { + let divergence_phase = divergence.frontier.phase; + block_cache = BlockCache::new( + divergence.original_parent_root, + divergence.original_parent_slot, + ); + let next_generation = self.allocate_generation(); + if let Some(frontier) = self.frontier_for_mut(divergence_phase) { + frontier.confirmed_empty_through = None; + frontier.id = FrontierId { + key: frontier.id.key, + generation: next_generation, + }; + } + debug_assert!( + !divergence + .confirming_peers + .contains(&divergence.non_connecting_peer), + "the peer that served the non-connecting block must never be excluded \ + alongside the peers that supplied the false coverage claim" + ); + segment_exclusions + .for_phase_mut(divergence_phase) + .extend(divergence.confirming_peers.clone()); + pending_conclusions.observe_divergence(divergence_phase, &divergence); + continue; + } if candidate_peers.is_empty() { let exhausted_since = *candidate_exhausted_since.get_or_insert_with(Instant::now); @@ -374,25 +821,121 @@ impl BlockRangeSyncer { if candidate_exhaustion_deserves_backoff(target.slot, head_slot) { self.next_segment_not_before = Some(Instant::now() + ZERO_PROGRESS_BACKOFF); } + pending_conclusions.observe( + phase, + observation.clone(), + RemoteNoProgressReason::CandidateExhausted, + HashSet::new(), + HashSet::new(), + ); break; } } else { candidate_exhausted_since = None; } + if restore_window_open { + if !block_cache.is_pristine_for_restore() { + restore_window_open = false; + } else if let Some(frontier) = self.frontier_for(phase) + && frontier.matches(&observation) + && let Some(coverage) = &frontier.confirmed_empty_through + { + debug_assert_eq!( + coverage.frontier_id, frontier.id, + "confirmed_empty_through must always belong to the frontier holding it" + ); + debug_assert_eq!( + coverage.origin_anchor.0, + block_cache.initial_parent_root(), + "a pristine cache being restored into must still share this coverage's original anchor root" + ); + let frontier_observation = observation_from_frontier(frontier); + let parent_root = block_cache.initial_parent_root(); + let covered_through_slot = coverage.end_slot_exclusive.saturating_sub(1); + let confirming_peers = coverage.confirming_peers.clone(); + let _ = block_cache.advance_empty_coverage( + frontier_observation, + parent_root, + covered_through_slot, + confirming_peers, + ); + restore_window_open = false; + } + } + + if let Some(frontier) = self.frontier_for_mut(phase) { + frontier.refresh_attempt_round(now); + } + + if recovery_window_open + && !recovery_decision_made_this_segment + && let Some(frontier) = self.frontier_for(phase) + && frontier.matches(&observation) + && frontier.tier() >= 2 + { + recovery_decision_made_this_segment = true; + recovery_window_open = false; + let recovery_round_not_before = frontier.recovery_round_not_before; + if recovery_round_not_before.is_none_or(|not_before| now >= not_before) { + let mut excluded = segment_exclusions.for_phase(phase).clone(); + excluded.extend(frontier.attempted_peers.iter().copied()); + match self + .run_recovery(&observation, &candidate_peers, &excluded) + .await? + { + RecoveryOutcome::AdvancedCoverage(advance) => { + block_cache.advance_empty_coverage( + observation.clone(), + advance.parent_root, + advance.covered_through_slot, + advance.confirming_peers.clone(), + )?; + self.merge_confirmed_empty_coverage(phase, &advance); + continue; + } + RecoveryOutcome::Seeded(seed) => { + block_cache = BlockCache::from_recovery_seed(seed)?; + continue; + } + RecoveryOutcome::NoProgress { + reason, + implicated_peers, + failed_candidates, + } => { + pending_conclusions.observe( + phase, + observation.clone(), + reason, + implicated_peers, + failed_candidates, + ); + } + } + } + } + let current_epoch = self .beacon_chain .store .lock() .await .get_current_store_epoch()?; + let frontier_tracked = self.frontier_for(phase).is_some(); let data_to_fetch = block_cache.data_to_fetch( target.slot, current_epoch, &required_columns, &candidate_peers, now, + frontier_tracked, ); + if !matches!( + data_to_fetch, + DataToFetch::DownloadsInProgress | DataToFetch::Finished + ) { + recovery_window_open = false; + } info!( "Forward sync status: Downloaded Blocks {}, Downloaded Blobs {}/{}, Stage {data_to_fetch}", block_cache.block_count(), @@ -415,6 +958,10 @@ impl BlockRangeSyncer { continue; }; + let frontier_id = self + .frontier_for(phase) + .as_ref() + .map(|frontier| frontier.id); block_cache.mark_block_range_in_progress(range); task_handles.push(DownloadTask::new_block_range( PeerRangeDownloader::start( @@ -425,6 +972,7 @@ impl BlockRangeSyncer { ), range, peer.peer_id, + frontier_id, )); } DataToFetch::DataColumnRange(range) => { @@ -603,6 +1151,16 @@ impl BlockRangeSyncer { finalized_phase_settled = true; continue; } + if block_cache.block_count() == 0 && block_cache.next_start_slot() < target.slot + { + pending_conclusions.observe( + phase, + observation.clone(), + RemoteNoProgressReason::SettledEmptyCoverage, + HashSet::new(), + HashSet::new(), + ); + } break; } } @@ -614,101 +1172,20 @@ impl BlockRangeSyncer { block_cache.downloaded_blob_count(), ); - let fulu_fork_epoch = beacon_network_spec().fulu_fork_epoch; - let mut imported_count: u64 = 0; - - // execute all the blocks downloaded - for BlockAndBlobBundle { - block, - blobs, - columns, - } in block_cache.get_blocks_and_blobs()? - { - info!("Processing block with slot {}", block.message.slot,); + let import_result = self.import_downloaded(block_cache).await; + let imported_count = match &import_result { + Ok(count) => *count, + Err(failure) => failure.imported_count, + }; - let (block, columns) = if block.message.body.blob_kzg_commitments.is_empty() { - ensure!( - blobs.is_empty(), - "Range-sync block without blob commitments had downloaded blob sidecars" - ); - ensure!( - columns.is_empty(), - "Range-sync block without blob commitments had downloaded data columns" - ); - (block, Vec::new()) - } else if compute_epoch_at_slot(block.message.slot) >= fulu_fork_epoch { - let required_columns = self - .beacon_chain - .store - .lock() - .await - .data_availability_checker - .required_columns() - .clone(); - let columns = columns - .into_values() - .filter(|column| required_columns.contains(&column.index)) - .collect::>(); - (block, columns) - } else { - let (blobs_provider, required_columns, verify_data_availability) = { - let store = self.beacon_chain.store.lock().await; - let network_spec = beacon_network_spec(); - ( - store.db.blobs_and_proofs_provider(), - store.data_availability_checker.required_columns().clone(), - is_data_availability_check_required( - compute_epoch_at_slot(block.message.slot), - store.get_current_store_epoch()?, - network_spec.fulu_fork_epoch, - network_spec.min_epochs_for_data_column_sidecars_requests, - ), - ) - }; - tokio::task::spawn_blocking(move || { - let columns = build_data_columns_from_blob_sidecars( - &block, - &blobs, - verify_data_availability, - )? - .into_iter() - .filter(|column| required_columns.contains(&column.index)) - .collect::>(); - for (identifier, sidecar) in blobs { - blobs_provider.insert(identifier, sidecar.into())?; - } - Ok::<_, anyhow::Error>((block, columns)) - }) - .await - .map_err(|err| anyhow!("Range-sync data-column task failed: {err}"))?? - }; + info!("All blocks processed successfully."); - match self.beacon_chain.process_block(block).await? { - BlockProcessingOutcome::Imported { .. } => {} - BlockProcessingOutcome::PendingAvailability { block_root } => { - for column in columns { - self.beacon_chain - .import_data_column_sidecar_if(column, |_| Ok(())) - .await?; - } - ensure!( - self.beacon_chain - .store - .lock() - .await - .db - .block_provider() - .get(block_root)? - .is_some(), - "Range-sync block {block_root} remained pending after processing its downloaded data" - ); - } - } - imported_count += 1; + if imported_count > 0 { + self.clear_tracker(); + } else if import_result.is_ok() { + self.commit_pending_conclusions(pending_conclusions, segment_exclusions); } - info!("All blocks processed successfully."); - let target_was_ahead = finalized_target_was_ahead || active_target .as_ref() @@ -717,8 +1194,147 @@ impl BlockRangeSyncer { self.next_segment_not_before = Some(Instant::now() + ZERO_PROGRESS_BACKOFF); } + import_result.map(|_| ()).map_err(|failure| failure.error) + } + + async fn import_downloaded(&mut self, block_cache: BlockCache) -> Result { + let bundles = block_cache + .get_blocks_and_blobs() + .map_err(|error| ImportFailure { + imported_count: 0, + error, + })?; + let mut imported_count = 0u64; + for bundle in bundles { + if let Err(error) = self.import_one(bundle).await { + return Err(ImportFailure { + imported_count, + error, + }); + } + imported_count += 1; + } + Ok(imported_count) + } + + async fn import_one(&mut self, bundle: BlockAndBlobBundle) -> anyhow::Result<()> { + let BlockAndBlobBundle { + block, + blobs, + columns, + source_peer, + } = bundle; + let fulu_fork_epoch = beacon_network_spec().fulu_fork_epoch; + info!("Processing block with slot {}", block.message.slot); + + let (block, columns) = if block.message.body.blob_kzg_commitments.is_empty() { + ensure!( + blobs.is_empty(), + "Range-sync block without blob commitments had downloaded blob sidecars" + ); + ensure!( + columns.is_empty(), + "Range-sync block without blob commitments had downloaded data columns" + ); + (block, Vec::new()) + } else if compute_epoch_at_slot(block.message.slot) >= fulu_fork_epoch { + let required_columns = self + .beacon_chain + .store + .lock() + .await + .data_availability_checker + .required_columns() + .clone(); + let columns = columns + .into_values() + .filter(|column| required_columns.contains(&column.index)) + .collect::>(); + (block, columns) + } else { + let (blobs_provider, required_columns, verify_data_availability) = { + let store = self.beacon_chain.store.lock().await; + let network_spec = beacon_network_spec(); + ( + store.db.blobs_and_proofs_provider(), + store.data_availability_checker.required_columns().clone(), + is_data_availability_check_required( + compute_epoch_at_slot(block.message.slot), + store.get_current_store_epoch()?, + network_spec.fulu_fork_epoch, + network_spec.min_epochs_for_data_column_sidecars_requests, + ), + ) + }; + tokio::task::spawn_blocking(move || { + let columns = build_data_columns_from_blob_sidecars( + &block, + &blobs, + verify_data_availability, + )? + .into_iter() + .filter(|column| required_columns.contains(&column.index)) + .collect::>(); + for (identifier, sidecar) in blobs { + blobs_provider.insert(identifier, sidecar.into())?; + } + Ok::<_, anyhow::Error>((block, columns)) + }) + .await + .map_err(|err| anyhow!("Range-sync data-column task failed: {err}"))?? + }; + + match self.beacon_chain.process_block(block).await? { + BlockProcessingOutcome::Imported { .. } => {} + BlockProcessingOutcome::PendingAvailability { block_root } => { + for column in columns { + self.beacon_chain + .import_data_column_sidecar_if(column, |_| Ok(())) + .await?; + } + ensure!( + self.beacon_chain + .store + .lock() + .await + .db + .block_provider() + .get(block_root)? + .is_some(), + "Range-sync block {block_root} remained pending after processing its downloaded data" + ); + } + } + self.peer_manager.record_processed_blocks(&source_peer, 1); Ok(()) } + + fn merge_confirmed_empty_coverage(&mut self, phase: SyncPhase, advance: &CoverageAdvance) { + if !advance.proven_empty { + return; + } + let Some(frontier) = self.frontier_for_mut(phase) else { + return; + }; + let coverage = EmptyCoverage { + start_slot: frontier.anchor_slot, + end_slot_exclusive: advance.covered_through_slot + 1, + }; + match &mut frontier.confirmed_empty_through { + Some(confirmed) => { + confirmed.merge(coverage, &advance.confirming_peers); + } + None => { + frontier.confirmed_empty_through = Some(ConfirmedEmptyCoverage { + frontier_id: frontier.id, + origin_anchor: (advance.parent_root, advance.parent_slot), + start_slot: coverage.start_slot, + end_slot_exclusive: coverage.end_slot_exclusive, + confirming_peers: advance.confirming_peers.clone(), + }); + } + } + } } fn group_by_block_root(identifiers: &[ColumnIdentifier]) -> Vec { @@ -748,6 +1364,7 @@ pub enum DownloadTask { handle: JoinHandle>>, range: Range, peer_id: PeerId, + frontier_id: Option, }, DataColumnRange { handle: JoinHandle>>, @@ -777,11 +1394,13 @@ impl DownloadTask { handle: JoinHandle>>, range: Range, peer_id: PeerId, + frontier_id: Option, ) -> Self { DownloadTask::BlockRange { handle, range, peer_id, + frontier_id, } } @@ -879,15 +1498,17 @@ fn poll_ready_tasks( tasks: &mut Vec, block_cache: &mut BlockCache, peer_manager: &mut PeerManager, + frontiers: &mut FrontierStore, required_columns: &HashSet, saw_empty_range: &mut bool, candidate_peers: &[PeerId], - now: Instant, -) -> anyhow::Result<()> { +) -> anyhow::Result { + let now = Instant::now(); let waker = noop_waker(); let mut context = Context::from_waker(&waker); let mut indexes_to_remove = vec![]; let fulu_fork_epoch = beacon_network_spec().fulu_fork_epoch; + let mut outcome = PollTasksOutcome::Settled; for index in (0..tasks.len()).rev() { let Some(task) = tasks.get_mut(index) else { @@ -899,6 +1520,7 @@ fn poll_ready_tasks( handle, range, peer_id, + frontier_id, } => { let pinned = Pin::new(handle); @@ -909,6 +1531,17 @@ fn poll_ready_tasks( peer_manager.mark_peer_as_idle(peer_id); let key = RequestKey::BlockRange(*range); + let matching_frontier = match frontier_id { + Some(id) => match frontiers.for_phase(id.key.phase).as_ref() { + Some(frontier) if frontier.id == *id => Some(frontier), + _ => { + outcome = PollTasksOutcome::StaleGeneration; + continue; + } + }, + None => None, + }; + let Some(blocks) = handle_stream_outcome(peer_manager, peer_id, result) else { block_cache.mark_attempted(key, *peer_id, candidate_peers, now); @@ -920,6 +1553,51 @@ fn poll_ready_tasks( info!("Received empty block range from peer: {peer_id}"); block_cache.clear_attempted(key); *saw_empty_range = true; + + if let Some(frontier) = matching_frontier { + let observation = observation_from_frontier(frontier); + let confirming_peers = HashSet::from([*peer_id]); + let parent_root = block_cache.initial_parent_root(); + let covered_through_slot = range.start_slot + range.count - 1; + if block_cache + .advance_empty_coverage( + observation.clone(), + parent_root, + covered_through_slot, + confirming_peers.clone(), + ) + .is_ok() + && let Some(stored) = + frontiers.for_phase_mut(observation.phase).as_mut() + { + match &mut stored.confirmed_empty_through { + Some(confirmed) => { + confirmed.merge( + EmptyCoverage { + start_slot: range.start_slot, + end_slot_exclusive: range.start_slot + + range.count, + }, + &confirming_peers, + ); + } + None => { + stored.confirmed_empty_through = + Some(ConfirmedEmptyCoverage { + frontier_id: stored.id, + origin_anchor: ( + parent_root, + observation.anchor_slot, + ), + start_slot: range.start_slot, + end_slot_exclusive: range.start_slot + + range.count, + confirming_peers, + }); + } + } + } + } continue; } @@ -948,18 +1626,43 @@ fn poll_ready_tasks( && compute_epoch_at_slot(block.message.slot) >= fulu_fork_epoch }); - if let Err(err) = block_cache.add_blocks(blocks, true) { - warn!("Failed to add downloaded blocks to cache: {err:?}"); - block_cache.mark_attempted(key, *peer_id, candidate_peers, now); - block_cache.push_retry_range(*range); - peer_manager.ban_peer( - peer_id, - BanReason::ProtocolError(format!("invalid block range: {err:?}")), - ); - } else { - block_cache.clear_attempted(key); - if needs_columns { - block_cache.push_column_range(*range); + match block_cache.add_blocks(blocks, true, *peer_id) { + Err(AddBlocksError::CoverageDivergence { + expected_parent, + actual_parent, + anchor, + }) => { + warn!( + "Coverage divergence: block from {peer_id} has parent \ + {actual_parent}, expected {expected_parent} -- rolling \ + back the optimistic coverage generation" + ); + outcome = PollTasksOutcome::CoverageDivergence(Box::new( + CoverageDivergenceOutcome { + original_parent_root: anchor.original_parent_root, + original_parent_slot: anchor.original_parent_slot, + frontier: anchor.frontier, + confirming_peers: anchor.confirming_peers, + non_connecting_peer: *peer_id, + }, + )); + } + Err(AddBlocksError::InvalidBatch(err)) => { + warn!("Failed to add downloaded blocks to cache: {err:?}"); + block_cache.mark_attempted(key, *peer_id, candidate_peers, now); + block_cache.push_retry_range(*range); + peer_manager.ban_peer( + peer_id, + BanReason::ProtocolError(format!( + "invalid block range: {err:?}" + )), + ); + } + Ok(()) => { + block_cache.clear_attempted(key); + if needs_columns { + block_cache.push_column_range(*range); + } } } } @@ -1163,7 +1866,7 @@ fn poll_ready_tasks( continue; } - if let Err(err) = block_cache.add_blocks(blocks, false) { + if let Err(err) = block_cache.add_blocks(blocks, false, *peer_id) { warn!("Failed to add downloaded blocks to cache: {err:?}"); } } @@ -1396,7 +2099,7 @@ fn poll_ready_tasks( tasks.remove(index); } - Ok(()) + Ok(outcome) } #[cfg(test)] @@ -1569,15 +2272,21 @@ mod tests { saw_empty_range: &mut bool, candidate_peers: &[PeerId], ) { + let mut finalized_frontier = None; + let mut head_frontier = None; for _ in 0..500 { + let mut frontiers = FrontierStore { + finalized: &mut finalized_frontier, + head: &mut head_frontier, + }; poll_ready_tasks( tasks, block_cache, peer_manager, + &mut frontiers, required_columns, saw_empty_range, candidate_peers, - Instant::now(), ) .expect("poll_ready_tasks should not error"); if tasks.is_empty() { @@ -1950,7 +2659,7 @@ mod tests { let mut block_cache = BlockCache::new(B256::ZERO, 0); block_cache - .add_blocks(vec![block], false) + .add_blocks(vec![block], false, peer_id) .expect("block should enter cache"); let handle = executor.spawn(async move { StreamOutcome::Complete(vec![sidecar]) }); @@ -2020,7 +2729,7 @@ mod tests { let mut block_cache = BlockCache::new(B256::ZERO, 0); block_cache - .add_blocks(vec![block], false) + .add_blocks(vec![block], false, peer_id) .expect("block should enter cache"); let handle = @@ -2087,7 +2796,7 @@ mod tests { let mut block_cache = BlockCache::new(B256::ZERO, 0); block_cache - .add_blocks(vec![block], false) + .add_blocks(vec![block], false, peer_id) .expect("block should enter cache"); let handle = @@ -2118,4 +2827,522 @@ mod tests { "a peer that returned a duplicate data column must be banned, not just marked idle" ); } + + fn test_frontier(id: FrontierId, anchor_root: B256, anchor_slot: u64) -> StuckFrontier { + StuckFrontier { + id, + anchor_root, + anchor_slot, + phase: id.key.phase, + scan_start_slot: id.key.scan_start_slot, + highest_observed_target: anchor_slot + 100, + consecutive_no_progress: 3, + attempted_peers: HashSet::new(), + cooldown_until: None, + recovery_round_not_before: None, + failed_candidates: HashSet::new(), + confirmed_empty_through: None, + } + } + + fn test_frontier_id(generation: u64) -> FrontierId { + FrontierId { + key: FrontierKey { + anchor_root: B256::ZERO, + phase: SyncPhase::Finalized, + scan_start_slot: 0, + }, + generation, + } + } + + #[test] + fn allocate_generation_never_repeats_across_interleaved_phase_resets() { + initialize_test_network_spec(); + let (_data_dir, beacon_chain) = test_beacon_chain(); + let executor = ReamExecutor::new().expect("executor should start"); + let (p2p_sender, _p2p_receiver) = tokio::sync::mpsc::unbounded_channel(); + let mut syncer = BlockRangeSyncer::new( + Arc::new(beacon_chain), + p2p_sender, + test_network_state(), + executor, + ); + + let mut seen = HashSet::new(); + for _ in 0..5 { + assert!(seen.insert(syncer.allocate_generation())); + assert!(seen.insert(syncer.allocate_generation())); + } + assert_eq!(seen.len(), 10, "every allocated generation must be unique"); + } + + #[test] + fn poll_ready_tasks_drops_a_stale_generation_block_range_result_without_peer_consequence() { + initialize_test_network_spec(); + let executor = ReamExecutor::new().expect("executor should start"); + let (mut peer_manager, peer_id) = test_peer_manager_with_one_peer(); + + let range = Range::new(11, 10); + let mut block_cache = BlockCache::new(B256::ZERO, 10); + block_cache.mark_block_range_in_progress(range); + let handle = executor.spawn(async move { + StreamOutcome::Complete(vec![SignedBeaconBlock { + message: BeaconBlock { + slot: 11, + ..Default::default() + }, + signature: Default::default(), + }]) + }); + let mut tasks = vec![DownloadTask::new_block_range( + handle, + range, + peer_id, + Some(test_frontier_id(0)), + )]; + + let mut finalized_frontier = Some(test_frontier(test_frontier_id(1), B256::ZERO, 10)); + let mut head_frontier = None; + let mut saw_empty_range = false; + let mut outcome = PollTasksOutcome::Settled; + for _ in 0..500 { + let mut frontiers = FrontierStore { + finalized: &mut finalized_frontier, + head: &mut head_frontier, + }; + outcome = poll_ready_tasks( + &mut tasks, + &mut block_cache, + &mut peer_manager, + &mut frontiers, + &HashSet::new(), + &mut saw_empty_range, + &[peer_id], + ) + .expect("poll_ready_tasks should not error"); + if tasks.is_empty() { + break; + } + std::thread::sleep(Duration::from_millis(2)); + } + + assert!(matches!(outcome, PollTasksOutcome::StaleGeneration)); + assert_eq!( + block_cache.block_count(), + 0, + "a stale-generation result must not be folded into the cache" + ); + assert!( + peer_manager.fetch_idle_peer_from(&[peer_id]).is_some(), + "a stale-generation result must not exclude or ban the peer that served it" + ); + } + + #[test] + fn ordinary_empty_range_fold_never_persists_coverage_when_the_live_cache_advance_fails() { + initialize_test_network_spec(); + let executor = ReamExecutor::new().expect("executor should start"); + let (mut peer_manager, peer_id) = test_peer_manager_with_one_peer(); + + let root = B256::ZERO; + let frontier_id = test_frontier_id(0); + let mut block_cache = BlockCache::new(root, 10); + block_cache + .add_blocks( + vec![SignedBeaconBlock { + message: BeaconBlock { + slot: 15, + parent_root: root, + ..Default::default() + }, + signature: Default::default(), + }], + true, + peer_id, + ) + .expect("a directly-connecting block should be accepted"); + + let range = Range::new(21, 10); + block_cache.mark_block_range_in_progress(range); + let handle = + executor.spawn(async move { StreamOutcome::Complete(Vec::::new()) }); + let mut tasks = vec![DownloadTask::new_block_range( + handle, + range, + peer_id, + Some(frontier_id), + )]; + + let mut finalized_frontier = Some(test_frontier(frontier_id, root, 10)); + let mut head_frontier = None; + let mut saw_empty_range = false; + for _ in 0..500 { + let mut frontiers = FrontierStore { + finalized: &mut finalized_frontier, + head: &mut head_frontier, + }; + poll_ready_tasks( + &mut tasks, + &mut block_cache, + &mut peer_manager, + &mut frontiers, + &HashSet::new(), + &mut saw_empty_range, + &[peer_id], + ) + .expect("poll_ready_tasks should not error"); + if tasks.is_empty() { + break; + } + std::thread::sleep(Duration::from_millis(2)); + } + + assert!( + finalized_frontier + .as_ref() + .expect("frontier should still exist") + .confirmed_empty_through + .is_none(), + "coverage must not be persisted when the live-cache advance itself failed -- doing \ + so anyway would assert something the cache never actually applied, and the range \ + isn't contiguous with what the cache holds (a real block sits before it)" + ); + } + + #[test] + fn coverage_divergence_implicates_the_confirming_peers_not_the_block_serving_peer() { + initialize_test_network_spec(); + let executor = ReamExecutor::new().expect("executor should start"); + let (mut peer_manager, block_serving_peer) = test_peer_manager_with_one_peer(); + let confirming_peer = PeerId::random(); + + let root = B256::ZERO; + let frontier_id = test_frontier_id(0); + let mut block_cache = BlockCache::new(root, 10); + block_cache + .advance_empty_coverage( + FrontierObservation { + anchor_root: root, + anchor_slot: 10, + phase: SyncPhase::Finalized, + scan_start_slot: 0, + target_slot: 200, + }, + root, + 20, + HashSet::from([confirming_peer]), + ) + .expect("advancing a pristine cache should succeed"); + + let range = Range::new(21, 10); + block_cache.mark_block_range_in_progress(range); + let wrong_parent = B256::repeat_byte(9); + let handle = executor.spawn(async move { + StreamOutcome::Complete(vec![SignedBeaconBlock { + message: BeaconBlock { + slot: 21, + parent_root: wrong_parent, + ..Default::default() + }, + signature: Default::default(), + }]) + }); + let mut tasks = vec![DownloadTask::new_block_range( + handle, + range, + block_serving_peer, + Some(frontier_id), + )]; + + let mut finalized_frontier = Some(test_frontier(frontier_id, root, 10)); + let mut head_frontier = None; + let mut saw_empty_range = false; + let mut outcome = PollTasksOutcome::Settled; + for _ in 0..500 { + let mut frontiers = FrontierStore { + finalized: &mut finalized_frontier, + head: &mut head_frontier, + }; + outcome = poll_ready_tasks( + &mut tasks, + &mut block_cache, + &mut peer_manager, + &mut frontiers, + &HashSet::new(), + &mut saw_empty_range, + &[block_serving_peer], + ) + .expect("poll_ready_tasks should not error"); + if tasks.is_empty() { + break; + } + std::thread::sleep(Duration::from_millis(2)); + } + + match outcome { + PollTasksOutcome::CoverageDivergence(divergence) => { + assert!(divergence.confirming_peers.contains(&confirming_peer)); + assert_eq!(divergence.non_connecting_peer, block_serving_peer); + assert!(!divergence.confirming_peers.contains(&block_serving_peer)); + } + _ => panic!("expected CoverageDivergence"), + } + assert!( + peer_manager + .fetch_idle_peer_from(&[block_serving_peer]) + .is_some(), + "the block-serving peer must not be banned -- it may be reporting the true chain" + ); + } + + #[test] + fn pending_conclusions_keep_finalized_and_head_evidence_independent() { + let mut pending = PendingConclusions::default(); + let finalized_observation = FrontierObservation { + anchor_root: B256::ZERO, + anchor_slot: 10, + phase: SyncPhase::Finalized, + scan_start_slot: 0, + target_slot: 100, + }; + let head_observation = FrontierObservation { + anchor_root: B256::ZERO, + anchor_slot: 10, + phase: SyncPhase::Head, + scan_start_slot: 0, + target_slot: 150, + }; + + pending.observe( + SyncPhase::Finalized, + finalized_observation, + RemoteNoProgressReason::SettledEmptyCoverage, + HashSet::new(), + HashSet::new(), + ); + pending.observe( + SyncPhase::Head, + head_observation, + RemoteNoProgressReason::CandidateExhausted, + HashSet::new(), + HashSet::new(), + ); + + assert!(pending.for_phase(SyncPhase::Finalized).is_some()); + assert!(pending.for_phase(SyncPhase::Head).is_some()); + assert_eq!( + pending + .for_phase(SyncPhase::Finalized) + .as_ref() + .expect("set above") + .reason, + RemoteNoProgressReason::SettledEmptyCoverage, + "a segment producing evidence for both phases must not let one silently overwrite the other" + ); + } + + #[test] + fn segment_exclusions_stay_scoped_to_their_own_phase() { + let mut exclusions = SegmentExclusions::default(); + let peer = PeerId::random(); + exclusions.for_phase_mut(SyncPhase::Finalized).insert(peer); + + assert!(exclusions.for_phase(SyncPhase::Finalized).contains(&peer)); + assert!( + !exclusions.for_phase(SyncPhase::Head).contains(&peer), + "a peer excluded for a false Finalized coverage claim must not also be excluded from Head" + ); + } + + #[test] + fn merge_confirmed_empty_coverage_never_persists_an_ancestor_walk_derived_advance() { + initialize_test_network_spec(); + let (_data_dir, beacon_chain) = test_beacon_chain(); + let executor = ReamExecutor::new().expect("executor should start"); + let (p2p_sender, _p2p_receiver) = tokio::sync::mpsc::unbounded_channel(); + let mut syncer = BlockRangeSyncer::new( + Arc::new(beacon_chain), + p2p_sender, + test_network_state(), + executor, + ); + + let root = B256::ZERO; + let frontier_id = test_frontier_id(0); + syncer.finalized_frontier = Some(test_frontier(frontier_id, root, 10)); + + let unproven = CoverageAdvance { + parent_root: B256::repeat_byte(7), + parent_slot: 500, + covered_through_slot: 500, + confirming_peers: HashSet::new(), + proven_empty: false, + }; + syncer.merge_confirmed_empty_coverage(SyncPhase::Finalized, &unproven); + assert!( + syncer + .finalized_frontier + .as_ref() + .expect("frontier should still exist") + .confirmed_empty_through + .is_none(), + "an unproven (ancestor-walk-derived) advance must never be persisted as confirmed-empty coverage" + ); + + let proven = CoverageAdvance { + parent_root: root, + parent_slot: 10, + covered_through_slot: 40, + confirming_peers: HashSet::new(), + proven_empty: true, + }; + syncer.merge_confirmed_empty_coverage(SyncPhase::Finalized, &proven); + assert!( + syncer + .finalized_frontier + .as_ref() + .expect("frontier should still exist") + .confirmed_empty_through + .is_some(), + "a proven-empty advance must be persisted" + ); + } + + #[test] + fn ordinary_empty_range_fold_does_not_skip_the_slot_immediately_after_the_checked_range() { + initialize_test_network_spec(); + let executor = ReamExecutor::new().expect("executor should start"); + let (mut peer_manager, peer_id) = test_peer_manager_with_one_peer(); + + let root = B256::ZERO; + let frontier_id = test_frontier_id(0); + let range = Range::new(11, 10); + let mut block_cache = BlockCache::new(root, 10); + block_cache.mark_block_range_in_progress(range); + let handle = + executor.spawn(async move { StreamOutcome::Complete(Vec::::new()) }); + let mut tasks = vec![DownloadTask::new_block_range( + handle, + range, + peer_id, + Some(frontier_id), + )]; + + let mut finalized_frontier = Some(test_frontier(frontier_id, root, 10)); + let mut head_frontier = None; + let mut saw_empty_range = false; + for _ in 0..500 { + let mut frontiers = FrontierStore { + finalized: &mut finalized_frontier, + head: &mut head_frontier, + }; + poll_ready_tasks( + &mut tasks, + &mut block_cache, + &mut peer_manager, + &mut frontiers, + &HashSet::new(), + &mut saw_empty_range, + &[peer_id], + ) + .expect("poll_ready_tasks should not error"); + if tasks.is_empty() { + break; + } + std::thread::sleep(Duration::from_millis(2)); + } + + assert_eq!( + block_cache.next_start_slot(), + 20, + "a [11, 21) empty response confirms up to slot 20 inclusive -- the cursor must \ + land there, not at 21, or the next dispatch (cursor + 1) would skip slot 21 \ + without ever checking it" + ); + let confirmed = finalized_frontier + .as_ref() + .expect("frontier should still exist") + .confirmed_empty_through + .as_ref() + .expect("an empty response under a tracked frontier must fold into persisted coverage"); + assert_eq!( + confirmed.end_slot_exclusive, 21, + "persisted coverage keeps its own exclusive-end convention independent of the \ + cache's inclusive cursor" + ); + } + + #[test] + fn cooldown_is_armed_once_not_re_extended_on_every_commit() { + initialize_test_network_spec(); + let (_data_dir, beacon_chain) = test_beacon_chain(); + let executor = ReamExecutor::new().expect("executor should start"); + let (p2p_sender, _p2p_receiver) = tokio::sync::mpsc::unbounded_channel(); + let mut syncer = BlockRangeSyncer::new( + Arc::new(beacon_chain), + p2p_sender, + test_network_state(), + executor, + ); + + let observation = FrontierObservation { + anchor_root: B256::ZERO, + anchor_slot: 10, + phase: SyncPhase::Finalized, + scan_start_slot: 0, + target_slot: 100, + }; + let peer = PeerId::random(); + let record = RemoteObservation { + observation: observation.clone(), + reason: RemoteNoProgressReason::ProbeNotFound, + implicated_peers: HashSet::from([peer]), + failed_candidates: HashSet::new(), + }; + + syncer.record_remote_no_progress(SyncPhase::Finalized, record.clone(), HashSet::new()); + let first_deadline = syncer + .finalized_frontier + .as_ref() + .expect("frontier should exist") + .cooldown_until + .expect("cooldown should be armed once a peer is attempted"); + + syncer.record_remote_no_progress(SyncPhase::Finalized, record, HashSet::new()); + let second_deadline = syncer + .finalized_frontier + .as_ref() + .expect("frontier should exist") + .cooldown_until + .expect("cooldown should still be armed"); + assert_eq!( + first_deadline, second_deadline, + "an already-armed cooldown must not be re-extended by a later commit, or it would \ + never elapse if segments repeat faster than the cooldown duration" + ); + } + + #[test] + fn refresh_attempt_round_clears_failed_candidates_alongside_attempted_peers() { + let mut frontier = test_frontier(test_frontier_id(0), B256::ZERO, 10); + frontier.attempted_peers.insert(PeerId::random()); + frontier.failed_candidates.insert(FailedRecoveryCandidate { + peer_id: PeerId::random(), + candidate_root: B256::repeat_byte(1), + target_bucket: 0, + }); + let now = Instant::now(); + frontier.cooldown_until = Some(now); + + frontier.refresh_attempt_round(now + Duration::from_secs(1)); + + assert!(frontier.attempted_peers.is_empty()); + assert!( + frontier.failed_candidates.is_empty(), + "failed_candidates must clear alongside attempted_peers on cooldown expiry, or a \ + candidate could stay excluded forever even after the peer cooldown reopens" + ); + assert!(frontier.cooldown_until.is_none()); + } } diff --git a/crates/networking/syncer/src/block_range/peer_manager.rs b/crates/networking/syncer/src/block_range/peer_manager.rs index a01ce8406..c404ed692 100644 --- a/crates/networking/syncer/src/block_range/peer_manager.rs +++ b/crates/networking/syncer/src/block_range/peer_manager.rs @@ -7,6 +7,7 @@ use std::{ use libp2p::PeerId; use ream_consensus_misc::constants::beacon::SLOTS_PER_EPOCH; use ream_p2p::network::beacon::{network_state::NetworkState, peer::CachedPeer}; +use ream_req_resp::beacon::messages::status::Status; use tracing::warn; /// How long a peer stays banned before it becomes eligible to rejoin the peer set. @@ -58,6 +59,8 @@ pub enum PeerStatus { pub struct PeerInfo { pub peer: CachedPeer, pub peer_status: PeerStatus, + pub processed_blocks: u64, + pub sync_requests_started: u64, } pub struct PeerManager { @@ -98,6 +101,8 @@ impl PeerManager { entry.insert(PeerInfo { peer: peer.clone(), peer_status: PeerStatus::Idle, + processed_blocks: 0, + sync_requests_started: 0, }); } } @@ -118,26 +123,32 @@ impl PeerManager { } } + fn reserve(&mut self, peer_id: &PeerId) -> Option { + let peer_info = self.peers.get_mut(peer_id)?; + if !matches!(peer_info.peer_status, PeerStatus::Idle) { + return None; + } + peer_info.peer_status = PeerStatus::Downloading; + peer_info.sync_requests_started += 1; + Some(peer_info.peer.clone()) + } + /// Fetches an idle peer from the peer set. /// /// Will set the peer status to `Downloading` if an idle peer is found. pub fn fetch_idle_peer(&mut self) -> Option { - for peer_info in self.peers.values_mut() { - if let PeerStatus::Idle = peer_info.peer_status { - peer_info.peer_status = PeerStatus::Downloading; - return Some(peer_info.peer.clone()); - } - } - None + let idle_peer_id = self + .peers + .iter() + .find(|(_, peer_info)| matches!(peer_info.peer_status, PeerStatus::Idle)) + .map(|(peer_id, _)| *peer_id)?; + self.reserve(&idle_peer_id) } pub fn fetch_idle_peer_from(&mut self, eligible: &[PeerId]) -> Option { for peer_id in eligible { - if let Some(peer_info) = self.peers.get_mut(peer_id) - && matches!(peer_info.peer_status, PeerStatus::Idle) - { - peer_info.peer_status = PeerStatus::Downloading; - return Some(peer_info.peer.clone()); + if let Some(peer) = self.reserve(peer_id) { + return Some(peer); } } None @@ -152,11 +163,8 @@ impl PeerManager { if excluded.contains(peer_id) { continue; } - if let Some(peer_info) = self.peers.get_mut(peer_id) - && matches!(peer_info.peer_status, PeerStatus::Idle) - { - peer_info.peer_status = PeerStatus::Downloading; - return Some(peer_info.peer.clone()); + if let Some(peer) = self.reserve(peer_id) { + return Some(peer); } } None @@ -184,6 +192,12 @@ impl PeerManager { } } + pub fn record_processed_blocks(&mut self, peer_id: &PeerId, count: u64) { + if let Some(peer_info) = self.peers.get_mut(peer_id) { + peer_info.processed_blocks += count; + } + } + pub fn best_finalized(&self, our_finalized_epoch: u64) -> TargetSelection { let mut votes: HashMap = HashMap::new(); let mut candidates: Vec<(PeerId, u64, u64)> = Vec::new(); @@ -269,6 +283,25 @@ impl PeerManager { } } + pub fn status_of(&self, peer_id: &PeerId) -> Option { + self.peers + .get(peer_id) + .and_then(|info| info.peer.status.clone()) + } + + pub fn exact_finalized_epoch_peers(&self, epoch: u64) -> Vec { + self.peers + .iter() + .filter(|(_, info)| { + info.peer + .status + .as_ref() + .is_some_and(|status| status.finalized_epoch == epoch) + }) + .map(|(peer_id, _)| *peer_id) + .collect() + } + pub fn peers_satisfying(&self, qualification: TargetQualification) -> Vec { self.peers .iter() @@ -328,6 +361,8 @@ mod tests { PeerInfo { peer, peer_status: PeerStatus::Idle, + processed_blocks: 0, + sync_requests_started: 0, }, ); } @@ -541,6 +576,8 @@ mod tests { PeerInfo { peer, peer_status: PeerStatus::Downloading, + processed_blocks: 0, + sync_requests_started: 0, }, ); @@ -556,6 +593,70 @@ mod tests { )); } + #[test] + fn reserving_a_peer_credits_a_started_sync_request_every_time_including_after_reuse() { + let mut peer_manager = PeerManager::new(test_network_state()); + let peer = test_peer(Status::default()); + let peer_id = peer.peer_id; + insert_idle(&mut peer_manager, peer); + + peer_manager + .fetch_idle_peer_from(&[peer_id]) + .expect("peer is idle"); + assert_eq!( + peer_manager + .peers + .get(&peer_id) + .expect("peer exists") + .sync_requests_started, + 1 + ); + + assert!(peer_manager.fetch_idle_peer_from(&[peer_id]).is_none()); + assert_eq!( + peer_manager + .peers + .get(&peer_id) + .expect("peer exists") + .sync_requests_started, + 1 + ); + + peer_manager.mark_peer_as_idle(&peer_id); + peer_manager + .fetch_idle_peer_from_excluding(&[peer_id], &HashSet::new()) + .expect("peer is idle again"); + assert_eq!( + peer_manager + .peers + .get(&peer_id) + .expect("peer exists") + .sync_requests_started, + 2 + ); + } + + #[test] + fn record_processed_blocks_accumulates_and_ignores_unknown_peers() { + let mut peer_manager = PeerManager::new(test_network_state()); + let peer = test_peer(Status::default()); + let peer_id = peer.peer_id; + insert_idle(&mut peer_manager, peer); + + peer_manager.record_processed_blocks(&peer_id, 3); + peer_manager.record_processed_blocks(&peer_id, 2); + assert_eq!( + peer_manager + .peers + .get(&peer_id) + .expect("peer exists") + .processed_blocks, + 5 + ); + + peer_manager.record_processed_blocks(&PeerId::random(), 1); + } + #[test] fn peers_satisfying_head_epoch_includes_a_peer_below_the_refined_max_slot() { let mut peer_manager = PeerManager::new(test_network_state()); diff --git a/crates/networking/syncer/src/block_range/recovery.rs b/crates/networking/syncer/src/block_range/recovery.rs new file mode 100644 index 000000000..d6519dd0c --- /dev/null +++ b/crates/networking/syncer/src/block_range/recovery.rs @@ -0,0 +1,844 @@ +use std::{collections::HashSet, time::Instant}; + +use alloy_primitives::B256; +use libp2p::PeerId; +use ream_consensus_beacon::electra::beacon_block::SignedBeaconBlock; +use ream_consensus_misc::{constants::beacon::SLOTS_PER_EPOCH, misc::compute_start_slot_at_epoch}; +use ream_fork_choice_beacon::store::Store; +use ream_storage::tables::{field::REDBField, table::REDBTable}; +use tokio::time::Duration; +use tracing::warn; +use tree_hash::TreeHash; + +use super::{ + BanReason, FailedRecoveryCandidate, FrontierObservation, PeerManager, RemoteNoProgressReason, + SyncPhase, + block_cache::validate_range_chain, + peer_range_downloader::{PeerRangeDownloader, PeerRootsDownloader, Range, StreamOutcome}, + target_bucket, +}; +use crate::block_range::BlockRangeSyncer; + +pub(super) const MAX_RECOVERY_PEERS_PER_ROUND: usize = 3; +pub(super) const MAX_PROBE_REQUESTS_PER_ROUND: usize = 10; +pub(super) const MAX_TOTAL_ANCESTOR_REQUESTS_PER_ROUND: u64 = 128; +pub(super) const MAX_RECOVERY_SEED_BLOCKS: usize = 48; +pub(super) const RECOVERY_ROUND_TIMEOUT: Duration = Duration::from_secs(60); + +struct RecoveryBudget { + deadline: Instant, + requests_used: u64, +} + +impl RecoveryBudget { + fn new() -> Self { + Self { + deadline: Instant::now() + RECOVERY_ROUND_TIMEOUT, + requests_used: 0, + } + } + + fn has_capacity(&self) -> bool { + Instant::now() < self.deadline && self.requests_used < MAX_TOTAL_ANCESTOR_REQUESTS_PER_ROUND + } + + fn consume(&mut self) { + self.requests_used += 1; + } +} + +pub(super) struct CoverageAdvance { + pub parent_root: B256, + pub parent_slot: u64, + pub covered_through_slot: u64, + pub confirming_peers: HashSet, + pub proven_empty: bool, +} + +pub(super) struct RecoverySeed { + pub(super) ancestor_root: B256, + pub(super) ancestor_slot: u64, + pub(super) forward_blocks: Vec, + pub(super) target_slot: u64, + pub(super) source_peer: PeerId, +} + +pub(super) enum RecoveryOutcome { + AdvancedCoverage(CoverageAdvance), + Seeded(RecoverySeed), + NoProgress { + reason: RemoteNoProgressReason, + implicated_peers: HashSet, + failed_candidates: HashSet, + }, +} + +fn no_progress( + reason: RemoteNoProgressReason, + implicated_peers: HashSet, +) -> anyhow::Result { + Ok(RecoveryOutcome::NoProgress { + reason, + implicated_peers, + failed_candidates: HashSet::new(), + }) +} + +fn budget_exhausted(implicated_peers: HashSet) -> anyhow::Result { + no_progress( + RemoteNoProgressReason::RecoveryBudgetExhausted, + implicated_peers, + ) +} + +fn is_processable_connection_point( + store: &Store, + parent_root: B256, + first_child_slot: u64, +) -> anyhow::Result { + if store.db.block_provider().get(parent_root)?.is_none() { + return Ok(false); + } + if store.db.state_provider().get(parent_root)?.is_none() { + return Ok(false); + } + let finalized_checkpoint = store.db.finalized_checkpoint_provider().get()?; + let finalized_slot = compute_start_slot_at_epoch(finalized_checkpoint.epoch); + if first_child_slot <= finalized_slot { + return Ok(false); + } + let checkpoint_block = store.get_checkpoint_block(parent_root, finalized_checkpoint.epoch)?; + Ok(checkpoint_block == finalized_checkpoint.root) +} + +enum ProbeStep { + Empty, + Found(Vec), + Failed, +} + +fn reserve_round_robin( + peer_manager: &mut PeerManager, + peers: &[PeerId], + excluded: &HashSet, + start_index: usize, +) -> Option<(PeerId, usize)> { + if peers.is_empty() { + return None; + } + for offset in 0..peers.len() { + let candidate = peers[(start_index + offset) % peers.len()]; + if excluded.contains(&candidate) { + continue; + } + if let Some(peer) = + peer_manager.fetch_idle_peer_from_excluding(&[candidate], &HashSet::new()) + { + return Some((peer.peer_id, offset + 1)); + } + } + None +} + +impl BlockRangeSyncer { + pub(super) async fn run_recovery( + &mut self, + observation: &FrontierObservation, + candidate_peers: &[PeerId], + excluded: &HashSet, + ) -> anyhow::Result { + let mut budget = RecoveryBudget::new(); + let tier = self + .frontier_for(observation.phase) + .as_ref() + .map(|frontier| frontier.tier()) + .unwrap_or(0); + + let baseline_end = self + .frontier_for(observation.phase) + .as_ref() + .and_then(|frontier| frontier.confirmed_empty_through.as_ref()) + .map(|coverage| coverage.end_slot_exclusive) + .unwrap_or(observation.anchor_slot + 1); + + let probe_peers: Vec = candidate_peers + .iter() + .filter(|peer_id| !excluded.contains(peer_id)) + .take(MAX_RECOVERY_PEERS_PER_ROUND) + .copied() + .collect(); + + if probe_peers.is_empty() { + return no_progress(RemoteNoProgressReason::ProbeNotFound, HashSet::new()); + } + + let mut implicated_peers: HashSet = HashSet::new(); + let mut cursor = baseline_end; + let mut confirmed_end: Option<(u64, HashSet)> = None; + let mut found: Option<(Vec, PeerId)> = None; + let mut failed_this_round: HashSet = HashSet::new(); + let mut next_peer_index = 0usize; + + for _ in 0..MAX_PROBE_REQUESTS_PER_ROUND { + if !budget.has_capacity() || cursor > observation.target_slot { + break; + } + let Some((peer_id, advance_by)) = reserve_round_robin( + &mut self.peer_manager, + &probe_peers, + &failed_this_round, + next_peer_index, + ) else { + break; + }; + next_peer_index = (next_peer_index + advance_by) % probe_peers.len(); + let window = SLOTS_PER_EPOCH.min(observation.target_slot.saturating_add(1) - cursor); + let range = Range::new(cursor, window); + budget.consume(); + + match self + .probe_range(peer_id, range, &mut implicated_peers) + .await + { + ProbeStep::Empty => { + match &mut confirmed_end { + Some((end, peers)) => { + *end = cursor + window; + peers.insert(peer_id); + } + None => confirmed_end = Some((cursor + window, HashSet::from([peer_id]))), + } + cursor += window; + } + ProbeStep::Found(blocks) => { + found = Some((blocks, peer_id)); + break; + } + ProbeStep::Failed => { + failed_this_round.insert(peer_id); + } + } + } + + if let Some((blocks, peer_id)) = found { + return self + .resolve_candidate( + observation, + blocks, + HashSet::from([peer_id]), + &probe_peers, + &mut budget, + implicated_peers, + ) + .await; + } + + if let Some((end, peers)) = confirmed_end + && end > baseline_end + { + return Ok(RecoveryOutcome::AdvancedCoverage(CoverageAdvance { + parent_root: observation.anchor_root, + parent_slot: observation.anchor_slot, + covered_through_slot: end - 1, + proven_empty: true, + confirming_peers: peers, + })); + } + + if tier < 3 { + return no_progress(RemoteNoProgressReason::ProbeNotFound, implicated_peers); + } + + self.run_fallback( + observation, + candidate_peers, + excluded, + &mut budget, + implicated_peers, + ) + .await + } + + async fn probe_range( + &mut self, + peer_id: PeerId, + range: Range, + implicated_peers: &mut HashSet, + ) -> ProbeStep { + let handle = PeerRangeDownloader::start( + peer_id, + self.p2p_sender.clone(), + self.executor.clone(), + range, + ); + let result = handle.await; + let idle = IdleOnDrop::new(&mut self.peer_manager, peer_id); + let outcome = match result { + Ok(Ok(outcome)) => outcome, + Ok(Err(err)) => { + warn!("Recovery probe task failed: {err}"); + implicated_peers.insert(peer_id); + return ProbeStep::Failed; + } + Err(err) => { + warn!("Recovery probe task panicked: {err}"); + implicated_peers.insert(peer_id); + return ProbeStep::Failed; + } + }; + drop(idle); + + match outcome { + StreamOutcome::Complete(blocks) if blocks.is_empty() => { + implicated_peers.insert(peer_id); + ProbeStep::Empty + } + StreamOutcome::Complete(blocks) => { + let range_end = range.start_slot + range.count; + let out_of_range = blocks.iter().any(|block| { + block.message.slot < range.start_slot || block.message.slot >= range_end + }); + if out_of_range || validate_range_chain(&blocks).is_err() { + self.peer_manager.ban_peer( + &peer_id, + BanReason::ProtocolError( + "block(s) outside the requested range or not a valid chain".to_string(), + ), + ); + return ProbeStep::Failed; + } + ProbeStep::Found(blocks) + } + StreamOutcome::Failed(_) => { + implicated_peers.insert(peer_id); + ProbeStep::Failed + } + } + } + + async fn run_fallback( + &mut self, + observation: &FrontierObservation, + candidate_peers: &[PeerId], + excluded: &HashSet, + budget: &mut RecoveryBudget, + implicated_peers: HashSet, + ) -> anyhow::Result { + if !budget.has_capacity() { + return budget_exhausted(implicated_peers); + } + match observation.phase { + SyncPhase::Head => { + let eligible: Vec = candidate_peers + .iter() + .filter(|peer_id| !excluded.contains(peer_id)) + .copied() + .collect(); + let Some(peer) = self + .peer_manager + .fetch_idle_peer_from_excluding(&eligible, &HashSet::new()) + else { + return no_progress(RemoteNoProgressReason::AncestorNotFound, implicated_peers); + }; + let peer_id = peer.peer_id; + let end_exclusive = observation.target_slot.saturating_add(1); + let window = SLOTS_PER_EPOCH.min(end_exclusive); + let start = end_exclusive.saturating_sub(window); + let range = Range::new(start, window.max(1)); + let mut implicated = implicated_peers; + budget.consume(); + match self.probe_range(peer_id, range, &mut implicated).await { + ProbeStep::Found(blocks) => { + self.resolve_candidate( + observation, + blocks, + HashSet::from([peer_id]), + &eligible, + budget, + implicated, + ) + .await + } + _ => no_progress(RemoteNoProgressReason::AncestorNotFound, implicated), + } + } + SyncPhase::Finalized => { + let epoch = observation.target_slot / SLOTS_PER_EPOCH; + let bucket = target_bucket(observation.target_slot); + let previously_failed = self + .frontier_for(observation.phase) + .as_ref() + .map(|frontier| frontier.failed_candidates.clone()) + .unwrap_or_default(); + + let exact_epoch_peers = self.peer_manager.exact_finalized_epoch_peers(epoch); + let ordered_peers = self.group_by_finalized_root_agreement(&exact_epoch_peers); + + let mut implicated = implicated_peers; + let mut failed_candidates = HashSet::new(); + + for peer_id in &ordered_peers { + let peer_id = *peer_id; + if !budget.has_capacity() { + break; + } + if excluded.contains(&peer_id) { + continue; + } + let Some(status) = self.peer_manager.status_of(&peer_id) else { + continue; + }; + let candidate = FailedRecoveryCandidate { + peer_id, + candidate_root: status.finalized_root, + target_bucket: bucket, + }; + if previously_failed.contains(&candidate) + || failed_candidates.contains(&candidate) + { + continue; + } + let Some(reserved) = self + .peer_manager + .fetch_idle_peer_from_excluding(&[peer_id], &HashSet::new()) + else { + continue; + }; + debug_assert_eq!(reserved.peer_id, peer_id); + + budget.consume(); + let Some(block) = self + .fetch_single_root(peer_id, status.finalized_root, &mut implicated) + .await + else { + failed_candidates.insert(candidate); + continue; + }; + + let required_backtrack_floor = block + .message + .slot + .saturating_sub(MAX_TOTAL_ANCESTOR_REQUESTS_PER_ROUND); + if status.earliest_available_slot > required_backtrack_floor { + implicated.insert(peer_id); + failed_candidates.insert(candidate); + continue; + } + + match self + .resolve_candidate( + observation, + vec![block], + HashSet::from([peer_id]), + &ordered_peers, + budget, + implicated, + ) + .await? + { + RecoveryOutcome::NoProgress { + reason, + implicated_peers: inner_implicated, + failed_candidates: mut inner_failed, + } => { + inner_failed.insert(candidate); + inner_failed.extend(failed_candidates.iter().copied()); + if reason == RemoteNoProgressReason::RecoveryBudgetExhausted { + return Ok(RecoveryOutcome::NoProgress { + reason, + implicated_peers: inner_implicated, + failed_candidates: inner_failed, + }); + } + implicated = inner_implicated; + failed_candidates = inner_failed; + } + other => return Ok(other), + } + } + + let reason = if budget.has_capacity() { + RemoteNoProgressReason::AncestorNotFound + } else { + RemoteNoProgressReason::RecoveryBudgetExhausted + }; + Ok(RecoveryOutcome::NoProgress { + reason, + implicated_peers: implicated, + failed_candidates, + }) + } + } + } + + fn group_by_finalized_root_agreement(&self, peers: &[PeerId]) -> Vec { + let mut by_root: std::collections::BTreeMap> = + std::collections::BTreeMap::new(); + for &peer_id in peers { + let Some(status) = self.peer_manager.status_of(&peer_id) else { + continue; + }; + by_root + .entry(status.finalized_root) + .or_default() + .push(peer_id); + } + for group in by_root.values_mut() { + group.sort(); + } + let mut groups: Vec<(B256, Vec)> = by_root.into_iter().collect(); + groups.sort_by(|(root_a, group_a), (root_b, group_b)| { + group_b.len().cmp(&group_a.len()).then(root_a.cmp(root_b)) + }); + groups.into_iter().flat_map(|(_, group)| group).collect() + } + + async fn fetch_single_root( + &mut self, + peer_id: PeerId, + root: B256, + implicated_peers: &mut HashSet, + ) -> Option { + let handle = PeerRootsDownloader::start( + peer_id, + self.p2p_sender.clone(), + self.executor.clone(), + vec![root], + ); + let result = handle.await; + let idle = IdleOnDrop::new(&mut self.peer_manager, peer_id); + let outcome = match result { + Ok(Ok(outcome)) => outcome, + _ => { + drop(idle); + implicated_peers.insert(peer_id); + return None; + } + }; + drop(idle); + + match outcome { + StreamOutcome::Complete(blocks) if blocks.len() == 1 => { + let block = &blocks[0]; + if block.message.tree_hash_root() == root { + Some(blocks.into_iter().next().expect("checked len == 1 above")) + } else { + self.peer_manager.ban_peer( + &peer_id, + BanReason::ProtocolError("returned an unrequested root".to_string()), + ); + None + } + } + StreamOutcome::Complete(_) => { + implicated_peers.insert(peer_id); + None + } + StreamOutcome::Failed(_) => { + implicated_peers.insert(peer_id); + None + } + } + } + + async fn resolve_candidate( + &mut self, + observation: &FrontierObservation, + blocks: Vec, + confirming_peers: HashSet, + fallback_peers: &[PeerId], + budget: &mut RecoveryBudget, + implicated_peers: HashSet, + ) -> anyhow::Result { + let mut blocks = blocks; + blocks.sort_by_key(|block| block.message.slot); + + let primary_peer = *confirming_peers + .iter() + .next() + .expect("resolve_candidate is always called with a non-empty confirming_peers set"); + + let leading_parent = blocks[0].message.parent_root; + let leading_slot = blocks[0].message.slot; + + let connected_at = { + let store = self.beacon_chain.store.lock().await; + if is_processable_connection_point(&store, leading_parent, leading_slot)? { + store + .db + .block_provider() + .get(leading_parent)? + .map(|block| block.message.slot) + } else { + None + } + }; + + if let Some(leading_parent_slot) = connected_at { + return self + .trim_and_seed( + observation, + (leading_parent, leading_parent_slot), + blocks, + confirming_peers, + implicated_peers, + primary_peer, + ) + .await; + } + + if !budget.has_capacity() { + return budget_exhausted(implicated_peers); + } + + let finalized_epoch_start = { + let store = self.beacon_chain.store.lock().await; + compute_start_slot_at_epoch(store.db.finalized_checkpoint_provider().get()?.epoch) + }; + + let mut chain: Vec = blocks; + let mut visited: HashSet = HashSet::from([leading_parent]); + let mut cursor_root = leading_parent; + let mut cursor_slot = leading_slot; + let mut implicated_peers = implicated_peers; + let ancestor_walk_peers: Vec = confirming_peers + .iter() + .copied() + .chain(fallback_peers.iter().copied()) + .collect::>() + .into_iter() + .collect(); + let mut next_peer_index = 0usize; + let mut hop_failed: HashSet = HashSet::new(); + + while budget.has_capacity() && cursor_slot > finalized_epoch_start { + let Some((peer_id, advance_by)) = reserve_round_robin( + &mut self.peer_manager, + &ancestor_walk_peers, + &hop_failed, + next_peer_index, + ) else { + return no_progress(RemoteNoProgressReason::AncestorNotFound, implicated_peers); + }; + next_peer_index = (next_peer_index + advance_by) % ancestor_walk_peers.len(); + budget.consume(); + let Some(cursor_block) = self + .fetch_single_root(peer_id, cursor_root, &mut implicated_peers) + .await + else { + hop_failed.insert(peer_id); + continue; + }; + hop_failed.clear(); + if cursor_block.message.slot >= cursor_slot { + self.peer_manager.ban_peer( + &peer_id, + BanReason::ProtocolError( + "ancestor walk did not strictly decrease slot".to_string(), + ), + ); + return no_progress(RemoteNoProgressReason::AncestorNotFound, implicated_peers); + } + let grandparent_root = cursor_block.message.parent_root; + let first_child_slot = cursor_block.message.slot; + let grandparent_slot = { + let store = self.beacon_chain.store.lock().await; + if is_processable_connection_point(&store, grandparent_root, first_child_slot)? { + store + .db + .block_provider() + .get(grandparent_root)? + .map(|block| block.message.slot) + } else { + None + } + }; + chain.insert(0, cursor_block); + if let Some(grandparent_slot) = grandparent_slot { + return self + .trim_and_seed( + observation, + (grandparent_root, grandparent_slot), + chain, + confirming_peers, + implicated_peers, + peer_id, + ) + .await; + } + if !visited.insert(grandparent_root) { + return no_progress(RemoteNoProgressReason::AncestorNotFound, implicated_peers); + } + cursor_root = grandparent_root; + cursor_slot = first_child_slot; + } + + budget_exhausted(implicated_peers) + } + + async fn trim_and_seed( + &mut self, + observation: &FrontierObservation, + ancestor: (B256, u64), + mut chain: Vec, + confirming_peers: HashSet, + implicated_peers: HashSet, + primary_peer: PeerId, + ) -> anyhow::Result { + let (ancestor_root, ancestor_slot) = ancestor; + chain.retain(|block| { + block.message.slot > ancestor_slot && block.message.slot <= observation.target_slot + }); + + let mut new_ancestor_root = ancestor_root; + let mut new_ancestor_slot = ancestor_slot; + { + let store = self.beacon_chain.store.lock().await; + while let Some(first) = chain.first() { + let first_root = first.message.tree_hash_root(); + let first_slot = first.message.slot; + let next_child_slot = chain.get(1).map_or(first_slot, |block| block.message.slot); + if is_processable_connection_point(&store, first_root, next_child_slot)? { + new_ancestor_root = first_root; + new_ancestor_slot = first_slot; + chain.remove(0); + } else { + break; + } + } + } + + if chain.is_empty() { + if new_ancestor_slot < observation.anchor_slot { + return no_progress(RemoteNoProgressReason::NoNewDescendants, implicated_peers); + } + return Ok(RecoveryOutcome::AdvancedCoverage(CoverageAdvance { + parent_root: new_ancestor_root, + parent_slot: new_ancestor_slot, + covered_through_slot: new_ancestor_slot, + proven_empty: false, + confirming_peers, + })); + } + + if chain.len() > MAX_RECOVERY_SEED_BLOCKS { + chain.truncate(MAX_RECOVERY_SEED_BLOCKS); + } + + Ok(RecoveryOutcome::Seeded(RecoverySeed { + ancestor_root: new_ancestor_root, + ancestor_slot: new_ancestor_slot, + forward_blocks: chain, + target_slot: observation.target_slot, + source_peer: primary_peer, + })) + } +} + +struct IdleOnDrop<'a> { + peer_manager: &'a mut super::PeerManager, + peer_id: PeerId, +} + +impl<'a> IdleOnDrop<'a> { + fn new(peer_manager: &'a mut super::PeerManager, peer_id: PeerId) -> Self { + Self { + peer_manager, + peer_id, + } + } +} + +impl Drop for IdleOnDrop<'_> { + fn drop(&mut self) { + self.peer_manager.mark_peer_as_idle(&self.peer_id); + } +} + +#[cfg(test)] +mod tests { + use std::{path::PathBuf, sync::Arc}; + + use discv5::{Enr, enr::CombinedKey}; + use libp2p::PeerId; + use parking_lot::RwLock; + use ream_p2p::network::beacon::{network_state::NetworkState, peer::CachedPeer}; + use ream_peer::{ConnectionState, Direction}; + use ream_req_resp::beacon::messages::{meta_data::GetMetaDataV3, status::Status}; + + use super::*; + + fn test_network_state() -> Arc { + let enr_key = CombinedKey::generate_secp256k1(); + Arc::new(NetworkState { + local_enr: RwLock::new(Enr::builder().build(&enr_key).expect("valid enr")), + peer_table: RwLock::new(std::collections::HashMap::new()), + meta_data: RwLock::new(GetMetaDataV3::default()), + status: RwLock::new(Status::default()), + data_dir: PathBuf::new(), + }) + } + + fn idle_peer_manager_with(peer_ids: &[PeerId]) -> PeerManager { + let network_state = test_network_state(); + for &peer_id in peer_ids { + let mut peer = CachedPeer::new( + peer_id, + None, + ConnectionState::Connected, + Direction::Outbound, + None, + ); + peer.status = Some(Status::default()); + network_state.peer_table.write().insert(peer_id, peer); + } + let mut peer_manager = PeerManager::new(network_state); + peer_manager.update_peer_set(); + peer_manager + } + + #[test] + fn recovery_budget_stops_capacity_at_the_request_cap_regardless_of_time_remaining() { + let mut budget = RecoveryBudget::new(); + assert!(budget.has_capacity()); + for _ in 0..MAX_TOTAL_ANCESTOR_REQUESTS_PER_ROUND { + assert!( + budget.has_capacity(), + "must still have capacity before the cap is reached" + ); + budget.consume(); + } + assert!( + !budget.has_capacity(), + "the shared budget must run out at exactly MAX_TOTAL_ANCESTOR_REQUESTS_PER_ROUND, \ + regardless of which phase (probe, fallback, or ancestor-walk hops) consumed it" + ); + } + + #[test] + fn recovery_budget_expires_on_deadline_even_with_requests_remaining() { + let mut budget = RecoveryBudget::new(); + budget.deadline = Instant::now(); + std::thread::sleep(Duration::from_millis(1)); + assert!( + !budget.has_capacity(), + "a round past its soft deadline must refuse further dispatch even if the request \ + count is nowhere near the cap -- fallback and ancestor-walk hops must respect this \ + the same way the forward probe already does" + ); + } + + #[test] + fn reserve_round_robin_skips_excluded_and_wraps_around() { + let peer_a = PeerId::random(); + let peer_b = PeerId::random(); + let mut peer_manager = idle_peer_manager_with(&[peer_a, peer_b]); + + let peers = [peer_a, peer_b]; + let excluded = HashSet::from([peer_a]); + let (selected, advance_by) = reserve_round_robin(&mut peer_manager, &peers, &excluded, 0) + .expect("peer_b should still be selectable"); + assert_eq!(selected, peer_b); + assert_eq!(advance_by, 2); + } +} From a0ef3f340407f57ddba707fbb52d0daada920247 Mon Sep 17 00:00:00 2001 From: vuonghuuhung Date: Fri, 4 Sep 2026 09:27:07 +0700 Subject: [PATCH 08/17] fix: adapt to upstream's P2PCallbackError after rebasing onto develop --- crates/networking/manager/src/unknown_parent_lookup.rs | 5 +++-- crates/networking/syncer/src/block_range/mod.rs | 8 ++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/crates/networking/manager/src/unknown_parent_lookup.rs b/crates/networking/manager/src/unknown_parent_lookup.rs index 99b5f82dc..beb160b2e 100644 --- a/crates/networking/manager/src/unknown_parent_lookup.rs +++ b/crates/networking/manager/src/unknown_parent_lookup.rs @@ -337,6 +337,7 @@ mod tests { use std::sync::Arc; use anyhow::anyhow; + use ream_p2p::network::beacon::channel::P2PCallbackError; use tokio::sync::mpsc::{self, UnboundedReceiver, error::TryRecvError}; use super::*; @@ -350,7 +351,7 @@ mod tests { receiver: &mut UnboundedReceiver, expected_peer: PeerId, expected_root: B256, - ) -> mpsc::Sender> { + ) -> mpsc::Sender> { let message = receiver.recv().await.expect("request should be sent"); let P2PMessage::Request(P2PRequest::BlockRoots { peer_id, @@ -490,7 +491,7 @@ mod tests { let callback = intercept_request(&mut receiver, peer_id, expected_root).await; callback - .send(Err(anyhow!("transport failed"))) + .send(Err(anyhow!("transport failed").into())) .await .expect("callback error should be delivered"); diff --git a/crates/networking/syncer/src/block_range/mod.rs b/crates/networking/syncer/src/block_range/mod.rs index 8de851ef1..4a3e81890 100644 --- a/crates/networking/syncer/src/block_range/mod.rs +++ b/crates/networking/syncer/src/block_range/mod.rs @@ -1347,15 +1347,15 @@ fn group_by_block_root(identifiers: &[ColumnIdentifier]) -> Vec Some(identifier), Err(err) => { warn!("Failed to build data column identifier for {block_root}: {err}"); None } - }, - ) + } + }) .collect() } From cbd3037cb3a5b86bc4399a160fdb8a5c7367bc3d Mon Sep 17 00:00:00 2001 From: vuonghuuhung Date: Fri, 4 Sep 2026 17:14:42 +0700 Subject: [PATCH 09/17] fix: naming convention --- crates/networking/syncer/src/block_range/mod.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/networking/syncer/src/block_range/mod.rs b/crates/networking/syncer/src/block_range/mod.rs index 4a3e81890..4b6bc127c 100644 --- a/crates/networking/syncer/src/block_range/mod.rs +++ b/crates/networking/syncer/src/block_range/mod.rs @@ -1175,7 +1175,7 @@ impl BlockRangeSyncer { let import_result = self.import_downloaded(block_cache).await; let imported_count = match &import_result { Ok(count) => *count, - Err(failure) => failure.imported_count, + Err(err) => err.imported_count, }; info!("All blocks processed successfully."); @@ -1194,22 +1194,22 @@ impl BlockRangeSyncer { self.next_segment_not_before = Some(Instant::now() + ZERO_PROGRESS_BACKOFF); } - import_result.map(|_| ()).map_err(|failure| failure.error) + import_result.map(|_| ()).map_err(|err| err.error) } async fn import_downloaded(&mut self, block_cache: BlockCache) -> Result { let bundles = block_cache .get_blocks_and_blobs() - .map_err(|error| ImportFailure { + .map_err(|err| ImportFailure { imported_count: 0, - error, + error: err, })?; let mut imported_count = 0u64; for bundle in bundles { - if let Err(error) = self.import_one(bundle).await { + if let Err(err) = self.import_one(bundle).await { return Err(ImportFailure { imported_count, - error, + error: err, }); } imported_count += 1; From f51ea3e8e95192f8f0ad516c8a92d6cd72787c9e Mon Sep 17 00:00:00 2001 From: vuonghuuhung Date: Mon, 14 Sep 2026 20:31:17 +0700 Subject: [PATCH 10/17] fix: harden range recovery and resolve mainnet interoperability --- Cargo.lock | 1 + bin/ream/src/main.rs | 2 + .../common/chain/beacon/src/beacon_chain.rs | 4 + .../execution/engine/src/engine_trait.rs | 2 - crates/common/execution/engine/src/lib.rs | 27 +- crates/common/execution/rpc_types/Cargo.toml | 4 +- .../rpc_types/src/execution_payload.rs | 74 ++- .../execution/rpc_types/src/get_payload.rs | 5 +- crates/common/fork_choice/beacon/src/store.rs | 31 +- .../gossipsub/validate/beacon_attestation.rs | 53 +- .../networking/p2p/src/network/beacon/mod.rs | 73 ++- .../req_resp/src/beacon/messages/mod.rs | 27 + crates/networking/req_resp/src/handler.rs | 2 +- .../req_resp/src/inbound_protocol.rs | 15 +- crates/networking/req_resp/src/messages.rs | 10 + .../req_resp/src/outbound_protocol.rs | 22 +- .../syncer/src/block_range/block_cache.rs | 227 +++++++- .../networking/syncer/src/block_range/mod.rs | 513 ++++++++++++++++-- .../syncer/src/block_range/peer_manager.rs | 253 ++++++++- .../src/block_range/peer_range_downloader.rs | 10 + .../syncer/src/block_range/recovery.rs | 16 +- .../src/tables/beacon/latest_messages.rs | 25 +- .../src/block_generator.rs | 17 +- 23 files changed, 1277 insertions(+), 136 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8bcbc1f3e..f97c7dbde 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6614,6 +6614,7 @@ dependencies = [ "ethereum_ssz_derive", "ream-consensus-misc", "serde", + "serde_json", "sha2", "ssz_types", "tree_hash", diff --git a/bin/ream/src/main.rs b/bin/ream/src/main.rs index 219a3ecfc..7219a79a9 100644 --- a/bin/ream/src/main.rs +++ b/bin/ream/src/main.rs @@ -635,6 +635,8 @@ async fn run_beacon_node_inner( }; let beacon_chain = Arc::new(beacon_chain); + beacon_chain.initialize_execution_forkchoice().await; + // Create network manager let mut manager_config = ManagerConfig::from(config); manager_config.gossipsub_history_length = gossipsub_history_length; diff --git a/crates/common/chain/beacon/src/beacon_chain.rs b/crates/common/chain/beacon/src/beacon_chain.rs index af62002e0..662be76d0 100644 --- a/crates/common/chain/beacon/src/beacon_chain.rs +++ b/crates/common/chain/beacon/src/beacon_chain.rs @@ -100,6 +100,10 @@ impl BeaconChain { self.block_import_sender.subscribe() } + pub async fn initialize_execution_forkchoice(&self) { + self.update_execution_forkchoice(true).await; + } + pub async fn process_block( &self, signed_block: SignedBeaconBlock, diff --git a/crates/common/execution/engine/src/engine_trait.rs b/crates/common/execution/engine/src/engine_trait.rs index 7e4dd699f..cacce81a3 100644 --- a/crates/common/execution/engine/src/engine_trait.rs +++ b/crates/common/execution/engine/src/engine_trait.rs @@ -6,8 +6,6 @@ use super::new_payload_request::NewPayloadRequest; #[async_trait] pub trait ExecutionApi { - /// Return ``True`` if and only if ``new_payload_request`` is valid with respect to - /// ``self.execution_state``. async fn verify_and_notify_new_payload( &self, new_payload_request: NewPayloadRequest, diff --git a/crates/common/execution/engine/src/lib.rs b/crates/common/execution/engine/src/lib.rs index b540a6b08..8c692d953 100644 --- a/crates/common/execution/engine/src/lib.rs +++ b/crates/common/execution/engine/src/lib.rs @@ -435,6 +435,13 @@ pub fn is_valid_versioned_hashes(new_payload_request: &NewPayloadRequest) -> any ) } +fn payload_status_allows_import(status: PayloadStatus) -> bool { + matches!( + status, + PayloadStatus::Valid | PayloadStatus::Syncing | PayloadStatus::Accepted + ) +} + fn get_execution_requests_list(execution_requests: &ExecutionRequests) -> Vec { let mut requests_list = vec![]; if !execution_requests.deposits.is_empty() { @@ -495,7 +502,9 @@ impl ExecutionApi for ExecutionEngine { return Ok(false); } - return Ok(self.notify_new_payload(new_payload_request).await? == PayloadStatus::Valid); + let status = self.notify_new_payload(new_payload_request).await?; + + Ok(payload_status_allows_import(status)) } async fn engine_get_blobs_v1( @@ -582,3 +591,19 @@ impl ExecutionApi for ExecutionEngine { result } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn syncing_and_accepted_payloads_are_not_treated_as_invalid() { + assert!(payload_status_allows_import(PayloadStatus::Valid)); + assert!(payload_status_allows_import(PayloadStatus::Syncing)); + assert!(payload_status_allows_import(PayloadStatus::Accepted)); + assert!(!payload_status_allows_import(PayloadStatus::Invalid)); + assert!(!payload_status_allows_import( + PayloadStatus::InvalidBlockHash + )); + } +} diff --git a/crates/common/execution/rpc_types/Cargo.toml b/crates/common/execution/rpc_types/Cargo.toml index b3387ecb4..9d33b5ae8 100644 --- a/crates/common/execution/rpc_types/Cargo.toml +++ b/crates/common/execution/rpc_types/Cargo.toml @@ -23,6 +23,8 @@ ssz_types.workspace = true tree_hash.workspace = true tree_hash_derive.workspace = true +[dev-dependencies] +serde_json.workspace = true + [lints] workspace = true - diff --git a/crates/common/execution/rpc_types/src/execution_payload.rs b/crates/common/execution/rpc_types/src/execution_payload.rs index 24b0416c2..dfcb1e4bd 100644 --- a/crates/common/execution/rpc_types/src/execution_payload.rs +++ b/crates/common/execution/rpc_types/src/execution_payload.rs @@ -11,6 +11,40 @@ use tree_hash_derive::TreeHash; use crate::electra::execution_payload::ExecutionPayload; +#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] +#[serde(rename_all = "camelCase")] +pub struct WithdrawalV1 { + #[serde(with = "serde_utils::u64_hex_be")] + pub index: u64, + #[serde(with = "serde_utils::u64_hex_be")] + pub validator_index: u64, + pub address: Address, + #[serde(with = "serde_utils::u64_hex_be")] + pub amount: u64, +} + +impl From for WithdrawalV1 { + fn from(value: Withdrawal) -> Self { + Self { + index: value.index, + validator_index: value.validator_index, + address: value.address, + amount: value.amount, + } + } +} + +impl From for Withdrawal { + fn from(value: WithdrawalV1) -> Self { + Self { + index: value.index, + validator_index: value.validator_index, + address: value.address, + amount: value.amount, + } + } +} + #[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] #[serde(rename_all = "camelCase")] pub struct ExecutionPayloadV3 { @@ -37,7 +71,7 @@ pub struct ExecutionPayloadV3 { pub block_hash: B256, #[serde(with = "list_of_hex_var_list")] pub transactions: VariableList, U1048576>, - pub withdrawals: VariableList, + pub withdrawals: VariableList, #[serde(with = "serde_utils::u64_hex_be")] pub blob_gas_used: u64, #[serde(with = "serde_utils::u64_hex_be")] @@ -61,9 +95,45 @@ impl From for ExecutionPayloadV3 { base_fee_per_gas: value.base_fee_per_gas, block_hash: value.block_hash, transactions: value.transactions, - withdrawals: value.withdrawals, + withdrawals: VariableList::new( + value + .withdrawals + .into_iter() + .map(WithdrawalV1::from) + .collect(), + ) + .expect("converting a U16-bounded withdrawal list preserves its length"), blob_gas_used: value.blob_gas_used, excess_blob_gas: value.excess_blob_gas, } } } + +#[cfg(test)] +mod tests { + use alloy_primitives::address; + use serde_json::json; + + use super::*; + + #[test] + fn withdrawal_uses_engine_api_hex_quantities() { + let withdrawal = WithdrawalV1 { + index: 15, + validator_index: 16, + address: address!("0000000000000000000000000000000000000001"), + amount: 17, + }; + + let encoded = serde_json::to_value(withdrawal).expect("withdrawal should serialize"); + assert_eq!(encoded["index"], json!("0xf")); + assert_eq!(encoded["validatorIndex"], json!("0x10")); + assert_eq!(encoded["amount"], json!("0x11")); + + let decoded: WithdrawalV1 = + serde_json::from_value(encoded).expect("Engine API withdrawal should deserialize"); + assert_eq!(decoded.index, 15); + assert_eq!(decoded.validator_index, 16); + assert_eq!(decoded.amount, 17); + } +} diff --git a/crates/common/execution/rpc_types/src/get_payload.rs b/crates/common/execution/rpc_types/src/get_payload.rs index 16558e526..29c83dfba 100644 --- a/crates/common/execution/rpc_types/src/get_payload.rs +++ b/crates/common/execution/rpc_types/src/get_payload.rs @@ -141,7 +141,10 @@ impl Payload { base_fee_per_gas: ep.base_fee_per_gas, block_hash: ep.block_hash, transactions: ep.transactions.clone(), - withdrawals: ep.withdrawals.clone(), + withdrawals: VariableList::new( + ep.withdrawals.iter().cloned().map(Into::into).collect(), + ) + .expect("converting a U16-bounded withdrawal list preserves its length"), blob_gas_used: ep.blob_gas_used, excess_blob_gas: ep.excess_blob_gas, } diff --git a/crates/common/fork_choice/beacon/src/store.rs b/crates/common/fork_choice/beacon/src/store.rs index bb92f5b90..71bc448dd 100644 --- a/crates/common/fork_choice/beacon/src/store.rs +++ b/crates/common/fork_choice/beacon/src/store.rs @@ -239,6 +239,11 @@ impl Store { return Ok(head); } + if children.len() == 1 { + head = *children[0]; + continue; + } + let mut weighted_children = children .into_iter() .map(|child| Ok((*child, self.get_weight(*child)?))) @@ -608,27 +613,25 @@ impl Store { } } - for index in &non_equivocating_attesting_indices { - if self.db.latest_messages_provider().get(*index)?.is_none() - || target.epoch - > self - .db - .latest_messages_provider() - .get(*index)? - .ok_or(anyhow!( - "Could not get expected latest message at index: {index}" - ))? - .epoch + let latest_messages = self.db.latest_messages_provider(); + let mut updates = Vec::new(); + for index in non_equivocating_attesting_indices { + if latest_messages + .get(index)? + .is_none_or(|message| target.epoch > message.epoch) { - self.db.latest_messages_provider().insert( - *index, + updates.push(( + index, LatestMessage { epoch: target.epoch, root: beacon_block_root, }, - )?; + )); } } + if !updates.is_empty() { + latest_messages.insert_batch(updates)?; + } Ok(()) } diff --git a/crates/networking/manager/src/gossipsub/validate/beacon_attestation.rs b/crates/networking/manager/src/gossipsub/validate/beacon_attestation.rs index f447df140..6206c41e1 100644 --- a/crates/networking/manager/src/gossipsub/validate/beacon_attestation.rs +++ b/crates/networking/manager/src/gossipsub/validate/beacon_attestation.rs @@ -5,7 +5,7 @@ use ream_consensus_beacon::{ electra::beacon_state::BeaconState, single_attestation::SingleAttestation, }; use ream_consensus_misc::{ - constants::beacon::DOMAIN_BEACON_ATTESTER, + constants::beacon::{DOMAIN_BEACON_ATTESTER, SLOTS_PER_EPOCH}, misc::{compute_epoch_at_slot, compute_signing_root}, }; use ream_storage::{ @@ -16,6 +16,12 @@ use ream_validator_beacon::attestation::compute_subnet_for_attestation; use super::result::ValidationResult; +const GOSSIP_ATTESTATION_SYNC_TOLERANCE: u64 = 2 * SLOTS_PER_EPOCH; + +fn is_too_far_behind_to_validate(head_slot: u64, attestation_slot: u64) -> bool { + attestation_slot.saturating_sub(head_slot) > GOSSIP_ATTESTATION_SYNC_TOLERANCE +} + pub async fn validate_beacon_attestation( attestation: &SingleAttestation, beacon_chain: &BeaconChain, @@ -25,12 +31,6 @@ pub async fn validate_beacon_attestation( let store = beacon_chain.store.lock().await; let head_root = store.get_head()?; - let mut state: BeaconState = store - .db - .state_provider() - .get(head_root)? - .ok_or_else(|| anyhow!("No beacon state found for head root: {head_root}"))?; - let current_slot = store.get_current_slot()?; // [IGNORE] attestation.data.slot is equal to or earlier than the current_slot (with a @@ -41,6 +41,25 @@ pub async fn validate_beacon_attestation( )); } + let head_slot = store + .db + .block_provider() + .get(head_root)? + .ok_or_else(|| anyhow!("No beacon block found for head root: {head_root}"))? + .message + .slot; + if is_too_far_behind_to_validate(head_slot, attestation.data.slot) { + return Ok(ValidationResult::Ignore( + "Local chain is syncing and too far behind to validate attestation".to_string(), + )); + } + + let mut state: BeaconState = store + .db + .state_provider() + .get(head_root)? + .ok_or_else(|| anyhow!("No beacon state found for head root: {head_root}"))?; + if state.slot < attestation.data.slot { state.process_slots(attestation.data.slot)?; } @@ -186,3 +205,23 @@ pub async fn validate_beacon_attestation( .put(attestation_key, ()); Ok(ValidationResult::Accept) } + +#[cfg(test)] +mod tests { + use super::{GOSSIP_ATTESTATION_SYNC_TOLERANCE, is_too_far_behind_to_validate}; + + #[test] + fn skips_attestation_validation_only_beyond_sync_tolerance() { + let head_slot = 1_000; + + assert!(!is_too_far_behind_to_validate( + head_slot, + head_slot + GOSSIP_ATTESTATION_SYNC_TOLERANCE + )); + assert!(is_too_far_behind_to_validate( + head_slot, + head_slot + GOSSIP_ATTESTATION_SYNC_TOLERANCE + 1 + )); + assert!(!is_too_far_behind_to_validate(head_slot, head_slot - 1)); + } +} diff --git a/crates/networking/p2p/src/network/beacon/mod.rs b/crates/networking/p2p/src/network/beacon/mod.rs index 8ef759538..bfb41674a 100644 --- a/crates/networking/p2p/src/network/beacon/mod.rs +++ b/crates/networking/p2p/src/network/beacon/mod.rs @@ -31,7 +31,7 @@ use libp2p_identity::{Keypair, PublicKey, secp256k1}; use network_state::NetworkState; use parking_lot::{Mutex, RwLock}; use peer::CachedPeer; -use ream_consensus_misc::constants::beacon::genesis_validators_root; +use ream_consensus_misc::constants::beacon::{SLOTS_PER_EPOCH, genesis_validators_root}; use ream_discv5::discovery::{Discovery, DiscoveryOutEvent, QueryType}; use ream_executor::ReamExecutor; use ream_metrics::set_peer_count; @@ -69,6 +69,15 @@ use crate::{ network::misc::{Executor, build_transport, peer_id_from_enr}, }; +fn status_is_plausible(status: &Status, current_epoch: u64) -> bool { + status.earliest_available_slot <= status.head_slot + && status.finalized_epoch <= current_epoch.saturating_add(1) + && status.head_slot + <= current_epoch + .saturating_add(2) + .saturating_mul(SLOTS_PER_EPOCH) +} + #[derive(NetworkBehaviour)] pub(crate) struct ReamBehaviour { pub identify: identify::Behaviour, @@ -454,7 +463,14 @@ impl Network { } } - let peer_count = peer_table.len(); + let active_peer_count = counts + .get(&ConnectionState::Connected) + .copied() + .unwrap_or_default() + + counts + .get(&ConnectionState::Connecting) + .copied() + .unwrap_or_default(); let peers_to_ping_count = self.peers_to_ping.len(); let seq_number = self.network_state.meta_data.read().seq_number; @@ -466,8 +482,8 @@ impl Network { warn!("Failed to update attestation subnet subscriptions: {err:?}"); } - if peer_count < TARGET_PEER_COUNT { - info!("Peer count is below target: {peer_count}, discovering more peers"); + if active_peer_count < TARGET_PEER_COUNT { + info!("Active peer count is below target: {active_peer_count}, discovering more peers"); self.swarm .behaviour_mut() .discovery @@ -866,16 +882,21 @@ impl Network { fn handle_status_req_resp_event(&mut self, peer_id: PeerId, status: Status) { if self.network_state.peer_table.read().get(&peer_id).is_some() { // We only want to have peers on the same network as us - let fork_digest = beacon_network_spec().fork_digest( - beacon_network_spec().current_epoch(), - genesis_validators_root(), - ); + let current_epoch = beacon_network_spec().current_epoch(); + let fork_digest = + beacon_network_spec().fork_digest(current_epoch, genesis_validators_root()); if status.fork_digest != fork_digest { warn!( "Peer {peer_id} is not on the same network as us, removing from peer table, fork_digest: {}, our fork_digest: {fork_digest}", status.fork_digest, ); self.network_state.peer_table.write().remove(&peer_id); + } else if !status_is_plausible(&status, current_epoch) { + warn!( + "Peer {peer_id} sent an impossible Status (finalized_epoch={}, head_slot={}, earliest_available_slot={}), removing from peer table", + status.finalized_epoch, status.head_slot, status.earliest_available_slot + ); + self.network_state.peer_table.write().remove(&peer_id); } else { self.network_state .peer_table @@ -957,6 +978,42 @@ mod tests { gossipsub::beacon::{configurations::GossipsubConfig, topics::GossipTopicKind}, }; + #[test] + fn status_sanity_rejects_impossible_history_and_future_progress() { + let current_epoch = 100; + assert!(status_is_plausible( + &Status { + finalized_epoch: 98, + head_slot: 100 * SLOTS_PER_EPOCH, + earliest_available_slot: 90 * SLOTS_PER_EPOCH, + ..Default::default() + }, + current_epoch + )); + assert!(!status_is_plausible( + &Status { + head_slot: 10, + earliest_available_slot: 11, + ..Default::default() + }, + current_epoch + )); + assert!(!status_is_plausible( + &Status { + finalized_epoch: current_epoch + 2, + ..Default::default() + }, + current_epoch + )); + assert!(!status_is_plausible( + &Status { + head_slot: (current_epoch + 3) * SLOTS_PER_EPOCH, + ..Default::default() + }, + current_epoch + )); + } + async fn create_network( socket_address: IpAddr, socket_port: u16, diff --git a/crates/networking/req_resp/src/beacon/messages/mod.rs b/crates/networking/req_resp/src/beacon/messages/mod.rs index 11d960e03..d8201ab99 100644 --- a/crates/networking/req_resp/src/beacon/messages/mod.rs +++ b/crates/networking/req_resp/src/beacon/messages/mod.rs @@ -8,6 +8,7 @@ pub mod status; use std::sync::Arc; +use alloy_primitives::aliases::B32; use blob_sidecars::{BlobSidecarsByRangeV1Request, BlobSidecarsByRootV1Request}; use blocks::{BeaconBlocksByRangeV2Request, BeaconBlocksByRootV2Request}; use data_column_sidecars::{DataColumnSidecarsByRangeV1Request, DataColumnSidecarsByRootV1Request}; @@ -18,6 +19,10 @@ use ream_consensus_beacon::{ blob_sidecar::BlobSidecar, data_column_sidecar::DataColumnSidecar, electra::beacon_block::SignedBeaconBlock, }; +use ream_consensus_misc::{ + constants::beacon::genesis_validators_root, misc::compute_epoch_at_slot, +}; +use ream_network_spec::networks::beacon_network_spec; use ssz_derive::{Decode, Encode}; use status::Status; @@ -143,3 +148,25 @@ pub enum BeaconResponseMessage { DataColumnSidecarsByRange(DataColumnSidecar), DataColumnSidecarsByRoot(DataColumnSidecar), } + +impl BeaconResponseMessage { + pub fn context_bytes(&self) -> Option { + let slot = match self { + Self::BeaconBlocksByRange(block) | Self::BeaconBlocksByRoot(block) => { + block.message.slot + } + Self::BlobSidecarsByRange(sidecar) | Self::BlobSidecarsByRoot(sidecar) => { + sidecar.signed_block_header.message.slot + } + Self::DataColumnSidecarsByRange(sidecar) | Self::DataColumnSidecarsByRoot(sidecar) => { + sidecar.signed_block_header.message.slot + } + Self::MetaData(_) | Self::Goodbye(_) | Self::Status(_) | Self::Ping(_) => return None, + }; + + Some( + beacon_network_spec() + .fork_digest(compute_epoch_at_slot(slot), genesis_validators_root()), + ) + } +} diff --git a/crates/networking/req_resp/src/handler.rs b/crates/networking/req_resp/src/handler.rs index 77a901a03..e15a59a46 100644 --- a/crates/networking/req_resp/src/handler.rs +++ b/crates/networking/req_resp/src/handler.rs @@ -225,7 +225,7 @@ impl ReqRespConnectionHandler { self.behaviour_events .push(HandlerEvent::Err(ReqRespMessageError::Outbound { request_id: info.request_id, - err: ReqRespError::InvalidData(format!("Dial upgrade error: {error:?}")), + err: ReqRespError::RawError(format!("Dial upgrade error: {error:?}")), })); } diff --git a/crates/networking/req_resp/src/inbound_protocol.rs b/crates/networking/req_resp/src/inbound_protocol.rs index f466a090f..01094bfeb 100644 --- a/crates/networking/req_resp/src/inbound_protocol.rs +++ b/crates/networking/req_resp/src/inbound_protocol.rs @@ -16,8 +16,6 @@ use libp2p::{ bytes::{Buf, BufMut}, core::UpgradeInfo, }; -use ream_consensus_misc::constants::beacon::{FULU_FORK_EPOCH, genesis_validators_root}; -use ream_network_spec::networks::beacon_network_spec; use snap::{read::FrameDecoder, write::FrameEncoder}; use ssz::{Decode, Encode}; use ssz_types::{VariableList, typenum::U256}; @@ -136,6 +134,10 @@ impl Encoder for InboundSSZSnappyCodec { fn encode(&mut self, item: RespMessage, dst: &mut BytesMut) -> Result<(), Self::Error> { dst.clear(); let response_code = item.as_response_code().expect("EndOfStream cannot be sent"); + let context_bytes = match &item { + RespMessage::Response(message) => message.context_bytes(), + RespMessage::Error(_) | RespMessage::EndOfStream => None, + }; dst.put_u8(u8::from(response_code)); let bytes = match item { @@ -163,9 +165,12 @@ impl Encoder for InboundSSZSnappyCodec { } if self.protocol.protocol.has_context_bytes() && response_code == ResponseCode::Success { - dst.extend( - beacon_network_spec().fork_digest(FULU_FORK_EPOCH, genesis_validators_root()), - ); + let context_bytes = context_bytes.ok_or_else(|| { + ReqRespError::InvalidData( + "Chunked beacon response does not provide fork context bytes".to_string(), + ) + })?; + dst.extend(context_bytes); } Uvi::::default().encode(bytes.len(), dst)?; diff --git a/crates/networking/req_resp/src/messages.rs b/crates/networking/req_resp/src/messages.rs index 638527611..89563fc5f 100644 --- a/crates/networking/req_resp/src/messages.rs +++ b/crates/networking/req_resp/src/messages.rs @@ -1,5 +1,6 @@ use std::sync::Arc; +use alloy_primitives::aliases::B32; use ssz_derive::{Decode, Encode}; use super::{ @@ -37,3 +38,12 @@ pub enum ResponseMessage { Beacon(Arc), Lean(Arc), } + +impl ResponseMessage { + pub fn context_bytes(&self) -> Option { + match self { + Self::Beacon(message) => message.context_bytes(), + Self::Lean(_) => None, + } + } +} diff --git a/crates/networking/req_resp/src/outbound_protocol.rs b/crates/networking/req_resp/src/outbound_protocol.rs index ed9ffeac4..f7ad722ed 100644 --- a/crates/networking/req_resp/src/outbound_protocol.rs +++ b/crates/networking/req_resp/src/outbound_protocol.rs @@ -18,8 +18,6 @@ use ream_consensus_beacon::{ electra::beacon_block::SignedBeaconBlock, }; use ream_consensus_lean::block::SignedBlock as ActiveBlock; -use ream_consensus_misc::constants::beacon::{FULU_FORK_EPOCH, genesis_validators_root}; -use ream_network_spec::networks::beacon_network_spec; use snap::{read::FrameDecoder, write::FrameEncoder}; use ssz::{Decode, Encode}; use ssz_types::{VariableList, typenum::U256}; @@ -156,15 +154,6 @@ impl Decoder for OutboundSSZSnappyCodec { src.advance(B32::len_bytes()); } - if let Some(context_bytes) = self.context_bytes - && context_bytes - != beacon_network_spec().fork_digest(FULU_FORK_EPOCH, genesis_validators_root()) - { - return Ok(Some(RespMessage::Error(ReqRespError::InvalidData( - "Invalid context bytes, we only support Electra".to_string(), - )))); - } - let length = match self.length { Some(cached_length) => cached_length, None => { @@ -191,7 +180,7 @@ impl Decoder for OutboundSSZSnappyCodec { Ok(_) => { src.advance(decoder.get_ref().position() as usize); self.length = None; - self.context_bytes = None; + let context_bytes = self.context_bytes.take(); if ResponseCode::Success == response_code { match self.protocol.protocol { SupportedProtocol::Beacon(beacon_supported_protocol) => { @@ -257,6 +246,15 @@ impl Decoder for OutboundSSZSnappyCodec { ) } }; + if let Some(context_bytes) = context_bytes + && response_message.context_bytes() != Some(context_bytes) + { + return Ok(Some(RespMessage::Error(ReqRespError::InvalidData( + format!( + "Response context bytes {context_bytes} do not match the response object's fork" + ), + )))); + } Ok(Some(RespMessage::Response(Box::new( ResponseMessage::Beacon(response_message.into()), )))) diff --git a/crates/networking/syncer/src/block_range/block_cache.rs b/crates/networking/syncer/src/block_range/block_cache.rs index 35e56cdf9..c477978b6 100644 --- a/crates/networking/syncer/src/block_range/block_cache.rs +++ b/crates/networking/syncer/src/block_range/block_cache.rs @@ -9,8 +9,12 @@ use libp2p::PeerId; use ream_chain_beacon::beacon_chain::is_data_availability_check_required; use ream_consensus_beacon::{ blob_sidecar::{BlobIdentifier, BlobSidecar}, - data_column_sidecar::{ColumnIdentifier, DataColumnSidecar}, + data_column_sidecar::{ + ColumnIdentifier, DataColumnSidecar, NUMBER_OF_COLUMNS, + get_data_column_sidecars_from_column_sidecar, + }, electra::beacon_block::SignedBeaconBlock, + matrix_entry::{das_context, recover_cells_and_kzg_proofs}, }; use ream_consensus_misc::misc::compute_epoch_at_slot; use ream_network_spec::networks::beacon_network_spec; @@ -18,6 +22,7 @@ use ream_polynomial_commitments::handlers::{ verify_blob_kzg_proof_batch, verify_data_column_sidecar_kzg_proofs, }; use ssz::Encode; +use tracing::warn; use tree_hash::TreeHash; use super::{FrontierObservation, MAX_BLOCKS_PER_REQUEST, peer_range_downloader::Range}; @@ -116,6 +121,7 @@ impl BlockAndBlobBundle { pub struct BlockCache { blocks_and_blobs: HashMap, + known_parent_roots: HashSet, current_cache_size: u64, initial_parent_root: B256, block_ranges_to_retry: Vec, @@ -135,6 +141,7 @@ impl BlockCache { pub fn new(initial_parent_root: B256, next_start_slot: u64) -> Self { Self { blocks_and_blobs: HashMap::new(), + known_parent_roots: HashSet::new(), current_cache_size: 0, initial_parent_root, block_ranges_to_retry: vec![], @@ -340,6 +347,51 @@ impl BlockCache { Ok(()) } + pub fn add_parent_blocks( + &mut self, + blocks: Vec, + source_peer: PeerId, + ) -> Result<(), AddBlocksError> { + for block in &blocks { + let root = block.message.tree_hash_root(); + let mut matching_children = self + .blocks_and_blobs + .values() + .filter(|child| child.block.message.parent_root == root) + .peekable(); + if matching_children.peek().is_none() { + return Err(anyhow::anyhow!( + "parent block {root} does not connect to any cached child" + ) + .into()); + } + for child in matching_children { + if block.message.slot >= child.block.message.slot { + return Err(anyhow::anyhow!( + "parent block {root} at slot {} is not older than child at slot {}", + block.message.slot, + child.block.message.slot + ) + .into()); + } + } + } + + self.add_blocks(blocks, false, source_peer) + } + + pub fn first_child_slot(&self, parent_root: B256) -> Option { + self.blocks_and_blobs + .values() + .filter(|bundle| bundle.block.message.parent_root == parent_root) + .map(|bundle| bundle.block.message.slot) + .min() + } + + pub fn mark_parent_root_known(&mut self, parent_root: B256) { + self.known_parent_roots.insert(parent_root); + } + pub fn add_blobs(&mut self, blobs: Vec) -> anyhow::Result<()> { let mut validated = Vec::with_capacity(blobs.len()); for blob_sidecar in blobs { @@ -420,6 +472,11 @@ impl BlockCache { .blocks_and_blobs .get(&block_root) .expect("presence just checked above"); + ensure!( + column.verify(), + "Malformed data column sidecar {} for block {block_root}", + column.index + ); ensure!( column.signed_block_header == bundle.block.signed_header(), "Data column sidecar {} does not belong to block {block_root}", @@ -439,6 +496,7 @@ impl BlockCache { validated.push((block_root, column)); } + let affected_roots: HashSet = validated.iter().map(|(root, _)| *root).collect(); for (block_root, column) in validated { let bundle = self .blocks_and_blobs @@ -449,6 +507,16 @@ impl BlockCache { .insert(ColumnIdentifier::new(block_root, column.index), column); } + for block_root in affected_roots { + let bundle = self + .blocks_and_blobs + .get_mut(&block_root) + .expect("affected block remains cached"); + if let Err(err) = reconstruct_required_columns(bundle, required_columns) { + warn!("Failed to reconstruct data columns for block {block_root}: {err:?}"); + } + } + Ok(()) } @@ -595,6 +663,10 @@ impl BlockCache { return DataToFetch::BlockRange(Range::new(start_slot, blocks_to_fill)); } + if !self.block_ranges_in_progress.is_empty() { + return DataToFetch::DownloadsInProgress; + } + if let Some(range) = self.take_schedulable_column_range(candidate_peers, now) { return DataToFetch::DataColumnRange(range); } @@ -674,6 +746,9 @@ impl BlockCache { .blocks_and_blobs .contains_key(&block.block.message.parent_root) && block.block.message.parent_root != self.initial_parent_root + && !self + .known_parent_roots + .contains(&block.block.message.parent_root) { missing_roots.push(block.block.message.parent_root); } @@ -777,6 +852,65 @@ impl BlockCache { } expected } + + pub fn block_slot(&self, block_root: B256) -> Option { + self.blocks_and_blobs + .get(&block_root) + .map(|bundle| bundle.block.message.slot) + } +} + +fn reconstruct_required_columns( + bundle: &mut BlockAndBlobBundle, + required_columns: &HashSet, +) -> anyhow::Result<()> { + if required_columns.iter().all(|index| { + bundle.columns.contains_key(&ColumnIdentifier::new( + bundle.block.message.tree_hash_root(), + *index, + )) + }) || bundle.columns.len() < (NUMBER_OF_COLUMNS / 2) as usize + { + return Ok(()); + } + + let available: Vec<&DataColumnSidecar> = bundle.columns.values().collect(); + let base = *available + .first() + .ok_or_else(|| anyhow::anyhow!("no data column is available for reconstruction"))?; + let blob_count = base.column.len(); + let mut cells_and_kzg_proofs = Vec::with_capacity(blob_count); + + for blob_index in 0..blob_count { + let mut column_indices = Vec::with_capacity(available.len()); + let mut cells = Vec::with_capacity(available.len()); + for sidecar in &available { + let cell = sidecar.column.get(blob_index).ok_or_else(|| { + anyhow::anyhow!("column {} is missing blob row {blob_index}", sidecar.index) + })?; + column_indices.push(sidecar.index); + cells.push(cell.clone()); + } + cells_and_kzg_proofs.push(recover_cells_and_kzg_proofs( + column_indices, + cells, + das_context(), + )?); + } + + let reconstructed = + get_data_column_sidecars_from_column_sidecar(base.clone(), cells_and_kzg_proofs) + .map_err(|err| anyhow::anyhow!("failed to assemble reconstructed columns: {err}"))?; + let block_root = bundle.block.message.tree_hash_root(); + for column in reconstructed { + if required_columns.contains(&column.index) { + bundle + .columns + .entry(ColumnIdentifier::new(block_root, column.index)) + .or_insert(column); + } + } + Ok(()) } #[derive(Debug, Clone, PartialEq, Eq)] @@ -856,6 +990,33 @@ mod tests { ); } + #[test] + fn in_flight_range_prevents_a_temporary_gap_from_becoming_a_parent_lookup() { + initialize_test_network_spec(); + let mut cache = BlockCache::new(B256::ZERO, 10); + let range = match cache.data_to_fetch(20, 0, &HashSet::new(), &[], Instant::now(), false) { + DataToFetch::BlockRange(range) => range, + other => panic!("expected initial block range, got {other:?}"), + }; + cache.mark_block_range_in_progress(range); + let later_block = SignedBeaconBlock { + message: BeaconBlock { + slot: 20, + parent_root: B256::repeat_byte(9), + ..Default::default() + }, + signature: Default::default(), + }; + cache + .add_blocks(vec![later_block], false, PeerId::random()) + .expect("later range result should enter cache"); + + assert_eq!( + cache.data_to_fetch(20, 0, &HashSet::new(), &[], Instant::now(), false), + DataToFetch::DownloadsInProgress + ); + } + #[test] fn data_to_fetch_requests_contiguous_non_empty_ranges() { initialize_test_network_spec(); @@ -1015,6 +1176,7 @@ mod tests { #[test] fn add_data_columns_ignores_extras_and_rejects_bad_proofs() { + initialize_test_network_spec(); use ream_consensus_beacon::data_column_sidecar::get_data_column_sidecars_from_block; let blob = Blob::default(); @@ -1043,7 +1205,7 @@ mod tests { let mut cache = BlockCache::new(B256::ZERO, 0); cache - .add_blocks(vec![block], false, PeerId::random()) + .add_blocks(vec![block.clone()], false, PeerId::random()) .expect("block should enter cache"); // Only column 0 was requested; extra columns in the response are ignored, not fatal. @@ -1068,10 +1230,32 @@ mod tests { .add_data_columns(vec![tampered], &required_columns) .is_err() ); + + let mut reconstruction_cache = BlockCache::new(B256::ZERO, 0); + reconstruction_cache + .add_blocks(vec![block], false, PeerId::random()) + .expect("block should enter reconstruction cache"); + let all_columns: HashSet = (0..NUMBER_OF_COLUMNS).collect(); + reconstruction_cache + .add_data_columns( + columns[..(NUMBER_OF_COLUMNS / 2) as usize].to_vec(), + &all_columns, + ) + .expect("the first half of valid columns should reconstruct the second half"); + assert_eq!( + reconstruction_cache + .blocks_and_blobs + .get(&block_root) + .expect("block should be cached") + .columns + .len(), + NUMBER_OF_COLUMNS as usize + ); } #[test] fn add_data_columns_is_atomic_a_bad_item_does_not_leave_earlier_items_mutated() { + initialize_test_network_spec(); use ream_consensus_beacon::data_column_sidecar::get_data_column_sidecars_from_block; let blob = Blob::default(); @@ -1280,6 +1464,7 @@ mod tests { #[test] fn add_data_columns_drops_a_column_for_an_unknown_block_without_erroring_or_banning() { + initialize_test_network_spec(); use ream_consensus_beacon::data_column_sidecar::get_data_column_sidecars_from_block; let blob = Blob::default(); @@ -1336,6 +1521,44 @@ mod tests { } } + #[test] + fn known_processable_parent_stops_missing_parent_requests() { + initialize_test_network_spec(); + let initial_root = B256::repeat_byte(1); + let known_parent = B256::repeat_byte(2); + let peer = PeerId::random(); + let mut cache = BlockCache::new(initial_root, 10); + cache + .add_blocks(vec![child_block(known_parent, 20)], true, peer) + .unwrap(); + + assert_eq!(cache.first_child_slot(known_parent), Some(20)); + assert_eq!(cache.get_missing_block_roots(), vec![known_parent]); + cache.mark_parent_root_known(known_parent); + assert!(cache.get_missing_block_roots().is_empty()); + } + + #[test] + fn parent_lookup_rejects_a_non_decreasing_ancestry() { + initialize_test_network_spec(); + let initial_root = B256::repeat_byte(1); + let peer = PeerId::random(); + let mut parent = child_block(B256::repeat_byte(3), 20); + let parent_root = parent.message.tree_hash_root(); + let child = child_block(parent_root, 20); + let mut cache = BlockCache::new(initial_root, 10); + cache.add_blocks(vec![child], true, peer).unwrap(); + + assert!(cache.add_parent_blocks(vec![parent.clone()], peer).is_err()); + parent.message.slot = 19; + let valid_parent_root = parent.message.tree_hash_root(); + let mut cache = BlockCache::new(initial_root, 10); + cache + .add_blocks(vec![child_block(valid_parent_root, 20)], true, peer) + .unwrap(); + cache.add_parent_blocks(vec![parent], peer).unwrap(); + } + #[test] fn advance_empty_coverage_opens_a_coverage_anchor_that_a_connecting_block_clears() { let root = B256::repeat_byte(1); diff --git a/crates/networking/syncer/src/block_range/mod.rs b/crates/networking/syncer/src/block_range/mod.rs index 4b6bc127c..7c1cfe6a8 100644 --- a/crates/networking/syncer/src/block_range/mod.rs +++ b/crates/networking/syncer/src/block_range/mod.rs @@ -32,20 +32,27 @@ use ream_consensus_beacon::{ electra::beacon_block::SignedBeaconBlock, matrix_entry::{compute_cells_and_kzg_proofs, das_context}, }; -use ream_consensus_misc::{constants::beacon::SLOTS_PER_EPOCH, misc::compute_epoch_at_slot}; +use ream_consensus_misc::{ + constants::beacon::SLOTS_PER_EPOCH, + misc::{compute_epoch_at_slot, compute_start_slot_at_epoch}, +}; use ream_executor::ReamExecutor; +use ream_fork_choice_beacon::store::Store; use ream_network_spec::networks::beacon_network_spec; use ream_p2p::network::beacon::{channel::P2PMessage, network_state::NetworkState}; use ream_polynomial_commitments::handlers::verify_blob_kzg_proof_batch; use ream_req_resp::{ MAX_CONCURRENT_REQUESTS, beacon::messages::data_column_sidecars::DataColumnsByRootIdentifier, - inbound_protocol::ResponseCode, + constants::MAX_REQUEST_BLOCKS_DENEB, inbound_protocol::ResponseCode, }; use ream_storage::tables::{ field::REDBField, table::{CustomTable, REDBTable}, }; -use recovery::{CoverageAdvance, RECOVERY_ROUND_TIMEOUT, RecoveryOutcome}; +use recovery::{ + CoverageAdvance, MAX_TOTAL_ANCESTOR_REQUESTS_PER_ROUND, RECOVERY_ROUND_TIMEOUT, + RecoveryOutcome, is_processable_connection_point, +}; use tokio::{sync::mpsc::UnboundedSender, task::JoinHandle, time::sleep}; use tracing::{info, warn}; use tree_hash::TreeHash; @@ -74,6 +81,47 @@ fn candidate_exhaustion_deserves_backoff(target_slot: u64, head_slot: u64) -> bo target_slot > head_slot } +fn parent_lookup_budget_exhausted( + started_at: Option, + requests_started: u64, + now: Instant, +) -> bool { + requests_started >= MAX_TOTAL_ANCESTOR_REQUESTS_PER_ROUND + || started_at + .is_some_and(|started| now.saturating_duration_since(started) >= RECOVERY_ROUND_TIMEOUT) +} + +fn stored_block_slot(store: &Store, root: B256) -> anyhow::Result> { + Ok(store + .db + .block_provider() + .get(root)? + .map(|block| block.message.slot)) +} + +fn sync_targets_confirm_caught_up( + latest_synced_slot: u64, + current_slot: u64, + finalized_selection: &TargetSelection, + head_selection: &TargetSelection, +) -> bool { + if matches!( + finalized_selection, + TargetSelection::Ready { target_slot, .. } if latest_synced_slot < *target_slot + ) { + return false; + } + + let clock_confirms_caught_up = + current_slot.saturating_sub(latest_synced_slot) <= SLOT_IMPORT_TOLERANCE; + match head_selection { + TargetSelection::Ready { target_slot, .. } => { + *target_slot <= latest_synced_slot && clock_confirms_caught_up + } + TargetSelection::NoQuorum => clock_confirms_caught_up, + } +} + /// Validates downloaded blob sidecars and derives the columns needed for data availability. /// An empty `blob_sidecars` map defers to the block-lookup coordinator instead of erroring, /// since range sync never fetches legacy blob sidecars for post-Fulu blocks. @@ -241,6 +289,10 @@ impl StuckFrontier { } } + fn needs_recovery(&self, observation: &FrontierObservation, next_start_slot: u64) -> bool { + self.matches(observation) && self.tier() >= 2 && next_start_slot < observation.target_slot + } + fn refresh_attempt_round(&mut self, now: Instant) { if let Some(cooldown_until) = self.cooldown_until && now >= cooldown_until @@ -574,12 +626,13 @@ impl BlockRangeSyncer { self.peer_manager.update_peer_set(); let store = self.beacon_chain.store.lock().await; - let latest_synced_slot = store - .db - .slot_index_provider() - .get_highest_slot() - .unwrap_or_default() - .unwrap_or(0); + let Ok(canonical_head_root) = store.get_head() else { + return false; + }; + let latest_synced_slot = match stored_block_slot(&store, canonical_head_root) { + Ok(Some(slot)) => slot, + _ => return false, + }; let Ok(finalized_epoch) = store.db.finalized_checkpoint_provider().get() else { return false; }; @@ -590,29 +643,16 @@ impl BlockRangeSyncer { drop(store); let finalized_selection = self.peer_manager.best_finalized(finalized_epoch); - let still_behind_finalized = matches!( - &finalized_selection, - TargetSelection::Ready { target_slot, .. } if latest_synced_slot < *target_slot - ); - if still_behind_finalized { - return false; - } - let our_head_epoch = latest_synced_slot / SLOTS_PER_EPOCH; let head_selection = self .peer_manager .best_non_finalized(MIN_SYNC_PEERS, our_head_epoch); - // The clock is ground truth peers can't fake. - let clock_confirms_caught_up = - current_slot.saturating_sub(latest_synced_slot) <= SLOT_IMPORT_TOLERANCE; - - let TargetSelection::Ready { target_slot, .. } = head_selection else { - return clock_confirms_caught_up; - }; - - let peers_report_caught_up = target_slot <= latest_synced_slot; - - peers_report_caught_up && clock_confirms_caught_up + sync_targets_confirm_caught_up( + latest_synced_slot, + current_slot, + &finalized_selection, + &head_selection, + ) } pub fn start(mut self) -> JoinHandle)>> { @@ -669,6 +709,9 @@ impl BlockRangeSyncer { let mut recovery_decision_made_this_segment = false; let mut pending_conclusions = PendingConclusions::default(); let mut segment_exclusions = SegmentExclusions::default(); + let mut parent_lookup_started_at: Option = None; + let mut parent_lookup_requests = 0u64; + let mut parent_lookup_peers = HashSet::new(); loop { self.peer_manager.update_peer_set(); @@ -854,12 +897,12 @@ impl BlockRangeSyncer { let parent_root = block_cache.initial_parent_root(); let covered_through_slot = coverage.end_slot_exclusive.saturating_sub(1); let confirming_peers = coverage.confirming_peers.clone(); - let _ = block_cache.advance_empty_coverage( + block_cache.advance_empty_coverage( frontier_observation, parent_root, covered_through_slot, confirming_peers, - ); + )?; restore_window_open = false; } } @@ -871,8 +914,7 @@ impl BlockRangeSyncer { if recovery_window_open && !recovery_decision_made_this_segment && let Some(frontier) = self.frontier_for(phase) - && frontier.matches(&observation) - && frontier.tier() >= 2 + && frontier.needs_recovery(&observation, block_cache.next_start_slot()) { recovery_decision_made_this_segment = true; recovery_window_open = false; @@ -978,9 +1020,23 @@ impl BlockRangeSyncer { DataToFetch::DataColumnRange(range) => { let key = RequestKey::ColumnRange(range); let excluded = block_cache.attempted_peers_for(key); + let expected_identifiers = + block_cache.expected_column_identifiers_in_range(range, &required_columns); + let requested_columns_and_slots: Vec<(u64, u64)> = expected_identifiers + .iter() + .filter_map(|identifier| { + block_cache + .block_slot(identifier.block_root) + .map(|slot| (identifier.index, slot)) + }) + .collect(); let Some(peer) = self .peer_manager - .fetch_idle_peer_from_excluding(&candidate_peers, &excluded) + .fetch_idle_peer_for_columns_from_excluding( + &candidate_peers, + &requested_columns_and_slots, + &excluded, + ) else { self.peer_manager.update_peer_set(); info!("No idle peers available for data column range sync."); @@ -989,8 +1045,28 @@ impl BlockRangeSyncer { continue; }; - let expected_known_identifiers = - block_cache.expected_column_identifiers_in_range(range, &required_columns); + let peer_columns: HashSet = expected_identifiers + .iter() + .filter_map(|identifier| { + let slot = block_cache.block_slot(identifier.block_root)?; + (self + .peer_manager + .peer_custodies_column(&peer.peer_id, identifier.index) + && self.peer_manager.peer_can_serve_slot(&peer.peer_id, slot)) + .then_some(identifier.index) + }) + .collect(); + let expected_known_identifiers: Vec = expected_identifiers + .into_iter() + .filter(|identifier| { + peer_columns.contains(&identifier.index) + && block_cache.block_slot(identifier.block_root).is_some_and( + |slot| { + self.peer_manager.peer_can_serve_slot(&peer.peer_id, slot) + }, + ) + }) + .collect(); block_cache.mark_column_range_in_progress(range); task_handles.push(DownloadTask::new_data_column_range( @@ -999,7 +1075,7 @@ impl BlockRangeSyncer { self.p2p_sender.clone(), self.executor.clone(), range, - required_columns.iter().copied().collect(), + peer_columns.into_iter().collect(), ), range, peer.peer_id, @@ -1007,6 +1083,60 @@ impl BlockRangeSyncer { )); } DataToFetch::MissingBlockRoots(mut block_roots) => { + let mut reached_finalized_boundary = false; + { + let store = self.beacon_chain.store.lock().await; + let finalized_slot = compute_start_slot_at_epoch( + store.db.finalized_checkpoint_provider().get()?.epoch, + ); + let mut unresolved = Vec::with_capacity(block_roots.len()); + for parent_root in block_roots { + let child_slot = + block_cache.first_child_slot(parent_root).ok_or_else(|| { + anyhow!("Missing cached child for parent root {parent_root}") + })?; + if is_processable_connection_point(&store, parent_root, child_slot)? { + block_cache.mark_parent_root_known(parent_root); + } else if child_slot <= finalized_slot { + reached_finalized_boundary = true; + } else { + unresolved.push(parent_root); + } + } + block_roots = unresolved; + } + + if reached_finalized_boundary { + pending_conclusions.observe( + phase, + observation.clone(), + RemoteNoProgressReason::AncestorNotFound, + parent_lookup_peers.clone(), + HashSet::new(), + ); + block_cache = BlockCache::new(head_root, head_slot); + break; + } + if block_roots.is_empty() { + continue; + } + + if parent_lookup_budget_exhausted( + parent_lookup_started_at, + parent_lookup_requests, + now, + ) { + pending_conclusions.observe( + phase, + observation.clone(), + RemoteNoProgressReason::RecoveryBudgetExhausted, + parent_lookup_peers.clone(), + HashSet::new(), + ); + block_cache = BlockCache::new(head_root, head_slot); + break; + } + let mut exhausted_this_tick = HashSet::new(); while !block_roots.is_empty() { let Some(peer) = self @@ -1031,10 +1161,22 @@ impl BlockRangeSyncer { exhausted_this_tick.insert(peer.peer_id); continue; } - let chunk: Vec = - assigned.into_iter().take(MAX_CONCURRENT_REQUESTS).collect(); + let remaining_budget = (MAX_TOTAL_ANCESTOR_REQUESTS_PER_ROUND + - parent_lookup_requests) + as usize; + let chunk: Vec = assigned + .into_iter() + .take(MAX_CONCURRENT_REQUESTS.min(remaining_budget)) + .collect(); + if chunk.is_empty() { + self.peer_manager.mark_peer_as_idle(&peer.peer_id); + break; + } block_cache.extend_block_roots_in_progress(&chunk); + parent_lookup_started_at.get_or_insert_with(Instant::now); + parent_lookup_requests += chunk.len() as u64; + parent_lookup_peers.insert(peer.peer_id); task_handles.push(DownloadTask::new_block_roots( PeerRootsDownloader::start( @@ -1096,9 +1238,21 @@ impl BlockRangeSyncer { DataToFetch::MissingDataColumnIdentifiers(mut identifiers) => { let mut exhausted_this_tick = HashSet::new(); while !identifiers.is_empty() { + let requested_columns_and_slots: Vec<(u64, u64)> = identifiers + .iter() + .filter_map(|identifier| { + block_cache + .block_slot(identifier.block_root) + .map(|slot| (identifier.index, slot)) + }) + .collect(); let Some(peer) = self .peer_manager - .fetch_idle_peer_from_excluding(&candidate_peers, &exhausted_this_tick) + .fetch_idle_peer_for_columns_from_excluding( + &candidate_peers, + &requested_columns_and_slots, + &exhausted_this_tick, + ) else { self.peer_manager.update_peer_set(); info!("No idle peers available for data column sync."); @@ -1108,9 +1262,17 @@ impl BlockRangeSyncer { let (assigned, remaining): (Vec, Vec) = identifiers.into_iter().partition(|identifier| { - !block_cache - .attempted_peers_for(RequestKey::Column(*identifier)) - .contains(&peer.peer_id) + self.peer_manager + .peer_custodies_column(&peer.peer_id, identifier.index) + && block_cache.block_slot(identifier.block_root).is_some_and( + |slot| { + self.peer_manager + .peer_can_serve_slot(&peer.peer_id, slot) + }, + ) + && !block_cache + .attempted_peers_for(RequestKey::Column(*identifier)) + .contains(&peer.peer_id) }); identifiers = remaining; if assigned.is_empty() { @@ -1118,8 +1280,10 @@ impl BlockRangeSyncer { exhausted_this_tick.insert(peer.peer_id); continue; } - let chunk: Vec = - assigned.into_iter().take(MAX_CONCURRENT_REQUESTS).collect(); + let chunk: Vec = assigned + .into_iter() + .take(MAX_REQUEST_BLOCKS_DENEB as usize) + .collect(); block_cache.extend_data_column_identifiers_in_progress(&chunk); @@ -1178,7 +1342,13 @@ impl BlockRangeSyncer { Err(err) => err.imported_count, }; - info!("All blocks processed successfully."); + match &import_result { + Ok(_) => info!("All blocks processed successfully."), + Err(err) => warn!( + "Block processing stopped after {} successful imports: {:?}", + err.imported_count, err.error + ), + } if imported_count > 0 { self.clear_tracker(); @@ -1866,8 +2036,22 @@ fn poll_ready_tasks( continue; } - if let Err(err) = block_cache.add_blocks(blocks, false, *peer_id) { + if let Err(err) = block_cache.add_parent_blocks(blocks, *peer_id) { warn!("Failed to add downloaded blocks to cache: {err:?}"); + for root in roots.iter() { + block_cache.mark_attempted( + RequestKey::BlockRoot(*root), + *peer_id, + candidate_peers, + now, + ); + } + peer_manager.ban_peer( + peer_id, + BanReason::ProtocolError(format!( + "returned an invalid parent block: {err}" + )), + ); } } Poll::Pending => {} @@ -2155,6 +2339,18 @@ mod tests { assert!(!candidate_exhaustion_deserves_backoff(50, 100)); } + #[test] + fn parent_lookup_is_bounded_by_both_requests_and_time() { + let now = Instant::now(); + assert!(!parent_lookup_budget_exhausted(Some(now), 127, now)); + assert!(parent_lookup_budget_exhausted(Some(now), 128, now)); + assert!(parent_lookup_budget_exhausted( + Some(now - RECOVERY_ROUND_TIMEOUT), + 1, + now + )); + } + #[test] fn range_blob_sidecars_build_valid_columns_and_reject_bad_proofs() { let blob = Blob::default(); @@ -2338,7 +2534,20 @@ mod tests { let (_data_dir, beacon_chain) = test_beacon_chain(); let seconds_per_slot = beacon_network_spec().seconds_per_slot(); let highest_slot = 100u64; - let highest_root = B256::repeat_byte(0x42); + let genesis_block = SignedBeaconBlock { + message: BeaconBlock::default(), + signature: Default::default(), + }; + let genesis_root = genesis_block.message.tree_hash_root(); + let highest_block = SignedBeaconBlock { + message: BeaconBlock { + slot: highest_slot, + parent_root: genesis_root, + ..Default::default() + }, + signature: Default::default(), + }; + let highest_root = highest_block.message.tree_hash_root(); { let store = beacon_chain.store.lock().await; @@ -2349,15 +2558,50 @@ mod tests { .expect("insert genesis time"); store .db - .slot_index_provider() - .insert(highest_slot, highest_root) - .expect("insert highest synced slot"); + .block_provider() + .insert(genesis_root, genesis_block) + .expect("insert genesis block"); + store + .db + .block_provider() + .insert(highest_root, highest_block) + .expect("insert canonical head block"); + store + .db + .unrealized_justifications_provider() + .insert( + genesis_root, + Checkpoint { + epoch: 0, + root: genesis_root, + }, + ) + .expect("insert genesis voting source"); + store + .db + .unrealized_justifications_provider() + .insert( + highest_root, + Checkpoint { + epoch: 0, + root: highest_root, + }, + ) + .expect("insert canonical head voting source"); + store + .db + .justified_checkpoint_provider() + .insert(Checkpoint { + epoch: 0, + root: highest_root, + }) + .expect("insert justified checkpoint"); store .db .finalized_checkpoint_provider() .insert(Checkpoint { epoch: 0, - root: B256::ZERO, + root: genesis_root, }) .expect("insert finalized checkpoint"); } @@ -2397,7 +2641,7 @@ mod tests { .await .db .time_provider() - .insert(highest_slot * seconds_per_slot) + .insert((highest_slot + SLOTS_PER_EPOCH) * seconds_per_slot) .expect("insert time"); assert!(syncer.is_synced_to_head_slot().await); @@ -2414,6 +2658,138 @@ mod tests { assert!(!syncer.is_synced_to_head_slot().await); } + #[test] + fn is_synced_to_head_slot_ignores_a_higher_noncanonical_stored_block() { + futures::executor::block_on( + is_synced_to_head_slot_ignores_a_higher_noncanonical_stored_block_inner(), + ); + } + + async fn is_synced_to_head_slot_ignores_a_higher_noncanonical_stored_block_inner() { + initialize_test_network_spec(); + let (_data_dir, beacon_chain) = test_beacon_chain(); + let seconds_per_slot = beacon_network_spec().seconds_per_slot(); + let canonical_slot = 100u64; + let side_slot = canonical_slot + SLOT_IMPORT_TOLERANCE + 10; + let genesis_block = SignedBeaconBlock { + message: BeaconBlock::default(), + signature: Default::default(), + }; + let genesis_root = genesis_block.message.tree_hash_root(); + let canonical_block = SignedBeaconBlock { + message: BeaconBlock { + slot: canonical_slot, + parent_root: genesis_root, + ..Default::default() + }, + signature: Default::default(), + }; + let canonical_root = canonical_block.message.tree_hash_root(); + let side_block = SignedBeaconBlock { + message: BeaconBlock { + slot: side_slot, + parent_root: B256::repeat_byte(0x55), + ..Default::default() + }, + signature: Default::default(), + }; + let side_root = side_block.message.tree_hash_root(); + + { + let store = beacon_chain.store.lock().await; + store.db.genesis_time_provider().insert(0).unwrap(); + store + .db + .block_provider() + .insert(genesis_root, genesis_block) + .unwrap(); + store + .db + .block_provider() + .insert(canonical_root, canonical_block) + .unwrap(); + store + .db + .unrealized_justifications_provider() + .insert( + genesis_root, + Checkpoint { + epoch: 0, + root: genesis_root, + }, + ) + .unwrap(); + store + .db + .unrealized_justifications_provider() + .insert( + canonical_root, + Checkpoint { + epoch: 0, + root: canonical_root, + }, + ) + .unwrap(); + store + .db + .block_provider() + .insert(side_root, side_block) + .unwrap(); + store + .db + .justified_checkpoint_provider() + .insert(Checkpoint { + epoch: 0, + root: canonical_root, + }) + .unwrap(); + store + .db + .finalized_checkpoint_provider() + .insert(Checkpoint { + epoch: 0, + root: genesis_root, + }) + .unwrap(); + store + .db + .time_provider() + .insert(side_slot * seconds_per_slot) + .unwrap(); + assert_eq!(store.get_head().unwrap(), canonical_root); + } + + let network_state = test_network_state(); + for _ in 0..MIN_SYNC_PEERS { + let peer_id = PeerId::random(); + let mut peer = CachedPeer::new( + peer_id, + None, + ConnectionState::Connected, + Direction::Outbound, + None, + ); + peer.status = Some(Status { + head_slot: side_slot, + head_root: side_root, + ..Default::default() + }); + network_state.peer_table.write().insert(peer_id, peer); + } + + let (p2p_sender, _p2p_receiver) = tokio::sync::mpsc::unbounded_channel(); + let mut syncer = BlockRangeSyncer::new( + Arc::new(beacon_chain), + p2p_sender, + network_state, + ReamExecutor::new().expect("executor should start"), + ); + assert!( + !syncer.is_synced_to_head_slot().await, + "a higher side-fork block in the slot index must not stand in for the canonical head" + ); + } + #[test] fn poll_ready_tasks_partial_block_roots_marks_only_the_omitted_root_attempted() { initialize_test_network_spec(); @@ -2431,6 +2807,20 @@ mod tests { let omitted_root = B256::repeat_byte(9); let mut block_cache = BlockCache::new(B256::ZERO, 0); + block_cache + .add_blocks( + vec![SignedBeaconBlock { + message: BeaconBlock { + slot: 2, + parent_root: delivered_root, + ..Default::default() + }, + signature: Default::default(), + }], + true, + peer_id, + ) + .expect("cached child should be accepted"); let handle = executor.spawn(async move { StreamOutcome::Complete(vec![delivered_block]) }); let mut tasks = vec![DownloadTask::new_block_roots( handle, @@ -2856,6 +3246,25 @@ mod tests { } } + #[test] + fn coverage_at_target_does_not_consume_the_segments_recovery_decision() { + let frontier = test_frontier(test_frontier_id(0), B256::ZERO, 10); + let observation = FrontierObservation { + anchor_root: B256::ZERO, + anchor_slot: 10, + phase: SyncPhase::Finalized, + scan_start_slot: 0, + target_slot: 110, + }; + + assert!(frontier.needs_recovery(&observation, 109)); + assert!( + !frontier.needs_recovery(&observation, 110), + "a restored Finalized frontier that already covers its target must leave the shared \ + recovery window available for a later Head frontier" + ); + } + #[test] fn allocate_generation_never_repeats_across_interleaved_phase_resets() { initialize_test_network_spec(); diff --git a/crates/networking/syncer/src/block_range/peer_manager.rs b/crates/networking/syncer/src/block_range/peer_manager.rs index c404ed692..6033d71eb 100644 --- a/crates/networking/syncer/src/block_range/peer_manager.rs +++ b/crates/networking/syncer/src/block_range/peer_manager.rs @@ -1,10 +1,14 @@ use std::{ + cmp::Ordering, collections::{HashMap, HashSet, hash_map::Entry}, sync::Arc, time::{Duration, Instant}, }; use libp2p::PeerId; +use ream_consensus_beacon::custody_group::{ + compute_columns_for_custody_group, get_custody_group_indices, +}; use ream_consensus_misc::constants::beacon::SLOTS_PER_EPOCH; use ream_p2p::network::beacon::{network_state::NetworkState, peer::CachedPeer}; use ream_req_resp::beacon::messages::status::Status; @@ -61,6 +65,7 @@ pub struct PeerInfo { pub peer_status: PeerStatus, pub processed_blocks: u64, pub sync_requests_started: u64, + custody_columns: Option>, } pub struct PeerManager { @@ -71,6 +76,31 @@ pub struct PeerManager { } impl PeerManager { + fn custody_columns(peer: &CachedPeer) -> Option> { + let enr = peer.enr.as_ref()?; + let metadata = peer.meta_data.as_ref()?; + let groups = get_custody_group_indices(enr.node_id(), metadata.custody_group_count).ok()?; + let mut columns = HashSet::new(); + for group in groups { + columns.extend(compute_columns_for_custody_group(group).ok()?); + } + Some(columns) + } + + fn peer_preference(peer_a: (&PeerId, &PeerInfo), peer_b: (&PeerId, &PeerInfo)) -> Ordering { + let (id_a, info_a) = peer_a; + let (id_b, info_b) = peer_b; + (info_a.sync_requests_started == 0) + .cmp(&(info_b.sync_requests_started == 0)) + .then(info_a.processed_blocks.cmp(&info_b.processed_blocks)) + .then( + info_b + .sync_requests_started + .cmp(&info_a.sync_requests_started), + ) + .then_with(|| id_b.to_bytes().cmp(&id_a.to_bytes())) + } + pub fn new(network_state: Arc) -> Self { Self { network_state, @@ -93,9 +123,11 @@ impl PeerManager { continue; } + let custody_columns = Self::custody_columns(peer); match self.peers.entry(peer.peer_id) { Entry::Occupied(mut entry) => { entry.get_mut().peer = peer.clone(); + entry.get_mut().custody_columns = custody_columns; } Entry::Vacant(entry) => { entry.insert(PeerInfo { @@ -103,6 +135,7 @@ impl PeerManager { peer_status: PeerStatus::Idle, processed_blocks: 0, sync_requests_started: 0, + custody_columns, }); } } @@ -140,18 +173,14 @@ impl PeerManager { let idle_peer_id = self .peers .iter() - .find(|(_, peer_info)| matches!(peer_info.peer_status, PeerStatus::Idle)) + .filter(|(_, peer_info)| matches!(peer_info.peer_status, PeerStatus::Idle)) + .max_by(|peer_a, peer_b| Self::peer_preference(*peer_a, *peer_b)) .map(|(peer_id, _)| *peer_id)?; self.reserve(&idle_peer_id) } pub fn fetch_idle_peer_from(&mut self, eligible: &[PeerId]) -> Option { - for peer_id in eligible { - if let Some(peer) = self.reserve(peer_id) { - return Some(peer); - } - } - None + self.fetch_idle_peer_from_excluding(eligible, &HashSet::new()) } pub fn fetch_idle_peer_from_excluding( @@ -159,15 +188,73 @@ impl PeerManager { eligible: &[PeerId], excluded: &HashSet, ) -> Option { - for peer_id in eligible { - if excluded.contains(peer_id) { - continue; - } - if let Some(peer) = self.reserve(peer_id) { - return Some(peer); - } - } - None + let peer_id = eligible + .iter() + .filter(|peer_id| !excluded.contains(peer_id)) + .filter_map(|peer_id| self.peers.get_key_value(peer_id)) + .filter(|(_, info)| matches!(info.peer_status, PeerStatus::Idle)) + .max_by(|peer_a, peer_b| Self::peer_preference(*peer_a, *peer_b)) + .map(|(peer_id, _)| *peer_id)?; + self.reserve(&peer_id) + } + + pub fn fetch_idle_peer_for_columns_from_excluding( + &mut self, + eligible: &[PeerId], + requested_columns: &[(u64, u64)], + excluded: &HashSet, + ) -> Option { + let coverage = |info: &PeerInfo| { + requested_columns + .iter() + .filter(|(column, slot)| { + Self::info_custodies_column(info, *column) + && Self::info_can_serve_slot(info, *slot) + }) + .count() + }; + let peer_id = eligible + .iter() + .filter(|peer_id| !excluded.contains(peer_id)) + .filter_map(|peer_id| self.peers.get_key_value(peer_id)) + .filter(|(_, info)| matches!(info.peer_status, PeerStatus::Idle)) + .filter(|(_, info)| coverage(info) > 0) + .max_by(|peer_a, peer_b| { + peer_a + .1 + .custody_columns + .is_some() + .cmp(&peer_b.1.custody_columns.is_some()) + .then_with(|| coverage(peer_a.1).cmp(&coverage(peer_b.1))) + .then_with(|| Self::peer_preference(*peer_a, *peer_b)) + }) + .map(|(peer_id, _)| *peer_id)?; + self.reserve(&peer_id) + } + + fn info_custodies_column(info: &PeerInfo, column: u64) -> bool { + info.custody_columns + .as_ref() + .is_none_or(|columns| columns.contains(&column)) + } + + fn info_can_serve_slot(info: &PeerInfo, slot: u64) -> bool { + info.peer + .status + .as_ref() + .is_none_or(|status| status.earliest_available_slot <= slot) + } + + pub fn peer_custodies_column(&self, peer_id: &PeerId, column: u64) -> bool { + self.peers + .get(peer_id) + .is_none_or(|info| Self::info_custodies_column(info, column)) + } + + pub fn peer_can_serve_slot(&self, peer_id: &PeerId, slot: u64) -> bool { + self.peers + .get(peer_id) + .is_none_or(|info| Self::info_can_serve_slot(info, slot)) } pub fn peer_counts(&self) -> String { @@ -206,6 +293,13 @@ impl PeerManager { let Some(status) = &peer_info.peer.status else { continue; }; + if status + .finalized_epoch + .checked_mul(SLOTS_PER_EPOCH) + .is_none() + { + continue; + } if status.finalized_epoch < our_finalized_epoch { continue; } @@ -230,13 +324,15 @@ impl PeerManager { eligible.sort_by(|a, b| b.1.cmp(&a.1).then(b.2.cmp(&a.2))); TargetSelection::Ready { - target_slot: winner_epoch * SLOTS_PER_EPOCH, + target_slot: winner_epoch + .checked_mul(SLOTS_PER_EPOCH) + .expect("overflowing finalized epochs were filtered above"), eligible_peers: eligible.into_iter().map(|(id, ..)| id).collect(), } } pub fn best_non_finalized(&self, min_peers: usize, our_head_epoch: u64) -> TargetSelection { - let our_head_slot = our_head_epoch * SLOTS_PER_EPOCH; + let our_head_slot = our_head_epoch.saturating_mul(SLOTS_PER_EPOCH); let mut epoch_votes: HashMap = HashMap::new(); let mut candidates: Vec<(PeerId, u64, u64)> = Vec::new(); @@ -272,7 +368,7 @@ impl PeerManager { .filter(|(_, head_slot)| head_slot / SLOTS_PER_EPOCH == target_epoch) .map(|(_, head_slot)| *head_slot) .max() - .unwrap_or(target_epoch * SLOTS_PER_EPOCH); + .unwrap_or_else(|| target_epoch.saturating_mul(SLOTS_PER_EPOCH)); let mut eligible = eligible; eligible.sort_by_key(|&(_, head_slot)| std::cmp::Reverse(head_slot)); @@ -289,6 +385,12 @@ impl PeerManager { .and_then(|info| info.peer.status.clone()) } + pub fn processed_blocks_of(&self, peer_id: &PeerId) -> u64 { + self.peers + .get(peer_id) + .map_or(0, |info| info.processed_blocks) + } + pub fn exact_finalized_epoch_peers(&self, epoch: u64) -> Vec { self.peers .iter() @@ -363,6 +465,7 @@ mod tests { peer_status: PeerStatus::Idle, processed_blocks: 0, sync_requests_started: 0, + custody_columns: None, }, ); } @@ -487,7 +590,7 @@ mod tests { } #[test] - fn fetch_idle_peer_from_returns_first_idle_in_given_order() { + fn fetch_idle_peer_from_prefers_never_used_then_productive_peers() { let mut peer_manager = PeerManager::new(test_network_state()); let first = test_peer(Status::default()); let second = test_peer(Status::default()); @@ -495,12 +598,21 @@ mod tests { let second_id = second.peer_id; insert_idle(&mut peer_manager, first); insert_idle(&mut peer_manager, second); + let first_info = peer_manager.peers.get_mut(&first_id).unwrap(); + first_info.sync_requests_started = 2; + first_info.processed_blocks = 20; let peer = peer_manager - .fetch_idle_peer_from(&[second_id, first_id]) + .fetch_idle_peer_from(&[first_id, second_id]) .expect("a peer should be found"); assert_eq!(peer.peer_id, second_id); + peer_manager.mark_peer_as_idle(&second_id); + let peer = peer_manager + .fetch_idle_peer_from(&[first_id, second_id]) + .expect("a peer should be found"); + assert_eq!(peer.peer_id, first_id, "productive used peer should win"); + assert!( peer_manager .fetch_idle_peer_from(&[PeerId::random()]) @@ -508,6 +620,20 @@ mod tests { ); } + #[test] + fn best_finalized_ignores_an_epoch_that_cannot_be_converted_to_a_slot() { + let mut peer_manager = PeerManager::new(test_network_state()); + insert_idle( + &mut peer_manager, + test_peer(Status { + finalized_epoch: u64::MAX, + ..Default::default() + }), + ); + + assert_eq!(peer_manager.best_finalized(0), TargetSelection::NoQuorum); + } + #[test] fn fetch_idle_peer_from_excluding_skips_excluded_peers() { let mut peer_manager = PeerManager::new(test_network_state()); @@ -524,6 +650,90 @@ mod tests { ); } + #[test] + fn data_column_selection_uses_the_peers_advertised_custody() { + let mut peer_manager = PeerManager::new(test_network_state()); + let wrong_peer = test_peer(Status::default()); + let right_peer = test_peer(Status::default()); + let wrong_id = wrong_peer.peer_id; + let right_id = right_peer.peer_id; + insert_idle(&mut peer_manager, wrong_peer); + insert_idle(&mut peer_manager, right_peer); + peer_manager + .peers + .get_mut(&wrong_id) + .unwrap() + .custody_columns = Some(HashSet::from([1, 2])); + peer_manager + .peers + .get_mut(&right_id) + .unwrap() + .custody_columns = Some(HashSet::from([7, 8])); + + let selected = peer_manager + .fetch_idle_peer_for_columns_from_excluding( + &[wrong_id, right_id], + &[(7, 0)], + &HashSet::new(), + ) + .expect("the peer responsible for column 7 should be selected"); + + assert_eq!(selected.peer_id, right_id); + } + + #[test] + fn data_column_selection_prefers_known_custody_over_unknown_metadata() { + let mut peer_manager = PeerManager::new(test_network_state()); + let unknown_peer = test_peer(Status::default()); + let known_peer = test_peer(Status::default()); + let unknown_id = unknown_peer.peer_id; + let known_id = known_peer.peer_id; + insert_idle(&mut peer_manager, unknown_peer); + insert_idle(&mut peer_manager, known_peer); + peer_manager + .peers + .get_mut(&known_id) + .unwrap() + .custody_columns = Some(HashSet::from([7])); + + let selected = peer_manager + .fetch_idle_peer_for_columns_from_excluding( + &[unknown_id, known_id], + &[(7, 100), (8, 100)], + &HashSet::new(), + ) + .expect("the peer with known coverage should win over unknown metadata"); + + assert_eq!(selected.peer_id, known_id); + } + + #[test] + fn data_column_selection_respects_earliest_available_slot() { + let mut peer_manager = PeerManager::new(test_network_state()); + let unavailable = test_peer(Status { + earliest_available_slot: 101, + ..Default::default() + }); + let available = test_peer(Status { + earliest_available_slot: 90, + ..Default::default() + }); + let unavailable_id = unavailable.peer_id; + let available_id = available.peer_id; + insert_idle(&mut peer_manager, unavailable); + insert_idle(&mut peer_manager, available); + + let selected = peer_manager + .fetch_idle_peer_for_columns_from_excluding( + &[unavailable_id, available_id], + &[(7, 100)], + &HashSet::new(), + ) + .expect("a peer retaining slot 100 should be selected"); + + assert_eq!(selected.peer_id, available_id); + } + #[test] fn peers_satisfying_distinguishes_finalized_and_head_qualification() { let mut peer_manager = PeerManager::new(test_network_state()); @@ -578,6 +788,7 @@ mod tests { peer_status: PeerStatus::Downloading, processed_blocks: 0, sync_requests_started: 0, + custody_columns: None, }, ); diff --git a/crates/networking/syncer/src/block_range/peer_range_downloader.rs b/crates/networking/syncer/src/block_range/peer_range_downloader.rs index c8c773de0..d6d6e7d8a 100644 --- a/crates/networking/syncer/src/block_range/peer_range_downloader.rs +++ b/crates/networking/syncer/src/block_range/peer_range_downloader.rs @@ -383,6 +383,16 @@ mod tests { )); } + #[test] + fn dial_upgrade_errors_are_transport_failures_not_invalid_peer_data() { + assert!(matches!( + classify_req_resp_error(ReqRespError::RawError( + "Dial upgrade error: Timeout".to_string() + )), + DownloadFailure::Transport(_) + )); + } + #[tokio::test] async fn drain_responses_classifies_remote_error_separately_from_invalid_data() { let (tx, rx) = mpsc::channel(10); diff --git a/crates/networking/syncer/src/block_range/recovery.rs b/crates/networking/syncer/src/block_range/recovery.rs index d6519dd0c..86c7831a0 100644 --- a/crates/networking/syncer/src/block_range/recovery.rs +++ b/crates/networking/syncer/src/block_range/recovery.rs @@ -91,7 +91,7 @@ fn budget_exhausted(implicated_peers: HashSet) -> anyhow::Result)> = by_root.into_iter().collect(); groups.sort_by(|(root_a, group_a), (root_b, group_b)| { - group_b.len().cmp(&group_a.len()).then(root_a.cmp(root_b)) + let score_a: u64 = group_a + .iter() + .map(|peer_id| self.peer_manager.processed_blocks_of(peer_id)) + .sum(); + let score_b: u64 = group_b + .iter() + .map(|peer_id| self.peer_manager.processed_blocks_of(peer_id)) + .sum(); + group_b + .len() + .cmp(&group_a.len()) + .then(score_b.cmp(&score_a)) + .then(root_a.cmp(root_b)) }); groups.into_iter().flat_map(|(_, group)| group).collect() } diff --git a/crates/storage/src/tables/beacon/latest_messages.rs b/crates/storage/src/tables/beacon/latest_messages.rs index 6c6198093..3db68a019 100644 --- a/crates/storage/src/tables/beacon/latest_messages.rs +++ b/crates/storage/src/tables/beacon/latest_messages.rs @@ -1,14 +1,35 @@ use std::sync::Arc; use ream_consensus_beacon::fork_choice::latest_message::LatestMessage; -use redb::{Database, TableDefinition}; +use redb::{Database, Durability, TableDefinition}; -use crate::tables::{ssz_encoder::SSZEncoding, table::REDBTable}; +use crate::{ + errors::StoreError, + tables::{ssz_encoder::SSZEncoding, table::REDBTable}, +}; pub struct LatestMessagesTable { pub db: Arc, } +impl LatestMessagesTable { + pub fn insert_batch( + &self, + entries: impl IntoIterator, + ) -> Result<(), StoreError> { + let mut write_txn = self.db.begin_write()?; + write_txn.set_durability(Durability::Immediate)?; + { + let mut table = write_txn.open_table(Self::TABLE_DEFINITION)?; + for (index, message) in entries { + table.insert(index, message)?; + } + } + write_txn.commit()?; + Ok(()) + } +} + /// Table definition for the Latest Message table /// /// Key: latest_messages diff --git a/testing/mock-execution-engine/src/block_generator.rs b/testing/mock-execution-engine/src/block_generator.rs index c34fa0256..ff3f7dc5b 100644 --- a/testing/mock-execution-engine/src/block_generator.rs +++ b/testing/mock-execution-engine/src/block_generator.rs @@ -325,7 +325,8 @@ pub fn payload_v3_to_execution_payload(payload: ExecutionPayloadV3) -> Execution base_fee_per_gas: payload.base_fee_per_gas, block_hash: payload.block_hash, transactions: payload.transactions, - withdrawals: payload.withdrawals, + withdrawals: VariableList::new(payload.withdrawals.into_iter().map(Into::into).collect()) + .expect("converting a U16-bounded withdrawal list preserves its length"), blob_gas_used: payload.blob_gas_used, excess_blob_gas: payload.excess_blob_gas, } @@ -352,7 +353,9 @@ pub fn test_payload_attributes(parent_beacon_block_root: B256) -> PayloadAttribu #[cfg(test)] mod tests { use alloy_primitives::B256; - use ream_execution_rpc_types::forkchoice_update::ForkchoiceStateV1; + use ream_execution_rpc_types::{ + execution_payload::WithdrawalV1, forkchoice_update::ForkchoiceStateV1, + }; use super::*; @@ -392,7 +395,15 @@ mod tests { let expected_timestamp = attrs.timestamp; let expected_prev_randao = attrs.prev_randao; let expected_fee_recipient = attrs.suggested_fee_recipient; - let expected_withdrawals = attrs.withdrawals.clone(); + let expected_withdrawals = VariableList::new( + attrs + .withdrawals + .clone() + .into_iter() + .map(WithdrawalV1::from) + .collect(), + ) + .expect("converting a U16-bounded withdrawal list preserves its length"); let mut generator = ExecutionBlockGenerator::new(genesis_hash); let response = generator From b3404991595abbf37baec4a35ad9a18eb4775d9f Mon Sep 17 00:00:00 2001 From: vuonghuuhung Date: Mon, 14 Sep 2026 21:26:46 +0700 Subject: [PATCH 11/17] fix: prevent duplicate data column imports during range sync --- .../manager/src/data_availability_fetch.rs | 24 ++++++++++++- crates/networking/manager/src/req_resp.rs | 35 ++++++++----------- .../req_resp/src/beacon/messages/mod.rs | 30 +++++++++------- .../networking/syncer/src/block_range/mod.rs | 20 ++++------- 4 files changed, 61 insertions(+), 48 deletions(-) diff --git a/crates/networking/manager/src/data_availability_fetch.rs b/crates/networking/manager/src/data_availability_fetch.rs index 989ea7859..98504aa28 100644 --- a/crates/networking/manager/src/data_availability_fetch.rs +++ b/crates/networking/manager/src/data_availability_fetch.rs @@ -271,6 +271,16 @@ pub async fn fetch_missing_columns( }; for sidecar in sidecars { + let still_pending = { + let store = beacon_chain.store.lock().await; + store + .data_availability_checker + .pending_block(&block_root) + .is_some() + }; + if !still_pending { + return ColumnFetchOutcome::Complete; + } if !missing.contains(&sidecar.index) { warn!( ?block_root, @@ -314,7 +324,19 @@ pub async fn fetch_missing_columns( .await { Ok(()) => {} - Err(err) => warn!(?block_root, ?err, "Failed to import fetched data column"), + Err(err) => { + let still_pending = { + let store = beacon_chain.store.lock().await; + store + .data_availability_checker + .pending_block(&block_root) + .is_some() + }; + if !still_pending { + return ColumnFetchOutcome::Complete; + } + warn!(?block_root, ?err, "Failed to import fetched data column"); + } } } diff --git a/crates/networking/manager/src/req_resp.rs b/crates/networking/manager/src/req_resp.rs index a6c9c18fa..1c559c470 100644 --- a/crates/networking/manager/src/req_resp.rs +++ b/crates/networking/manager/src/req_resp.rs @@ -2,19 +2,13 @@ use std::sync::Arc; use libp2p::{PeerId, swarm::ConnectionId}; use ream_consensus_beacon::{blob_sidecar::BlobIdentifier, data_column_sidecar::ColumnIdentifier}; +use ream_network_spec::networks::beacon::beacon_network_spec; use ream_p2p::network::beacon::network_state::NetworkState; -use ream_req_resp::{ - beacon::messages::{ - BeaconRequestMessage, BeaconResponseMessage, - blob_sidecars::{BlobSidecarsByRangeV1Request, BlobSidecarsByRootV1Request}, - blocks::{BeaconBlocksByRangeV2Request, BeaconBlocksByRootV2Request}, - data_column_sidecars::{ - DataColumnSidecarsByRangeV1Request, DataColumnSidecarsByRootV1Request, - }, - }, - constants::{ - MAX_REQUEST_BLOCKS, MAX_REQUEST_BLOCKS_DENEB, MAX_REQUEST_DATA_COLUMN_SIDECARS_PER_COLUMN, - }, +use ream_req_resp::beacon::messages::{ + BeaconRequestMessage, BeaconResponseMessage, + blob_sidecars::{BlobSidecarsByRangeV1Request, BlobSidecarsByRootV1Request}, + blocks::{BeaconBlocksByRangeV2Request, BeaconBlocksByRootV2Request}, + data_column_sidecars::{DataColumnSidecarsByRangeV1Request, DataColumnSidecarsByRootV1Request}, }; use ream_storage::{ db::beacon::BeaconDB, @@ -57,12 +51,13 @@ pub async fn handle_req_resp_message( count, .. }) => { - if count > MAX_REQUEST_BLOCKS { + let max_request_blocks = beacon_network_spec().max_request_blocks; + if count > max_request_blocks { p2p_sender.send_invalid_request( peer_id, connection_id, stream_id, - &format!("Requested count {count} exceeds MAX_REQUEST_BLOCKS"), + &format!("Requested count {count} exceeds maximum {max_request_blocks}"), ); return; } @@ -141,12 +136,13 @@ pub async fn handle_req_resp_message( start_slot, count, }) => { - if count > MAX_REQUEST_BLOCKS_DENEB { + let max_request_blocks = beacon_network_spec().max_request_blocks_deneb; + if count > max_request_blocks { p2p_sender.send_invalid_request( peer_id, connection_id, stream_id, - &format!("Requested count {count} exceeds MAX_REQUEST_BLOCKS_DENEB"), + &format!("Requested count {count} exceeds maximum {max_request_blocks}"), ); return; } @@ -289,14 +285,13 @@ pub async fn handle_req_resp_message( count, columns, }) => { - if count > MAX_REQUEST_DATA_COLUMN_SIDECARS_PER_COLUMN { + let max_request_blocks = beacon_network_spec().max_request_blocks_deneb; + if count > max_request_blocks { p2p_sender.send_invalid_request( peer_id, connection_id, stream_id, - &format!( - "Requested count {count} exceeds MAX_REQUEST_DATA_COLUMN_SIDECARS_PER_COLUMN" - ), + &format!("Requested count {count} exceeds maximum {max_request_blocks}"), ); return; } diff --git a/crates/networking/req_resp/src/beacon/messages/mod.rs b/crates/networking/req_resp/src/beacon/messages/mod.rs index d8201ab99..45e411f2d 100644 --- a/crates/networking/req_resp/src/beacon/messages/mod.rs +++ b/crates/networking/req_resp/src/beacon/messages/mod.rs @@ -27,13 +27,7 @@ use ssz_derive::{Decode, Encode}; use status::Status; use super::protocol_id::BeaconSupportedProtocol; -use crate::{ - constants::{ - MAX_BLOBS_PER_BLOCK, MAX_REQUEST_BLOB_SIDECARS, MAX_REQUEST_BLOCKS, - MAX_REQUEST_DATA_COLUMN_SIDECARS_PER_COLUMN, - }, - protocol_id::{ProtocolId, SupportedProtocol}, -}; +use crate::protocol_id::{ProtocolId, SupportedProtocol}; #[derive(Debug, Clone, PartialEq, Eq, Encode, Decode)] #[ssz(enum_behaviour = "transparent")] @@ -105,6 +99,7 @@ impl BeaconRequestMessage { } pub fn max_response_chunks(&self) -> u64 { + let network_spec = beacon_network_spec(); match self { BeaconRequestMessage::MetaData(_) | BeaconRequestMessage::Goodbye(_) @@ -112,20 +107,29 @@ impl BeaconRequestMessage { | BeaconRequestMessage::Ping(_) => 1, BeaconRequestMessage::BeaconBlocksByRange(request) => { - request.count.min(MAX_REQUEST_BLOCKS) + request.count.min(network_spec.max_request_blocks) } BeaconRequestMessage::BeaconBlocksByRoot(request) => request.inner.len() as u64, - BeaconRequestMessage::BlobSidecarsByRange(request) => request - .count - .saturating_mul(MAX_BLOBS_PER_BLOCK) - .min(MAX_REQUEST_BLOB_SIDECARS), + BeaconRequestMessage::BlobSidecarsByRange(request) => { + let max_blobs_per_block = network_spec + .blob_schedule + .iter() + .map(|parameters| parameters.max_blobs_per_block) + .max() + .unwrap_or(network_spec.max_blobs_per_block_electra) + .max(network_spec.max_blobs_per_block_electra); + request + .count + .min(network_spec.max_request_blocks_deneb) + .saturating_mul(max_blobs_per_block) + } BeaconRequestMessage::BlobSidecarsByRoot(request) => request.inner.len() as u64, BeaconRequestMessage::DataColumnSidecarsByRange(request) => { let num_columns = request.columns.len() as u64; request .count + .min(network_spec.max_request_blocks_deneb) .saturating_mul(num_columns) - .min(MAX_REQUEST_DATA_COLUMN_SIDECARS_PER_COLUMN.saturating_mul(num_columns)) } BeaconRequestMessage::DataColumnSidecarsByRoot(request) => { request.inner.iter().map(|id| id.columns.len() as u64).sum() diff --git a/crates/networking/syncer/src/block_range/mod.rs b/crates/networking/syncer/src/block_range/mod.rs index 7c1cfe6a8..d2446370f 100644 --- a/crates/networking/syncer/src/block_range/mod.rs +++ b/crates/networking/syncer/src/block_range/mod.rs @@ -1454,23 +1454,15 @@ impl BlockRangeSyncer { .map_err(|err| anyhow!("Range-sync data-column task failed: {err}"))?? }; + for column in columns { + self.beacon_chain + .import_data_column_sidecar_if(column, |_| Ok(())) + .await?; + } match self.beacon_chain.process_block(block).await? { BlockProcessingOutcome::Imported { .. } => {} BlockProcessingOutcome::PendingAvailability { block_root } => { - for column in columns { - self.beacon_chain - .import_data_column_sidecar_if(column, |_| Ok(())) - .await?; - } - ensure!( - self.beacon_chain - .store - .lock() - .await - .db - .block_provider() - .get(block_root)? - .is_some(), + bail!( "Range-sync block {block_root} remained pending after processing its downloaded data" ); } From d08a42cb3e3d77b7a68511ca02a8fa900b1c1ec7 Mon Sep 17 00:00:00 2001 From: vuonghuuhung Date: Tue, 15 Sep 2026 19:47:20 +0700 Subject: [PATCH 12/17] fix: address range sync recovery review feedback --- Cargo.lock | 1 + crates/common/execution/engine/src/lib.rs | 11 +- crates/networking/manager/src/service.rs | 2 +- crates/networking/syncer/Cargo.toml | 1 + .../networking/syncer/src/block_range/mod.rs | 376 +++++++++++++++--- .../syncer/src/block_range/peer_manager.rs | 27 ++ .../syncer/src/block_range/recovery.rs | 122 ++++-- 7 files changed, 460 insertions(+), 80 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f97c7dbde..2f574bf15 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7194,6 +7194,7 @@ dependencies = [ "ream-req-resp", "ream-storage", "ream-sync-committee-pool", + "snap", "tempfile", "tokio", "tracing", diff --git a/crates/common/execution/engine/src/lib.rs b/crates/common/execution/engine/src/lib.rs index 8c692d953..369e87f55 100644 --- a/crates/common/execution/engine/src/lib.rs +++ b/crates/common/execution/engine/src/lib.rs @@ -436,10 +436,7 @@ pub fn is_valid_versioned_hashes(new_payload_request: &NewPayloadRequest) -> any } fn payload_status_allows_import(status: PayloadStatus) -> bool { - matches!( - status, - PayloadStatus::Valid | PayloadStatus::Syncing | PayloadStatus::Accepted - ) + matches!(status, PayloadStatus::Valid) } fn get_execution_requests_list(execution_requests: &ExecutionRequests) -> Vec { @@ -597,10 +594,10 @@ mod tests { use super::*; #[test] - fn syncing_and_accepted_payloads_are_not_treated_as_invalid() { + fn only_valid_payload_status_allows_import() { assert!(payload_status_allows_import(PayloadStatus::Valid)); - assert!(payload_status_allows_import(PayloadStatus::Syncing)); - assert!(payload_status_allows_import(PayloadStatus::Accepted)); + assert!(!payload_status_allows_import(PayloadStatus::Syncing)); + assert!(!payload_status_allows_import(PayloadStatus::Accepted)); assert!(!payload_status_allows_import(PayloadStatus::Invalid)); assert!(!payload_status_allows_import( PayloadStatus::InvalidBlockHash diff --git a/crates/networking/manager/src/service.rs b/crates/networking/manager/src/service.rs index 2e4a3d162..0ca557810 100644 --- a/crates/networking/manager/src/service.rs +++ b/crates/networking/manager/src/service.rs @@ -542,7 +542,7 @@ impl NetworkManagerService { match result { Ok(Ok((mut block_range_syncer, sync_result))) => { if let Err(err) = sync_result { - warn!("Block range sync segment failed: {err:?}"); + warn!("Block range sync failed: {err:?}"); } if block_range_syncer.is_synced_to_head_slot().await { idle_syncer = Some(block_range_syncer); diff --git a/crates/networking/syncer/Cargo.toml b/crates/networking/syncer/Cargo.toml index 6819d3c97..7e0d3e947 100644 --- a/crates/networking/syncer/Cargo.toml +++ b/crates/networking/syncer/Cargo.toml @@ -46,6 +46,7 @@ ream-execution-rpc-types.workspace = true ream-operation-pool.workspace = true ream-peer.workspace = true ream-sync-committee-pool.workspace = true +snap.workspace = true tempfile.workspace = true [lints] diff --git a/crates/networking/syncer/src/block_range/mod.rs b/crates/networking/syncer/src/block_range/mod.rs index d2446370f..bcc4c44fe 100644 --- a/crates/networking/syncer/src/block_range/mod.rs +++ b/crates/networking/syncer/src/block_range/mod.rs @@ -81,6 +81,13 @@ fn candidate_exhaustion_deserves_backoff(target_slot: u64, head_slot: u64) -> bo target_slot > head_slot } +fn finalized_target_reached(selection: &TargetSelection, next_start_slot: u64) -> bool { + matches!( + selection, + TargetSelection::Ready { target_slot, .. } if next_start_slot >= *target_slot + ) +} + fn parent_lookup_budget_exhausted( started_at: Option, requests_started: u64, @@ -462,12 +469,12 @@ impl PendingConclusions { } #[derive(Default)] -struct SegmentExclusions { +struct RangeExclusions { finalized: HashSet, head: HashSet, } -impl SegmentExclusions { +impl RangeExclusions { fn for_phase(&self, phase: SyncPhase) -> &HashSet { match phase { SyncPhase::Finalized => &self.finalized, @@ -488,7 +495,7 @@ pub struct BlockRangeSyncer { pub peer_manager: PeerManager, pub p2p_sender: UnboundedSender, pub executor: ReamExecutor, - next_segment_not_before: Option, + next_range_not_before: Option, next_generation: u64, finalized_frontier: Option, head_frontier: Option, @@ -506,7 +513,7 @@ impl BlockRangeSyncer { p2p_sender, peer_manager: PeerManager::new(network_state), executor, - next_segment_not_before: None, + next_range_not_before: None, next_generation: 0, finalized_frontier: None, head_frontier: None, @@ -561,7 +568,7 @@ impl BlockRangeSyncer { fn commit_pending_conclusions( &mut self, pending: PendingConclusions, - exclusions: SegmentExclusions, + exclusions: RangeExclusions, ) { for phase in [SyncPhase::Finalized, SyncPhase::Head] { if let Some(record) = pending.for_phase(phase).clone() { @@ -605,7 +612,7 @@ impl BlockRangeSyncer { fn clear_tracker(&mut self) { self.finalized_frontier = None; self.head_frontier = None; - self.next_segment_not_before = None; + self.next_range_not_before = None; } fn reconcile_not_ahead(&mut self, observation: &FrontierObservation) { @@ -658,16 +665,16 @@ impl BlockRangeSyncer { pub fn start(mut self) -> JoinHandle)>> { let executor = self.executor.clone(); executor.spawn(async move { - let result = self.run_segment().await; + let result = self.run_range_sync().await; (self, result) }) } - async fn run_segment(&mut self) -> anyhow::Result<()> { - if let Some(not_before) = self.next_segment_not_before.take() + async fn run_range_sync(&mut self) -> anyhow::Result<()> { + if let Some(not_before) = self.next_range_not_before.take() && let Some(remaining) = not_before.checked_duration_since(Instant::now()) { - info!("Backing off {remaining:?} after the previous segment made no progress..."); + info!("Backing off {remaining:?} after the previous range made no progress..."); sleep(remaining).await; } @@ -706,9 +713,9 @@ impl BlockRangeSyncer { let scan_start_slot = head_slot; let mut restore_window_open = true; let mut recovery_window_open = true; - let mut recovery_decision_made_this_segment = false; + let mut recovery_decision_made_this_range = false; let mut pending_conclusions = PendingConclusions::default(); - let mut segment_exclusions = SegmentExclusions::default(); + let mut range_exclusions = RangeExclusions::default(); let mut parent_lookup_started_at: Option = None; let mut parent_lookup_requests = 0u64; let mut parent_lookup_peers = HashSet::new(); @@ -734,15 +741,12 @@ impl BlockRangeSyncer { if active_target.is_none() { if phase == SyncPhase::Finalized { + if finalized_target_reached(&selection, block_cache.next_start_slot()) { + phase = SyncPhase::Head; + continue; + } match &selection { - TargetSelection::Ready { - target_slot, - eligible_peers, - } => { - if block_cache.next_start_slot() >= *target_slot { - phase = SyncPhase::Head; - continue; - } + TargetSelection::Ready { eligible_peers, .. } => { if eligible_peers.len() < MIN_SYNC_PEERS { info!( "Finalized target not yet confirmed by enough peers ({} < {MIN_SYNC_PEERS}), waiting...", @@ -753,14 +757,15 @@ impl BlockRangeSyncer { } } TargetSelection::NoQuorum => { - phase = SyncPhase::Head; + info!("No finalized sync target yet, waiting for peers..."); + sleep(SLEEP_DURATION).await; continue; } } } else if let TargetSelection::NoQuorum = &selection { if finalized_phase_settled || block_cache.block_count() > 0 { info!( - "No head-phase sync target after the finalized phase settled; ending this segment." + "No head-phase sync target after the finalized phase settled; ending this range." ); break; } @@ -805,7 +810,12 @@ impl BlockRangeSyncer { }; self.reconcile_not_ahead(&observation); - let candidate_peers = self.peer_manager.peers_satisfying(target.qualification); + let candidate_peers: Vec = self + .peer_manager + .peers_satisfying(target.qualification) + .into_iter() + .filter(|peer_id| self.peer_manager.peer_head_reaches(peer_id, target.slot)) + .collect(); let now = Instant::now(); let poll_outcome = { @@ -845,7 +855,7 @@ impl BlockRangeSyncer { "the peer that served the non-connecting block must never be excluded \ alongside the peers that supplied the false coverage claim" ); - segment_exclusions + range_exclusions .for_phase_mut(divergence_phase) .extend(divergence.confirming_peers.clone()); pending_conclusions.observe_divergence(divergence_phase, &divergence); @@ -859,10 +869,10 @@ impl BlockRangeSyncer { !task_handles.is_empty(), ) { info!( - "No peer has qualified for the active sync target for over {CANDIDATE_EXHAUSTION_TIMEOUT:?}; ending this segment." + "No peer has qualified for the active sync target for over {CANDIDATE_EXHAUSTION_TIMEOUT:?}; ending this range." ); if candidate_exhaustion_deserves_backoff(target.slot, head_slot) { - self.next_segment_not_before = Some(Instant::now() + ZERO_PROGRESS_BACKOFF); + self.next_range_not_before = Some(Instant::now() + ZERO_PROGRESS_BACKOFF); } pending_conclusions.observe( phase, @@ -912,15 +922,15 @@ impl BlockRangeSyncer { } if recovery_window_open - && !recovery_decision_made_this_segment + && !recovery_decision_made_this_range && let Some(frontier) = self.frontier_for(phase) && frontier.needs_recovery(&observation, block_cache.next_start_slot()) { - recovery_decision_made_this_segment = true; + recovery_decision_made_this_range = true; recovery_window_open = false; let recovery_round_not_before = frontier.recovery_round_not_before; if recovery_round_not_before.is_none_or(|not_before| now >= not_before) { - let mut excluded = segment_exclusions.for_phase(phase).clone(); + let mut excluded = range_exclusions.for_phase(phase).clone(); excluded.extend(frontier.attempted_peers.iter().copied()); match self .run_recovery(&observation, &candidate_peers, &excluded) @@ -1331,7 +1341,7 @@ impl BlockRangeSyncer { } info!( - "Block range sync completed a segment successfully with {} blocks and {} blobs.", + "Block range sync completed a range successfully with {} blocks and {} blobs.", block_cache.block_count(), block_cache.downloaded_blob_count(), ); @@ -1353,7 +1363,7 @@ impl BlockRangeSyncer { if imported_count > 0 { self.clear_tracker(); } else if import_result.is_ok() { - self.commit_pending_conclusions(pending_conclusions, segment_exclusions); + self.commit_pending_conclusions(pending_conclusions, range_exclusions); } let target_was_ahead = finalized_target_was_ahead @@ -1361,7 +1371,7 @@ impl BlockRangeSyncer { .as_ref() .is_some_and(|target| target.slot > head_slot); if target_was_ahead && saw_empty_range && imported_count == 0 { - self.next_segment_not_before = Some(Instant::now() + ZERO_PROGRESS_BACKOFF); + self.next_range_not_before = Some(Instant::now() + ZERO_PROGRESS_BACKOFF); } import_result.map(|_| ()).map_err(|err| err.error) @@ -1713,9 +1723,22 @@ fn poll_ready_tasks( if blocks.is_empty() { info!("Received empty block range from peer: {peer_id}"); - block_cache.clear_attempted(key); *saw_empty_range = true; + let range_end = range.start_slot.saturating_add(range.count); + let reaches_head_target = matching_frontier.is_some_and(|frontier| { + frontier.phase == SyncPhase::Head + && range.start_slot <= frontier.highest_observed_target + && range_end > frontier.highest_observed_target + }); + if reaches_head_target { + block_cache.mark_attempted(key, *peer_id, candidate_peers, now); + block_cache.push_retry_range(*range); + continue; + } + + block_cache.clear_attempted(key); + if let Some(frontier) = matching_frontier { let observation = observation_from_frontier(frontier); let confirming_peers = HashSet::from([*peer_id]); @@ -2286,7 +2309,8 @@ mod tests { use kzg::{G1, eip_4844::compute_blob_kzg_proof_raw}; use parking_lot::RwLock; use ream_consensus_beacon::{ - data_column_sidecar::NUMBER_OF_COLUMNS, electra::beacon_block::BeaconBlock, + data_column_sidecar::NUMBER_OF_COLUMNS, + electra::{beacon_block::BeaconBlock, beacon_state::BeaconState}, }; use ream_consensus_misc::{ checkpoint::Checkpoint, @@ -2295,11 +2319,18 @@ mod tests { use ream_execution_rpc_types::get_blobs::{Blob, BlobAndProofV1}; use ream_network_spec::networks::beacon::initialize_test_network_spec; use ream_operation_pool::OperationPool; - use ream_p2p::network::beacon::peer::CachedPeer; + use ream_p2p::network::beacon::{ + channel::{P2PCallbackResponse, P2PRequest}, + peer::CachedPeer, + }; use ream_peer::{ConnectionState, Direction}; - use ream_req_resp::beacon::messages::{meta_data::GetMetaDataV3, status::Status}; + use ream_req_resp::beacon::messages::{ + BeaconResponseMessage, meta_data::GetMetaDataV3, status::Status, + }; use ream_storage::{db::ReamDB, tables::field::REDBField}; use ream_sync_committee_pool::SyncCommitteePool; + use snap::raw::Decoder; + use ssz::Decode; use tempfile::TempDir; use super::*; @@ -2331,6 +2362,25 @@ mod tests { assert!(!candidate_exhaustion_deserves_backoff(50, 100)); } + #[test] + fn finalized_phase_waits_on_no_quorum_and_advances_only_after_reaching_a_target() { + assert!(!finalized_target_reached(&TargetSelection::NoQuorum, 100)); + assert!(!finalized_target_reached( + &TargetSelection::Ready { + target_slot: 101, + eligible_peers: Vec::new(), + }, + 100, + )); + assert!(finalized_target_reached( + &TargetSelection::Ready { + target_slot: 100, + eligible_peers: Vec::new(), + }, + 100, + )); + } + #[test] fn parent_lookup_is_bounded_by_both_requests_and_time() { let now = Instant::now(); @@ -2423,6 +2473,17 @@ mod tests { (data_dir, beacon_chain) } + fn test_beacon_state() -> BeaconState { + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "../../../testing/gossip-validation/tests/assets/sepolia/states/grandparent_state_9552074.ssz_snappy", + ); + let compressed = std::fs::read(path).expect("test beacon state should be readable"); + let bytes = Decoder::new() + .decompress_vec(&compressed) + .expect("test beacon state should decompress"); + BeaconState::from_ssz_bytes(&bytes).expect("test beacon state should decode") + } + fn test_network_state() -> Arc { let enr_key = CombinedKey::generate_secp256k1(); Arc::new(NetworkState { @@ -2485,12 +2546,12 @@ mod tests { panic!("task did not complete within the test timeout"); } - /// `start()` must hand `self` back out even on an ordinary segment error, not just success, + /// `start()` must hand `self` back out even on an ordinary range error, not just success, /// or the caller loses the syncer (and its warmed-up peer table) and can never retry. #[test] - fn start_returns_self_after_a_failed_segment() { + fn start_returns_self_after_a_failed_range() { initialize_test_network_spec(); - // Empty DB has no highest synced slot, so run_segment fails immediately. + // Empty DB has no highest synced slot, so run_range_sync fails immediately. let (_data_dir, beacon_chain) = test_beacon_chain(); let executor = ReamExecutor::new().expect("executor should start"); let (p2p_sender, _p2p_receiver) = tokio::sync::mpsc::unbounded_channel(); @@ -2507,9 +2568,165 @@ mod tests { .expect("task should not panic") .expect("task should not be cancelled by shutdown"); - assert!( - sync_result.is_err(), - "expected the empty-DB segment to fail" + assert!(sync_result.is_err(), "expected the empty-DB range to fail"); + } + + #[test] + fn head_fork_recovery_walks_by_root_to_the_common_ancestor_without_gossip() { + futures::executor::block_on( + head_fork_recovery_walks_by_root_to_the_common_ancestor_without_gossip_inner(), + ); + } + + async fn head_fork_recovery_walks_by_root_to_the_common_ancestor_without_gossip_inner() { + initialize_test_network_spec(); + let (_data_dir, beacon_chain) = test_beacon_chain(); + + let common_block = SignedBeaconBlock { + message: BeaconBlock::default(), + signature: Default::default(), + }; + let common_root = common_block.message.tree_hash_root(); + { + let store = beacon_chain.store.lock().await; + store + .db + .block_provider() + .insert(common_root, common_block) + .expect("common block should be stored"); + store + .db + .state_provider() + .insert(common_root, test_beacon_state()) + .expect("common state should be stored"); + store + .db + .finalized_checkpoint_provider() + .insert(Checkpoint { + epoch: 0, + root: common_root, + }) + .expect("finalized checkpoint should be stored"); + } + + let mut remote_blocks = HashMap::new(); + let mut remote_parent = common_root; + for slot in 1..=12 { + let block = SignedBeaconBlock { + message: BeaconBlock { + slot, + parent_root: remote_parent, + ..Default::default() + }, + signature: Default::default(), + }; + remote_parent = block.message.tree_hash_root(); + remote_blocks.insert(remote_parent, block); + } + let remote_head = remote_parent; + let local_anchor = SignedBeaconBlock { + message: BeaconBlock { + slot: 11, + proposer_index: 999, + parent_root: common_root, + ..Default::default() + }, + signature: Default::default(), + } + .message + .tree_hash_root(); + + let peer_id = PeerId::random(); + let network_state = test_network_state(); + let mut peer = CachedPeer::new( + peer_id, + None, + ConnectionState::Connected, + Direction::Outbound, + None, + ); + peer.status = Some(Status { + finalized_root: common_root, + finalized_epoch: 0, + head_root: remote_head, + head_slot: 12, + earliest_available_slot: 0, + ..Default::default() + }); + network_state.peer_table.write().insert(peer_id, peer); + + let executor = ReamExecutor::new().expect("executor should start"); + let (p2p_sender, mut p2p_receiver) = tokio::sync::mpsc::unbounded_channel(); + let responder = executor.spawn(async move { + while let Some(message) = p2p_receiver.recv().await { + let P2PMessage::Request(P2PRequest::BlockRoots { + roots, callback, .. + }) = message + else { + panic!("head-fork recovery should use only by-root requests"); + }; + for root in roots { + if let Some(block) = remote_blocks.get(&root) { + callback + .send(Ok(P2PCallbackResponse::ResponseMessage(Arc::new( + BeaconResponseMessage::BeaconBlocksByRoot(block.clone()), + )))) + .await + .expect("response receiver should remain open"); + } + } + callback + .send(Ok(P2PCallbackResponse::EndOfStream)) + .await + .expect("response receiver should remain open"); + } + }); + + let mut syncer = + BlockRangeSyncer::new(Arc::new(beacon_chain), p2p_sender, network_state, executor); + syncer.peer_manager.update_peer_set(); + let frontier_id = FrontierId { + key: FrontierKey { + anchor_root: local_anchor, + phase: SyncPhase::Head, + scan_start_slot: 11, + }, + generation: syncer.allocate_generation(), + }; + let mut frontier = test_frontier(frontier_id, local_anchor, 11); + frontier.highest_observed_target = 12; + frontier.consecutive_no_progress = 6; + syncer.head_frontier = Some(frontier); + let observation = FrontierObservation { + anchor_root: local_anchor, + anchor_slot: 11, + phase: SyncPhase::Head, + scan_start_slot: 11, + target_slot: 12, + }; + + let outcome = syncer + .run_recovery(&observation, &[peer_id], &HashSet::new()) + .await + .expect("head-fork recovery should complete"); + responder.abort(); + + let RecoveryOutcome::Seeded(seed) = outcome else { + panic!("expected a recovery seed after reaching the common ancestor"); + }; + assert_eq!(seed.ancestor_root, common_root); + assert_eq!(seed.ancestor_slot, 0); + assert_eq!(seed.source_peer, peer_id); + assert_eq!( + seed.forward_blocks + .iter() + .map(|block| block.message.slot) + .collect::>(), + (1..=12).collect::>() + ); + assert_ne!( + seed.forward_blocks[10].message.tree_hash_root(), + local_anchor ); } @@ -3239,7 +3456,7 @@ mod tests { } #[test] - fn coverage_at_target_does_not_consume_the_segments_recovery_decision() { + fn coverage_at_target_does_not_consume_the_ranges_recovery_decision() { let frontier = test_frontier(test_frontier_id(0), B256::ZERO, 10); let observation = FrontierObservation { anchor_root: B256::ZERO, @@ -3411,6 +3628,71 @@ mod tests { ); } + #[test] + fn empty_head_target_range_is_retried_instead_of_becoming_coverage() { + initialize_test_network_spec(); + let executor = ReamExecutor::new().expect("executor should start"); + let (mut peer_manager, peer_id) = test_peer_manager_with_one_peer(); + + let root = B256::ZERO; + let frontier_id = FrontierId { + key: FrontierKey { + anchor_root: root, + phase: SyncPhase::Head, + scan_start_slot: 100, + }, + generation: 0, + }; + let mut frontier = test_frontier(frontier_id, root, 100); + frontier.highest_observed_target = 110; + let range = Range::new(101, 10); + let key = RequestKey::BlockRange(range); + let mut block_cache = BlockCache::new(root, 100); + block_cache.mark_block_range_in_progress(range); + let handle = + executor.spawn(async move { StreamOutcome::Complete(Vec::::new()) }); + let mut tasks = vec![DownloadTask::new_block_range( + handle, + range, + peer_id, + Some(frontier_id), + )]; + + let mut finalized_frontier = None; + let mut head_frontier = Some(frontier); + let mut saw_empty_range = false; + for _ in 0..500 { + let mut frontiers = FrontierStore { + finalized: &mut finalized_frontier, + head: &mut head_frontier, + }; + poll_ready_tasks( + &mut tasks, + &mut block_cache, + &mut peer_manager, + &mut frontiers, + &HashSet::new(), + &mut saw_empty_range, + &[peer_id], + ) + .expect("poll_ready_tasks should not error"); + if tasks.is_empty() { + break; + } + std::thread::sleep(Duration::from_millis(2)); + } + + assert!( + head_frontier + .as_ref() + .expect("head frontier should remain") + .confirmed_empty_through + .is_none() + ); + assert!(block_cache.attempted_peers_for(key).contains(&peer_id)); + assert_eq!(block_cache.next_start_slot(), 100); + } + #[test] fn coverage_divergence_implicates_the_confirming_peers_not_the_block_serving_peer() { initialize_test_network_spec(); @@ -3539,13 +3821,13 @@ mod tests { .expect("set above") .reason, RemoteNoProgressReason::SettledEmptyCoverage, - "a segment producing evidence for both phases must not let one silently overwrite the other" + "a range producing evidence for both phases must not let one silently overwrite the other" ); } #[test] - fn segment_exclusions_stay_scoped_to_their_own_phase() { - let mut exclusions = SegmentExclusions::default(); + fn range_exclusions_stay_scoped_to_their_own_phase() { + let mut exclusions = RangeExclusions::default(); let peer = PeerId::random(); exclusions.for_phase_mut(SyncPhase::Finalized).insert(peer); @@ -3720,7 +4002,7 @@ mod tests { assert_eq!( first_deadline, second_deadline, "an already-armed cooldown must not be re-extended by a later commit, or it would \ - never elapse if segments repeat faster than the cooldown duration" + never elapse if ranges repeat faster than the cooldown duration" ); } diff --git a/crates/networking/syncer/src/block_range/peer_manager.rs b/crates/networking/syncer/src/block_range/peer_manager.rs index 6033d71eb..12b5d0a40 100644 --- a/crates/networking/syncer/src/block_range/peer_manager.rs +++ b/crates/networking/syncer/src/block_range/peer_manager.rs @@ -385,6 +385,13 @@ impl PeerManager { .and_then(|info| info.peer.status.clone()) } + pub fn peer_head_reaches(&self, peer_id: &PeerId, slot: u64) -> bool { + self.peers + .get(peer_id) + .and_then(|info| info.peer.status.as_ref()) + .is_some_and(|status| status.head_slot >= slot) + } + pub fn processed_blocks_of(&self, peer_id: &PeerId) -> u64 { self.peers .get(peer_id) @@ -883,4 +890,24 @@ mod tests { vec![id] ); } + + #[test] + fn peer_head_reaches_filters_peers_that_cannot_serve_the_refined_target() { + let mut peer_manager = PeerManager::new(test_network_state()); + let lower = test_peer(Status { + head_slot: 98, + ..Default::default() + }); + let target = test_peer(Status { + head_slot: 100, + ..Default::default() + }); + let lower_id = lower.peer_id; + let target_id = target.peer_id; + insert_idle(&mut peer_manager, lower); + insert_idle(&mut peer_manager, target); + + assert!(!peer_manager.peer_head_reaches(&lower_id, 100)); + assert!(peer_manager.peer_head_reaches(&target_id, 100)); + } } diff --git a/crates/networking/syncer/src/block_range/recovery.rs b/crates/networking/syncer/src/block_range/recovery.rs index 86c7831a0..0ee092bcd 100644 --- a/crates/networking/syncer/src/block_range/recovery.rs +++ b/crates/networking/syncer/src/block_range/recovery.rs @@ -179,8 +179,13 @@ impl BlockRangeSyncer { let mut failed_this_round: HashSet = HashSet::new(); let mut next_peer_index = 0usize; + let probe_end_exclusive = match observation.phase { + SyncPhase::Finalized => observation.target_slot.saturating_add(1), + SyncPhase::Head => observation.target_slot, + }; + for _ in 0..MAX_PROBE_REQUESTS_PER_ROUND { - if !budget.has_capacity() || cursor > observation.target_slot { + if !budget.has_capacity() || cursor >= probe_end_exclusive { break; } let Some((peer_id, advance_by)) = reserve_round_robin( @@ -192,7 +197,7 @@ impl BlockRangeSyncer { break; }; next_peer_index = (next_peer_index + advance_by) % probe_peers.len(); - let window = SLOTS_PER_EPOCH.min(observation.target_slot.saturating_add(1) - cursor); + let window = SLOTS_PER_EPOCH.min(probe_end_exclusive - cursor); let range = Range::new(cursor, window); budget.consume(); @@ -329,38 +334,71 @@ impl BlockRangeSyncer { } match observation.phase { SyncPhase::Head => { - let eligible: Vec = candidate_peers - .iter() - .filter(|peer_id| !excluded.contains(peer_id)) - .copied() - .collect(); - let Some(peer) = self - .peer_manager - .fetch_idle_peer_from_excluding(&eligible, &HashSet::new()) - else { - return no_progress(RemoteNoProgressReason::AncestorNotFound, implicated_peers); - }; - let peer_id = peer.peer_id; - let end_exclusive = observation.target_slot.saturating_add(1); - let window = SLOTS_PER_EPOCH.min(end_exclusive); - let start = end_exclusive.saturating_sub(window); - let range = Range::new(start, window.max(1)); let mut implicated = implicated_peers; - budget.consume(); - match self.probe_range(peer_id, range, &mut implicated).await { - ProbeStep::Found(blocks) => { - self.resolve_candidate( + let eligible = + self.group_by_head_root_agreement(candidate_peers, observation.target_slot); + + for &peer_id in &eligible { + if !budget.has_capacity() { + break; + } + if excluded.contains(&peer_id) { + continue; + } + let Some(status) = self.peer_manager.status_of(&peer_id) else { + continue; + }; + let Some(reserved) = self + .peer_manager + .fetch_idle_peer_from_excluding(&[peer_id], &HashSet::new()) + else { + continue; + }; + debug_assert_eq!(reserved.peer_id, peer_id); + + budget.consume(); + let Some(block) = self + .fetch_single_root(peer_id, status.head_root, &mut implicated) + .await + else { + continue; + }; + if block.message.slot != observation.target_slot { + implicated.insert(peer_id); + continue; + } + + match self + .resolve_candidate( observation, - blocks, + vec![block], HashSet::from([peer_id]), &eligible, budget, implicated, ) - .await + .await? + { + RecoveryOutcome::NoProgress { + reason, + implicated_peers, + .. + } => { + if reason == RemoteNoProgressReason::RecoveryBudgetExhausted { + return no_progress(reason, implicated_peers); + } + implicated = implicated_peers; + } + other => return Ok(other), } - _ => no_progress(RemoteNoProgressReason::AncestorNotFound, implicated), } + + let reason = if budget.has_capacity() { + RemoteNoProgressReason::AncestorNotFound + } else { + RemoteNoProgressReason::RecoveryBudgetExhausted + }; + no_progress(reason, implicated) } SyncPhase::Finalized => { let epoch = observation.target_slot / SLOTS_PER_EPOCH; @@ -505,6 +543,40 @@ impl BlockRangeSyncer { groups.into_iter().flat_map(|(_, group)| group).collect() } + fn group_by_head_root_agreement(&self, peers: &[PeerId], target_slot: u64) -> Vec { + let mut by_root: std::collections::BTreeMap> = + std::collections::BTreeMap::new(); + for &peer_id in peers { + let Some(status) = self.peer_manager.status_of(&peer_id) else { + continue; + }; + if status.head_slot != target_slot { + continue; + } + by_root.entry(status.head_root).or_default().push(peer_id); + } + for group in by_root.values_mut() { + group.sort(); + } + let mut groups: Vec<(B256, Vec)> = by_root.into_iter().collect(); + groups.sort_by(|(root_a, group_a), (root_b, group_b)| { + let score_a: u64 = group_a + .iter() + .map(|peer_id| self.peer_manager.processed_blocks_of(peer_id)) + .sum(); + let score_b: u64 = group_b + .iter() + .map(|peer_id| self.peer_manager.processed_blocks_of(peer_id)) + .sum(); + group_b + .len() + .cmp(&group_a.len()) + .then(score_b.cmp(&score_a)) + .then(root_a.cmp(root_b)) + }); + groups.into_iter().flat_map(|(_, group)| group).collect() + } + async fn fetch_single_root( &mut self, peer_id: PeerId, From 276e1910d44231e3bdf5bc381824922f64210c0c Mon Sep 17 00:00:00 2001 From: vuonghuuhung Date: Tue, 15 Sep 2026 23:28:22 +0700 Subject: [PATCH 13/17] test: add head fork recovery e2e --- bin/ream/src/main.rs | 343 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 343 insertions(+) diff --git a/bin/ream/src/main.rs b/bin/ream/src/main.rs index 7219a79a9..bce4bca96 100644 --- a/bin/ream/src/main.rs +++ b/bin/ream/src/main.rs @@ -1870,6 +1870,51 @@ mod tests { } } + async fn wait_for_connected_beacon_peers(http_port: u16, minimum: u64) { + let start = Instant::now(); + let timeout_duration = Duration::from_secs(60); + loop { + let peer_count = wait_for_beacon_json(http_port, "/eth/v1/node/peer_count").await; + let connected = peer_count_connected(&peer_count); + if connected >= minimum { + return; + } + + assert!( + start.elapsed() < timeout_duration, + "Timed out waiting for beacon node on port {http_port} to connect to {minimum} peers; latest response: {peer_count:?}" + ); + sleep(Duration::from_millis(500)).await; + } + } + + async fn beacon_block_root_at_slot(http_port: u16, slot: u64) -> String { + let response = + wait_for_beacon_json(http_port, &format!("/eth/v1/beacon/blocks/{slot}/root")).await; + response["data"]["root"] + .as_str() + .unwrap_or_else(|| panic!("block root response missing root: {response:?}")) + .to_string() + } + + async fn wait_for_head_to_match(http_port: u16, target_slot: u64, target_root: &str) { + let start = Instant::now(); + let timeout_duration = Duration::from_secs(180); + loop { + let head = wait_for_beacon_json(http_port, "/eth/v1/beacon/headers").await; + let (slot, root) = head_slot_and_root(&head); + if slot == target_slot && root == target_root { + return; + } + + assert!( + start.elapsed() < timeout_duration, + "Timed out waiting for beacon node on port {http_port} to reach ({target_slot}, {target_root}); latest head: ({slot}, {root})" + ); + sleep(Duration::from_secs(1)).await; + } + } + async fn wait_for_matching_heads_all(http_ports: &[u16]) -> (u64, String) { assert!( !http_ports.is_empty(), @@ -2676,6 +2721,304 @@ mod tests { ); } + #[test] + #[serial] + fn test_beacon_node_recovers_head_fork_via_range_sync() { + init_test_tracing(); + + let port_offset = beacon_port_offset(); + let mut node_a_config = beacon_node_config_from_args(port_offset, None); + initialize_beacon_e2e_network_spec(node_a_config.network.clone()); + let public_keys = beacon_e2e_public_keys(); + let (genesis_state, genesis_block) = build_dev_genesis(&public_keys); + let genesis_execution_block_hash = genesis_state.latest_execution_payload_header.block_hash; + + let node_a_db = create_beacon_test_node_db("beacon_head_fork_recovery", 1); + let node_b1_db = create_beacon_test_node_db("beacon_head_fork_recovery", 2); + let node_b2_db = create_beacon_test_node_db("beacon_head_fork_recovery", 3); + let node_b3_db = create_beacon_test_node_db("beacon_head_fork_recovery", 4); + let genesis_validators_root = + seed_beacon_test_db(&node_a_db, genesis_state.clone(), &genesis_block); + for db in [&node_b1_db, &node_b2_db, &node_b3_db] { + let peer_genesis_root = seed_beacon_test_db(db, genesis_state.clone(), &genesis_block); + assert_eq!( + genesis_validators_root, peer_genesis_root, + "head-fork e2e nodes must share genesis" + ); + } + initialize_beacon_e2e_genesis_root(genesis_validators_root); + + let control_executor = ReamExecutor::new().unwrap(); + let node_a_executor = ReamExecutor::new().unwrap(); + let node_a_restart_executor = ReamExecutor::new().unwrap(); + let node_b1_executor = ReamExecutor::new().unwrap(); + let node_b2_executor = ReamExecutor::new().unwrap(); + let node_b3_executor = ReamExecutor::new().unwrap(); + let node_b1_restart_executor = ReamExecutor::new().unwrap(); + let node_b2_restart_executor = ReamExecutor::new().unwrap(); + let node_b3_restart_executor = ReamExecutor::new().unwrap(); + let validator_a_executor = ReamExecutor::new().unwrap(); + let validator_b_executor = ReamExecutor::new().unwrap(); + + let node_a_executor_handle = node_a_executor.clone(); + let node_a_restart_executor_handle = node_a_restart_executor.clone(); + let node_b1_executor_handle = node_b1_executor.clone(); + let node_b2_executor_handle = node_b2_executor.clone(); + let node_b3_executor_handle = node_b3_executor.clone(); + let node_b1_restart_executor_handle = node_b1_restart_executor.clone(); + let node_b2_restart_executor_handle = node_b2_restart_executor.clone(); + let node_b3_restart_executor_handle = node_b3_restart_executor.clone(); + let validator_a_executor_handle = validator_a_executor.clone(); + let validator_b_executor_handle = validator_b_executor.clone(); + let node_a_db_for_restart = node_a_db.clone(); + let node_b1_db_for_restart = node_b1_db.clone(); + let node_b2_db_for_restart = node_b2_db.clone(); + let node_b3_db_for_restart = node_b3_db.clone(); + + let result = control_executor.clone().runtime().block_on(async move { + let test_dir = temp_dir().join(format!( + "{APP_NAME}_beacon_head_fork_recovery_{}", + SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("System time is before UNIX epoch") + .as_nanos() + )); + fs::create_dir_all(&test_dir).expect("Failed to create beacon e2e test dir"); + let jwt_secret_path = test_dir.join("jwt.hex"); + fs::write( + &jwt_secret_path, + "0x2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a", + ) + .expect("Failed to write mock EL JWT secret"); + + let mock_execution_server = MockExecutionServer::start(genesis_execution_block_hash); + let execution_endpoint = mock_execution_server.url(); + + node_a_config.disable_discovery = true; + node_a_config.execution_endpoint = Some(execution_endpoint.clone()); + node_a_config.execution_jwt_secret = Some(jwt_secret_path.clone()); + let node_a_http_port = node_a_config.http_port; + let node_a_handle = + spawn_beacon_test_node(node_a_config, node_a_db, node_a_executor_handle.clone()); + wait_for_beacon_identity(node_a_http_port).await; + + let mut node_b1_config = beacon_node_config_from_args(port_offset + 1, None); + node_b1_config.disable_discovery = true; + node_b1_config.execution_endpoint = Some(execution_endpoint.clone()); + node_b1_config.execution_jwt_secret = Some(jwt_secret_path.clone()); + let node_b1_http_port = node_b1_config.http_port; + let node_b1_handle = + spawn_beacon_test_node(node_b1_config, node_b1_db, node_b1_executor_handle.clone()); + let node_b1_identity = wait_for_beacon_identity(node_b1_http_port).await; + let node_b1_enr = node_b1_identity["data"]["enr"] + .as_str() + .expect("identity response should contain ENR") + .to_string(); + + let mut node_b2_config = + beacon_node_config_from_args(port_offset + 2, Some(node_b1_enr.clone())); + node_b2_config.disable_discovery = true; + node_b2_config.execution_endpoint = Some(execution_endpoint.clone()); + node_b2_config.execution_jwt_secret = Some(jwt_secret_path.clone()); + let node_b2_http_port = node_b2_config.http_port; + let node_b2_handle = + spawn_beacon_test_node(node_b2_config, node_b2_db, node_b2_executor_handle.clone()); + wait_for_beacon_identity(node_b2_http_port).await; + + let mut node_b3_config = + beacon_node_config_from_args(port_offset + 3, Some(node_b1_enr.clone())); + node_b3_config.disable_discovery = true; + node_b3_config.execution_endpoint = Some(execution_endpoint.clone()); + node_b3_config.execution_jwt_secret = Some(jwt_secret_path.clone()); + let node_b3_http_port = node_b3_config.http_port; + let node_b3_handle = + spawn_beacon_test_node(node_b3_config, node_b3_db, node_b3_executor_handle.clone()); + wait_for_beacon_identity(node_b3_http_port).await; + + wait_for_connected_beacon_peers(node_b1_http_port, 2).await; + wait_for_connected_beacon_peers(node_b2_http_port, 1).await; + wait_for_connected_beacon_peers(node_b3_http_port, 1).await; + + let fork_a_dir = test_dir.join("fork_a"); + let fork_b_dir = test_dir.join("fork_b"); + let (fork_a_keystores, fork_a_password) = + write_validator_keystores(&fork_a_dir, 0, 0..BEACON_E2E_VALIDATOR_COUNT); + let (fork_b_keystores, fork_b_password) = + write_validator_keystores(&fork_b_dir, 0, 0..BEACON_E2E_VALIDATOR_COUNT); + let validator_a_config = validator_node_config_from_args( + node_a_http_port, + &fork_a_keystores, + &fork_a_password, + ); + let mut validator_b_config = validator_node_config_from_args( + node_b1_http_port, + &fork_b_keystores, + &fork_b_password, + ); + validator_b_config.suggested_fee_recipient = + "0x0000000000000000000000000000000000000002" + .parse() + .expect("alternate fee recipient should parse"); + let validator_a_handle = + spawn_validator_test_node(validator_a_config, validator_a_executor_handle.clone()); + let validator_b_handle = + spawn_validator_test_node(validator_b_config, validator_b_executor_handle.clone()); + + let first_a = wait_for_head_slot_at_least(node_a_http_port, 1).await; + let first_b = wait_for_head_slot_at_least(node_b1_http_port, 1).await; + let fork_slot = first_a.0.max(first_b.0) + 4; + wait_for_head_slot_at_least(node_a_http_port, fork_slot).await; + wait_for_head_slot_at_least(node_b1_http_port, fork_slot).await; + + shutdown_validator_test_node(&validator_a_executor_handle, validator_a_handle).await; + let node_a_head = wait_for_head_slot_at_least(node_a_http_port, fork_slot).await; + shutdown_beacon_test_node(&node_a_executor_handle, node_a_handle).await; + + wait_for_head_slot_at_least(node_b1_http_port, node_a_head.0 + 4).await; + shutdown_validator_test_node(&validator_b_executor_handle, validator_b_handle).await; + let node_b_head = wait_for_matching_heads_all(&[ + node_b1_http_port, + node_b2_http_port, + node_b3_http_port, + ]) + .await; + + let node_b_root_at_fork_slot = + beacon_block_root_at_slot(node_b1_http_port, node_a_head.0).await; + assert_ne!( + node_a_head.1, node_b_root_at_fork_slot, + "isolated validators should have produced different roots at slot {}", + node_a_head.0 + ); + + shutdown_beacon_test_node(&node_b3_executor_handle, node_b3_handle).await; + shutdown_beacon_test_node(&node_b2_executor_handle, node_b2_handle).await; + shutdown_beacon_test_node(&node_b1_executor_handle, node_b1_handle).await; + + let mut node_b1_restart_config = beacon_node_config_from_args(port_offset + 5, None); + node_b1_restart_config.disable_discovery = true; + node_b1_restart_config.execution_endpoint = Some(execution_endpoint.clone()); + node_b1_restart_config.execution_jwt_secret = Some(jwt_secret_path.clone()); + let node_b1_restart_http_port = node_b1_restart_config.http_port; + let node_b1_restart_handle = spawn_beacon_test_node( + node_b1_restart_config, + node_b1_db_for_restart, + node_b1_restart_executor_handle.clone(), + ); + let node_b1_restart_identity = + wait_for_beacon_identity(node_b1_restart_http_port).await; + let node_b1_restart_enr = node_b1_restart_identity["data"]["enr"] + .as_str() + .expect("identity response should contain ENR") + .to_string(); + + let mut node_b2_restart_config = + beacon_node_config_from_args(port_offset + 6, Some(node_b1_restart_enr.clone())); + node_b2_restart_config.disable_discovery = true; + node_b2_restart_config.execution_endpoint = Some(execution_endpoint.clone()); + node_b2_restart_config.execution_jwt_secret = Some(jwt_secret_path.clone()); + let node_b2_restart_http_port = node_b2_restart_config.http_port; + let node_b2_restart_handle = spawn_beacon_test_node( + node_b2_restart_config, + node_b2_db_for_restart, + node_b2_restart_executor_handle.clone(), + ); + let node_b2_restart_identity = + wait_for_beacon_identity(node_b2_restart_http_port).await; + let node_b2_restart_enr = node_b2_restart_identity["data"]["enr"] + .as_str() + .expect("identity response should contain ENR") + .to_string(); + + let mut node_b3_restart_config = + beacon_node_config_from_args(port_offset + 7, Some(node_b1_restart_enr.clone())); + node_b3_restart_config.disable_discovery = true; + node_b3_restart_config.execution_endpoint = Some(execution_endpoint.clone()); + node_b3_restart_config.execution_jwt_secret = Some(jwt_secret_path.clone()); + let node_b3_restart_http_port = node_b3_restart_config.http_port; + let node_b3_restart_handle = spawn_beacon_test_node( + node_b3_restart_config, + node_b3_db_for_restart, + node_b3_restart_executor_handle.clone(), + ); + let node_b3_restart_identity = + wait_for_beacon_identity(node_b3_restart_http_port).await; + let node_b3_restart_enr = node_b3_restart_identity["data"]["enr"] + .as_str() + .expect("identity response should contain ENR") + .to_string(); + + wait_for_connected_beacon_peers(node_b1_restart_http_port, 2).await; + wait_for_connected_beacon_peers(node_b2_restart_http_port, 1).await; + wait_for_connected_beacon_peers(node_b3_restart_http_port, 1).await; + let restarted_peer_head = wait_for_matching_heads_all(&[ + node_b1_restart_http_port, + node_b2_restart_http_port, + node_b3_restart_http_port, + ]) + .await; + assert_eq!(node_b_head, restarted_peer_head); + + let bootnodes = [ + node_b1_restart_enr, + node_b2_restart_enr, + node_b3_restart_enr, + ] + .join(","); + let mut node_a_restart_config = + beacon_node_config_from_args(port_offset + 8, Some(bootnodes)); + node_a_restart_config.disable_discovery = true; + node_a_restart_config.execution_endpoint = Some(execution_endpoint); + node_a_restart_config.execution_jwt_secret = Some(jwt_secret_path); + let node_a_restart_http_port = node_a_restart_config.http_port; + let node_a_restart_handle = spawn_beacon_test_node( + node_a_restart_config, + node_a_db_for_restart, + node_a_restart_executor_handle.clone(), + ); + wait_for_beacon_identity(node_a_restart_http_port).await; + wait_for_connected_beacon_peers(node_a_restart_http_port, 3).await; + wait_for_head_to_match(node_a_restart_http_port, node_b_head.0, &node_b_head.1).await; + + let recovered_head = + wait_for_beacon_json(node_a_restart_http_port, "/eth/v1/beacon/headers").await; + let recovered_head = head_slot_and_root(&recovered_head); + + shutdown_beacon_test_node(&node_a_restart_executor_handle, node_a_restart_handle).await; + shutdown_beacon_test_node(&node_b3_restart_executor_handle, node_b3_restart_handle) + .await; + shutdown_beacon_test_node(&node_b2_restart_executor_handle, node_b2_restart_handle) + .await; + shutdown_beacon_test_node(&node_b1_restart_executor_handle, node_b1_restart_handle) + .await; + mock_execution_server.stop().await; + + (node_a_head, node_b_head, recovered_head) + }); + + validator_b_executor.shutdown_runtime(); + validator_a_executor.shutdown_runtime(); + node_b3_executor.shutdown_runtime(); + node_b2_executor.shutdown_runtime(); + node_b1_executor.shutdown_runtime(); + node_b3_restart_executor.shutdown_runtime(); + node_b2_restart_executor.shutdown_runtime(); + node_b1_restart_executor.shutdown_runtime(); + node_a_restart_executor.shutdown_runtime(); + node_a_executor.shutdown_runtime(); + + assert_eq!( + result.1, result.2, + "restarted node should converge to peer head" + ); + info!( + fork_head = ?result.0, + target_head = ?result.1, + recovered_head = ?result.2, + "Beacon head-fork range-sync e2e test completed" + ); + } + fn run_beacon_nodes_propagate_data_column_sidecars(wait_for_finality: bool) { init_test_tracing(); From 2a14f4946c7ea2b7bd9d6f7e2bfefdcdb426bd9b Mon Sep 17 00:00:00 2001 From: vuonghuuhung Date: Wed, 16 Sep 2026 00:27:47 +0700 Subject: [PATCH 14/17] test: cover phase 3 fork recovery --- bin/ream/Cargo.toml | 1 + bin/ream/src/main.rs | 59 +++++++++- .../syncer/src/block_range/block_cache.rs | 28 +++++ .../networking/syncer/src/block_range/mod.rs | 101 ++++++++++++++++-- 4 files changed, 179 insertions(+), 10 deletions(-) diff --git a/bin/ream/Cargo.toml b/bin/ream/Cargo.toml index 473e72c22..3956d013c 100644 --- a/bin/ream/Cargo.toml +++ b/bin/ream/Cargo.toml @@ -25,6 +25,7 @@ devnet5 = [ "ream-p2p/devnet5", "ream-rpc-lean/devnet5", "ream-storage/devnet5", + "ream-syncer/devnet5", "ream-validator-lean/devnet5", ] jemalloc = ["dep:tikv-jemallocator"] diff --git a/bin/ream/src/main.rs b/bin/ream/src/main.rs index bce4bca96..6b8b3f7b7 100644 --- a/bin/ream/src/main.rs +++ b/bin/ream/src/main.rs @@ -1897,9 +1897,48 @@ mod tests { .to_string() } + fn remove_beacon_slot_indexes(db: &ReamDB, start_slot: u64, end_slot: u64) { + let beacon_db = db + .init_beacon_db() + .expect("unable to open Ream Beacon Database"); + let mut removed = 0; + + for slot in start_slot..=end_slot { + let Some(block_root) = beacon_db + .slot_index_provider() + .remove(slot) + .expect("slot index removal should succeed") + else { + continue; + }; + assert!( + beacon_db + .block_provider() + .get(block_root) + .expect("block lookup should succeed") + .is_some(), + "removing a slot index must retain the block for by-root recovery" + ); + removed += 1; + } + + assert!( + removed > 0, + "at least one block-range slot index must be removed" + ); + assert!( + beacon_db + .slot_index_provider() + .get(end_slot) + .expect("slot index lookup should succeed") + .is_none(), + "the target block must be unavailable through block-by-range" + ); + } + async fn wait_for_head_to_match(http_port: u16, target_slot: u64, target_root: &str) { let start = Instant::now(); - let timeout_duration = Duration::from_secs(180); + let timeout_duration = Duration::from_secs(300); loop { let head = wait_for_beacon_json(http_port, "/eth/v1/beacon/headers").await; let (slot, root) = head_slot_and_root(&head); @@ -2902,7 +2941,7 @@ mod tests { let node_b1_restart_http_port = node_b1_restart_config.http_port; let node_b1_restart_handle = spawn_beacon_test_node( node_b1_restart_config, - node_b1_db_for_restart, + node_b1_db_for_restart.clone(), node_b1_restart_executor_handle.clone(), ); let node_b1_restart_identity = @@ -2920,7 +2959,7 @@ mod tests { let node_b2_restart_http_port = node_b2_restart_config.http_port; let node_b2_restart_handle = spawn_beacon_test_node( node_b2_restart_config, - node_b2_db_for_restart, + node_b2_db_for_restart.clone(), node_b2_restart_executor_handle.clone(), ); let node_b2_restart_identity = @@ -2938,7 +2977,7 @@ mod tests { let node_b3_restart_http_port = node_b3_restart_config.http_port; let node_b3_restart_handle = spawn_beacon_test_node( node_b3_restart_config, - node_b3_db_for_restart, + node_b3_db_for_restart.clone(), node_b3_restart_executor_handle.clone(), ); let node_b3_restart_identity = @@ -2959,12 +2998,24 @@ mod tests { .await; assert_eq!(node_b_head, restarted_peer_head); + for db in [ + &node_b1_db_for_restart, + &node_b2_db_for_restart, + &node_b3_db_for_restart, + ] { + remove_beacon_slot_indexes(db, node_a_head.0 + 1, node_b_head.0); + } + let bootnodes = [ node_b1_restart_enr, node_b2_restart_enr, node_b3_restart_enr, ] .join(","); + let _range_sync_timing = ream_syncer::block_range::override_range_sync_timing( + Duration::from_millis(20), + Duration::from_millis(20), + ); let mut node_a_restart_config = beacon_node_config_from_args(port_offset + 8, Some(bootnodes)); node_a_restart_config.disable_discovery = true; diff --git a/crates/networking/syncer/src/block_range/block_cache.rs b/crates/networking/syncer/src/block_range/block_cache.rs index c477978b6..4810e1ceb 100644 --- a/crates/networking/syncer/src/block_range/block_cache.rs +++ b/crates/networking/syncer/src/block_range/block_cache.rs @@ -650,10 +650,18 @@ impl BlockCache { return DataToFetch::DownloadsInProgress; } + let has_retry_ranges = !self.block_ranges_to_retry.is_empty(); if let Some(range) = self.take_schedulable_retry_range(candidate_peers, now) { return DataToFetch::BlockRange(range); } + if has_retry_ranges + && !candidate_peers.is_empty() + && self.block_ranges_in_progress.is_empty() + { + return DataToFetch::Finished; + } + let estimated_blocks_to_fetch = self.estimated_blocks_to_fetch(); if estimated_blocks_to_fetch > 0 && self.next_start_slot < target_slot { let blocks_to_fill = estimated_blocks_to_fetch @@ -990,6 +998,26 @@ mod tests { ); } + #[test] + fn data_to_fetch_finishes_after_all_candidates_exhaust_a_retry_range() { + initialize_test_network_spec(); + let mut cache = BlockCache::new(B256::ZERO, 10); + let range = Range::new(11, 4); + let peer_a = PeerId::random(); + let peer_b = PeerId::random(); + let candidates = [peer_a, peer_b]; + let now = Instant::now(); + + cache.push_retry_range(range); + cache.mark_attempted(RequestKey::BlockRange(range), peer_a, &candidates, now); + cache.mark_attempted(RequestKey::BlockRange(range), peer_b, &candidates, now); + + assert_eq!( + cache.data_to_fetch(14, 0, &HashSet::new(), &candidates, now, true), + DataToFetch::Finished + ); + } + #[test] fn in_flight_range_prevents_a_temporary_gap_from_becoming_a_parent_lookup() { initialize_test_network_spec(); diff --git a/crates/networking/syncer/src/block_range/mod.rs b/crates/networking/syncer/src/block_range/mod.rs index bcc4c44fe..48eb90faf 100644 --- a/crates/networking/syncer/src/block_range/mod.rs +++ b/crates/networking/syncer/src/block_range/mod.rs @@ -11,6 +11,9 @@ use std::{ time::{Duration, Instant}, }; +#[cfg(feature = "devnet5")] +use std::sync::atomic::{AtomicU64, Ordering}; + use alloy_primitives::B256; use anyhow::{anyhow, bail, ensure}; use block_cache::{AddBlocksError, BlockAndBlobBundle, BlockCache, DataToFetch, RequestKey}; @@ -69,6 +72,61 @@ const SLOT_IMPORT_TOLERANCE: u64 = 32; const ZERO_PROGRESS_BACKOFF: Duration = Duration::from_secs(30); +#[cfg(feature = "devnet5")] +static ZERO_PROGRESS_BACKOFF_OVERRIDE_MS: AtomicU64 = AtomicU64::new(u64::MAX); + +#[cfg(feature = "devnet5")] +static DOWNLOAD_POLL_DELAY_OVERRIDE_MS: AtomicU64 = AtomicU64::new(u64::MAX); + +#[cfg(feature = "devnet5")] +pub struct RangeSyncTimingOverride { + previous_backoff_ms: u64, + previous_poll_delay_ms: u64, +} + +#[cfg(feature = "devnet5")] +impl Drop for RangeSyncTimingOverride { + fn drop(&mut self) { + ZERO_PROGRESS_BACKOFF_OVERRIDE_MS.store(self.previous_backoff_ms, Ordering::Relaxed); + DOWNLOAD_POLL_DELAY_OVERRIDE_MS.store(self.previous_poll_delay_ms, Ordering::Relaxed); + } +} + +#[cfg(feature = "devnet5")] +pub fn override_range_sync_timing( + zero_progress_backoff: Duration, + download_poll_delay: Duration, +) -> RangeSyncTimingOverride { + RangeSyncTimingOverride { + previous_backoff_ms: ZERO_PROGRESS_BACKOFF_OVERRIDE_MS + .swap(zero_progress_backoff.as_millis() as u64, Ordering::Relaxed), + previous_poll_delay_ms: DOWNLOAD_POLL_DELAY_OVERRIDE_MS + .swap(download_poll_delay.as_millis() as u64, Ordering::Relaxed), + } +} + +fn zero_progress_backoff() -> Duration { + #[cfg(feature = "devnet5")] + { + let override_ms = ZERO_PROGRESS_BACKOFF_OVERRIDE_MS.load(Ordering::Relaxed); + if override_ms != u64::MAX { + return Duration::from_millis(override_ms); + } + } + ZERO_PROGRESS_BACKOFF +} + +fn download_poll_delay() -> Duration { + #[cfg(feature = "devnet5")] + { + let override_ms = DOWNLOAD_POLL_DELAY_OVERRIDE_MS.load(Ordering::Relaxed); + if override_ms != u64::MAX { + return Duration::from_millis(override_ms); + } + } + Duration::from_secs(10) +} + const RECOVERY_ATTEMPT_COOLDOWN: Duration = Duration::from_secs(30); const CANDIDATE_EXHAUSTION_TIMEOUT: Duration = Duration::from_secs(60); @@ -81,6 +139,17 @@ fn candidate_exhaustion_deserves_backoff(target_slot: u64, head_slot: u64) -> bo target_slot > head_slot } +fn range_settled_without_progress( + block_count: u64, + next_start_slot: u64, + target_slot: u64, + head_slot: u64, + saw_empty_range: bool, +) -> bool { + block_count == 0 + && (next_start_slot < target_slot || (target_slot > head_slot && saw_empty_range)) +} + fn finalized_target_reached(selection: &TargetSelection, next_start_slot: u64) -> bool { matches!( selection, @@ -706,6 +775,7 @@ impl BlockRangeSyncer { let mut phase = SyncPhase::Finalized; let mut active_target: Option = None; let mut saw_empty_range = false; + let mut saw_empty_range_in_completed_phase = false; let mut finalized_phase_settled = false; let mut finalized_target_was_ahead = false; let mut candidate_exhausted_since: Option = None; @@ -742,6 +812,8 @@ impl BlockRangeSyncer { if active_target.is_none() { if phase == SyncPhase::Finalized { if finalized_target_reached(&selection, block_cache.next_start_slot()) { + saw_empty_range_in_completed_phase |= saw_empty_range; + saw_empty_range = false; phase = SyncPhase::Head; continue; } @@ -872,7 +944,7 @@ impl BlockRangeSyncer { "No peer has qualified for the active sync target for over {CANDIDATE_EXHAUSTION_TIMEOUT:?}; ending this range." ); if candidate_exhaustion_deserves_backoff(target.slot, head_slot) { - self.next_range_not_before = Some(Instant::now() + ZERO_PROGRESS_BACKOFF); + self.next_range_not_before = Some(Instant::now() + zero_progress_backoff()); } pending_conclusions.observe( phase, @@ -1314,19 +1386,26 @@ impl BlockRangeSyncer { "Waiting for ongoing downloads to complete... {}", self.peer_manager.peer_counts() ); - sleep(Duration::from_secs(10)).await; + sleep(download_poll_delay()).await; } DataToFetch::Finished => { if phase == SyncPhase::Finalized && block_cache.next_start_slot() >= target.slot { finalized_target_was_ahead = target.slot > head_slot; + saw_empty_range_in_completed_phase |= saw_empty_range; + saw_empty_range = false; phase = SyncPhase::Head; active_target = None; finalized_phase_settled = true; continue; } - if block_cache.block_count() == 0 && block_cache.next_start_slot() < target.slot - { + if range_settled_without_progress( + block_cache.block_count(), + block_cache.next_start_slot(), + target.slot, + head_slot, + saw_empty_range, + ) { pending_conclusions.observe( phase, observation.clone(), @@ -1370,8 +1449,11 @@ impl BlockRangeSyncer { || active_target .as_ref() .is_some_and(|target| target.slot > head_slot); - if target_was_ahead && saw_empty_range && imported_count == 0 { - self.next_range_not_before = Some(Instant::now() + ZERO_PROGRESS_BACKOFF); + if target_was_ahead + && (saw_empty_range_in_completed_phase || saw_empty_range) + && imported_count == 0 + { + self.next_range_not_before = Some(Instant::now() + zero_progress_backoff()); } import_result.map(|_| ()).map_err(|err| err.error) @@ -2362,6 +2444,13 @@ mod tests { assert!(!candidate_exhaustion_deserves_backoff(50, 100)); } + #[test] + fn empty_range_at_reserved_target_counts_as_no_progress() { + assert!(range_settled_without_progress(0, 100, 100, 90, true)); + assert!(!range_settled_without_progress(0, 100, 100, 90, false)); + assert!(!range_settled_without_progress(1, 100, 100, 90, true)); + } + #[test] fn finalized_phase_waits_on_no_quorum_and_advances_only_after_reaching_a_target() { assert!(!finalized_target_reached(&TargetSelection::NoQuorum, 100)); From a6d4ecc50a2e772b34a7ce650dd977ab21d99c8c Mon Sep 17 00:00:00 2001 From: vuonghuuhung Date: Wed, 16 Sep 2026 00:35:42 +0700 Subject: [PATCH 15/17] style: format range sync --- crates/networking/syncer/src/block_range/mod.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/networking/syncer/src/block_range/mod.rs b/crates/networking/syncer/src/block_range/mod.rs index 48eb90faf..8b7b86ad9 100644 --- a/crates/networking/syncer/src/block_range/mod.rs +++ b/crates/networking/syncer/src/block_range/mod.rs @@ -3,6 +3,8 @@ mod peer_manager; mod peer_range_downloader; mod recovery; +#[cfg(feature = "devnet5")] +use std::sync::atomic::{AtomicU64, Ordering}; use std::{ collections::HashSet, pin::Pin, @@ -11,9 +13,6 @@ use std::{ time::{Duration, Instant}, }; -#[cfg(feature = "devnet5")] -use std::sync::atomic::{AtomicU64, Ordering}; - use alloy_primitives::B256; use anyhow::{anyhow, bail, ensure}; use block_cache::{AddBlocksError, BlockAndBlobBundle, BlockCache, DataToFetch, RequestKey}; From 0b2303515c92937dd15acdfa76a08d31045510ef Mon Sep 17 00:00:00 2001 From: vuonghuuhung Date: Wed, 16 Sep 2026 09:38:04 +0700 Subject: [PATCH 16/17] chore: simplify range sync timing override --- bin/ream/Cargo.toml | 3 +- bin/ream/src/main.rs | 4 -- crates/networking/syncer/Cargo.toml | 1 + .../networking/syncer/src/block_range/mod.rs | 67 +++---------------- 4 files changed, 13 insertions(+), 62 deletions(-) diff --git a/bin/ream/Cargo.toml b/bin/ream/Cargo.toml index 3956d013c..faab590b8 100644 --- a/bin/ream/Cargo.toml +++ b/bin/ream/Cargo.toml @@ -25,7 +25,6 @@ devnet5 = [ "ream-p2p/devnet5", "ream-rpc-lean/devnet5", "ream-storage/devnet5", - "ream-syncer/devnet5", "ream-validator-lean/devnet5", ] jemalloc = ["dep:tikv-jemallocator"] @@ -100,7 +99,7 @@ ream-validator-lean.workspace = true ream-bls.workspace = true ream-mock-execution-engine.workspace = true ream-req-resp.workspace = true -ream-syncer.workspace = true +ream-syncer = { workspace = true, features = ["test-utils"] } reqwest.workspace = true serial_test = "3.4.0" sha2.workspace = true diff --git a/bin/ream/src/main.rs b/bin/ream/src/main.rs index 6b8b3f7b7..37064eada 100644 --- a/bin/ream/src/main.rs +++ b/bin/ream/src/main.rs @@ -3012,10 +3012,6 @@ mod tests { node_b3_restart_enr, ] .join(","); - let _range_sync_timing = ream_syncer::block_range::override_range_sync_timing( - Duration::from_millis(20), - Duration::from_millis(20), - ); let mut node_a_restart_config = beacon_node_config_from_args(port_offset + 8, Some(bootnodes)); node_a_restart_config.disable_discovery = true; diff --git a/crates/networking/syncer/Cargo.toml b/crates/networking/syncer/Cargo.toml index 7e0d3e947..47b480c4a 100644 --- a/crates/networking/syncer/Cargo.toml +++ b/crates/networking/syncer/Cargo.toml @@ -12,6 +12,7 @@ version.workspace = true [features] default = [] devnet5 = ["ream-storage/devnet5", "ream-p2p/devnet5"] +test-utils = [] [dependencies] alloy-primitives.workspace = true diff --git a/crates/networking/syncer/src/block_range/mod.rs b/crates/networking/syncer/src/block_range/mod.rs index 8b7b86ad9..83dc13922 100644 --- a/crates/networking/syncer/src/block_range/mod.rs +++ b/crates/networking/syncer/src/block_range/mod.rs @@ -3,8 +3,6 @@ mod peer_manager; mod peer_range_downloader; mod recovery; -#[cfg(feature = "devnet5")] -use std::sync::atomic::{AtomicU64, Ordering}; use std::{ collections::HashSet, pin::Pin, @@ -71,59 +69,14 @@ const SLOT_IMPORT_TOLERANCE: u64 = 32; const ZERO_PROGRESS_BACKOFF: Duration = Duration::from_secs(30); -#[cfg(feature = "devnet5")] -static ZERO_PROGRESS_BACKOFF_OVERRIDE_MS: AtomicU64 = AtomicU64::new(u64::MAX); - -#[cfg(feature = "devnet5")] -static DOWNLOAD_POLL_DELAY_OVERRIDE_MS: AtomicU64 = AtomicU64::new(u64::MAX); - -#[cfg(feature = "devnet5")] -pub struct RangeSyncTimingOverride { - previous_backoff_ms: u64, - previous_poll_delay_ms: u64, -} - -#[cfg(feature = "devnet5")] -impl Drop for RangeSyncTimingOverride { - fn drop(&mut self) { - ZERO_PROGRESS_BACKOFF_OVERRIDE_MS.store(self.previous_backoff_ms, Ordering::Relaxed); - DOWNLOAD_POLL_DELAY_OVERRIDE_MS.store(self.previous_poll_delay_ms, Ordering::Relaxed); - } -} - -#[cfg(feature = "devnet5")] -pub fn override_range_sync_timing( - zero_progress_backoff: Duration, - download_poll_delay: Duration, -) -> RangeSyncTimingOverride { - RangeSyncTimingOverride { - previous_backoff_ms: ZERO_PROGRESS_BACKOFF_OVERRIDE_MS - .swap(zero_progress_backoff.as_millis() as u64, Ordering::Relaxed), - previous_poll_delay_ms: DOWNLOAD_POLL_DELAY_OVERRIDE_MS - .swap(download_poll_delay.as_millis() as u64, Ordering::Relaxed), - } +#[cfg(feature = "test-utils")] +fn range_sync_delay(_: Duration) -> Duration { + Duration::from_millis(20) } -fn zero_progress_backoff() -> Duration { - #[cfg(feature = "devnet5")] - { - let override_ms = ZERO_PROGRESS_BACKOFF_OVERRIDE_MS.load(Ordering::Relaxed); - if override_ms != u64::MAX { - return Duration::from_millis(override_ms); - } - } - ZERO_PROGRESS_BACKOFF -} - -fn download_poll_delay() -> Duration { - #[cfg(feature = "devnet5")] - { - let override_ms = DOWNLOAD_POLL_DELAY_OVERRIDE_MS.load(Ordering::Relaxed); - if override_ms != u64::MAX { - return Duration::from_millis(override_ms); - } - } - Duration::from_secs(10) +#[cfg(not(feature = "test-utils"))] +fn range_sync_delay(delay: Duration) -> Duration { + delay } const RECOVERY_ATTEMPT_COOLDOWN: Duration = Duration::from_secs(30); @@ -943,7 +896,8 @@ impl BlockRangeSyncer { "No peer has qualified for the active sync target for over {CANDIDATE_EXHAUSTION_TIMEOUT:?}; ending this range." ); if candidate_exhaustion_deserves_backoff(target.slot, head_slot) { - self.next_range_not_before = Some(Instant::now() + zero_progress_backoff()); + self.next_range_not_before = + Some(Instant::now() + range_sync_delay(ZERO_PROGRESS_BACKOFF)); } pending_conclusions.observe( phase, @@ -1385,7 +1339,7 @@ impl BlockRangeSyncer { "Waiting for ongoing downloads to complete... {}", self.peer_manager.peer_counts() ); - sleep(download_poll_delay()).await; + sleep(range_sync_delay(Duration::from_secs(10))).await; } DataToFetch::Finished => { if phase == SyncPhase::Finalized && block_cache.next_start_slot() >= target.slot @@ -1452,7 +1406,8 @@ impl BlockRangeSyncer { && (saw_empty_range_in_completed_phase || saw_empty_range) && imported_count == 0 { - self.next_range_not_before = Some(Instant::now() + zero_progress_backoff()); + self.next_range_not_before = + Some(Instant::now() + range_sync_delay(ZERO_PROGRESS_BACKOFF)); } import_result.map(|_| ()).map_err(|err| err.error) From fdb34665e4c4225db9a76e4e4a959b1c533c075f Mon Sep 17 00:00:00 2001 From: vuonghuuhung Date: Wed, 16 Sep 2026 10:59:26 +0700 Subject: [PATCH 17/17] fix: reconsider finalized sync when head has no quorum --- .../networking/syncer/src/block_range/mod.rs | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/crates/networking/syncer/src/block_range/mod.rs b/crates/networking/syncer/src/block_range/mod.rs index 83dc13922..68b926a7c 100644 --- a/crates/networking/syncer/src/block_range/mod.rs +++ b/crates/networking/syncer/src/block_range/mod.rs @@ -109,6 +109,10 @@ fn finalized_target_reached(selection: &TargetSelection, next_start_slot: u64) - ) } +fn should_reconsider_finalized(finalized_phase_settled: bool, block_count: u64) -> bool { + !finalized_phase_settled && block_count == 0 +} + fn parent_lookup_budget_exhausted( started_at: Option, requests_started: u64, @@ -787,14 +791,19 @@ impl BlockRangeSyncer { } } } else if let TargetSelection::NoQuorum = &selection { - if finalized_phase_settled || block_cache.block_count() > 0 { + if !should_reconsider_finalized( + finalized_phase_settled, + block_cache.block_count(), + ) { info!( "No head-phase sync target after the finalized phase settled; ending this range." ); break; } - info!("No sync target yet, waiting for peers..."); + info!("No head-phase sync target; reconsidering finalized sync..."); sleep(SLEEP_DURATION).await; + phase = SyncPhase::Finalized; + candidate_exhausted_since = None; continue; } } @@ -2424,6 +2433,13 @@ mod tests { )); } + #[test] + fn head_no_quorum_reconsiders_an_unsettled_finalized_phase() { + assert!(should_reconsider_finalized(false, 0)); + assert!(!should_reconsider_finalized(true, 0)); + assert!(!should_reconsider_finalized(false, 1)); + } + #[test] fn parent_lookup_is_bounded_by_both_requests_and_time() { let now = Instant::now();