diff --git a/Cargo.lock b/Cargo.lock index 080764023..8bcbc1f3e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7170,23 +7170,30 @@ version = "0.1.0" dependencies = [ "alloy-primitives", "anyhow", + "discv5", "ethereum_ssz", "futures", "kzg", "libp2p", "libp2p-identity", "libp2p-mplex", + "parking_lot", "ream-bls", "ream-chain-beacon", "ream-consensus-beacon", "ream-consensus-misc", "ream-execution-rpc-types", "ream-executor", + "ream-fork-choice-beacon", "ream-network-spec", + "ream-operation-pool", "ream-p2p", + "ream-peer", "ream-polynomial-commitments", "ream-req-resp", "ream-storage", + "ream-sync-committee-pool", + "tempfile", "tokio", "tracing", "tree_hash", diff --git a/crates/networking/manager/src/p2p_sender.rs b/crates/networking/manager/src/p2p_sender.rs index 7f559be22..b1386b8ae 100644 --- a/crates/networking/manager/src/p2p_sender.rs +++ b/crates/networking/manager/src/p2p_sender.rs @@ -90,4 +90,23 @@ impl P2PSender { warn!("Failed to send error response: {err}"); } } + + pub fn send_invalid_request( + &self, + peer_id: PeerId, + connection_id: ConnectionId, + stream_id: u64, + error: &str, + ) { + if let Err(err) = self.0.send(P2PMessage::Response(P2PResponse { + peer_id, + connection_id, + stream_id, + message: Box::new(RespMessage::Error(ReqRespError::InvalidData( + error.to_string(), + ))), + })) { + warn!("Failed to send invalid-request response: {err}"); + } + } } diff --git a/crates/networking/manager/src/req_resp.rs b/crates/networking/manager/src/req_resp.rs index e5ef0eb6e..a6c9c18fa 100644 --- a/crates/networking/manager/src/req_resp.rs +++ b/crates/networking/manager/src/req_resp.rs @@ -3,11 +3,18 @@ use std::sync::Arc; use libp2p::{PeerId, swarm::ConnectionId}; use ream_consensus_beacon::{blob_sidecar::BlobIdentifier, data_column_sidecar::ColumnIdentifier}; use ream_p2p::network::beacon::network_state::NetworkState; -use ream_req_resp::beacon::messages::{ - BeaconRequestMessage, BeaconResponseMessage, - blob_sidecars::{BlobSidecarsByRangeV1Request, BlobSidecarsByRootV1Request}, - blocks::{BeaconBlocksByRangeV2Request, BeaconBlocksByRootV2Request}, - data_column_sidecars::{DataColumnSidecarsByRangeV1Request, DataColumnSidecarsByRootV1Request}, +use ream_req_resp::{ + beacon::messages::{ + BeaconRequestMessage, BeaconResponseMessage, + blob_sidecars::{BlobSidecarsByRangeV1Request, BlobSidecarsByRootV1Request}, + blocks::{BeaconBlocksByRangeV2Request, BeaconBlocksByRootV2Request}, + data_column_sidecars::{ + DataColumnSidecarsByRangeV1Request, DataColumnSidecarsByRootV1Request, + }, + }, + constants::{ + MAX_REQUEST_BLOCKS, MAX_REQUEST_BLOCKS_DENEB, MAX_REQUEST_DATA_COLUMN_SIDECARS_PER_COLUMN, + }, }; use ream_storage::{ db::beacon::BeaconDB, @@ -50,16 +57,41 @@ pub async fn handle_req_resp_message( count, .. }) => { - for slot in start_slot..start_slot + count { - let Ok(Some(block_root)) = ream_db.slot_index_provider().get(slot) else { - trace!("No block root found for slot {slot}"); - p2p_sender.send_error_response( - peer_id, - connection_id, - stream_id, - &format!("No block root found for slot {slot}"), - ); - return; + if count > MAX_REQUEST_BLOCKS { + p2p_sender.send_invalid_request( + peer_id, + connection_id, + stream_id, + &format!("Requested count {count} exceeds MAX_REQUEST_BLOCKS"), + ); + return; + } + let Some(end_slot_exclusive) = start_slot.checked_add(count) else { + p2p_sender.send_invalid_request( + peer_id, + connection_id, + stream_id, + &format!("start_slot {start_slot} + count {count} overflows"), + ); + return; + }; + + for slot in start_slot..end_slot_exclusive { + let block_root = match ream_db.slot_index_provider().get(slot) { + Ok(Some(block_root)) => block_root, + Ok(None) => { + trace!("No block root found for slot {slot}"); + continue; + } + Err(err) => { + p2p_sender.send_error_response( + peer_id, + connection_id, + stream_id, + &format!("Failed to read slot index for slot {slot}: {err:?}"), + ); + return; + } }; let Ok(Some(block)) = ream_db.block_provider().get(block_root) else { trace!("No block found for root {block_root}"); @@ -109,16 +141,41 @@ pub async fn handle_req_resp_message( start_slot, count, }) => { - for slot in start_slot..start_slot + count { - let Ok(Some(block_root)) = ream_db.slot_index_provider().get(slot) else { - trace!("No block root found for slot {slot}"); - p2p_sender.send_error_response( - peer_id, - connection_id, - stream_id, - &format!("No block root found for slot {slot}"), - ); - return; + if count > MAX_REQUEST_BLOCKS_DENEB { + p2p_sender.send_invalid_request( + peer_id, + connection_id, + stream_id, + &format!("Requested count {count} exceeds MAX_REQUEST_BLOCKS_DENEB"), + ); + return; + } + let Some(end_slot_exclusive) = start_slot.checked_add(count) else { + p2p_sender.send_invalid_request( + peer_id, + connection_id, + stream_id, + &format!("start_slot {start_slot} + count {count} overflows"), + ); + return; + }; + + for slot in start_slot..end_slot_exclusive { + let block_root = match ream_db.slot_index_provider().get(slot) { + Ok(Some(block_root)) => block_root, + Ok(None) => { + trace!("No block root found for slot {slot}"); + continue; + } + Err(err) => { + p2p_sender.send_error_response( + peer_id, + connection_id, + stream_id, + &format!("Failed to read slot index for slot {slot}: {err:?}"), + ); + return; + } }; let Ok(Some(block)) = ream_db.block_provider().get(block_root) else { trace!("No block found for root {block_root}"); @@ -232,10 +289,43 @@ pub async fn handle_req_resp_message( count, columns, }) => { - for slot in start_slot..start_slot + count { - let Ok(Some(block_root)) = ream_db.slot_index_provider().get(slot) else { - trace!("No block root found for slot {slot}"); - continue; + if count > MAX_REQUEST_DATA_COLUMN_SIDECARS_PER_COLUMN { + p2p_sender.send_invalid_request( + peer_id, + connection_id, + stream_id, + &format!( + "Requested count {count} exceeds MAX_REQUEST_DATA_COLUMN_SIDECARS_PER_COLUMN" + ), + ); + return; + } + let Some(end_slot_exclusive) = start_slot.checked_add(count) else { + p2p_sender.send_invalid_request( + peer_id, + connection_id, + stream_id, + &format!("start_slot {start_slot} + count {count} overflows"), + ); + return; + }; + + for slot in start_slot..end_slot_exclusive { + let block_root = match ream_db.slot_index_provider().get(slot) { + Ok(Some(block_root)) => block_root, + Ok(None) => { + trace!("No block root found for slot {slot}"); + continue; + } + Err(err) => { + p2p_sender.send_error_response( + peer_id, + connection_id, + stream_id, + &format!("Failed to read slot index for slot {slot}: {err:?}"), + ); + return; + } }; for &column_index in &columns { diff --git a/crates/networking/manager/src/service.rs b/crates/networking/manager/src/service.rs index 6de6eccf4..2e4a3d162 100644 --- a/crates/networking/manager/src/service.rs +++ b/crates/networking/manager/src/service.rs @@ -8,7 +8,8 @@ use alloy_primitives::B256; use libp2p::PeerId; use ream_chain_beacon::beacon_chain::BeaconChain; use ream_consensus_misc::{ - constants::beacon::NUM_CUSTODY_GROUPS, misc::compute_start_slot_at_epoch, + constants::beacon::{NUM_CUSTODY_GROUPS, SLOTS_PER_EPOCH}, + misc::compute_start_slot_at_epoch, }; use ream_discv5::{ config::DiscoveryConfig, @@ -37,7 +38,10 @@ use ream_syncer::{ block_range::BlockRangeSyncer, unknown_parent_lookups::{MAX_LOOKUPS, UnknownBlockMeta, UnknownParentLookupCoordinator}, }; -use tokio::{sync::mpsc, time::interval}; +use tokio::{ + sync::mpsc, + time::{interval, sleep}, +}; use tracing::{error, info, warn}; use tree_hash::TreeHash; @@ -107,6 +111,7 @@ pub struct NetworkManagerService { pub ream_db: BeaconDB, pub cached_db: Arc, pub sync_committee_pool: Arc, + pub executor: ReamExecutor, } struct ReconciledBlockLookupState { @@ -274,6 +279,7 @@ impl NetworkManagerService { ream_db, cached_db, sync_committee_pool, + executor, }) } @@ -290,9 +296,13 @@ impl NetworkManagerService { cached_db, network_state, block_range_syncer, + executor, .. } = self; + let mut resync_check_interval = interval(Duration::from_secs( + SLOTS_PER_EPOCH * beacon_network_spec().seconds_per_slot(), + )); let mut interval = interval(Duration::from_secs( beacon_network_spec().seconds_per_slot(), )); @@ -314,6 +324,8 @@ impl NetworkManagerService { let mut syncer_handle = block_range_syncer.start(); // Avoid polling a completed JoinHandle after the syncer has caught up. let mut syncer_active = true; + // Kept alive (not dropped) once synced, so the periodic re-check can restart it later. + let mut idle_syncer: Option = None; loop { tokio::select! { // Drive unknown-parent lookup actions and results. @@ -527,31 +539,43 @@ impl NetworkManagerService { // Restart range sync until the finalized target is reached. result = &mut syncer_handle, if syncer_active => { syncer_active = false; - let joined_result = match result { - Ok(joined_result) => joined_result, - Err(err) => { - error!("Block range syncer failed to join task: {err}"); - continue; + match result { + Ok(Ok((mut block_range_syncer, sync_result))) => { + if let Err(err) = sync_result { + warn!("Block range sync segment failed: {err:?}"); + } + if block_range_syncer.is_synced_to_head_slot().await { + idle_syncer = Some(block_range_syncer); + } else { + syncer_handle = block_range_syncer.start(); + syncer_active = true; + } } - }; - - let thread_result = match joined_result { - Ok(result) => result, - Err(err) => { - error!("Block range syncer thread failed: {err}"); - continue; + Ok(Err(err)) => { + // Executor shutdown cancelled the task; do not re-arm. + error!("Block range syncer task cancelled: {err}"); } - }; - - let block_range_syncer = match thread_result { - Ok(syncer) => syncer, Err(err) => { - error!("Block range syncer failed to start: {err}"); - continue; + error!("Block range syncer task panicked, reconstructing: {err}"); + sleep(Duration::from_secs(5)).await; + let block_range_syncer = BlockRangeSyncer::new( + beacon_chain.clone(), + p2p_sender.0.clone(), + network_state.clone(), + executor.clone(), + ); + syncer_handle = block_range_syncer.start(); + syncer_active = true; } - }; - - if !block_range_syncer.is_synced_to_finalized_slot().await { + } + } + _ = resync_check_interval.tick(), if !syncer_active && idle_syncer.is_some() => { + let mut block_range_syncer = idle_syncer + .take() + .expect("checked by the select guard above"); + if block_range_syncer.is_synced_to_head_slot().await { + idle_syncer = Some(block_range_syncer); + } else { syncer_handle = block_range_syncer.start(); syncer_active = true; } diff --git a/crates/networking/manager/src/unknown_parent_lookup.rs b/crates/networking/manager/src/unknown_parent_lookup.rs index 99b5f82dc..beb160b2e 100644 --- a/crates/networking/manager/src/unknown_parent_lookup.rs +++ b/crates/networking/manager/src/unknown_parent_lookup.rs @@ -337,6 +337,7 @@ mod tests { use std::sync::Arc; use anyhow::anyhow; + use ream_p2p::network::beacon::channel::P2PCallbackError; use tokio::sync::mpsc::{self, UnboundedReceiver, error::TryRecvError}; use super::*; @@ -350,7 +351,7 @@ mod tests { receiver: &mut UnboundedReceiver, expected_peer: PeerId, expected_root: B256, - ) -> mpsc::Sender> { + ) -> mpsc::Sender> { let message = receiver.recv().await.expect("request should be sent"); let P2PMessage::Request(P2PRequest::BlockRoots { peer_id, @@ -490,7 +491,7 @@ mod tests { let callback = intercept_request(&mut receiver, peer_id, expected_root).await; callback - .send(Err(anyhow!("transport failed"))) + .send(Err(anyhow!("transport failed").into())) .await .expect("callback error should be delivered"); diff --git a/crates/networking/p2p/src/network/beacon/channel.rs b/crates/networking/p2p/src/network/beacon/channel.rs index d7ff2bd2e..abb6c3dcd 100644 --- a/crates/networking/p2p/src/network/beacon/channel.rs +++ b/crates/networking/p2p/src/network/beacon/channel.rs @@ -11,6 +11,7 @@ use ream_req_resp::{ beacon::messages::{ BeaconResponseMessage, data_column_sidecars::DataColumnsByRootIdentifier, status::Status, }, + error::ReqRespError, handler::RespMessage, }; use tokio::sync::mpsc; @@ -24,6 +25,14 @@ pub enum P2PCallbackResponse { EndOfStream, } +#[derive(thiserror::Error, Debug)] +pub enum P2PCallbackError { + #[error(transparent)] + ReqResp(#[from] ReqRespError), + #[error(transparent)] + Other(#[from] anyhow::Error), +} + pub enum P2PMessage { Request(P2PRequest), Response(P2PResponse), @@ -44,22 +53,29 @@ pub enum P2PRequest { peer_id: PeerId, start: u64, count: u64, - callback: mpsc::Sender>, + callback: mpsc::Sender>, }, BlockRoots { peer_id: PeerId, roots: Vec, - callback: mpsc::Sender>, + callback: mpsc::Sender>, }, BlobIdentifiers { peer_id: PeerId, blob_identifiers: Vec, - callback: mpsc::Sender>, + callback: mpsc::Sender>, + }, + DataColumnRange { + peer_id: PeerId, + start: u64, + count: u64, + columns: Vec, + callback: mpsc::Sender>, }, DataColumnIdentifiers { peer_id: PeerId, column_identifiers: Vec, - callback: mpsc::Sender>, + callback: mpsc::Sender>, }, } diff --git a/crates/networking/p2p/src/network/beacon/mod.rs b/crates/networking/p2p/src/network/beacon/mod.rs index f3dcbfaaf..8ef759538 100644 --- a/crates/networking/p2p/src/network/beacon/mod.rs +++ b/crates/networking/p2p/src/network/beacon/mod.rs @@ -12,7 +12,7 @@ use std::{ }; use anyhow::anyhow; -use channel::{P2PCallbackResponse, P2PMessage, P2PRequest, P2PResponse}; +use channel::{P2PCallbackError, P2PCallbackResponse, P2PMessage, P2PRequest, P2PResponse}; use delay_map::{HashMapDelay, HashSetDelay}; use discv5::Enr; use libp2p::{ @@ -43,7 +43,9 @@ use ream_req_resp::{ BeaconRequestMessage, BeaconResponseMessage, blob_sidecars::BlobSidecarsByRootV1Request, blocks::{BeaconBlocksByRangeV2Request, BeaconBlocksByRootV2Request}, - data_column_sidecars::DataColumnSidecarsByRootV1Request, + data_column_sidecars::{ + DataColumnSidecarsByRangeV1Request, DataColumnSidecarsByRootV1Request, + }, meta_data::GetMetaDataV3, ping::Ping, status::Status, @@ -52,6 +54,7 @@ use ream_req_resp::{ handler::{ReqRespMessageError, ReqRespMessageReceived, RespMessage}, messages::{RequestMessage, ResponseMessage}, }; +use ssz_types::VariableList; use tokio::{ sync::mpsc::{self, UnboundedReceiver, UnboundedSender}, time::interval, @@ -106,7 +109,7 @@ pub struct Network { peer_id: PeerId, swarm: Swarm, subscribed_topics: Arc>>, - callbacks: HashMapDelay>>, + callbacks: HashMapDelay>>, request_id: u64, network_state: Arc, peers_to_ping: HashSetDelay, @@ -364,6 +367,19 @@ impl Network { warn!("Failed to send error response: {err:?}"); } }, + P2PRequest::DataColumnRange { peer_id, start, count, columns, callback } => { + let request = DataColumnSidecarsByRangeV1Request { + start_slot: start, + count, + columns: VariableList::new(columns) + .expect("Too many columns were requested"), + }; + if let Some(request_id) = self.send_request(peer_id, BeaconRequestMessage::DataColumnSidecarsByRange(request)) { + self.callbacks.insert(request_id, callback); + } else if let Err(err) = callback.send(Ok(P2PCallbackResponse::Disconnected)).await { + warn!("Failed to send error response: {err:?}"); + } + }, P2PRequest::DataColumnIdentifiers { peer_id, column_identifiers, callback } => { if let Some(request_id) = self.send_request(peer_id, BeaconRequestMessage::DataColumnSidecarsByRoot(DataColumnSidecarsByRootV1Request::new(column_identifiers))) { self.callbacks.insert(request_id, callback); @@ -671,9 +687,9 @@ impl Network { let message = match message { Ok(message) => message, Err(err) => { - if let ReqRespMessageError::Outbound { request_id, .. } = &err - && let Some(callback) = self.callbacks.get(request_id) - && let Err(err) = callback.send(Err(anyhow!("{err:?}"))).await + if let ReqRespMessageError::Outbound { request_id, err } = err + && let Some(callback) = self.callbacks.remove(&request_id) + && let Err(err) = callback.send(Err(P2PCallbackError::ReqResp(err))).await { warn!("Failed to send error response: {err:?}"); } diff --git a/crates/networking/req_resp/src/beacon/messages/mod.rs b/crates/networking/req_resp/src/beacon/messages/mod.rs index 515c81c81..11d960e03 100644 --- a/crates/networking/req_resp/src/beacon/messages/mod.rs +++ b/crates/networking/req_resp/src/beacon/messages/mod.rs @@ -110,14 +110,17 @@ impl BeaconRequestMessage { request.count.min(MAX_REQUEST_BLOCKS) } BeaconRequestMessage::BeaconBlocksByRoot(request) => request.inner.len() as u64, - BeaconRequestMessage::BlobSidecarsByRange(request) => { - (request.count * MAX_BLOBS_PER_BLOCK).min(MAX_REQUEST_BLOB_SIDECARS) - } + BeaconRequestMessage::BlobSidecarsByRange(request) => request + .count + .saturating_mul(MAX_BLOBS_PER_BLOCK) + .min(MAX_REQUEST_BLOB_SIDECARS), BeaconRequestMessage::BlobSidecarsByRoot(request) => request.inner.len() as u64, BeaconRequestMessage::DataColumnSidecarsByRange(request) => { let num_columns = request.columns.len() as u64; - (request.count * num_columns) - .min(MAX_REQUEST_DATA_COLUMN_SIDECARS_PER_COLUMN * num_columns) + request + .count + .saturating_mul(num_columns) + .min(MAX_REQUEST_DATA_COLUMN_SIDECARS_PER_COLUMN.saturating_mul(num_columns)) } BeaconRequestMessage::DataColumnSidecarsByRoot(request) => { request.inner.iter().map(|id| id.columns.len() as u64).sum() diff --git a/crates/networking/req_resp/src/error.rs b/crates/networking/req_resp/src/error.rs index c41a20944..6213e0387 100644 --- a/crates/networking/req_resp/src/error.rs +++ b/crates/networking/req_resp/src/error.rs @@ -1,6 +1,6 @@ use std::io::{self}; -use ssz_types::{VariableList, typenum::U256}; +use crate::inbound_protocol::ResponseCode; #[derive(thiserror::Error, Debug)] pub enum ReqRespError { @@ -13,6 +13,9 @@ pub enum ReqRespError { #[error("Invalid data {0}")] InvalidData(String), + #[error("Remote error [{code:?}]: {message}")] + RemoteError { code: ResponseCode, message: String }, + #[error("Incomplete stream")] IncompleteStream, @@ -34,10 +37,3 @@ impl From for ReqRespError { ReqRespError::InvalidData(format!("Failed to decode ssz: {err:?}")) } } - -impl From> for ReqRespError { - fn from(err: VariableList) -> Self { - let err = String::from_utf8(Vec::from(err)).unwrap_or("Invalid UTF-8".to_string()); - ReqRespError::InvalidData(format!("ReqResp error message from peer: {err:?}")) - } -} diff --git a/crates/networking/req_resp/src/handler.rs b/crates/networking/req_resp/src/handler.rs index 8b3dc9cfe..77a901a03 100644 --- a/crates/networking/req_resp/src/handler.rs +++ b/crates/networking/req_resp/src/handler.rs @@ -66,6 +66,7 @@ impl RespMessage { | ReqRespError::Anyhow(_) | ReqRespError::IoError(_) => Some(ResponseCode::ServerError), ReqRespError::InvalidData(_) => Some(ResponseCode::InvalidRequest), + ReqRespError::RemoteError { code, .. } => Some(*code), ReqRespError::Disconnected | ReqRespError::StreamTimedOut(_) | ReqRespError::TokioTimedOut(_) => Some(ResponseCode::ResourceUnavailable), diff --git a/crates/networking/req_resp/src/outbound_protocol.rs b/crates/networking/req_resp/src/outbound_protocol.rs index 75365c8a9..ed9ffeac4 100644 --- a/crates/networking/req_resp/src/outbound_protocol.rs +++ b/crates/networking/req_resp/src/outbound_protocol.rs @@ -285,9 +285,19 @@ impl Decoder for OutboundSSZSnappyCodec { } } } else { - Ok(Some(RespMessage::Error( - VariableList::::from_ssz_bytes(&buf).map(ReqRespError::from).map_err(|err| anyhow!("OutboundSSZSnappyCodec::decode: protocol: {:?}, response_code: {response_code:?}, err: {err:?}", self.protocol.protocol))?, - ))) + let message = match VariableList::::from_ssz_bytes(&buf) { + Ok(bytes) => String::from_utf8(Vec::from(bytes)) + .unwrap_or_else(|_| "".to_string()), + Err(err) => { + return Ok(Some(RespMessage::Error(ReqRespError::InvalidData( + format!("Failed to decode error body: {err:?}"), + )))); + } + }; + Ok(Some(RespMessage::Error(ReqRespError::RemoteError { + code: response_code, + message, + }))) } } Err(err) => match err.kind() { diff --git a/crates/networking/syncer/Cargo.toml b/crates/networking/syncer/Cargo.toml index 9ba1f3566..6819d3c97 100644 --- a/crates/networking/syncer/Cargo.toml +++ b/crates/networking/syncer/Cargo.toml @@ -31,6 +31,7 @@ ream-chain-beacon.workspace = true ream-consensus-beacon.workspace = true ream-consensus-misc.workspace = true ream-executor.workspace = true +ream-fork-choice-beacon.workspace = true ream-network-spec.workspace = true ream-p2p.workspace = true ream-polynomial-commitments.workspace = true @@ -38,8 +39,14 @@ ream-req-resp.workspace = true ream-storage.workspace = true [dev-dependencies] +discv5.workspace = true kzg.workspace = true +parking_lot.workspace = true ream-execution-rpc-types.workspace = true +ream-operation-pool.workspace = true +ream-peer.workspace = true +ream-sync-committee-pool.workspace = true +tempfile.workspace = true [lints] workspace = true diff --git a/crates/networking/syncer/src/block_range/block_cache.rs b/crates/networking/syncer/src/block_range/block_cache.rs index e36ec7c2b..35e56cdf9 100644 --- a/crates/networking/syncer/src/block_range/block_cache.rs +++ b/crates/networking/syncer/src/block_range/block_cache.rs @@ -1,29 +1,115 @@ -use std::collections::{HashMap, HashSet}; +use std::{ + collections::{HashMap, HashSet}, + time::{Duration, Instant}, +}; use alloy_primitives::B256; use anyhow::{bail, ensure}; +use libp2p::PeerId; use ream_chain_beacon::beacon_chain::is_data_availability_check_required; use ream_consensus_beacon::{ blob_sidecar::{BlobIdentifier, BlobSidecar}, + data_column_sidecar::{ColumnIdentifier, DataColumnSidecar}, electra::beacon_block::SignedBeaconBlock, }; use ream_consensus_misc::misc::compute_epoch_at_slot; use ream_network_spec::networks::beacon_network_spec; +use ream_polynomial_commitments::handlers::{ + verify_blob_kzg_proof_batch, verify_data_column_sidecar_kzg_proofs, +}; use ssz::Encode; use tree_hash::TreeHash; -use super::{MAX_BLOCKS_PER_REQUEST, peer_range_downloader::Range}; +use super::{FrontierObservation, MAX_BLOCKS_PER_REQUEST, peer_range_downloader::Range}; + +const ATTEMPT_COOLDOWN: Duration = Duration::from_secs(30); + +#[derive(Debug, Clone)] +pub struct CoverageAnchor { + pub frontier: FrontierObservation, + pub original_parent_root: B256, + pub original_parent_slot: u64, + pub covered_through_slot: u64, + pub confirming_peers: HashSet, +} + +#[derive(Debug)] +pub enum AddBlocksError { + InvalidBatch(anyhow::Error), + CoverageDivergence { + expected_parent: B256, + actual_parent: B256, + anchor: Box, + }, +} + +impl std::fmt::Display for AddBlocksError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + AddBlocksError::InvalidBatch(err) => write!(f, "invalid batch: {err:?}"), + AddBlocksError::CoverageDivergence { + expected_parent, + actual_parent, + .. + } => write!( + f, + "coverage divergence: expected parent {expected_parent}, got {actual_parent}" + ), + } + } +} + +impl From for AddBlocksError { + fn from(err: anyhow::Error) -> Self { + AddBlocksError::InvalidBatch(err) + } +} + +pub(super) fn validate_range_chain(blocks: &[SignedBeaconBlock]) -> anyhow::Result<()> { + for (index, block) in blocks.iter().enumerate().rev() { + if index > 0 { + ensure!( + block.message.parent_root == blocks[index - 1].message.tree_hash_root(), + "Block at index {index} has a parent root that does not match the previous block's tree hash root", + ); + ensure!( + block.message.slot > blocks[index - 1].message.slot, + "Block at index {index} does not have a strictly increasing slot", + ); + } + } + Ok(()) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum RequestKey { + BlockRange(Range), + ColumnRange(Range), + BlockRoot(B256), + Blob(BlobIdentifier), + Column(ColumnIdentifier), +} + +#[derive(Default)] +struct AttemptState { + attempted_peers: HashSet, + cooldown_until: Option, +} pub struct BlockAndBlobBundle { pub block: SignedBeaconBlock, pub blobs: HashMap, + pub columns: HashMap, + pub source_peer: PeerId, } impl BlockAndBlobBundle { - pub fn new(block: SignedBeaconBlock) -> Self { + pub fn new(block: SignedBeaconBlock, source_peer: PeerId) -> Self { Self { block, blobs: HashMap::new(), + columns: HashMap::new(), + source_peer, } } } @@ -35,8 +121,14 @@ pub struct BlockCache { block_ranges_to_retry: Vec, initial_slot: u64, next_start_slot: u64, + block_ranges_in_progress: HashSet, block_roots_in_progress: HashSet, blob_identifiers_in_progress: HashSet, + column_ranges_to_fetch: Vec, + column_ranges_in_progress: HashSet, + data_column_identifiers_in_progress: HashSet, + attempts: HashMap, + coverage_anchor: Option, } impl BlockCache { @@ -48,25 +140,192 @@ impl BlockCache { block_ranges_to_retry: vec![], initial_slot: next_start_slot, next_start_slot, + block_ranges_in_progress: HashSet::new(), block_roots_in_progress: HashSet::new(), blob_identifiers_in_progress: HashSet::new(), + column_ranges_to_fetch: vec![], + column_ranges_in_progress: HashSet::new(), + data_column_identifiers_in_progress: HashSet::new(), + attempts: HashMap::new(), + coverage_anchor: None, } } + pub fn from_recovery_seed(seed: super::recovery::RecoverySeed) -> anyhow::Result { + let super::recovery::RecoverySeed { + ancestor_root, + ancestor_slot, + forward_blocks, + target_slot, + source_peer, + } = seed; + + ensure!( + !forward_blocks.is_empty(), + "a recovery seed must contain at least one new block" + ); + validate_range_chain(&forward_blocks)?; + + let first = &forward_blocks[0]; + ensure!( + first.message.parent_root == ancestor_root, + "recovery seed's first block does not connect to the ancestor root" + ); + ensure!( + first.message.slot > ancestor_slot, + "recovery seed's first block is not after the ancestor slot" + ); + + let mut seen_roots = HashSet::new(); + for block in &forward_blocks { + ensure!( + block.message.slot <= target_slot, + "recovery seed block at slot {} exceeds target_slot {target_slot}", + block.message.slot + ); + ensure!( + seen_roots.insert(block.message.tree_hash_root()), + "duplicate root in recovery seed" + ); + } + + let tip_slot = forward_blocks + .last() + .expect("checked non-empty above") + .message + .slot; + + let mut cache = BlockCache::new(ancestor_root, ancestor_slot); + cache + .add_blocks(forward_blocks, true, source_peer) + .map_err(|err| anyhow::anyhow!("{err}"))?; + cache.next_start_slot = tip_slot; + Ok(cache) + } + + fn is_pristine_for_advance(&self) -> bool { + self.blocks_and_blobs.is_empty() + && self.block_ranges_to_retry.is_empty() + && self.block_ranges_in_progress.is_empty() + && self.block_roots_in_progress.is_empty() + && self.blob_identifiers_in_progress.is_empty() + && self.column_ranges_to_fetch.is_empty() + && self.column_ranges_in_progress.is_empty() + && self.data_column_identifiers_in_progress.is_empty() + } + + pub fn is_pristine_for_restore(&self) -> bool { + self.coverage_anchor.is_none() && self.is_pristine_for_advance() + } + + pub fn advance_empty_coverage( + &mut self, + frontier: FrontierObservation, + parent_root: B256, + covered_through_slot: u64, + confirming_peers: HashSet, + ) -> anyhow::Result<()> { + match &mut self.coverage_anchor { + Some(anchor) => { + anchor.covered_through_slot = covered_through_slot; + anchor.confirming_peers.extend(confirming_peers); + } + None => { + ensure!( + self.is_pristine_for_advance(), + "advance_empty_coverage requires a pristine cache to start a new generation" + ); + self.coverage_anchor = Some(CoverageAnchor { + frontier, + original_parent_root: self.initial_parent_root, + original_parent_slot: self.initial_slot, + covered_through_slot, + confirming_peers, + }); + } + } + + self.initial_parent_root = parent_root; + self.next_start_slot = covered_through_slot; + self.initial_slot = covered_through_slot; + Ok(()) + } + + fn refresh_attempt_round(&mut self, key: &RequestKey, now: Instant) { + if let Some(state) = self.attempts.get_mut(key) + && let Some(cooldown_until) = state.cooldown_until + && now >= cooldown_until + { + state.attempted_peers.clear(); + state.cooldown_until = None; + } + } + + fn is_schedulable( + &mut self, + key: RequestKey, + candidate_peers: &[PeerId], + now: Instant, + ) -> bool { + self.refresh_attempt_round(&key, now); + match self.attempts.get(&key) { + None => true, + Some(state) => candidate_peers + .iter() + .any(|peer_id| !state.attempted_peers.contains(peer_id)), + } + } + + pub fn attempted_peers_for(&self, key: RequestKey) -> HashSet { + self.attempts + .get(&key) + .map(|state| state.attempted_peers.clone()) + .unwrap_or_default() + } + + pub fn mark_attempted( + &mut self, + key: RequestKey, + peer_id: PeerId, + candidate_peers: &[PeerId], + now: Instant, + ) { + let state = self.attempts.entry(key).or_default(); + state.attempted_peers.insert(peer_id); + if candidate_peers + .iter() + .all(|peer_id| state.attempted_peers.contains(peer_id)) + { + state.cooldown_until = Some(now + ATTEMPT_COOLDOWN); + } + } + + pub fn clear_attempted(&mut self, key: RequestKey) { + self.attempts.remove(&key); + } + pub fn add_blocks( &mut self, blocks: Vec, is_range: bool, - ) -> anyhow::Result<()> { + source_peer: PeerId, + ) -> Result<(), AddBlocksError> { // Ensure that all blocks form a chain if is_range { - for (index, block) in blocks.iter().enumerate().rev() { - if index > 0 { - ensure!( - block.message.parent_root == blocks[index - 1].message.tree_hash_root(), - "Block at index {index} has a parent root that does not match the previous block's tree hash root", - ); + validate_range_chain(&blocks)?; + + if self.blocks_and_blobs.is_empty() + && let Some(anchor) = &self.coverage_anchor + && let Some(first) = blocks.first() + { + if first.message.parent_root != self.initial_parent_root { + return Err(AddBlocksError::CoverageDivergence { + expected_parent: self.initial_parent_root, + actual_parent: first.message.parent_root, + anchor: Box::new(anchor.clone()), + }); } + self.coverage_anchor = None; } } @@ -74,7 +333,7 @@ impl BlockCache { self.current_cache_size += block.as_ssz_bytes().len() as u64; self.blocks_and_blobs.insert( block.message.tree_hash_root(), - BlockAndBlobBundle::new(block), + BlockAndBlobBundle::new(block, source_peer), ); } @@ -82,20 +341,112 @@ impl BlockCache { } pub fn add_blobs(&mut self, blobs: Vec) -> anyhow::Result<()> { + let mut validated = Vec::with_capacity(blobs.len()); for blob_sidecar in blobs { let block_root = blob_sidecar.signed_block_header.message.tree_hash_root(); - - if let Some(bundle) = self.blocks_and_blobs.get_mut(&block_root) { - bundle.blobs.insert( - BlobIdentifier { - block_root, - index: blob_sidecar.index, - }, - blob_sidecar, - ); - } else { + let Some(bundle) = self.blocks_and_blobs.get(&block_root) else { bail!("Block root {block_root} not found in cache, this should be impossible"); + }; + + ensure!( + blob_sidecar.signed_block_header == bundle.block.signed_header(), + "Blob sidecar {} does not belong to block {block_root}", + blob_sidecar.index + ); + let commitments = &bundle.block.message.body.blob_kzg_commitments; + let index = blob_sidecar.index as usize; + ensure!( + index < commitments.len(), + "Blob sidecar index {} out of range for block {block_root}", + blob_sidecar.index + ); + ensure!( + blob_sidecar.kzg_commitment == commitments[index], + "Blob sidecar {} commitment does not match block {block_root}", + blob_sidecar.index + ); + ensure!( + blob_sidecar.verify_blob_sidecar_inclusion_proof(), + "Invalid inclusion proof for blob {} of block {block_root}", + blob_sidecar.index + ); + ensure!( + verify_blob_kzg_proof_batch( + std::slice::from_ref(&blob_sidecar.blob), + std::slice::from_ref(&blob_sidecar.kzg_commitment), + std::slice::from_ref(&blob_sidecar.kzg_proof), + )?, + "Invalid KZG proof for blob {} of block {block_root}", + blob_sidecar.index + ); + + validated.push((block_root, blob_sidecar)); + } + + for (block_root, blob_sidecar) in validated { + let bundle = self + .blocks_and_blobs + .get_mut(&block_root) + .expect("presence already checked in the validation pass above"); + bundle.blobs.insert( + BlobIdentifier { + block_root, + index: blob_sidecar.index, + }, + blob_sidecar, + ); + } + + Ok(()) + } + + pub fn add_data_columns( + &mut self, + columns: Vec, + required_columns: &HashSet, + ) -> anyhow::Result<()> { + let mut validated = Vec::with_capacity(columns.len()); + for column in columns { + if !required_columns.contains(&column.index) { + continue; } + + let block_root = column.signed_block_header.message.tree_hash_root(); + if !self.blocks_and_blobs.contains_key(&block_root) { + continue; + } + + let bundle = self + .blocks_and_blobs + .get(&block_root) + .expect("presence just checked above"); + ensure!( + column.signed_block_header == bundle.block.signed_header(), + "Data column sidecar {} does not belong to block {block_root}", + column.index + ); + ensure!( + column.verify_inclusion_proof(), + "Invalid inclusion proof for column {} of block {block_root}", + column.index + ); + ensure!( + verify_data_column_sidecar_kzg_proofs(&column)?, + "Invalid KZG proof for column {} of block {block_root}", + column.index + ); + + validated.push((block_root, column)); + } + + for (block_root, column) in validated { + let bundle = self + .blocks_and_blobs + .get_mut(&block_root) + .expect("presence already checked in the validation pass above"); + bundle + .columns + .insert(ColumnIdentifier::new(block_root, column.index), column); } Ok(()) @@ -121,6 +472,28 @@ impl BlockCache { } } + pub fn push_column_range(&mut self, range: Range) { + self.column_ranges_to_fetch.push(range); + } + + pub fn mark_column_range_in_progress(&mut self, range: Range) { + self.column_ranges_in_progress.insert(range); + } + + pub fn remove_column_range_in_progress(&mut self, range: &Range) { + self.column_ranges_in_progress.remove(range); + } + + pub fn extend_data_column_identifiers_in_progress(&mut self, identifiers: &[ColumnIdentifier]) { + self.data_column_identifiers_in_progress.extend(identifiers); + } + + pub fn remove_data_column_identifiers_in_progress(&mut self, identifiers: &[ColumnIdentifier]) { + for identifier in identifiers { + self.data_column_identifiers_in_progress.remove(identifier); + } + } + pub fn block_count(&self) -> u64 { self.blocks_and_blobs.len() as u64 } @@ -139,6 +512,14 @@ impl BlockCache { .sum() } + pub fn next_start_slot(&self) -> u64 { + self.next_start_slot + } + + pub fn initial_parent_root(&self) -> B256 { + self.initial_parent_root + } + pub fn estimated_blocks_to_fetch(&self) -> u64 { if self.next_start_slot.saturating_sub(self.initial_slot) > 30 { return 0; @@ -151,29 +532,96 @@ impl BlockCache { self.block_ranges_to_retry.push(range); } - pub fn data_to_fetch(&mut self, finalized_slot: u64, current_epoch: u64) -> DataToFetch { - match self.block_ranges_to_retry.pop() { - Some(range) => return DataToFetch::BlockRange(range), - None => { - let estimated_blocks_to_fetch = self.estimated_blocks_to_fetch(); - if estimated_blocks_to_fetch > 0 && self.next_start_slot < finalized_slot { - let blocks_to_fill = estimated_blocks_to_fetch - .min(MAX_BLOCKS_PER_REQUEST.min(finalized_slot - self.next_start_slot)); - let start_slot = self.next_start_slot + 1; - self.next_start_slot += blocks_to_fill; - return DataToFetch::BlockRange(Range::new(start_slot, blocks_to_fill)); - } + pub fn mark_block_range_in_progress(&mut self, range: Range) { + self.block_ranges_in_progress.insert(range); + } + + pub fn remove_block_range_in_progress(&mut self, range: &Range) { + self.block_ranges_in_progress.remove(range); + } + + fn take_schedulable_retry_range( + &mut self, + candidate_peers: &[PeerId], + now: Instant, + ) -> Option { + let ranges = self.block_ranges_to_retry.clone(); + for (index, range) in ranges.iter().enumerate().rev() { + if self.is_schedulable(RequestKey::BlockRange(*range), candidate_peers, now) { + return Some(self.block_ranges_to_retry.remove(index)); } } + None + } + + fn take_schedulable_column_range( + &mut self, + candidate_peers: &[PeerId], + now: Instant, + ) -> Option { + let ranges = self.column_ranges_to_fetch.clone(); + for (index, range) in ranges.iter().enumerate().rev() { + if self.is_schedulable(RequestKey::ColumnRange(*range), candidate_peers, now) { + return Some(self.column_ranges_to_fetch.remove(index)); + } + } + None + } + + pub fn data_to_fetch( + &mut self, + target_slot: u64, + current_epoch: u64, + required_columns: &HashSet, + candidate_peers: &[PeerId], + now: Instant, + frontier_tracked: bool, + ) -> DataToFetch { + let single_flight = frontier_tracked || self.coverage_anchor.is_some(); + if single_flight && !self.block_ranges_in_progress.is_empty() { + return DataToFetch::DownloadsInProgress; + } + + if let Some(range) = self.take_schedulable_retry_range(candidate_peers, now) { + return DataToFetch::BlockRange(range); + } + + let estimated_blocks_to_fetch = self.estimated_blocks_to_fetch(); + if estimated_blocks_to_fetch > 0 && self.next_start_slot < target_slot { + let blocks_to_fill = estimated_blocks_to_fetch + .min(MAX_BLOCKS_PER_REQUEST.min(target_slot - self.next_start_slot)); + let start_slot = self.next_start_slot + 1; + self.next_start_slot += blocks_to_fill; + return DataToFetch::BlockRange(Range::new(start_slot, blocks_to_fill)); + } + + if let Some(range) = self.take_schedulable_column_range(candidate_peers, now) { + return DataToFetch::DataColumnRange(range); + } let mut block_roots_left_to_fetch = self.get_missing_block_roots(); let missing_block_roots_len = block_roots_left_to_fetch.len(); - block_roots_left_to_fetch.retain(|root| !self.block_roots_in_progress.contains(root)); + block_roots_left_to_fetch.retain(|root| { + !self.block_roots_in_progress.contains(root) + && self.is_schedulable(RequestKey::BlockRoot(*root), candidate_peers, now) + }); let mut blob_identifiers_left_to_fetch = self.get_missing_blob_identifiers(current_epoch); let missing_blob_identifiers_len = blob_identifiers_left_to_fetch.len(); - blob_identifiers_left_to_fetch - .retain(|blob_identifier| !self.blob_identifiers_in_progress.contains(blob_identifier)); + blob_identifiers_left_to_fetch.retain(|blob_identifier| { + !self.blob_identifiers_in_progress.contains(blob_identifier) + && self.is_schedulable(RequestKey::Blob(*blob_identifier), candidate_peers, now) + }); + + let mut data_column_identifiers_left_to_fetch = + self.get_missing_data_column_identifiers(current_epoch, required_columns); + let missing_data_column_identifiers_len = data_column_identifiers_left_to_fetch.len(); + data_column_identifiers_left_to_fetch.retain(|identifier| { + !self + .data_column_identifiers_in_progress + .contains(identifier) + && self.is_schedulable(RequestKey::Column(*identifier), candidate_peers, now) + }); if !block_roots_left_to_fetch.is_empty() { return DataToFetch::MissingBlockRoots(block_roots_left_to_fetch); @@ -183,7 +631,20 @@ impl BlockCache { return DataToFetch::MissingBlobIdentifiers(blob_identifiers_left_to_fetch); } - if missing_block_roots_len > 0 || missing_blob_identifiers_len > 0 { + if !data_column_identifiers_left_to_fetch.is_empty() { + return DataToFetch::MissingDataColumnIdentifiers( + data_column_identifiers_left_to_fetch, + ); + } + + if missing_block_roots_len > 0 + || missing_blob_identifiers_len > 0 + || missing_data_column_identifiers_len > 0 + || !self.block_ranges_in_progress.is_empty() + || !self.column_ranges_in_progress.is_empty() + || !self.block_ranges_to_retry.is_empty() + || !self.column_ranges_to_fetch.is_empty() + { return DataToFetch::DownloadsInProgress; } @@ -255,13 +716,76 @@ impl BlockCache { } missing_roots } + + /// ByRoot fallback for columns a range fetch missed (e.g. root-fetched blocks). + fn get_missing_data_column_identifiers( + &self, + current_epoch: u64, + required_columns: &HashSet, + ) -> Vec { + let network_spec = beacon_network_spec(); + let mut missing_identifiers = Vec::new(); + for block in self.blocks_and_blobs.values() { + if block.block.message.body.blob_kzg_commitments.is_empty() { + continue; + } + + if !is_data_availability_check_required( + compute_epoch_at_slot(block.block.message.slot), + current_epoch, + network_spec.fulu_fork_epoch, + network_spec.min_epochs_for_data_column_sidecars_requests, + ) { + continue; + } + + if compute_epoch_at_slot(block.block.message.slot) < network_spec.fulu_fork_epoch { + continue; + } + + let block_root = block.block.message.tree_hash_root(); + for column_index in required_columns { + let identifier = ColumnIdentifier::new(block_root, *column_index); + if block.columns.contains_key(&identifier) { + continue; + } + missing_identifiers.push(identifier); + } + } + missing_identifiers + } + + pub fn expected_column_identifiers_in_range( + &self, + range: Range, + required_columns: &HashSet, + ) -> Vec { + let range_end = range.start_slot + range.count; + let mut expected = Vec::new(); + for block in self.blocks_and_blobs.values() { + if block.block.message.slot < range.start_slot || block.block.message.slot >= range_end + { + continue; + } + let block_root = block.block.message.tree_hash_root(); + for column_index in required_columns { + let identifier = ColumnIdentifier::new(block_root, *column_index); + if !block.columns.contains_key(&identifier) { + expected.push(identifier); + } + } + } + expected + } } #[derive(Debug, Clone, PartialEq, Eq)] pub enum DataToFetch { BlockRange(Range), + DataColumnRange(Range), MissingBlockRoots(Vec), MissingBlobIdentifiers(Vec), + MissingDataColumnIdentifiers(Vec), DownloadsInProgress, Finished, } @@ -270,12 +794,16 @@ impl std::fmt::Display for DataToFetch { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { DataToFetch::BlockRange(range) => write!(f, "BlockRange({range:?})"), + DataToFetch::DataColumnRange(range) => write!(f, "DataColumnRange({range:?})"), DataToFetch::MissingBlockRoots(roots) => { write!(f, "MissingBlockRoots({})", roots.len()) } DataToFetch::MissingBlobIdentifiers(identifiers) => { write!(f, "MissingBlobIdentifiers({})", identifiers.len()) } + DataToFetch::MissingDataColumnIdentifiers(identifiers) => { + write!(f, "MissingDataColumnIdentifiers({})", identifiers.len()) + } DataToFetch::DownloadsInProgress => write!(f, "DownloadsInProgress"), DataToFetch::Finished => write!(f, "Finished"), } @@ -284,20 +812,48 @@ impl std::fmt::Display for DataToFetch { #[cfg(test)] mod tests { - use ream_consensus_beacon::electra::beacon_block::BeaconBlock; + use ream_consensus_beacon::{ + electra::beacon_block::BeaconBlock, + matrix_entry::{compute_cells_and_kzg_proofs, das_context}, + }; use ream_consensus_misc::{ misc::compute_start_slot_at_epoch, polynomial_commitments::kzg_commitment::KZGCommitment, }; + use ream_execution_rpc_types::get_blobs::Blob; use ream_network_spec::networks::beacon::initialize_test_network_spec; use super::*; #[test] - fn data_to_fetch_finishes_at_finalized_slot() { + fn data_to_fetch_finishes_at_target_slot() { initialize_test_network_spec(); let mut cache = BlockCache::new(B256::ZERO, 10); - assert_eq!(cache.data_to_fetch(10, 0), DataToFetch::Finished); + assert_eq!( + cache.data_to_fetch(10, 0, &HashSet::new(), &[], Instant::now(), false), + DataToFetch::Finished + ); + } + + /// `Finished` must not fire while a `BlockRange` is still in flight, or the download (and + /// its peer) gets silently orphaned. + #[test] + fn data_to_fetch_waits_for_in_flight_block_ranges_before_finishing() { + initialize_test_network_spec(); + let mut cache = BlockCache::new(B256::ZERO, 10); + let range = Range::new(11, 10); + cache.mark_block_range_in_progress(range); + + assert_eq!( + cache.data_to_fetch(10, 0, &HashSet::new(), &[], Instant::now(), false), + DataToFetch::DownloadsInProgress + ); + + cache.remove_block_range_in_progress(&range); + assert_eq!( + cache.data_to_fetch(10, 0, &HashSet::new(), &[], Instant::now(), false), + DataToFetch::Finished + ); } #[test] @@ -306,18 +862,21 @@ mod tests { let mut cache = BlockCache::new(B256::ZERO, 10); assert_eq!( - cache.data_to_fetch(25, 0), + cache.data_to_fetch(25, 0, &HashSet::new(), &[], Instant::now(), false), DataToFetch::BlockRange(Range::new(11, 10)) ); assert_eq!( - cache.data_to_fetch(25, 0), + cache.data_to_fetch(25, 0, &HashSet::new(), &[], Instant::now(), false), DataToFetch::BlockRange(Range::new(21, 5)) ); - assert_eq!(cache.data_to_fetch(25, 0), DataToFetch::Finished); + assert_eq!( + cache.data_to_fetch(25, 0, &HashSet::new(), &[], Instant::now(), false), + DataToFetch::Finished + ); } #[test] - fn blob_fetching_uses_the_data_availability_retention_boundary() { + fn post_fulu_blob_commitments_never_request_legacy_blob_identifiers() { initialize_test_network_spec(); let network_spec = beacon_network_spec(); let current_epoch = network_spec.fulu_fork_epoch @@ -348,21 +907,544 @@ mod tests { let boundary_slot = compute_start_slot_at_epoch(boundary_epoch); let mut retained = BlockCache::new(parent_root, boundary_slot); retained - .add_blocks(vec![block_with_blob(boundary_slot)], false) + .add_blocks( + vec![block_with_blob(boundary_slot)], + false, + PeerId::random(), + ) .expect("boundary block should enter cache"); - assert!(matches!( - retained.data_to_fetch(boundary_slot, current_epoch), - DataToFetch::MissingBlobIdentifiers(identifiers) if identifiers.len() == 1 - )); + assert_eq!( + retained.data_to_fetch( + boundary_slot, + current_epoch, + &HashSet::new(), + &[], + Instant::now(), + false + ), + DataToFetch::Finished + ); let expired_slot = compute_start_slot_at_epoch(boundary_epoch - 1); let mut expired = BlockCache::new(parent_root, expired_slot); expired - .add_blocks(vec![block_with_blob(expired_slot)], false) + .add_blocks(vec![block_with_blob(expired_slot)], false, PeerId::random()) .expect("expired block should enter cache"); assert_eq!( - expired.data_to_fetch(expired_slot, current_epoch), + expired.data_to_fetch( + expired_slot, + current_epoch, + &HashSet::new(), + &[], + Instant::now(), + false + ), DataToFetch::Finished ); } + + #[test] + fn post_fulu_blob_commitments_request_required_data_column_identifiers() { + initialize_test_network_spec(); + let network_spec = beacon_network_spec(); + let current_epoch = network_spec.fulu_fork_epoch + + network_spec.min_epochs_for_data_column_sidecars_requests + + 10; + let boundary_epoch = + current_epoch - network_spec.min_epochs_for_data_column_sidecars_requests; + let parent_root = B256::repeat_byte(1); + let boundary_slot = compute_start_slot_at_epoch(boundary_epoch); + + let mut block = SignedBeaconBlock { + message: BeaconBlock { + slot: boundary_slot, + parent_root, + ..Default::default() + }, + signature: Default::default(), + }; + block + .message + .body + .blob_kzg_commitments + .push(KZGCommitment::empty_for_testing()) + .expect("one commitment should fit"); + let block_root = block.message.tree_hash_root(); + + let mut cache = BlockCache::new(parent_root, boundary_slot); + cache + .add_blocks(vec![block], false, PeerId::random()) + .expect("boundary block should enter cache"); + + let required_columns = HashSet::from([1, 5]); + match cache.data_to_fetch( + boundary_slot, + current_epoch, + &required_columns, + &[], + Instant::now(), + false, + ) { + DataToFetch::MissingDataColumnIdentifiers(identifiers) => { + let mut identifiers = identifiers; + identifiers.sort(); + assert_eq!( + identifiers, + vec![ + ColumnIdentifier::new(block_root, 1), + ColumnIdentifier::new(block_root, 5), + ] + ); + } + other => panic!("expected MissingDataColumnIdentifiers, got {other:?}"), + } + + // Unneeded columns never count as missing. + assert_eq!( + cache.data_to_fetch( + boundary_slot, + current_epoch, + &HashSet::new(), + &[], + Instant::now(), + false + ), + DataToFetch::Finished + ); + } + + #[test] + fn add_data_columns_ignores_extras_and_rejects_bad_proofs() { + use ream_consensus_beacon::data_column_sidecar::get_data_column_sidecars_from_block; + + let blob = Blob::default(); + let blob_bytes = blob.to_fixed_bytes(); + let raw_commitment = das_context() + .blob_to_kzg_commitment(&blob_bytes) + .expect("test blob should produce a commitment"); + let commitment = KZGCommitment(raw_commitment); + + let mut block = SignedBeaconBlock { + message: Default::default(), + signature: Default::default(), + }; + block + .message + .body + .blob_kzg_commitments + .push(commitment) + .expect("one commitment should fit"); + let block_root = block.message.tree_hash_root(); + + let cells_and_kzg_proofs = compute_cells_and_kzg_proofs(&blob, das_context()) + .expect("test blob should produce cells and proofs"); + let columns = get_data_column_sidecars_from_block(&block, vec![cells_and_kzg_proofs]) + .expect("test block should produce data columns"); + + let mut cache = BlockCache::new(B256::ZERO, 0); + cache + .add_blocks(vec![block], false, PeerId::random()) + .expect("block should enter cache"); + + // Only column 0 was requested; extra columns in the response are ignored, not fatal. + let required_columns = HashSet::from([0]); + cache + .add_data_columns(columns.clone(), &required_columns) + .expect("valid, requested column should be accepted"); + assert_eq!( + cache + .blocks_and_blobs + .get(&block_root) + .expect("block should be cached") + .columns + .len(), + 1 + ); + + let mut tampered = columns[0].clone(); + tampered.kzg_proofs[0][0] ^= 1; + assert!( + cache + .add_data_columns(vec![tampered], &required_columns) + .is_err() + ); + } + + #[test] + fn add_data_columns_is_atomic_a_bad_item_does_not_leave_earlier_items_mutated() { + use ream_consensus_beacon::data_column_sidecar::get_data_column_sidecars_from_block; + + let blob = Blob::default(); + let blob_bytes = blob.to_fixed_bytes(); + let raw_commitment = das_context() + .blob_to_kzg_commitment(&blob_bytes) + .expect("test blob should produce a commitment"); + let commitment = KZGCommitment(raw_commitment); + + let mut block = SignedBeaconBlock { + message: Default::default(), + signature: Default::default(), + }; + block + .message + .body + .blob_kzg_commitments + .push(commitment) + .expect("one commitment should fit"); + + let cells_and_kzg_proofs = compute_cells_and_kzg_proofs(&blob, das_context()) + .expect("test blob should produce cells and proofs"); + let columns = get_data_column_sidecars_from_block(&block, vec![cells_and_kzg_proofs]) + .expect("test block should produce data columns"); + let block_root = block.message.tree_hash_root(); + + let mut cache = BlockCache::new(B256::ZERO, 0); + cache + .add_blocks(vec![block], false, PeerId::random()) + .expect("block should enter cache"); + + let mut tampered = columns[1].clone(); + tampered.kzg_proofs[0][0] ^= 1; + let required_columns = HashSet::from([0, 1]); + + let result = cache.add_data_columns(vec![columns[0].clone(), tampered], &required_columns); + assert!(result.is_err(), "the batch as a whole must fail"); + + assert!( + cache + .blocks_and_blobs + .get(&block_root) + .expect("block should still be cached") + .columns + .is_empty(), + "a failed batch must not partially mutate the cache" + ); + } + + #[test] + fn is_schedulable_false_only_when_every_candidate_is_attempted() { + let mut cache = BlockCache::new(B256::ZERO, 0); + let key = RequestKey::BlockRoot(B256::repeat_byte(1)); + let peer_a = PeerId::random(); + let peer_b = PeerId::random(); + let candidates = [peer_a, peer_b]; + let now = Instant::now(); + + assert!(cache.is_schedulable(key, &candidates, now)); + + cache.mark_attempted(key, peer_a, &candidates, now); + assert!(cache.is_schedulable(key, &candidates, now)); + + cache.mark_attempted(key, peer_b, &candidates, now); + assert!(!cache.is_schedulable(key, &candidates, now)); + } + + #[test] + fn cooldown_expiring_resets_the_attempted_set_for_a_fresh_round() { + let mut cache = BlockCache::new(B256::ZERO, 0); + let key = RequestKey::BlockRoot(B256::repeat_byte(1)); + let peer_a = PeerId::random(); + let candidates = [peer_a]; + let now = Instant::now(); + + cache.mark_attempted(key, peer_a, &candidates, now); + assert!(!cache.is_schedulable(key, &candidates, now)); + + assert!(!cache.is_schedulable(key, &candidates, now + Duration::from_secs(1))); + + assert!(cache.is_schedulable( + key, + &candidates, + now + ATTEMPT_COOLDOWN + Duration::from_secs(1) + )); + } + + #[test] + fn a_new_peer_makes_an_exhausted_key_schedulable_without_waiting_for_cooldown() { + let mut cache = BlockCache::new(B256::ZERO, 0); + let key = RequestKey::BlockRoot(B256::repeat_byte(1)); + let peer_a = PeerId::random(); + let peer_b = PeerId::random(); + let now = Instant::now(); + + cache.mark_attempted(key, peer_a, &[peer_a], now); + assert!(!cache.is_schedulable(key, &[peer_a], now)); + + assert!(cache.is_schedulable(key, &[peer_a, peer_b], now)); + } + + #[test] + fn clear_attempted_starts_a_clean_round() { + let mut cache = BlockCache::new(B256::ZERO, 0); + let key = RequestKey::BlockRoot(B256::repeat_byte(1)); + let peer_a = PeerId::random(); + let now = Instant::now(); + + cache.mark_attempted(key, peer_a, &[peer_a], now); + assert!(!cache.is_schedulable(key, &[peer_a], now)); + + cache.clear_attempted(key); + assert!(cache.is_schedulable(key, &[peer_a], now)); + assert!(cache.attempted_peers_for(key).is_empty()); + } + + #[test] + fn take_schedulable_retry_range_skips_backed_off_ranges_but_keeps_them_queued() { + let mut cache = BlockCache::new(B256::ZERO, 0); + let backed_off = Range::new(1, 10); + let schedulable = Range::new(11, 10); + let peer_a = PeerId::random(); + let now = Instant::now(); + + cache.push_retry_range(backed_off); + cache.push_retry_range(schedulable); + cache.mark_attempted(RequestKey::BlockRange(backed_off), peer_a, &[peer_a], now); + + assert_eq!( + cache.take_schedulable_retry_range(&[peer_a], now), + Some(schedulable) + ); + assert_eq!(cache.block_ranges_to_retry, vec![backed_off]); + assert_eq!(cache.take_schedulable_retry_range(&[peer_a], now), None); + } + + #[test] + fn an_exhausted_backed_off_root_does_not_starve_other_tiers_or_cause_false_finished() { + initialize_test_network_spec(); + let parent_root = B256::repeat_byte(1); + let mut block = SignedBeaconBlock { + message: BeaconBlock { + slot: 1, + parent_root, + ..Default::default() + }, + signature: Default::default(), + }; + block.message.parent_root = B256::repeat_byte(2); + let mut cache = BlockCache::new(parent_root, 1); + cache + .add_blocks(vec![block], false, PeerId::random()) + .expect("block should enter cache"); + + let peer_a = PeerId::random(); + let now = Instant::now(); + let missing_root = B256::repeat_byte(2); + + cache.mark_attempted(RequestKey::BlockRoot(missing_root), peer_a, &[peer_a], now); + + assert_eq!( + cache.data_to_fetch(1, 0, &HashSet::new(), &[peer_a], now, false), + DataToFetch::DownloadsInProgress + ); + } + + #[test] + fn data_to_fetch_filters_out_only_the_exhausted_root_not_the_whole_tier() { + initialize_test_network_spec(); + let parent_root = B256::repeat_byte(1); + let root_a = B256::repeat_byte(2); + let root_b = B256::repeat_byte(3); + let block1 = SignedBeaconBlock { + message: BeaconBlock { + slot: 1, + parent_root: root_a, + ..Default::default() + }, + signature: Default::default(), + }; + let block2 = SignedBeaconBlock { + message: BeaconBlock { + slot: 2, + parent_root: root_b, + ..Default::default() + }, + signature: Default::default(), + }; + + let mut cache = BlockCache::new(parent_root, 2); + cache + .add_blocks(vec![block1, block2], false, PeerId::random()) + .expect("blocks should enter cache"); + + let peer_a = PeerId::random(); + let now = Instant::now(); + cache.mark_attempted(RequestKey::BlockRoot(root_a), peer_a, &[peer_a], now); + + match cache.data_to_fetch(2, 0, &HashSet::new(), &[peer_a], now, false) { + DataToFetch::MissingBlockRoots(roots) => { + assert_eq!(roots, vec![root_b]); + } + other => panic!("expected MissingBlockRoots, got {other:?}"), + } + } + + #[test] + fn add_data_columns_drops_a_column_for_an_unknown_block_without_erroring_or_banning() { + use ream_consensus_beacon::data_column_sidecar::get_data_column_sidecars_from_block; + + let blob = Blob::default(); + let blob_bytes = blob.to_fixed_bytes(); + let raw_commitment = das_context() + .blob_to_kzg_commitment(&blob_bytes) + .expect("test blob should produce a commitment"); + let commitment = KZGCommitment(raw_commitment); + + let mut block = SignedBeaconBlock { + message: Default::default(), + signature: Default::default(), + }; + block + .message + .body + .blob_kzg_commitments + .push(commitment) + .expect("one commitment should fit"); + + let cells_and_kzg_proofs = compute_cells_and_kzg_proofs(&blob, das_context()) + .expect("test blob should produce cells and proofs"); + let columns = get_data_column_sidecars_from_block(&block, vec![cells_and_kzg_proofs]) + .expect("test block should produce data columns"); + + let mut cache = BlockCache::new(B256::ZERO, 0); + let required_columns = HashSet::from([0]); + let result = cache.add_data_columns(vec![columns[0].clone()], &required_columns); + + assert!( + result.is_ok(), + "a column for a block not yet in the cache must be dropped, not erroring the batch" + ); + } + + fn test_observation(anchor_root: B256, anchor_slot: u64) -> FrontierObservation { + FrontierObservation { + anchor_root, + anchor_slot, + phase: super::super::SyncPhase::Finalized, + scan_start_slot: anchor_slot, + target_slot: anchor_slot + 100, + } + } + + fn child_block(parent_root: B256, slot: u64) -> SignedBeaconBlock { + SignedBeaconBlock { + message: BeaconBlock { + slot, + parent_root, + ..Default::default() + }, + signature: Default::default(), + } + } + + #[test] + fn advance_empty_coverage_opens_a_coverage_anchor_that_a_connecting_block_clears() { + let root = B256::repeat_byte(1); + let mut cache = BlockCache::new(root, 10); + assert!(cache.coverage_anchor.is_none()); + + cache + .advance_empty_coverage(test_observation(root, 10), root, 20, HashSet::new()) + .expect("advancing a pristine cache should succeed"); + assert!( + cache.coverage_anchor.is_some(), + "a coverage advance must open an anchor pending confirmation" + ); + assert_eq!(cache.next_start_slot(), 20); + assert_eq!( + cache.initial_parent_root(), + root, + "an empty span never changes the anchor root" + ); + + cache + .add_blocks(vec![child_block(root, 21)], true, PeerId::random()) + .expect("a directly-connecting block must be accepted"); + assert!( + cache.coverage_anchor.is_none(), + "a connecting first block after the advance must clear the anchor" + ); + } + + #[test] + fn a_non_connecting_block_after_a_coverage_advance_is_reported_as_divergence_not_a_missing_parent() + { + let root = B256::repeat_byte(1); + let mut cache = BlockCache::new(root, 10); + cache + .advance_empty_coverage(test_observation(root, 10), root, 20, HashSet::new()) + .expect("advancing a pristine cache should succeed"); + + let wrong_parent = B256::repeat_byte(2); + let result = cache.add_blocks(vec![child_block(wrong_parent, 21)], true, PeerId::random()); + + match result { + Err(AddBlocksError::CoverageDivergence { + expected_parent, + actual_parent, + .. + }) => { + assert_eq!(expected_parent, root); + assert_eq!(actual_parent, wrong_parent); + } + other => panic!("expected CoverageDivergence, got {other:?}"), + } + assert_eq!( + cache.block_count(), + 0, + "a divergent batch must not be committed to the cache" + ); + } + + #[test] + fn advance_empty_coverage_extends_within_the_same_generation_without_losing_the_original_anchor() + { + let root = B256::repeat_byte(1); + let mut cache = BlockCache::new(root, 10); + cache + .advance_empty_coverage( + test_observation(root, 10), + root, + 20, + HashSet::from([PeerId::random()]), + ) + .expect("first advance should succeed"); + let second_peer = PeerId::random(); + cache + .advance_empty_coverage( + test_observation(root, 10), + root, + 30, + HashSet::from([second_peer]), + ) + .expect("a second advance within the same generation should succeed"); + + assert_eq!(cache.next_start_slot(), 30); + let anchor = cache + .coverage_anchor + .as_ref() + .expect("anchor should still be open"); + assert_eq!( + anchor.original_parent_root, root, + "the original pre-generation anchor must survive a second advance" + ); + assert_eq!(anchor.covered_through_slot, 30); + assert!(anchor.confirming_peers.contains(&second_peer)); + } + + #[test] + fn single_flight_blocks_a_second_block_range_while_a_frontier_is_tracked() { + let mut untracked = BlockCache::new(B256::ZERO, 10); + untracked.mark_block_range_in_progress(Range::new(11, 10)); + assert_ne!( + untracked.data_to_fetch(100, 0, &HashSet::new(), &[], Instant::now(), false), + DataToFetch::DownloadsInProgress, + ); + + let mut tracked = BlockCache::new(B256::ZERO, 10); + tracked.mark_block_range_in_progress(Range::new(11, 10)); + assert_eq!( + tracked.data_to_fetch(100, 0, &HashSet::new(), &[], Instant::now(), true), + DataToFetch::DownloadsInProgress, + ); + } } diff --git a/crates/networking/syncer/src/block_range/mod.rs b/crates/networking/syncer/src/block_range/mod.rs index 9c5609496..4b6bc127c 100644 --- a/crates/networking/syncer/src/block_range/mod.rs +++ b/crates/networking/syncer/src/block_range/mod.rs @@ -1,37 +1,51 @@ mod block_cache; mod peer_manager; mod peer_range_downloader; +mod recovery; use std::{ + collections::HashSet, pin::Pin, sync::Arc, task::{Context, Poll}, - time::Duration, + time::{Duration, Instant}, }; use alloy_primitives::B256; use anyhow::{anyhow, bail, ensure}; -use block_cache::{BlockAndBlobBundle, BlockCache, DataToFetch}; +use block_cache::{AddBlocksError, BlockAndBlobBundle, BlockCache, DataToFetch, RequestKey}; use futures::task::noop_waker; use libp2p::PeerId; -use peer_manager::PeerManager; -use peer_range_downloader::{PeerBlobIdentifierDownloader, PeerRootsDownloader}; +use peer_manager::{BanReason, MIN_SYNC_PEERS, PeerManager, TargetQualification, TargetSelection}; +use peer_range_downloader::{ + DownloadFailure, PeerBlobIdentifierDownloader, PeerDataColumnIdentifierDownloader, + PeerDataColumnRangeDownloader, PeerRootsDownloader, StreamOutcome, +}; use ream_chain_beacon::beacon_chain::{ BeaconChain, BlockProcessingOutcome, is_data_availability_check_required, }; use ream_consensus_beacon::{ blob_sidecar::{BlobIdentifier, BlobSidecar}, - data_column_sidecar::{DataColumnSidecar, get_data_column_sidecars_from_block}, + data_column_sidecar::{ + ColumnIdentifier, DataColumnSidecar, get_data_column_sidecars_from_block, + }, electra::beacon_block::SignedBeaconBlock, matrix_entry::{compute_cells_and_kzg_proofs, das_context}, }; -use ream_consensus_misc::misc::compute_epoch_at_slot; +use ream_consensus_misc::{constants::beacon::SLOTS_PER_EPOCH, misc::compute_epoch_at_slot}; use ream_executor::ReamExecutor; use ream_network_spec::networks::beacon_network_spec; use ream_p2p::network::beacon::{channel::P2PMessage, network_state::NetworkState}; use ream_polynomial_commitments::handlers::verify_blob_kzg_proof_batch; -use ream_req_resp::MAX_CONCURRENT_REQUESTS; -use ream_storage::tables::table::{CustomTable, REDBTable}; +use ream_req_resp::{ + MAX_CONCURRENT_REQUESTS, beacon::messages::data_column_sidecars::DataColumnsByRootIdentifier, + inbound_protocol::ResponseCode, +}; +use ream_storage::tables::{ + field::REDBField, + table::{CustomTable, REDBTable}, +}; +use recovery::{CoverageAdvance, RECOVERY_ROUND_TIMEOUT, RecoveryOutcome}; use tokio::{sync::mpsc::UnboundedSender, task::JoinHandle, time::sleep}; use tracing::{info, warn}; use tree_hash::TreeHash; @@ -42,13 +56,33 @@ const MAX_BLOBS_PER_REQUEST: usize = 6; const MAX_BLOCKS_PER_REQUEST: u64 = 10; const SLEEP_DURATION: Duration = Duration::from_secs(5); +/// Max slots behind wall-clock to still count as synced (like Lighthouse's +/// `SLOT_IMPORT_TOLERANCE`). A thin/early peer sample can be fooled; the clock can't. +const SLOT_IMPORT_TOLERANCE: u64 = 32; + +const ZERO_PROGRESS_BACKOFF: Duration = Duration::from_secs(30); + +const RECOVERY_ATTEMPT_COOLDOWN: Duration = Duration::from_secs(30); + +const CANDIDATE_EXHAUSTION_TIMEOUT: Duration = Duration::from_secs(60); + +fn should_abort_for_candidate_exhaustion(elapsed: Duration, tasks_in_flight: bool) -> bool { + elapsed >= CANDIDATE_EXHAUSTION_TIMEOUT && !tasks_in_flight +} + +fn candidate_exhaustion_deserves_backoff(target_slot: u64, head_slot: u64) -> bool { + target_slot > head_slot +} + /// Validates downloaded blob sidecars and derives the columns needed for data availability. +/// An empty `blob_sidecars` map defers to the block-lookup coordinator instead of erroring, +/// since range sync never fetches legacy blob sidecars for post-Fulu blocks. fn build_data_columns_from_blob_sidecars( block: &SignedBeaconBlock, blob_sidecars: &std::collections::HashMap, verify_data_availability: bool, ) -> anyhow::Result> { - if !verify_data_availability { + if !verify_data_availability || blob_sidecars.is_empty() { return Ok(Vec::new()); } @@ -78,6 +112,10 @@ fn build_data_columns_from_blob_sidecars( sidecar.kzg_commitment == *expected_commitment, "Blob sidecar {index} commitment does not match block {block_root}" ); + ensure!( + sidecar.verify_blob_sidecar_inclusion_proof(), + "Invalid inclusion proof for blob sidecar {index} of block {block_root}" + ); blobs.push(sidecar.blob.clone()); proofs.push(sidecar.kzg_proof); @@ -96,11 +134,312 @@ fn build_data_columns_from_blob_sidecars( .map_err(|err| anyhow!("Failed to build data columns for block {block_root}: {err}")) } +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +enum SyncPhase { + Finalized, + Head, +} + +#[derive(Debug, Clone)] +struct ActivePhaseTarget { + slot: u64, + qualification: TargetQualification, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct FrontierKey { + anchor_root: B256, + phase: SyncPhase, + scan_start_slot: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct FrontierId { + key: FrontierKey, + generation: u64, +} + +#[derive(Debug, Clone)] +struct FrontierObservation { + anchor_root: B256, + anchor_slot: u64, + phase: SyncPhase, + scan_start_slot: u64, + target_slot: u64, +} + +#[derive(Debug, Clone, Copy)] +struct EmptyCoverage { + start_slot: u64, + end_slot_exclusive: u64, +} + +#[derive(Debug, Clone)] +struct ConfirmedEmptyCoverage { + frontier_id: FrontierId, + origin_anchor: (B256, u64), + start_slot: u64, + end_slot_exclusive: u64, + confirming_peers: HashSet, +} + +impl ConfirmedEmptyCoverage { + fn merge(&mut self, coverage: EmptyCoverage, confirming_peers: &HashSet) -> bool { + if coverage.start_slot <= self.end_slot_exclusive + && coverage.end_slot_exclusive >= self.start_slot + { + self.end_slot_exclusive = self.end_slot_exclusive.max(coverage.end_slot_exclusive); + self.start_slot = self.start_slot.min(coverage.start_slot); + self.confirming_peers + .extend(confirming_peers.iter().copied()); + true + } else { + false + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +struct FailedRecoveryCandidate { + peer_id: PeerId, + candidate_root: B256, + target_bucket: u64, +} + +fn target_bucket(target_slot: u64) -> u64 { + target_slot / SLOTS_PER_EPOCH +} + +#[derive(Debug, Clone)] +struct StuckFrontier { + id: FrontierId, + anchor_root: B256, + anchor_slot: u64, + phase: SyncPhase, + scan_start_slot: u64, + highest_observed_target: u64, + consecutive_no_progress: u32, + attempted_peers: HashSet, + cooldown_until: Option, + recovery_round_not_before: Option, + failed_candidates: HashSet, + confirmed_empty_through: Option, +} + +impl StuckFrontier { + fn matches(&self, observation: &FrontierObservation) -> bool { + self.anchor_root == observation.anchor_root + && self.phase == observation.phase + && self.scan_start_slot == observation.scan_start_slot + } + + fn tier(&self) -> u8 { + match self.consecutive_no_progress { + 0..=2 => 0, + 3..=5 => 2, + _ => 3, + } + } + + fn refresh_attempt_round(&mut self, now: Instant) { + if let Some(cooldown_until) = self.cooldown_until + && now >= cooldown_until + { + self.attempted_peers.clear(); + self.failed_candidates.clear(); + self.cooldown_until = None; + } + } +} + +fn qualification_for(phase: SyncPhase, target_slot: u64) -> TargetQualification { + match phase { + SyncPhase::Finalized => TargetQualification::FinalizedEpoch(target_slot / SLOTS_PER_EPOCH), + SyncPhase::Head => TargetQualification::HeadEpoch(target_slot / SLOTS_PER_EPOCH), + } +} + +fn observation_from_frontier(frontier: &StuckFrontier) -> FrontierObservation { + FrontierObservation { + anchor_root: frontier.anchor_root, + anchor_slot: frontier.anchor_slot, + phase: frontier.phase, + scan_start_slot: frontier.scan_start_slot, + target_slot: frontier.highest_observed_target, + } +} + +struct FrontierStore<'a> { + finalized: &'a mut Option, + head: &'a mut Option, +} + +impl<'a> FrontierStore<'a> { + fn for_phase(&self, phase: SyncPhase) -> &Option { + match phase { + SyncPhase::Finalized => self.finalized, + SyncPhase::Head => self.head, + } + } + + fn for_phase_mut(&mut self, phase: SyncPhase) -> &mut Option { + match phase { + SyncPhase::Finalized => self.finalized, + SyncPhase::Head => self.head, + } + } +} + +struct CoverageDivergenceOutcome { + original_parent_root: B256, + original_parent_slot: u64, + frontier: FrontierObservation, + confirming_peers: HashSet, + non_connecting_peer: PeerId, +} + +struct ImportFailure { + imported_count: u64, + error: anyhow::Error, +} + +enum PollTasksOutcome { + Settled, + StaleGeneration, + CoverageDivergence(Box), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum RemoteNoProgressReason { + CandidateExhausted, + SettledEmptyCoverage, + ProbeNotFound, + UnconnectedProbe, + AncestorNotFound, + NoNewDescendants, + RecoveryBudgetExhausted, +} + +impl RemoteNoProgressReason { + fn precedence(self) -> u8 { + match self { + RemoteNoProgressReason::CandidateExhausted => 6, + RemoteNoProgressReason::RecoveryBudgetExhausted => 5, + RemoteNoProgressReason::AncestorNotFound | RemoteNoProgressReason::UnconnectedProbe => { + 4 + } + RemoteNoProgressReason::ProbeNotFound => 3, + RemoteNoProgressReason::NoNewDescendants => 2, + RemoteNoProgressReason::SettledEmptyCoverage => 1, + } + } +} + +#[derive(Debug, Clone)] +struct RemoteObservation { + observation: FrontierObservation, + reason: RemoteNoProgressReason, + implicated_peers: HashSet, + failed_candidates: HashSet, +} + +#[derive(Default)] +struct PendingConclusions { + finalized: Option, + head: Option, +} + +impl PendingConclusions { + fn for_phase(&self, phase: SyncPhase) -> &Option { + match phase { + SyncPhase::Finalized => &self.finalized, + SyncPhase::Head => &self.head, + } + } + + fn slot_mut(&mut self, phase: SyncPhase) -> &mut Option { + match phase { + SyncPhase::Finalized => &mut self.finalized, + SyncPhase::Head => &mut self.head, + } + } + + fn observe( + &mut self, + phase: SyncPhase, + observation: FrontierObservation, + reason: RemoteNoProgressReason, + implicated_peers: HashSet, + failed_candidates: HashSet, + ) { + let slot = self.slot_mut(phase); + match slot { + None => { + *slot = Some(RemoteObservation { + observation, + reason, + implicated_peers, + failed_candidates, + }); + } + Some(existing) => { + let target_slot = existing + .observation + .target_slot + .max(observation.target_slot); + if reason.precedence() >= existing.reason.precedence() { + existing.reason = reason; + existing.observation = observation; + } + existing.observation.target_slot = target_slot; + existing.implicated_peers.extend(implicated_peers); + existing.failed_candidates.extend(failed_candidates); + } + } + } + + fn observe_divergence(&mut self, phase: SyncPhase, divergence: &CoverageDivergenceOutcome) { + self.observe( + phase, + divergence.frontier.clone(), + RemoteNoProgressReason::UnconnectedProbe, + divergence.confirming_peers.clone(), + HashSet::new(), + ); + } +} + +#[derive(Default)] +struct SegmentExclusions { + finalized: HashSet, + head: HashSet, +} + +impl SegmentExclusions { + fn for_phase(&self, phase: SyncPhase) -> &HashSet { + match phase { + SyncPhase::Finalized => &self.finalized, + SyncPhase::Head => &self.head, + } + } + + fn for_phase_mut(&mut self, phase: SyncPhase) -> &mut HashSet { + match phase { + SyncPhase::Finalized => &mut self.finalized, + SyncPhase::Head => &mut self.head, + } + } +} + pub struct BlockRangeSyncer { pub beacon_chain: Arc, pub peer_manager: PeerManager, pub p2p_sender: UnboundedSender, pub executor: ReamExecutor, + next_segment_not_before: Option, + next_generation: u64, + finalized_frontier: Option, + head_frontier: Option, } impl BlockRangeSyncer { @@ -115,271 +454,972 @@ impl BlockRangeSyncer { p2p_sender, peer_manager: PeerManager::new(network_state), executor, + next_segment_not_before: None, + next_generation: 0, + finalized_frontier: None, + head_frontier: None, } } - pub async fn is_synced_to_finalized_slot(&self) -> bool { - let finalized_slot = self.peer_manager.finalized_slot(); - let latest_synced_slot = self - .beacon_chain - .store - .lock() - .await + fn allocate_generation(&mut self) -> u64 { + let generation = self.next_generation; + self.next_generation += 1; + generation + } + + fn frontier_for(&self, phase: SyncPhase) -> &Option { + match phase { + SyncPhase::Finalized => &self.finalized_frontier, + SyncPhase::Head => &self.head_frontier, + } + } + + fn frontier_for_mut(&mut self, phase: SyncPhase) -> &mut Option { + match phase { + SyncPhase::Finalized => &mut self.finalized_frontier, + SyncPhase::Head => &mut self.head_frontier, + } + } + + fn new_frontier(&mut self, observation: &FrontierObservation) -> StuckFrontier { + let key = FrontierKey { + anchor_root: observation.anchor_root, + phase: observation.phase, + scan_start_slot: observation.scan_start_slot, + }; + StuckFrontier { + id: FrontierId { + key, + generation: self.allocate_generation(), + }, + anchor_root: observation.anchor_root, + anchor_slot: observation.anchor_slot, + phase: observation.phase, + scan_start_slot: observation.scan_start_slot, + highest_observed_target: observation.target_slot, + consecutive_no_progress: 1, + attempted_peers: HashSet::new(), + cooldown_until: None, + recovery_round_not_before: None, + failed_candidates: HashSet::new(), + confirmed_empty_through: None, + } + } + + fn commit_pending_conclusions( + &mut self, + pending: PendingConclusions, + exclusions: SegmentExclusions, + ) { + for phase in [SyncPhase::Finalized, SyncPhase::Head] { + if let Some(record) = pending.for_phase(phase).clone() { + self.record_remote_no_progress(phase, record, exclusions.for_phase(phase).clone()); + } + } + } + + fn record_remote_no_progress( + &mut self, + phase: SyncPhase, + record: RemoteObservation, + excluded: HashSet, + ) { + let observation = &record.observation; + let mut frontier = match self.frontier_for(phase).clone() { + Some(existing) + if existing.matches(observation) + && observation.target_slot >= existing.highest_observed_target => + { + let mut existing = existing; + existing.consecutive_no_progress = + existing.consecutive_no_progress.saturating_add(1); + existing.highest_observed_target = observation.target_slot; + existing + } + _ => self.new_frontier(observation), + }; + frontier.attempted_peers.extend(record.implicated_peers); + frontier.attempted_peers.extend(excluded); + frontier.failed_candidates.extend(record.failed_candidates); + if !frontier.attempted_peers.is_empty() && frontier.cooldown_until.is_none() { + frontier.cooldown_until = Some(Instant::now() + RECOVERY_ATTEMPT_COOLDOWN); + } + if record.reason == RemoteNoProgressReason::RecoveryBudgetExhausted { + frontier.recovery_round_not_before = Some(Instant::now() + RECOVERY_ROUND_TIMEOUT); + } + *self.frontier_for_mut(phase) = Some(frontier); + } + + fn clear_tracker(&mut self) { + self.finalized_frontier = None; + self.head_frontier = None; + self.next_segment_not_before = None; + } + + fn reconcile_not_ahead(&mut self, observation: &FrontierObservation) { + let phase = observation.phase; + let should_clear = match self.frontier_for(phase) { + Some(frontier) if frontier.anchor_root != observation.anchor_root => true, + Some(frontier) => { + frontier.matches(observation) && observation.target_slot <= frontier.anchor_slot + } + None => false, + }; + if should_clear { + *self.frontier_for_mut(phase) = None; + } + } + + pub async fn is_synced_to_head_slot(&mut self) -> bool { + self.peer_manager.update_peer_set(); + + let store = self.beacon_chain.store.lock().await; + let latest_synced_slot = store .db .slot_index_provider() .get_highest_slot() .unwrap_or_default() .unwrap_or(0); + let Ok(finalized_epoch) = store.db.finalized_checkpoint_provider().get() else { + return false; + }; + let finalized_epoch = finalized_epoch.epoch; + let Ok(current_slot) = store.get_current_slot() else { + return false; + }; + drop(store); + + let finalized_selection = self.peer_manager.best_finalized(finalized_epoch); + let still_behind_finalized = matches!( + &finalized_selection, + TargetSelection::Ready { target_slot, .. } if latest_synced_slot < *target_slot + ); + if still_behind_finalized { + return false; + } + + let our_head_epoch = latest_synced_slot / SLOTS_PER_EPOCH; + let head_selection = self + .peer_manager + .best_non_finalized(MIN_SYNC_PEERS, our_head_epoch); + // The clock is ground truth peers can't fake. + let clock_confirms_caught_up = + current_slot.saturating_sub(latest_synced_slot) <= SLOT_IMPORT_TOLERANCE; - finalized_slot <= Some(latest_synced_slot) + let TargetSelection::Ready { target_slot, .. } = head_selection else { + return clock_confirms_caught_up; + }; + + let peers_report_caught_up = target_slot <= latest_synced_slot; + + peers_report_caught_up && clock_confirms_caught_up } - pub fn start(mut self) -> JoinHandle>> { + pub fn start(mut self) -> JoinHandle)>> { let executor = self.executor.clone(); executor.spawn(async move { - let Some(latest_synced_root) = self - .beacon_chain - .store - .lock() - .await - .db - .slot_index_provider() - .get_highest_root() - .map_err(|err| anyhow!("Failed to get highest root: {err}"))? - else { - bail!("No synced root found in the database"); - }; + let result = self.run_segment().await; + (self, result) + }) + } + + async fn run_segment(&mut self) -> anyhow::Result<()> { + if let Some(not_before) = self.next_segment_not_before.take() + && let Some(remaining) = not_before.checked_duration_since(Instant::now()) + { + info!("Backing off {remaining:?} after the previous segment made no progress..."); + sleep(remaining).await; + } + + let store = self.beacon_chain.store.lock().await; + let head_root = store + .get_head() + .map_err(|err| anyhow!("Failed to get canonical head: {err}"))?; + let head_slot = store + .db + .block_provider() + .get(head_root) + .map_err(|err| anyhow!("Failed to load head block: {err}"))? + .ok_or_else(|| anyhow!("Head block {head_root} not found"))? + .message + .slot; + let finalized_epoch = store + .db + .finalized_checkpoint_provider() + .get() + .map_err(|err| anyhow!("Failed to get finalized checkpoint: {err}"))? + .epoch; + drop(store); + let our_head_epoch = head_slot / SLOTS_PER_EPOCH; + + // phase 1: download majority of blocks from ranges + let mut block_cache = BlockCache::new(head_root, head_slot); + let mut task_handles = vec![]; + + let mut phase = SyncPhase::Finalized; + let mut active_target: Option = None; + let mut saw_empty_range = false; + let mut finalized_phase_settled = false; + let mut finalized_target_was_ahead = false; + let mut candidate_exhausted_since: Option = None; - let Some(latest_synced_slot) = self + let scan_start_slot = head_slot; + let mut restore_window_open = true; + let mut recovery_window_open = true; + let mut recovery_decision_made_this_segment = false; + let mut pending_conclusions = PendingConclusions::default(); + let mut segment_exclusions = SegmentExclusions::default(); + + loop { + self.peer_manager.update_peer_set(); + + let required_columns = self .beacon_chain .store .lock() .await - .db - .slot_index_provider() - .get_highest_slot() - .map_err(|err| anyhow!("Failed to get highest slot: {err}"))? - else { - bail!("No synced slot found in the database"); + .data_availability_checker + .required_columns() + .clone(); + + let selection = match phase { + SyncPhase::Finalized => self.peer_manager.best_finalized(finalized_epoch), + SyncPhase::Head => self + .peer_manager + .best_non_finalized(MIN_SYNC_PEERS, our_head_epoch), }; - // phase 1: download majority of blocks from ranges - let mut block_cache = - BlockCache::new(latest_synced_root, latest_synced_slot); - let mut task_handles = vec![]; - loop { - poll_ready_tasks(&mut task_handles, &mut block_cache, &mut self.peer_manager)?; - - let finalized_slot = match self.peer_manager.finalized_slot() { - Some(finalized_slot) => finalized_slot, - None => { - warn!("No peers available to determine finalized slot, retrying..."); - sleep(SLEEP_DURATION).await; - self.peer_manager.update_peer_set(); - continue; + if active_target.is_none() { + if phase == SyncPhase::Finalized { + match &selection { + TargetSelection::Ready { + target_slot, + eligible_peers, + } => { + if block_cache.next_start_slot() >= *target_slot { + phase = SyncPhase::Head; + continue; + } + if eligible_peers.len() < MIN_SYNC_PEERS { + info!( + "Finalized target not yet confirmed by enough peers ({} < {MIN_SYNC_PEERS}), waiting...", + eligible_peers.len() + ); + sleep(SLEEP_DURATION).await; + continue; + } + } + TargetSelection::NoQuorum => { + phase = SyncPhase::Head; + continue; + } + } + } else if let TargetSelection::NoQuorum = &selection { + if finalized_phase_settled || block_cache.block_count() > 0 { + info!( + "No head-phase sync target after the finalized phase settled; ending this segment." + ); + break; } + info!("No sync target yet, waiting for peers..."); + sleep(SLEEP_DURATION).await; + continue; + } + } + + if let TargetSelection::Ready { + target_slot, + eligible_peers, + } = &selection + { + let sufficient = match phase { + SyncPhase::Finalized => eligible_peers.len() >= MIN_SYNC_PEERS, + SyncPhase::Head => true, }; + if sufficient { + let qualification = qualification_for(phase, *target_slot); + active_target = Some(match active_target { + Some(existing) if existing.slot > *target_slot => existing, + _ => ActivePhaseTarget { + slot: *target_slot, + qualification, + }, + }); + } + } - let current_epoch = self - .beacon_chain - .store - .lock() - .await - .get_current_store_epoch()?; - let data_to_fetch = block_cache.data_to_fetch(finalized_slot, current_epoch); - info!( - "Forward sync status: Downloaded Blocks {}, Downloaded Blobs {}/{}, Stage {data_to_fetch}", - block_cache.block_count(), - block_cache.downloaded_blob_count(), - block_cache.blob_count(), - ); + let Some(target) = active_target.clone() else { + sleep(SLEEP_DURATION).await; + continue; + }; - match data_to_fetch { - DataToFetch::BlockRange(range) => { - let Some(peer) = self.peer_manager.fetch_idle_peer() else { - self.peer_manager.update_peer_set(); - info!("No idle peers available for block range sync."); - sleep(SLEEP_DURATION).await; - continue; - }; + let observation = FrontierObservation { + anchor_root: head_root, + anchor_slot: head_slot, + phase, + scan_start_slot, + target_slot: target.slot, + }; + self.reconcile_not_ahead(&observation); - task_handles.push(DownloadTask::new_block_range( - PeerRangeDownloader::start( - peer.peer_id, - self.p2p_sender.clone(), - self.executor.clone(), - range, - ), - range, - peer.peer_id, - )); - } - DataToFetch::MissingBlockRoots(block_roots) => { - for block_roots_chunk in block_roots.chunks(MAX_CONCURRENT_REQUESTS) { - let Some(peer) = self.peer_manager.fetch_idle_peer() else { - self.peer_manager.update_peer_set(); - info!("No idle peers available for block roots sync."); - sleep(SLEEP_DURATION).await; - break; - }; - block_cache.extend_block_roots_in_progress(block_roots_chunk); - - task_handles.push(DownloadTask::new_block_roots( - PeerRootsDownloader::start( - peer.peer_id, - self.p2p_sender.clone(), - self.executor.clone(), - block_roots_chunk.to_vec(), - ), - block_roots_chunk.to_vec(), - peer.peer_id, - )); - } - } - DataToFetch::MissingBlobIdentifiers(blob_identifiers) => { - for blob_identifiers_chunk in blob_identifiers.chunks(MAX_BLOBS_PER_REQUEST) { - let Some(peer) = self.peer_manager.fetch_idle_peer() else { - self.peer_manager.update_peer_set(); - info!("No idle peers available for blob sync. {}", self.peer_manager.peer_counts()); - sleep(SLEEP_DURATION).await; - break; - }; + let candidate_peers = self.peer_manager.peers_satisfying(target.qualification); - block_cache.extend_blob_identifiers_in_progress(blob_identifiers_chunk); + let now = Instant::now(); + let poll_outcome = { + let mut frontiers = FrontierStore { + finalized: &mut self.finalized_frontier, + head: &mut self.head_frontier, + }; + poll_ready_tasks( + &mut task_handles, + &mut block_cache, + &mut self.peer_manager, + &mut frontiers, + &required_columns, + &mut saw_empty_range, + &candidate_peers, + )? + }; - task_handles.push(DownloadTask::new_blob_identifiers( - PeerBlobIdentifierDownloader::start( - peer.peer_id, - self.p2p_sender.clone(), - self.executor.clone(), - blob_identifiers_chunk.to_vec(), - ), - blob_identifiers_chunk.to_vec(), - peer.peer_id, - )); - } - } - DataToFetch::DownloadsInProgress => { - info!("Waiting for ongoing downloads to complete... {}", self.peer_manager.peer_counts()); - sleep(Duration::from_secs(10)).await; + if let PollTasksOutcome::CoverageDivergence(divergence) = poll_outcome { + let divergence_phase = divergence.frontier.phase; + block_cache = BlockCache::new( + divergence.original_parent_root, + divergence.original_parent_slot, + ); + let next_generation = self.allocate_generation(); + if let Some(frontier) = self.frontier_for_mut(divergence_phase) { + frontier.confirmed_empty_through = None; + frontier.id = FrontierId { + key: frontier.id.key, + generation: next_generation, + }; + } + debug_assert!( + !divergence + .confirming_peers + .contains(&divergence.non_connecting_peer), + "the peer that served the non-connecting block must never be excluded \ + alongside the peers that supplied the false coverage claim" + ); + segment_exclusions + .for_phase_mut(divergence_phase) + .extend(divergence.confirming_peers.clone()); + pending_conclusions.observe_divergence(divergence_phase, &divergence); + continue; + } + + if candidate_peers.is_empty() { + let exhausted_since = *candidate_exhausted_since.get_or_insert_with(Instant::now); + if should_abort_for_candidate_exhaustion( + exhausted_since.elapsed(), + !task_handles.is_empty(), + ) { + info!( + "No peer has qualified for the active sync target for over {CANDIDATE_EXHAUSTION_TIMEOUT:?}; ending this segment." + ); + if candidate_exhaustion_deserves_backoff(target.slot, head_slot) { + self.next_segment_not_before = Some(Instant::now() + ZERO_PROGRESS_BACKOFF); } - DataToFetch::Finished => break, + pending_conclusions.observe( + phase, + observation.clone(), + RemoteNoProgressReason::CandidateExhausted, + HashSet::new(), + HashSet::new(), + ); + break; } + } else { + candidate_exhausted_since = None; } - info!("Block range sync completed a segment successfully with {} blocks and {} blobs.", - block_cache.block_count(), - block_cache.downloaded_blob_count(), - ); + if restore_window_open { + if !block_cache.is_pristine_for_restore() { + restore_window_open = false; + } else if let Some(frontier) = self.frontier_for(phase) + && frontier.matches(&observation) + && let Some(coverage) = &frontier.confirmed_empty_through + { + debug_assert_eq!( + coverage.frontier_id, frontier.id, + "confirmed_empty_through must always belong to the frontier holding it" + ); + debug_assert_eq!( + coverage.origin_anchor.0, + block_cache.initial_parent_root(), + "a pristine cache being restored into must still share this coverage's original anchor root" + ); + let frontier_observation = observation_from_frontier(frontier); + let parent_root = block_cache.initial_parent_root(); + let covered_through_slot = coverage.end_slot_exclusive.saturating_sub(1); + let confirming_peers = coverage.confirming_peers.clone(); + let _ = block_cache.advance_empty_coverage( + frontier_observation, + parent_root, + covered_through_slot, + confirming_peers, + ); + restore_window_open = false; + } + } - // execute all the blocks downloaded - for BlockAndBlobBundle { block, blobs } in block_cache.get_blocks_and_blobs()? { - info!("Processing block with slot {}", - block.message.slot, - ); + if let Some(frontier) = self.frontier_for_mut(phase) { + frontier.refresh_attempt_round(now); + } - let (block, columns) = if block.message.body.blob_kzg_commitments.is_empty() { - ensure!( - blobs.is_empty(), - "Range-sync block without blob commitments had downloaded blob sidecars" - ); - (block, Vec::new()) - } else { - let (blobs_provider, required_columns, verify_data_availability) = { - let store = self.beacon_chain.store.lock().await; - let network_spec = beacon_network_spec(); - ( - store.db.blobs_and_proofs_provider(), - store.data_availability_checker.required_columns().clone(), - is_data_availability_check_required( - compute_epoch_at_slot(block.message.slot), - store.get_current_store_epoch()?, - network_spec.fulu_fork_epoch, - network_spec.min_epochs_for_data_column_sidecars_requests, - ), - ) - }; - tokio::task::spawn_blocking(move || { - let columns = build_data_columns_from_blob_sidecars( - &block, - &blobs, - verify_data_availability, - )? - .into_iter() - .filter(|column| required_columns.contains(&column.index)) - .collect::>(); - for (identifier, sidecar) in blobs { - blobs_provider.insert(identifier, sidecar.into())?; + if recovery_window_open + && !recovery_decision_made_this_segment + && let Some(frontier) = self.frontier_for(phase) + && frontier.matches(&observation) + && frontier.tier() >= 2 + { + recovery_decision_made_this_segment = true; + recovery_window_open = false; + let recovery_round_not_before = frontier.recovery_round_not_before; + if recovery_round_not_before.is_none_or(|not_before| now >= not_before) { + let mut excluded = segment_exclusions.for_phase(phase).clone(); + excluded.extend(frontier.attempted_peers.iter().copied()); + match self + .run_recovery(&observation, &candidate_peers, &excluded) + .await? + { + RecoveryOutcome::AdvancedCoverage(advance) => { + block_cache.advance_empty_coverage( + observation.clone(), + advance.parent_root, + advance.covered_through_slot, + advance.confirming_peers.clone(), + )?; + self.merge_confirmed_empty_coverage(phase, &advance); + continue; } - Ok::<_, anyhow::Error>((block, columns)) - }) - .await - .map_err(|err| anyhow!("Range-sync data-column task failed: {err}"))?? - }; - - match self.beacon_chain.process_block(block).await? { - BlockProcessingOutcome::Imported { .. } => {} - BlockProcessingOutcome::PendingAvailability { block_root } => { - for column in columns { - self.beacon_chain - .import_data_column_sidecar_if(column, |_| Ok(())) - .await?; + RecoveryOutcome::Seeded(seed) => { + block_cache = BlockCache::from_recovery_seed(seed)?; + continue; + } + RecoveryOutcome::NoProgress { + reason, + implicated_peers, + failed_candidates, + } => { + pending_conclusions.observe( + phase, + observation.clone(), + reason, + implicated_peers, + failed_candidates, + ); } - ensure!( - self.beacon_chain - .store - .lock() - .await - .db - .block_provider() - .get(block_root)? - .is_some(), - "Range-sync block {block_root} remained pending after processing its downloaded data" - ); } } } - info!("All blocks processed successfully."); + let current_epoch = self + .beacon_chain + .store + .lock() + .await + .get_current_store_epoch()?; + let frontier_tracked = self.frontier_for(phase).is_some(); + let data_to_fetch = block_cache.data_to_fetch( + target.slot, + current_epoch, + &required_columns, + &candidate_peers, + now, + frontier_tracked, + ); + if !matches!( + data_to_fetch, + DataToFetch::DownloadsInProgress | DataToFetch::Finished + ) { + recovery_window_open = false; + } + info!( + "Forward sync status: Downloaded Blocks {}, Downloaded Blobs {}/{}, Stage {data_to_fetch}", + block_cache.block_count(), + block_cache.downloaded_blob_count(), + block_cache.blob_count(), + ); - Ok(self) - }) - } -} + match data_to_fetch { + DataToFetch::BlockRange(range) => { + let key = RequestKey::BlockRange(range); + let excluded = block_cache.attempted_peers_for(key); + let Some(peer) = self + .peer_manager + .fetch_idle_peer_from_excluding(&candidate_peers, &excluded) + else { + self.peer_manager.update_peer_set(); + info!("No idle peers available for block range sync."); + block_cache.push_retry_range(range); + sleep(SLEEP_DURATION).await; + continue; + }; + + let frontier_id = self + .frontier_for(phase) + .as_ref() + .map(|frontier| frontier.id); + block_cache.mark_block_range_in_progress(range); + task_handles.push(DownloadTask::new_block_range( + PeerRangeDownloader::start( + peer.peer_id, + self.p2p_sender.clone(), + self.executor.clone(), + range, + ), + range, + peer.peer_id, + frontier_id, + )); + } + DataToFetch::DataColumnRange(range) => { + let key = RequestKey::ColumnRange(range); + let excluded = block_cache.attempted_peers_for(key); + let Some(peer) = self + .peer_manager + .fetch_idle_peer_from_excluding(&candidate_peers, &excluded) + else { + self.peer_manager.update_peer_set(); + info!("No idle peers available for data column range sync."); + block_cache.push_column_range(range); + sleep(SLEEP_DURATION).await; + continue; + }; + + let expected_known_identifiers = + block_cache.expected_column_identifiers_in_range(range, &required_columns); + + block_cache.mark_column_range_in_progress(range); + task_handles.push(DownloadTask::new_data_column_range( + PeerDataColumnRangeDownloader::start( + peer.peer_id, + self.p2p_sender.clone(), + self.executor.clone(), + range, + required_columns.iter().copied().collect(), + ), + range, + peer.peer_id, + expected_known_identifiers, + )); + } + DataToFetch::MissingBlockRoots(mut block_roots) => { + let mut exhausted_this_tick = HashSet::new(); + while !block_roots.is_empty() { + let Some(peer) = self + .peer_manager + .fetch_idle_peer_from_excluding(&candidate_peers, &exhausted_this_tick) + else { + self.peer_manager.update_peer_set(); + info!("No idle peers available for block roots sync."); + sleep(SLEEP_DURATION).await; + break; + }; + + let (assigned, remaining): (Vec, Vec) = + block_roots.into_iter().partition(|root| { + !block_cache + .attempted_peers_for(RequestKey::BlockRoot(*root)) + .contains(&peer.peer_id) + }); + block_roots = remaining; + if assigned.is_empty() { + self.peer_manager.mark_peer_as_idle(&peer.peer_id); + exhausted_this_tick.insert(peer.peer_id); + continue; + } + let chunk: Vec = + assigned.into_iter().take(MAX_CONCURRENT_REQUESTS).collect(); + + block_cache.extend_block_roots_in_progress(&chunk); + + task_handles.push(DownloadTask::new_block_roots( + PeerRootsDownloader::start( + peer.peer_id, + self.p2p_sender.clone(), + self.executor.clone(), + chunk.clone(), + ), + chunk, + peer.peer_id, + )); + } + } + DataToFetch::MissingBlobIdentifiers(mut blob_identifiers) => { + let mut exhausted_this_tick = HashSet::new(); + while !blob_identifiers.is_empty() { + let Some(peer) = self + .peer_manager + .fetch_idle_peer_from_excluding(&candidate_peers, &exhausted_this_tick) + else { + self.peer_manager.update_peer_set(); + info!( + "No idle peers available for blob sync. {}", + self.peer_manager.peer_counts() + ); + sleep(SLEEP_DURATION).await; + break; + }; + + let (assigned, remaining): (Vec, Vec) = + blob_identifiers.into_iter().partition(|identifier| { + !block_cache + .attempted_peers_for(RequestKey::Blob(*identifier)) + .contains(&peer.peer_id) + }); + blob_identifiers = remaining; + if assigned.is_empty() { + self.peer_manager.mark_peer_as_idle(&peer.peer_id); + exhausted_this_tick.insert(peer.peer_id); + continue; + } + let chunk: Vec = + assigned.into_iter().take(MAX_BLOBS_PER_REQUEST).collect(); + + block_cache.extend_blob_identifiers_in_progress(&chunk); + + task_handles.push(DownloadTask::new_blob_identifiers( + PeerBlobIdentifierDownloader::start( + peer.peer_id, + self.p2p_sender.clone(), + self.executor.clone(), + chunk.clone(), + ), + chunk, + peer.peer_id, + )); + } + } + DataToFetch::MissingDataColumnIdentifiers(mut identifiers) => { + let mut exhausted_this_tick = HashSet::new(); + while !identifiers.is_empty() { + let Some(peer) = self + .peer_manager + .fetch_idle_peer_from_excluding(&candidate_peers, &exhausted_this_tick) + else { + self.peer_manager.update_peer_set(); + info!("No idle peers available for data column sync."); + sleep(SLEEP_DURATION).await; + break; + }; + + let (assigned, remaining): (Vec, Vec) = + identifiers.into_iter().partition(|identifier| { + !block_cache + .attempted_peers_for(RequestKey::Column(*identifier)) + .contains(&peer.peer_id) + }); + identifiers = remaining; + if assigned.is_empty() { + self.peer_manager.mark_peer_as_idle(&peer.peer_id); + exhausted_this_tick.insert(peer.peer_id); + continue; + } + let chunk: Vec = + assigned.into_iter().take(MAX_CONCURRENT_REQUESTS).collect(); + + block_cache.extend_data_column_identifiers_in_progress(&chunk); + + task_handles.push(DownloadTask::new_data_column_identifiers( + PeerDataColumnIdentifierDownloader::start( + peer.peer_id, + self.p2p_sender.clone(), + self.executor.clone(), + group_by_block_root(&chunk), + ), + chunk, + peer.peer_id, + )); + } + } + DataToFetch::DownloadsInProgress => { + info!( + "Waiting for ongoing downloads to complete... {}", + self.peer_manager.peer_counts() + ); + sleep(Duration::from_secs(10)).await; + } + DataToFetch::Finished => { + if phase == SyncPhase::Finalized && block_cache.next_start_slot() >= target.slot + { + finalized_target_was_ahead = target.slot > head_slot; + phase = SyncPhase::Head; + active_target = None; + finalized_phase_settled = true; + continue; + } + if block_cache.block_count() == 0 && block_cache.next_start_slot() < target.slot + { + pending_conclusions.observe( + phase, + observation.clone(), + RemoteNoProgressReason::SettledEmptyCoverage, + HashSet::new(), + HashSet::new(), + ); + } + break; + } + } + } + + info!( + "Block range sync completed a segment successfully with {} blocks and {} blobs.", + block_cache.block_count(), + block_cache.downloaded_blob_count(), + ); + + let import_result = self.import_downloaded(block_cache).await; + let imported_count = match &import_result { + Ok(count) => *count, + Err(err) => err.imported_count, + }; + + info!("All blocks processed successfully."); + + if imported_count > 0 { + self.clear_tracker(); + } else if import_result.is_ok() { + self.commit_pending_conclusions(pending_conclusions, segment_exclusions); + } + + let target_was_ahead = finalized_target_was_ahead + || active_target + .as_ref() + .is_some_and(|target| target.slot > head_slot); + if target_was_ahead && saw_empty_range && imported_count == 0 { + self.next_segment_not_before = Some(Instant::now() + ZERO_PROGRESS_BACKOFF); + } + + import_result.map(|_| ()).map_err(|err| err.error) + } + + async fn import_downloaded(&mut self, block_cache: BlockCache) -> Result { + let bundles = block_cache + .get_blocks_and_blobs() + .map_err(|err| ImportFailure { + imported_count: 0, + error: err, + })?; + let mut imported_count = 0u64; + for bundle in bundles { + if let Err(err) = self.import_one(bundle).await { + return Err(ImportFailure { + imported_count, + error: err, + }); + } + imported_count += 1; + } + Ok(imported_count) + } + + async fn import_one(&mut self, bundle: BlockAndBlobBundle) -> anyhow::Result<()> { + let BlockAndBlobBundle { + block, + blobs, + columns, + source_peer, + } = bundle; + let fulu_fork_epoch = beacon_network_spec().fulu_fork_epoch; + info!("Processing block with slot {}", block.message.slot); + + let (block, columns) = if block.message.body.blob_kzg_commitments.is_empty() { + ensure!( + blobs.is_empty(), + "Range-sync block without blob commitments had downloaded blob sidecars" + ); + ensure!( + columns.is_empty(), + "Range-sync block without blob commitments had downloaded data columns" + ); + (block, Vec::new()) + } else if compute_epoch_at_slot(block.message.slot) >= fulu_fork_epoch { + let required_columns = self + .beacon_chain + .store + .lock() + .await + .data_availability_checker + .required_columns() + .clone(); + let columns = columns + .into_values() + .filter(|column| required_columns.contains(&column.index)) + .collect::>(); + (block, columns) + } else { + let (blobs_provider, required_columns, verify_data_availability) = { + let store = self.beacon_chain.store.lock().await; + let network_spec = beacon_network_spec(); + ( + store.db.blobs_and_proofs_provider(), + store.data_availability_checker.required_columns().clone(), + is_data_availability_check_required( + compute_epoch_at_slot(block.message.slot), + store.get_current_store_epoch()?, + network_spec.fulu_fork_epoch, + network_spec.min_epochs_for_data_column_sidecars_requests, + ), + ) + }; + tokio::task::spawn_blocking(move || { + let columns = build_data_columns_from_blob_sidecars( + &block, + &blobs, + verify_data_availability, + )? + .into_iter() + .filter(|column| required_columns.contains(&column.index)) + .collect::>(); + for (identifier, sidecar) in blobs { + blobs_provider.insert(identifier, sidecar.into())?; + } + Ok::<_, anyhow::Error>((block, columns)) + }) + .await + .map_err(|err| anyhow!("Range-sync data-column task failed: {err}"))?? + }; + + match self.beacon_chain.process_block(block).await? { + BlockProcessingOutcome::Imported { .. } => {} + BlockProcessingOutcome::PendingAvailability { block_root } => { + for column in columns { + self.beacon_chain + .import_data_column_sidecar_if(column, |_| Ok(())) + .await?; + } + ensure!( + self.beacon_chain + .store + .lock() + .await + .db + .block_provider() + .get(block_root)? + .is_some(), + "Range-sync block {block_root} remained pending after processing its downloaded data" + ); + } + } + self.peer_manager.record_processed_blocks(&source_peer, 1); + Ok(()) + } + + fn merge_confirmed_empty_coverage(&mut self, phase: SyncPhase, advance: &CoverageAdvance) { + if !advance.proven_empty { + return; + } + let Some(frontier) = self.frontier_for_mut(phase) else { + return; + }; + let coverage = EmptyCoverage { + start_slot: frontier.anchor_slot, + end_slot_exclusive: advance.covered_through_slot + 1, + }; + match &mut frontier.confirmed_empty_through { + Some(confirmed) => { + confirmed.merge(coverage, &advance.confirming_peers); + } + None => { + frontier.confirmed_empty_through = Some(ConfirmedEmptyCoverage { + frontier_id: frontier.id, + origin_anchor: (advance.parent_root, advance.parent_slot), + start_slot: coverage.start_slot, + end_slot_exclusive: coverage.end_slot_exclusive, + confirming_peers: advance.confirming_peers.clone(), + }); + } + } + } +} + +fn group_by_block_root(identifiers: &[ColumnIdentifier]) -> Vec { + let mut by_root: std::collections::HashMap> = std::collections::HashMap::new(); + for identifier in identifiers { + by_root + .entry(identifier.block_root) + .or_default() + .push(identifier.index); + } + by_root + .into_iter() + .filter_map(|(block_root, columns)| { + match DataColumnsByRootIdentifier::new(block_root, columns) { + Ok(identifier) => Some(identifier), + Err(err) => { + warn!("Failed to build data column identifier for {block_root}: {err}"); + None + } + } + }) + .collect() +} pub enum DownloadTask { BlockRange { - handle: JoinHandle>>>, + handle: JoinHandle>>, + range: Range, + peer_id: PeerId, + frontier_id: Option, + }, + DataColumnRange { + handle: JoinHandle>>, range: Range, peer_id: PeerId, + expected_known_identifiers: Vec, }, BlockRoots { - handle: JoinHandle>>>, + handle: JoinHandle>>, roots: Vec, peer_id: PeerId, }, BlobIdentifiers { - handle: JoinHandle>>>, + handle: JoinHandle>>, blob_identifiers: Vec, peer_id: PeerId, }, + DataColumnIdentifiers { + handle: JoinHandle>>, + identifiers: Vec, + peer_id: PeerId, + }, } impl DownloadTask { pub fn new_block_range( - handle: JoinHandle>>>, + handle: JoinHandle>>, range: Range, peer_id: PeerId, + frontier_id: Option, ) -> Self { DownloadTask::BlockRange { handle, range, peer_id, + frontier_id, + } + } + + pub fn new_data_column_range( + handle: JoinHandle>>, + range: Range, + peer_id: PeerId, + expected_known_identifiers: Vec, + ) -> Self { + DownloadTask::DataColumnRange { + handle, + range, + peer_id, + expected_known_identifiers, } } pub fn new_block_roots( - handle: JoinHandle>>>, + handle: JoinHandle>>, roots: Vec, peer_id: PeerId, ) -> Self { @@ -391,7 +1431,7 @@ impl DownloadTask { } pub fn new_blob_identifiers( - handle: JoinHandle>>>, + handle: JoinHandle>>, blob_identifiers: Vec, peer_id: PeerId, ) -> Self { @@ -401,16 +1441,74 @@ impl DownloadTask { peer_id, } } + + pub fn new_data_column_identifiers( + handle: JoinHandle>>, + identifiers: Vec, + peer_id: PeerId, + ) -> Self { + DownloadTask::DataColumnIdentifiers { + handle, + identifiers, + peer_id, + } + } +} + +fn handle_stream_outcome( + peer_manager: &mut PeerManager, + peer_id: &PeerId, + result: Result>, tokio::task::JoinError>, +) -> Option> { + let outcome = match result { + Ok(Ok(outcome)) => outcome, + Ok(Err(err)) => { + warn!("Forward fill task failed: {err}"); + return None; + } + Err(err) => { + warn!("Forward fill task panicked: {err}"); + return None; + } + }; + + match outcome { + StreamOutcome::Complete(items) => Some(items), + StreamOutcome::Failed(DownloadFailure::Transport(detail)) => { + info!("Transport failure from peer {peer_id}, will retry with someone else: {detail}"); + None + } + StreamOutcome::Failed(DownloadFailure::InvalidData(detail)) => { + warn!("Invalid data from peer {peer_id}: {detail}"); + peer_manager.ban_peer(peer_id, BanReason::ProtocolError(detail)); + None + } + StreamOutcome::Failed(DownloadFailure::RemoteError { code, message }) => { + if code == ResponseCode::InvalidRequest { + warn!("Peer {peer_id} reported InvalidRequest (possible local bug): {message}"); + } else { + info!("Peer {peer_id} declined the request ({code:?}): {message}"); + } + None + } + } } fn poll_ready_tasks( tasks: &mut Vec, block_cache: &mut BlockCache, peer_manager: &mut PeerManager, -) -> anyhow::Result<()> { + frontiers: &mut FrontierStore, + required_columns: &HashSet, + saw_empty_range: &mut bool, + candidate_peers: &[PeerId], +) -> anyhow::Result { + let now = Instant::now(); let waker = noop_waker(); let mut context = Context::from_waker(&waker); let mut indexes_to_remove = vec![]; + let fulu_fork_epoch = beacon_network_spec().fulu_fork_epoch; + let mut outcome = PollTasksOutcome::Settled; for index in (0..tasks.len()).rev() { let Some(task) = tasks.get_mut(index) else { @@ -422,48 +1520,273 @@ fn poll_ready_tasks( handle, range, peer_id, + frontier_id, } => { let pinned = Pin::new(handle); match pinned.poll(&mut context) { - Poll::Ready(Ok(blocks_result)) => { + Poll::Ready(result) => { indexes_to_remove.push(index); + block_cache.remove_block_range_in_progress(range); peer_manager.mark_peer_as_idle(peer_id); - let blocks = match blocks_result { - Ok(blocks) => blocks, - Err(err) => { - warn!("Failed to fetch blocks from peer: {err:?}"); - block_cache.push_retry_range(*range); - continue; - } + let key = RequestKey::BlockRange(*range); + + let matching_frontier = match frontier_id { + Some(id) => match frontiers.for_phase(id.key.phase).as_ref() { + Some(frontier) if frontier.id == *id => Some(frontier), + _ => { + outcome = PollTasksOutcome::StaleGeneration; + continue; + } + }, + None => None, }; - let blocks = match blocks { - Ok(blocks) => blocks, - Err(err) => { - block_cache.push_retry_range(*range); - peer_manager - .ban_peer(peer_id, format!("Failed to fetch blocks: {err:?}")); - continue; - } + let Some(blocks) = handle_stream_outcome(peer_manager, peer_id, result) + else { + block_cache.mark_attempted(key, *peer_id, candidate_peers, now); + block_cache.push_retry_range(*range); + continue; }; if blocks.is_empty() { - warn!("Received empty block range from peer: {peer_id}"); - block_cache.push_retry_range(*range); - peer_manager - .ban_peer(peer_id, "Received empty block range".to_string()); + info!("Received empty block range from peer: {peer_id}"); + block_cache.clear_attempted(key); + *saw_empty_range = true; + + if let Some(frontier) = matching_frontier { + let observation = observation_from_frontier(frontier); + let confirming_peers = HashSet::from([*peer_id]); + let parent_root = block_cache.initial_parent_root(); + let covered_through_slot = range.start_slot + range.count - 1; + if block_cache + .advance_empty_coverage( + observation.clone(), + parent_root, + covered_through_slot, + confirming_peers.clone(), + ) + .is_ok() + && let Some(stored) = + frontiers.for_phase_mut(observation.phase).as_mut() + { + match &mut stored.confirmed_empty_through { + Some(confirmed) => { + confirmed.merge( + EmptyCoverage { + start_slot: range.start_slot, + end_slot_exclusive: range.start_slot + + range.count, + }, + &confirming_peers, + ); + } + None => { + stored.confirmed_empty_through = + Some(ConfirmedEmptyCoverage { + frontier_id: stored.id, + origin_anchor: ( + parent_root, + observation.anchor_slot, + ), + start_slot: range.start_slot, + end_slot_exclusive: range.start_slot + + range.count, + confirming_peers, + }); + } + } + } + } continue; } - if let Err(err) = block_cache.add_blocks(blocks, true) { - warn!("Failed to add downloaded blocks to cache: {err:?}"); + let range_end = range.start_slot + range.count; + let out_of_range = blocks.iter().any(|block| { + block.message.slot < range.start_slot || block.message.slot >= range_end + }); + if out_of_range { + warn!( + "Peer {peer_id} returned block(s) outside the requested range {range:?}" + ); + block_cache.mark_attempted(key, *peer_id, candidate_peers, now); block_cache.push_retry_range(*range); + peer_manager.ban_peer( + peer_id, + BanReason::ProtocolError( + "block(s) outside the requested range".to_string(), + ), + ); + continue; + } + + let needs_columns = !required_columns.is_empty() + && blocks.iter().any(|block| { + !block.message.body.blob_kzg_commitments.is_empty() + && compute_epoch_at_slot(block.message.slot) >= fulu_fork_epoch + }); + + match block_cache.add_blocks(blocks, true, *peer_id) { + Err(AddBlocksError::CoverageDivergence { + expected_parent, + actual_parent, + anchor, + }) => { + warn!( + "Coverage divergence: block from {peer_id} has parent \ + {actual_parent}, expected {expected_parent} -- rolling \ + back the optimistic coverage generation" + ); + outcome = PollTasksOutcome::CoverageDivergence(Box::new( + CoverageDivergenceOutcome { + original_parent_root: anchor.original_parent_root, + original_parent_slot: anchor.original_parent_slot, + frontier: anchor.frontier, + confirming_peers: anchor.confirming_peers, + non_connecting_peer: *peer_id, + }, + )); + } + Err(AddBlocksError::InvalidBatch(err)) => { + warn!("Failed to add downloaded blocks to cache: {err:?}"); + block_cache.mark_attempted(key, *peer_id, candidate_peers, now); + block_cache.push_retry_range(*range); + peer_manager.ban_peer( + peer_id, + BanReason::ProtocolError(format!( + "invalid block range: {err:?}" + )), + ); + } + Ok(()) => { + block_cache.clear_attempted(key); + if needs_columns { + block_cache.push_column_range(*range); + } + } } } - Poll::Ready(Err(err)) => { - warn!("Forward fill task failed: {err}"); + Poll::Pending => {} + } + } + DownloadTask::DataColumnRange { + handle, + range, + peer_id, + expected_known_identifiers, + } => { + let pinned = Pin::new(handle); + + match pinned.poll(&mut context) { + Poll::Ready(result) => { indexes_to_remove.push(index); + block_cache.remove_column_range_in_progress(range); + peer_manager.mark_peer_as_idle(peer_id); + let key = RequestKey::ColumnRange(*range); + + let Some(columns) = handle_stream_outcome(peer_manager, peer_id, result) + else { + block_cache.mark_attempted(key, *peer_id, candidate_peers, now); + for identifier in expected_known_identifiers.iter() { + block_cache.mark_attempted( + RequestKey::Column(*identifier), + *peer_id, + candidate_peers, + now, + ); + } + continue; + }; + + if columns.is_empty() { + info!("Received empty data column range from peer: {peer_id}"); + block_cache.clear_attempted(key); + for identifier in expected_known_identifiers.iter() { + block_cache.mark_attempted( + RequestKey::Column(*identifier), + *peer_id, + candidate_peers, + now, + ); + } + *saw_empty_range = true; + continue; + } + + let range_end = range.start_slot + range.count; + let mut seen_identifiers = HashSet::new(); + let out_of_shape = columns.iter().any(|column| { + let slot = column.signed_block_header.message.slot; + let identifier = ColumnIdentifier::new( + column.signed_block_header.message.tree_hash_root(), + column.index, + ); + slot < range.start_slot + || slot >= range_end + || !required_columns.contains(&column.index) + || !seen_identifiers.insert(identifier) + }); + if out_of_shape { + warn!( + "Peer {peer_id} returned data column(s) outside the requested range/columns, or a duplicate, for range {range:?}" + ); + block_cache.clear_attempted(key); + for identifier in expected_known_identifiers.iter() { + block_cache.mark_attempted( + RequestKey::Column(*identifier), + *peer_id, + candidate_peers, + now, + ); + } + peer_manager.ban_peer( + peer_id, + BanReason::ProtocolError( + "data column(s) outside the requested range/columns, or a duplicate" + .to_string(), + ), + ); + continue; + } + + let returned: HashSet = columns + .iter() + .map(|column| { + ColumnIdentifier::new( + column.signed_block_header.message.tree_hash_root(), + column.index, + ) + }) + .collect(); + + if let Err(err) = block_cache.add_data_columns(columns, required_columns) { + warn!("Failed to add downloaded data columns to cache: {err:?}"); + block_cache.clear_attempted(key); + for identifier in expected_known_identifiers.iter() { + block_cache.mark_attempted( + RequestKey::Column(*identifier), + *peer_id, + candidate_peers, + now, + ); + } + peer_manager + .ban_peer(peer_id, BanReason::InvalidProof(format!("{err:?}"))); + } else { + block_cache.clear_attempted(key); + for identifier in expected_known_identifiers.iter() { + if returned.contains(identifier) { + block_cache.clear_attempted(RequestKey::Column(*identifier)); + } else { + block_cache.mark_attempted( + RequestKey::Column(*identifier), + *peer_id, + candidate_peers, + now, + ); + } + } + } } Poll::Pending => {} } @@ -476,45 +1799,77 @@ fn poll_ready_tasks( let pinned = Pin::new(handle); match pinned.poll(&mut context) { - Poll::Ready(Ok(blocks_result)) => { + Poll::Ready(result) => { indexes_to_remove.push(index); block_cache.remove_block_roots_in_progress(roots); peer_manager.mark_peer_as_idle(peer_id); - let blocks = match blocks_result { - Ok(blocks) => blocks, - Err(err) => { - warn!("Failed to fetch blocks from peer: {err:?}"); - continue; + + let Some(blocks) = handle_stream_outcome(peer_manager, peer_id, result) + else { + for root in roots.iter() { + block_cache.mark_attempted( + RequestKey::BlockRoot(*root), + *peer_id, + candidate_peers, + now, + ); } + continue; }; - let blocks = match blocks { - Ok(blocks) => blocks, - Err(err) => { - warn!("Failed to fetch blocks from roots: {err:?}"); - peer_manager.ban_peer( - peer_id, - format!("Failed to fetch blocks from receipts: {err:?}"), + let requested: HashSet = roots.iter().copied().collect(); + let mut seen_roots = HashSet::new(); + let has_unexpected = blocks.iter().any(|block| { + let root = block.message.tree_hash_root(); + !requested.contains(&root) || !seen_roots.insert(root) + }); + if has_unexpected { + warn!( + "Peer {peer_id} returned an unrequested or duplicate root for a BlockRoots request" + ); + for root in roots.iter() { + block_cache.mark_attempted( + RequestKey::BlockRoot(*root), + *peer_id, + candidate_peers, + now, ); - continue; } - }; + peer_manager.ban_peer( + peer_id, + BanReason::ProtocolError( + "returned block(s) with an unrequested root".to_string(), + ), + ); + continue; + } + + let returned: HashSet = blocks + .iter() + .map(|block| block.message.tree_hash_root()) + .collect(); + for root in roots.iter() { + if returned.contains(root) { + block_cache.clear_attempted(RequestKey::BlockRoot(*root)); + } else { + block_cache.mark_attempted( + RequestKey::BlockRoot(*root), + *peer_id, + candidate_peers, + now, + ); + } + } if blocks.is_empty() { warn!("Received empty block roots from peer: {peer_id}"); - peer_manager - .ban_peer(peer_id, "Received empty block roots".to_string()); continue; } - if let Err(err) = block_cache.add_blocks(blocks, false) { + if let Err(err) = block_cache.add_blocks(blocks, false, *peer_id) { warn!("Failed to add downloaded blocks to cache: {err:?}"); } } - Poll::Ready(Err(err)) => { - warn!("Forward fill task failed: {err}"); - indexes_to_remove.push(index); - } Poll::Pending => {} } } @@ -526,44 +1881,213 @@ fn poll_ready_tasks( let pinned = Pin::new(handle); match pinned.poll(&mut context) { - Poll::Ready(Ok(blob_sidecars_result)) => { + Poll::Ready(result) => { indexes_to_remove.push(index); block_cache.remove_blob_identifiers_in_progress(blob_identifiers); peer_manager.mark_peer_as_idle(peer_id); - let blob_sidecars = match blob_sidecars_result { - Ok(blob_sidecars) => blob_sidecars, - Err(err) => { - warn!("Failed to fetch blobs from peer: {err:?}"); - continue; - } - }; - let blob_sidecars = match blob_sidecars { - Ok(blob_sidecars) => blob_sidecars, - Err(err) => { - warn!("Failed to fetch blobs from identifiers: {err:?}"); - peer_manager.ban_peer( - peer_id, - format!("Failed to fetch blobs from identifiers: {err:?}"), + let Some(blob_sidecars) = + handle_stream_outcome(peer_manager, peer_id, result) + else { + for identifier in blob_identifiers.iter() { + block_cache.mark_attempted( + RequestKey::Blob(*identifier), + *peer_id, + candidate_peers, + now, ); - continue; } + continue; }; if blob_sidecars.is_empty() { warn!("Received empty blob identifiers from peer: {peer_id}"); - peer_manager - .ban_peer(peer_id, "Received empty blob identifiers".to_string()); + for identifier in blob_identifiers.iter() { + block_cache.mark_attempted( + RequestKey::Blob(*identifier), + *peer_id, + candidate_peers, + now, + ); + } + continue; + } + + let requested: HashSet = + blob_identifiers.iter().copied().collect(); + let mut seen_identifiers = HashSet::new(); + let has_unexpected = blob_sidecars.iter().any(|sidecar| { + let identifier = BlobIdentifier { + block_root: sidecar.signed_block_header.message.tree_hash_root(), + index: sidecar.index, + }; + !requested.contains(&identifier) || !seen_identifiers.insert(identifier) + }); + if has_unexpected { + warn!( + "Peer {peer_id} returned an unrequested or duplicate blob sidecar" + ); + for identifier in blob_identifiers.iter() { + block_cache.mark_attempted( + RequestKey::Blob(*identifier), + *peer_id, + candidate_peers, + now, + ); + } + peer_manager.ban_peer( + peer_id, + BanReason::ProtocolError( + "returned unrequested blob sidecar(s)".to_string(), + ), + ); continue; } + let returned: HashSet = blob_sidecars + .iter() + .map(|sidecar| BlobIdentifier { + block_root: sidecar.signed_block_header.message.tree_hash_root(), + index: sidecar.index, + }) + .collect(); + if let Err(err) = block_cache.add_blobs(blob_sidecars) { warn!("Failed to add downloaded blobs to cache: {err:?}"); + for identifier in blob_identifiers.iter() { + block_cache.mark_attempted( + RequestKey::Blob(*identifier), + *peer_id, + candidate_peers, + now, + ); + } + peer_manager + .ban_peer(peer_id, BanReason::InvalidProof(format!("{err:?}"))); + } else { + for identifier in blob_identifiers.iter() { + if returned.contains(identifier) { + block_cache.clear_attempted(RequestKey::Blob(*identifier)); + } else { + block_cache.mark_attempted( + RequestKey::Blob(*identifier), + *peer_id, + candidate_peers, + now, + ); + } + } } } - Poll::Ready(Err(err)) => { - warn!("Forward fill task failed: {err}"); + Poll::Pending => {} + } + } + DownloadTask::DataColumnIdentifiers { + handle, + identifiers, + peer_id, + } => { + let pinned = Pin::new(handle); + + match pinned.poll(&mut context) { + Poll::Ready(result) => { indexes_to_remove.push(index); + block_cache.remove_data_column_identifiers_in_progress(identifiers); + peer_manager.mark_peer_as_idle(peer_id); + + let Some(columns) = handle_stream_outcome(peer_manager, peer_id, result) + else { + for identifier in identifiers.iter() { + block_cache.mark_attempted( + RequestKey::Column(*identifier), + *peer_id, + candidate_peers, + now, + ); + } + continue; + }; + + if columns.is_empty() { + warn!("Received empty data column identifiers from peer: {peer_id}"); + for identifier in identifiers.iter() { + block_cache.mark_attempted( + RequestKey::Column(*identifier), + *peer_id, + candidate_peers, + now, + ); + } + continue; + } + + let requested: HashSet = + identifiers.iter().copied().collect(); + let mut seen_identifiers = HashSet::new(); + let has_unexpected = columns.iter().any(|column| { + let identifier = ColumnIdentifier::new( + column.signed_block_header.message.tree_hash_root(), + column.index, + ); + !requested.contains(&identifier) || !seen_identifiers.insert(identifier) + }); + if has_unexpected { + warn!( + "Peer {peer_id} returned an unrequested or duplicate data column" + ); + for identifier in identifiers.iter() { + block_cache.mark_attempted( + RequestKey::Column(*identifier), + *peer_id, + candidate_peers, + now, + ); + } + peer_manager.ban_peer( + peer_id, + BanReason::ProtocolError( + "returned unrequested data column(s)".to_string(), + ), + ); + continue; + } + + let returned: HashSet = columns + .iter() + .map(|column| { + ColumnIdentifier::new( + column.signed_block_header.message.tree_hash_root(), + column.index, + ) + }) + .collect(); + + if let Err(err) = block_cache.add_data_columns(columns, required_columns) { + warn!("Failed to add downloaded data columns to cache: {err:?}"); + for identifier in identifiers.iter() { + block_cache.mark_attempted( + RequestKey::Column(*identifier), + *peer_id, + candidate_peers, + now, + ); + } + peer_manager + .ban_peer(peer_id, BanReason::InvalidProof(format!("{err:?}"))); + } else { + for identifier in identifiers.iter() { + if returned.contains(identifier) { + block_cache.clear_attempted(RequestKey::Column(*identifier)); + } else { + block_cache.mark_attempted( + RequestKey::Column(*identifier), + *peer_id, + candidate_peers, + now, + ); + } + } + } } Poll::Pending => {} } @@ -575,22 +2099,62 @@ fn poll_ready_tasks( tasks.remove(index); } - Ok(()) + Ok(outcome) } #[cfg(test)] mod tests { - use std::collections::HashMap; + use std::{collections::HashMap, path::PathBuf}; + use discv5::{Enr, enr::CombinedKey}; use kzg::{G1, eip_4844::compute_blob_kzg_proof_raw}; - use ream_consensus_beacon::data_column_sidecar::NUMBER_OF_COLUMNS; - use ream_consensus_misc::polynomial_commitments::{ - kzg_commitment::KZGCommitment, kzg_proof::KZGProof, + use parking_lot::RwLock; + use ream_consensus_beacon::{ + data_column_sidecar::NUMBER_OF_COLUMNS, electra::beacon_block::BeaconBlock, + }; + use ream_consensus_misc::{ + checkpoint::Checkpoint, + polynomial_commitments::{kzg_commitment::KZGCommitment, kzg_proof::KZGProof}, }; use ream_execution_rpc_types::get_blobs::{Blob, BlobAndProofV1}; + use ream_network_spec::networks::beacon::initialize_test_network_spec; + use ream_operation_pool::OperationPool; + use ream_p2p::network::beacon::peer::CachedPeer; + use ream_peer::{ConnectionState, Direction}; + use ream_req_resp::beacon::messages::{meta_data::GetMetaDataV3, status::Status}; + use ream_storage::{db::ReamDB, tables::field::REDBField}; + use ream_sync_committee_pool::SyncCommitteePool; + use tempfile::TempDir; use super::*; + #[test] + fn should_abort_for_candidate_exhaustion_waits_for_the_timeout() { + assert!(!should_abort_for_candidate_exhaustion( + CANDIDATE_EXHAUSTION_TIMEOUT - Duration::from_secs(1), + false, + )); + assert!(should_abort_for_candidate_exhaustion( + CANDIDATE_EXHAUSTION_TIMEOUT, + false, + )); + } + + #[test] + fn should_abort_for_candidate_exhaustion_never_fires_with_a_task_still_in_flight() { + assert!(!should_abort_for_candidate_exhaustion( + CANDIDATE_EXHAUSTION_TIMEOUT + Duration::from_secs(60), + true, + )); + } + + #[test] + fn candidate_exhaustion_deserves_backoff_only_when_the_target_was_ahead() { + assert!(candidate_exhaustion_deserves_backoff(100, 50)); + assert!(!candidate_exhaustion_deserves_backoff(50, 50)); + assert!(!candidate_exhaustion_deserves_backoff(50, 100)); + } + #[test] fn range_blob_sidecars_build_valid_columns_and_reject_bad_proofs() { let blob = Blob::default(); @@ -630,7 +2194,11 @@ mod tests { .expect("expired blocks should not require blob sidecars") .is_empty() ); - assert!(build_data_columns_from_blob_sidecars(&block, &HashMap::new(), true).is_err()); + assert!( + build_data_columns_from_blob_sidecars(&block, &HashMap::new(), true) + .expect("blocks awaiting column fetch should not error") + .is_empty() + ); let columns = build_data_columns_from_blob_sidecars(&block, &sidecars, true) .expect("valid blobs should produce data columns"); @@ -641,5 +2209,1140 @@ mod tests { .expect("test sidecar should exist") .kzg_proof[0] ^= 1; assert!(build_data_columns_from_blob_sidecars(&block, &sidecars, true).is_err()); + + let wrong_identifier = BlobIdentifier::new(B256::repeat_byte(0xAB), 0); + let mismatched = HashMap::from([( + wrong_identifier, + sidecars.remove(&identifier).expect("sidecar should exist"), + )]); + assert!(build_data_columns_from_blob_sidecars(&block, &mismatched, true).is_err()); + } + + /// Kept alongside the chain so the dir isn't dropped (and cleaned up) until the test ends. + fn test_beacon_chain() -> (TempDir, BeaconChain) { + let data_dir = tempfile::tempdir().expect("tempdir should be created"); + let beacon_db = ReamDB::new(data_dir.path().to_path_buf()) + .expect("ReamDB should init") + .init_beacon_db() + .expect("beacon DB tables should init"); + let beacon_chain = BeaconChain::new( + beacon_db, + Arc::new(OperationPool::default()), + Arc::new(SyncCommitteePool::default()), + None, + None, + ); + (data_dir, beacon_chain) + } + + fn test_network_state() -> Arc { + let enr_key = CombinedKey::generate_secp256k1(); + Arc::new(NetworkState { + local_enr: RwLock::new(Enr::builder().build(&enr_key).expect("valid enr")), + peer_table: RwLock::new(HashMap::new()), + meta_data: RwLock::new(GetMetaDataV3::default()), + status: RwLock::new(Status::default()), + data_dir: PathBuf::new(), + }) + } + + fn test_peer_manager_with_one_peer() -> (PeerManager, PeerId) { + let network_state = test_network_state(); + let peer_id = PeerId::random(); + let mut peer = CachedPeer::new( + peer_id, + None, + ConnectionState::Connected, + Direction::Outbound, + None, + ); + peer.status = Some(Status::default()); + network_state.peer_table.write().insert(peer_id, peer); + + let mut peer_manager = PeerManager::new(network_state); + peer_manager.update_peer_set(); + (peer_manager, peer_id) + } + + fn poll_until_done( + tasks: &mut Vec, + block_cache: &mut BlockCache, + peer_manager: &mut PeerManager, + required_columns: &HashSet, + saw_empty_range: &mut bool, + candidate_peers: &[PeerId], + ) { + let mut finalized_frontier = None; + let mut head_frontier = None; + for _ in 0..500 { + let mut frontiers = FrontierStore { + finalized: &mut finalized_frontier, + head: &mut head_frontier, + }; + poll_ready_tasks( + tasks, + block_cache, + peer_manager, + &mut frontiers, + required_columns, + saw_empty_range, + candidate_peers, + ) + .expect("poll_ready_tasks should not error"); + if tasks.is_empty() { + return; + } + std::thread::sleep(Duration::from_millis(2)); + } + panic!("task did not complete within the test timeout"); + } + + /// `start()` must hand `self` back out even on an ordinary segment error, not just success, + /// or the caller loses the syncer (and its warmed-up peer table) and can never retry. + #[test] + fn start_returns_self_after_a_failed_segment() { + initialize_test_network_spec(); + // Empty DB has no highest synced slot, so run_segment fails immediately. + let (_data_dir, beacon_chain) = test_beacon_chain(); + let executor = ReamExecutor::new().expect("executor should start"); + let (p2p_sender, _p2p_receiver) = tokio::sync::mpsc::unbounded_channel(); + + let syncer = BlockRangeSyncer::new( + Arc::new(beacon_chain), + p2p_sender, + test_network_state(), + executor, + ); + + // ReamExecutor owns its own runtime; #[tokio::test] would make dropping it panic. + let (_syncer, sync_result) = futures::executor::block_on(syncer.start()) + .expect("task should not panic") + .expect("task should not be cancelled by shutdown"); + + assert!( + sync_result.is_err(), + "expected the empty-DB segment to fail" + ); + } + + /// A lone/early peer must not be enough to call the node synced. The wall clock has to + /// agree too, or one thin sample could "vote" the node caught up while it's still far behind. + #[test] + fn is_synced_to_head_slot_requires_wall_clock_agreement() { + // ReamExecutor owns its own runtime; #[tokio::test] would make dropping it panic. + futures::executor::block_on(is_synced_to_head_slot_requires_wall_clock_agreement_inner()); + } + + async fn is_synced_to_head_slot_requires_wall_clock_agreement_inner() { + initialize_test_network_spec(); + let (_data_dir, beacon_chain) = test_beacon_chain(); + let seconds_per_slot = beacon_network_spec().seconds_per_slot(); + let highest_slot = 100u64; + let highest_root = B256::repeat_byte(0x42); + + { + let store = beacon_chain.store.lock().await; + store + .db + .genesis_time_provider() + .insert(0) + .expect("insert genesis time"); + store + .db + .slot_index_provider() + .insert(highest_slot, highest_root) + .expect("insert highest synced slot"); + store + .db + .finalized_checkpoint_provider() + .insert(Checkpoint { + epoch: 0, + root: B256::ZERO, + }) + .expect("insert finalized checkpoint"); + } + + let network_state = test_network_state(); + for _ in 0..MIN_SYNC_PEERS { + let peer_id = PeerId::random(); + let mut peer = CachedPeer::new( + peer_id, + None, + ConnectionState::Connected, + Direction::Outbound, + None, + ); + peer.status = Some(Status { + head_slot: highest_slot, + head_root: highest_root, + ..Default::default() + }); + network_state.peer_table.write().insert(peer_id, peer); + } + + let (p2p_sender, _p2p_receiver) = tokio::sync::mpsc::unbounded_channel(); + let mut syncer = BlockRangeSyncer::new( + Arc::new(beacon_chain), + p2p_sender, + network_state, + ReamExecutor::new().expect("executor should start"), + ); + syncer.peer_manager.update_peer_set(); + + // Clock agrees with the peer: genuinely caught up. + syncer + .beacon_chain + .store + .lock() + .await + .db + .time_provider() + .insert(highest_slot * seconds_per_slot) + .expect("insert time"); + assert!(syncer.is_synced_to_head_slot().await); + + // Clock disagrees: must not report synced, even though the peer still does. + syncer + .beacon_chain + .store + .lock() + .await + .db + .time_provider() + .insert((highest_slot + 10_000) * seconds_per_slot) + .expect("insert time"); + assert!(!syncer.is_synced_to_head_slot().await); + } + + #[test] + fn poll_ready_tasks_partial_block_roots_marks_only_the_omitted_root_attempted() { + initialize_test_network_spec(); + let executor = ReamExecutor::new().expect("executor should start"); + let (mut peer_manager, peer_id) = test_peer_manager_with_one_peer(); + + let delivered_block = SignedBeaconBlock { + message: BeaconBlock { + slot: 1, + ..Default::default() + }, + signature: Default::default(), + }; + let delivered_root = delivered_block.message.tree_hash_root(); + let omitted_root = B256::repeat_byte(9); + + let mut block_cache = BlockCache::new(B256::ZERO, 0); + let handle = executor.spawn(async move { StreamOutcome::Complete(vec![delivered_block]) }); + let mut tasks = vec![DownloadTask::new_block_roots( + handle, + vec![delivered_root, omitted_root], + peer_id, + )]; + + poll_until_done( + &mut tasks, + &mut block_cache, + &mut peer_manager, + &HashSet::new(), + &mut false, + &[peer_id], + ); + + assert!( + block_cache + .attempted_peers_for(RequestKey::BlockRoot(delivered_root)) + .is_empty(), + "a delivered root must not be marked as a failed attempt" + ); + assert!( + block_cache + .attempted_peers_for(RequestKey::BlockRoot(omitted_root)) + .contains(&peer_id), + "the omitted root must be attributed to this peer, so a retry excludes it" + ); + } + + #[test] + fn poll_ready_tasks_unrequested_block_root_bans_peer_and_discards_whole_batch() { + initialize_test_network_spec(); + let executor = ReamExecutor::new().expect("executor should start"); + let (mut peer_manager, peer_id) = test_peer_manager_with_one_peer(); + + let requested_block = SignedBeaconBlock { + message: BeaconBlock { + slot: 1, + ..Default::default() + }, + signature: Default::default(), + }; + let requested_root = requested_block.message.tree_hash_root(); + let unrequested_block = SignedBeaconBlock { + message: BeaconBlock { + slot: 2, + ..Default::default() + }, + signature: Default::default(), + }; + + let mut block_cache = BlockCache::new(B256::ZERO, 0); + let handle = executor.spawn(async move { + StreamOutcome::Complete(vec![requested_block, unrequested_block]) + }); + let mut tasks = vec![DownloadTask::new_block_roots( + handle, + vec![requested_root], + peer_id, + )]; + + poll_until_done( + &mut tasks, + &mut block_cache, + &mut peer_manager, + &HashSet::new(), + &mut false, + &[peer_id], + ); + + assert_eq!( + block_cache.block_count(), + 0, + "a response with any unrequested item is atomically discarded -- even the \ + requested, otherwise-valid item in the same batch must not be committed" + ); + assert!( + block_cache + .attempted_peers_for(RequestKey::BlockRoot(requested_root)) + .contains(&peer_id), + "the requested root must still be attributed to this peer's failed attempt" + ); + assert!( + peer_manager.fetch_idle_peer_from(&[peer_id]).is_none(), + "a peer that smuggled in an unrequested item must be banned, not just marked idle" + ); + } + + #[test] + fn poll_ready_tasks_duplicate_block_root_bans_peer_and_discards_whole_batch() { + initialize_test_network_spec(); + let executor = ReamExecutor::new().expect("executor should start"); + let (mut peer_manager, peer_id) = test_peer_manager_with_one_peer(); + + let block_a = SignedBeaconBlock { + message: BeaconBlock { + slot: 1, + ..Default::default() + }, + signature: Default::default(), + }; + let requested_root = block_a.message.tree_hash_root(); + let block_b = SignedBeaconBlock { + message: BeaconBlock { + slot: 1, + ..Default::default() + }, + signature: Default::default(), + }; + + let mut block_cache = BlockCache::new(B256::ZERO, 0); + let handle = executor.spawn(async move { StreamOutcome::Complete(vec![block_a, block_b]) }); + let mut tasks = vec![DownloadTask::new_block_roots( + handle, + vec![requested_root], + peer_id, + )]; + + poll_until_done( + &mut tasks, + &mut block_cache, + &mut peer_manager, + &HashSet::new(), + &mut false, + &[peer_id], + ); + + assert_eq!( + block_cache.block_count(), + 0, + "a response returning the same requested root twice is atomically discarded" + ); + assert!( + block_cache + .attempted_peers_for(RequestKey::BlockRoot(requested_root)) + .contains(&peer_id), + "the requested root must still be attributed to this peer's failed attempt" + ); + assert!( + peer_manager.fetch_idle_peer_from(&[peer_id]).is_none(), + "a peer that returned a duplicate item must be banned, not just marked idle" + ); + } + + #[test] + fn poll_ready_tasks_empty_data_column_range_falls_through_to_by_root_and_excludes_peer() { + initialize_test_network_spec(); + let executor = ReamExecutor::new().expect("executor should start"); + let (mut peer_manager, peer_id) = test_peer_manager_with_one_peer(); + + let range = Range::new(1, 10); + let identifier = ColumnIdentifier::new(B256::repeat_byte(3), 0); + let mut block_cache = BlockCache::new(B256::ZERO, 0); + let handle = + executor.spawn(async move { StreamOutcome::Complete(Vec::::new()) }); + let mut tasks = vec![DownloadTask::new_data_column_range( + handle, + range, + peer_id, + vec![identifier], + )]; + + let mut saw_empty_range = false; + poll_until_done( + &mut tasks, + &mut block_cache, + &mut peer_manager, + &HashSet::new(), + &mut saw_empty_range, + &[peer_id], + ); + + assert!(saw_empty_range); + assert!( + block_cache + .attempted_peers_for(RequestKey::Column(identifier)) + .contains(&peer_id), + "the expected identifier must fall through to the by-root fallback, excluding this peer" + ); + assert!( + block_cache + .attempted_peers_for(RequestKey::ColumnRange(range)) + .is_empty(), + "an empty range response is accepted (settled), not held as a pending attempt" + ); + } + + #[test] + fn poll_ready_tasks_invalid_blob_proof_bans_peer_and_marks_it_attempted() { + initialize_test_network_spec(); + let executor = ReamExecutor::new().expect("executor should start"); + let (mut peer_manager, peer_id) = test_peer_manager_with_one_peer(); + + let blob = Blob::default(); + let blob_bytes = blob.to_fixed_bytes(); + let raw_commitment = das_context() + .blob_to_kzg_commitment(&blob_bytes) + .expect("test blob should produce a commitment"); + let commitment = KZGCommitment(raw_commitment); + let proof = KZGProof::from( + compute_blob_kzg_proof_raw( + blob_bytes, + raw_commitment, + ream_polynomial_commitments::trusted_setup::blst_settings(), + ) + .expect("test blob should produce a proof") + .to_bytes(), + ); + let mut block = SignedBeaconBlock { + message: Default::default(), + signature: Default::default(), + }; + block + .message + .body + .blob_kzg_commitments + .push(commitment) + .expect("one commitment should fit"); + let block_root = block.message.tree_hash_root(); + let identifier = BlobIdentifier::new(block_root, 0); + let mut sidecar = block + .blob_sidecar(BlobAndProofV1 { blob, proof }, 0) + .expect("test sidecar should be constructed"); + sidecar.kzg_proof[0] ^= 1; + + let mut block_cache = BlockCache::new(B256::ZERO, 0); + block_cache + .add_blocks(vec![block], false, peer_id) + .expect("block should enter cache"); + + let handle = executor.spawn(async move { StreamOutcome::Complete(vec![sidecar]) }); + let mut tasks = vec![DownloadTask::new_blob_identifiers( + handle, + vec![identifier], + peer_id, + )]; + + poll_until_done( + &mut tasks, + &mut block_cache, + &mut peer_manager, + &HashSet::new(), + &mut false, + &[peer_id], + ); + + assert!( + block_cache + .attempted_peers_for(RequestKey::Blob(identifier)) + .contains(&peer_id), + "the dependency must be attributed to this peer so a retry picks someone else" + ); + assert!( + peer_manager.fetch_idle_peer_from(&[peer_id]).is_none(), + "a peer that served an invalid blob proof must be banned, not just marked idle" + ); + } + + #[test] + fn poll_ready_tasks_duplicate_blob_identifier_bans_peer_and_discards_whole_batch() { + initialize_test_network_spec(); + let executor = ReamExecutor::new().expect("executor should start"); + let (mut peer_manager, peer_id) = test_peer_manager_with_one_peer(); + + let blob = Blob::default(); + let blob_bytes = blob.to_fixed_bytes(); + let raw_commitment = das_context() + .blob_to_kzg_commitment(&blob_bytes) + .expect("test blob should produce a commitment"); + let commitment = KZGCommitment(raw_commitment); + let proof = KZGProof::from( + compute_blob_kzg_proof_raw( + blob_bytes, + raw_commitment, + ream_polynomial_commitments::trusted_setup::blst_settings(), + ) + .expect("test blob should produce a proof") + .to_bytes(), + ); + let mut block = SignedBeaconBlock { + message: Default::default(), + signature: Default::default(), + }; + block + .message + .body + .blob_kzg_commitments + .push(commitment) + .expect("one commitment should fit"); + let block_root = block.message.tree_hash_root(); + let identifier = BlobIdentifier::new(block_root, 0); + let sidecar = block + .blob_sidecar(BlobAndProofV1 { blob, proof }, 0) + .expect("test sidecar should be constructed"); + + let mut block_cache = BlockCache::new(B256::ZERO, 0); + block_cache + .add_blocks(vec![block], false, peer_id) + .expect("block should enter cache"); + + let handle = + executor.spawn(async move { StreamOutcome::Complete(vec![sidecar.clone(), sidecar]) }); + let mut tasks = vec![DownloadTask::new_blob_identifiers( + handle, + vec![identifier], + peer_id, + )]; + + poll_until_done( + &mut tasks, + &mut block_cache, + &mut peer_manager, + &HashSet::new(), + &mut false, + &[peer_id], + ); + + assert!( + block_cache + .attempted_peers_for(RequestKey::Blob(identifier)) + .contains(&peer_id), + "the requested identifier must still be attributed to this peer's failed attempt" + ); + assert!( + peer_manager.fetch_idle_peer_from(&[peer_id]).is_none(), + "a peer that returned a duplicate blob sidecar must be banned, not just marked idle" + ); + } + + #[test] + fn poll_ready_tasks_duplicate_data_column_bans_peer_and_discards_whole_batch() { + use ream_consensus_beacon::data_column_sidecar::get_data_column_sidecars_from_block; + + initialize_test_network_spec(); + let executor = ReamExecutor::new().expect("executor should start"); + let (mut peer_manager, peer_id) = test_peer_manager_with_one_peer(); + + let blob = Blob::default(); + let blob_bytes = blob.to_fixed_bytes(); + let raw_commitment = das_context() + .blob_to_kzg_commitment(&blob_bytes) + .expect("test blob should produce a commitment"); + let commitment = KZGCommitment(raw_commitment); + let mut block = SignedBeaconBlock { + message: Default::default(), + signature: Default::default(), + }; + block + .message + .body + .blob_kzg_commitments + .push(commitment) + .expect("one commitment should fit"); + let block_root = block.message.tree_hash_root(); + + let cells_and_kzg_proofs = compute_cells_and_kzg_proofs(&blob, das_context()) + .expect("test blob should produce cells and proofs"); + let columns = get_data_column_sidecars_from_block(&block, vec![cells_and_kzg_proofs]) + .expect("test block should produce data columns"); + let column = columns[0].clone(); + let identifier = ColumnIdentifier::new(block_root, column.index); + + let mut block_cache = BlockCache::new(B256::ZERO, 0); + block_cache + .add_blocks(vec![block], false, peer_id) + .expect("block should enter cache"); + + let handle = + executor.spawn(async move { StreamOutcome::Complete(vec![column.clone(), column]) }); + let mut tasks = vec![DownloadTask::new_data_column_identifiers( + handle, + vec![identifier], + peer_id, + )]; + + poll_until_done( + &mut tasks, + &mut block_cache, + &mut peer_manager, + &HashSet::from([identifier.index]), + &mut false, + &[peer_id], + ); + + assert!( + block_cache + .attempted_peers_for(RequestKey::Column(identifier)) + .contains(&peer_id), + "the requested identifier must still be attributed to this peer's failed attempt" + ); + assert!( + peer_manager.fetch_idle_peer_from(&[peer_id]).is_none(), + "a peer that returned a duplicate data column must be banned, not just marked idle" + ); + } + + fn test_frontier(id: FrontierId, anchor_root: B256, anchor_slot: u64) -> StuckFrontier { + StuckFrontier { + id, + anchor_root, + anchor_slot, + phase: id.key.phase, + scan_start_slot: id.key.scan_start_slot, + highest_observed_target: anchor_slot + 100, + consecutive_no_progress: 3, + attempted_peers: HashSet::new(), + cooldown_until: None, + recovery_round_not_before: None, + failed_candidates: HashSet::new(), + confirmed_empty_through: None, + } + } + + fn test_frontier_id(generation: u64) -> FrontierId { + FrontierId { + key: FrontierKey { + anchor_root: B256::ZERO, + phase: SyncPhase::Finalized, + scan_start_slot: 0, + }, + generation, + } + } + + #[test] + fn allocate_generation_never_repeats_across_interleaved_phase_resets() { + initialize_test_network_spec(); + let (_data_dir, beacon_chain) = test_beacon_chain(); + let executor = ReamExecutor::new().expect("executor should start"); + let (p2p_sender, _p2p_receiver) = tokio::sync::mpsc::unbounded_channel(); + let mut syncer = BlockRangeSyncer::new( + Arc::new(beacon_chain), + p2p_sender, + test_network_state(), + executor, + ); + + let mut seen = HashSet::new(); + for _ in 0..5 { + assert!(seen.insert(syncer.allocate_generation())); + assert!(seen.insert(syncer.allocate_generation())); + } + assert_eq!(seen.len(), 10, "every allocated generation must be unique"); + } + + #[test] + fn poll_ready_tasks_drops_a_stale_generation_block_range_result_without_peer_consequence() { + initialize_test_network_spec(); + let executor = ReamExecutor::new().expect("executor should start"); + let (mut peer_manager, peer_id) = test_peer_manager_with_one_peer(); + + let range = Range::new(11, 10); + let mut block_cache = BlockCache::new(B256::ZERO, 10); + block_cache.mark_block_range_in_progress(range); + let handle = executor.spawn(async move { + StreamOutcome::Complete(vec![SignedBeaconBlock { + message: BeaconBlock { + slot: 11, + ..Default::default() + }, + signature: Default::default(), + }]) + }); + let mut tasks = vec![DownloadTask::new_block_range( + handle, + range, + peer_id, + Some(test_frontier_id(0)), + )]; + + let mut finalized_frontier = Some(test_frontier(test_frontier_id(1), B256::ZERO, 10)); + let mut head_frontier = None; + let mut saw_empty_range = false; + let mut outcome = PollTasksOutcome::Settled; + for _ in 0..500 { + let mut frontiers = FrontierStore { + finalized: &mut finalized_frontier, + head: &mut head_frontier, + }; + outcome = poll_ready_tasks( + &mut tasks, + &mut block_cache, + &mut peer_manager, + &mut frontiers, + &HashSet::new(), + &mut saw_empty_range, + &[peer_id], + ) + .expect("poll_ready_tasks should not error"); + if tasks.is_empty() { + break; + } + std::thread::sleep(Duration::from_millis(2)); + } + + assert!(matches!(outcome, PollTasksOutcome::StaleGeneration)); + assert_eq!( + block_cache.block_count(), + 0, + "a stale-generation result must not be folded into the cache" + ); + assert!( + peer_manager.fetch_idle_peer_from(&[peer_id]).is_some(), + "a stale-generation result must not exclude or ban the peer that served it" + ); + } + + #[test] + fn ordinary_empty_range_fold_never_persists_coverage_when_the_live_cache_advance_fails() { + initialize_test_network_spec(); + let executor = ReamExecutor::new().expect("executor should start"); + let (mut peer_manager, peer_id) = test_peer_manager_with_one_peer(); + + let root = B256::ZERO; + let frontier_id = test_frontier_id(0); + let mut block_cache = BlockCache::new(root, 10); + block_cache + .add_blocks( + vec![SignedBeaconBlock { + message: BeaconBlock { + slot: 15, + parent_root: root, + ..Default::default() + }, + signature: Default::default(), + }], + true, + peer_id, + ) + .expect("a directly-connecting block should be accepted"); + + let range = Range::new(21, 10); + block_cache.mark_block_range_in_progress(range); + let handle = + executor.spawn(async move { StreamOutcome::Complete(Vec::::new()) }); + let mut tasks = vec![DownloadTask::new_block_range( + handle, + range, + peer_id, + Some(frontier_id), + )]; + + let mut finalized_frontier = Some(test_frontier(frontier_id, root, 10)); + let mut head_frontier = None; + let mut saw_empty_range = false; + for _ in 0..500 { + let mut frontiers = FrontierStore { + finalized: &mut finalized_frontier, + head: &mut head_frontier, + }; + poll_ready_tasks( + &mut tasks, + &mut block_cache, + &mut peer_manager, + &mut frontiers, + &HashSet::new(), + &mut saw_empty_range, + &[peer_id], + ) + .expect("poll_ready_tasks should not error"); + if tasks.is_empty() { + break; + } + std::thread::sleep(Duration::from_millis(2)); + } + + assert!( + finalized_frontier + .as_ref() + .expect("frontier should still exist") + .confirmed_empty_through + .is_none(), + "coverage must not be persisted when the live-cache advance itself failed -- doing \ + so anyway would assert something the cache never actually applied, and the range \ + isn't contiguous with what the cache holds (a real block sits before it)" + ); + } + + #[test] + fn coverage_divergence_implicates_the_confirming_peers_not_the_block_serving_peer() { + initialize_test_network_spec(); + let executor = ReamExecutor::new().expect("executor should start"); + let (mut peer_manager, block_serving_peer) = test_peer_manager_with_one_peer(); + let confirming_peer = PeerId::random(); + + let root = B256::ZERO; + let frontier_id = test_frontier_id(0); + let mut block_cache = BlockCache::new(root, 10); + block_cache + .advance_empty_coverage( + FrontierObservation { + anchor_root: root, + anchor_slot: 10, + phase: SyncPhase::Finalized, + scan_start_slot: 0, + target_slot: 200, + }, + root, + 20, + HashSet::from([confirming_peer]), + ) + .expect("advancing a pristine cache should succeed"); + + let range = Range::new(21, 10); + block_cache.mark_block_range_in_progress(range); + let wrong_parent = B256::repeat_byte(9); + let handle = executor.spawn(async move { + StreamOutcome::Complete(vec![SignedBeaconBlock { + message: BeaconBlock { + slot: 21, + parent_root: wrong_parent, + ..Default::default() + }, + signature: Default::default(), + }]) + }); + let mut tasks = vec![DownloadTask::new_block_range( + handle, + range, + block_serving_peer, + Some(frontier_id), + )]; + + let mut finalized_frontier = Some(test_frontier(frontier_id, root, 10)); + let mut head_frontier = None; + let mut saw_empty_range = false; + let mut outcome = PollTasksOutcome::Settled; + for _ in 0..500 { + let mut frontiers = FrontierStore { + finalized: &mut finalized_frontier, + head: &mut head_frontier, + }; + outcome = poll_ready_tasks( + &mut tasks, + &mut block_cache, + &mut peer_manager, + &mut frontiers, + &HashSet::new(), + &mut saw_empty_range, + &[block_serving_peer], + ) + .expect("poll_ready_tasks should not error"); + if tasks.is_empty() { + break; + } + std::thread::sleep(Duration::from_millis(2)); + } + + match outcome { + PollTasksOutcome::CoverageDivergence(divergence) => { + assert!(divergence.confirming_peers.contains(&confirming_peer)); + assert_eq!(divergence.non_connecting_peer, block_serving_peer); + assert!(!divergence.confirming_peers.contains(&block_serving_peer)); + } + _ => panic!("expected CoverageDivergence"), + } + assert!( + peer_manager + .fetch_idle_peer_from(&[block_serving_peer]) + .is_some(), + "the block-serving peer must not be banned -- it may be reporting the true chain" + ); + } + + #[test] + fn pending_conclusions_keep_finalized_and_head_evidence_independent() { + let mut pending = PendingConclusions::default(); + let finalized_observation = FrontierObservation { + anchor_root: B256::ZERO, + anchor_slot: 10, + phase: SyncPhase::Finalized, + scan_start_slot: 0, + target_slot: 100, + }; + let head_observation = FrontierObservation { + anchor_root: B256::ZERO, + anchor_slot: 10, + phase: SyncPhase::Head, + scan_start_slot: 0, + target_slot: 150, + }; + + pending.observe( + SyncPhase::Finalized, + finalized_observation, + RemoteNoProgressReason::SettledEmptyCoverage, + HashSet::new(), + HashSet::new(), + ); + pending.observe( + SyncPhase::Head, + head_observation, + RemoteNoProgressReason::CandidateExhausted, + HashSet::new(), + HashSet::new(), + ); + + assert!(pending.for_phase(SyncPhase::Finalized).is_some()); + assert!(pending.for_phase(SyncPhase::Head).is_some()); + assert_eq!( + pending + .for_phase(SyncPhase::Finalized) + .as_ref() + .expect("set above") + .reason, + RemoteNoProgressReason::SettledEmptyCoverage, + "a segment producing evidence for both phases must not let one silently overwrite the other" + ); + } + + #[test] + fn segment_exclusions_stay_scoped_to_their_own_phase() { + let mut exclusions = SegmentExclusions::default(); + let peer = PeerId::random(); + exclusions.for_phase_mut(SyncPhase::Finalized).insert(peer); + + assert!(exclusions.for_phase(SyncPhase::Finalized).contains(&peer)); + assert!( + !exclusions.for_phase(SyncPhase::Head).contains(&peer), + "a peer excluded for a false Finalized coverage claim must not also be excluded from Head" + ); + } + + #[test] + fn merge_confirmed_empty_coverage_never_persists_an_ancestor_walk_derived_advance() { + initialize_test_network_spec(); + let (_data_dir, beacon_chain) = test_beacon_chain(); + let executor = ReamExecutor::new().expect("executor should start"); + let (p2p_sender, _p2p_receiver) = tokio::sync::mpsc::unbounded_channel(); + let mut syncer = BlockRangeSyncer::new( + Arc::new(beacon_chain), + p2p_sender, + test_network_state(), + executor, + ); + + let root = B256::ZERO; + let frontier_id = test_frontier_id(0); + syncer.finalized_frontier = Some(test_frontier(frontier_id, root, 10)); + + let unproven = CoverageAdvance { + parent_root: B256::repeat_byte(7), + parent_slot: 500, + covered_through_slot: 500, + confirming_peers: HashSet::new(), + proven_empty: false, + }; + syncer.merge_confirmed_empty_coverage(SyncPhase::Finalized, &unproven); + assert!( + syncer + .finalized_frontier + .as_ref() + .expect("frontier should still exist") + .confirmed_empty_through + .is_none(), + "an unproven (ancestor-walk-derived) advance must never be persisted as confirmed-empty coverage" + ); + + let proven = CoverageAdvance { + parent_root: root, + parent_slot: 10, + covered_through_slot: 40, + confirming_peers: HashSet::new(), + proven_empty: true, + }; + syncer.merge_confirmed_empty_coverage(SyncPhase::Finalized, &proven); + assert!( + syncer + .finalized_frontier + .as_ref() + .expect("frontier should still exist") + .confirmed_empty_through + .is_some(), + "a proven-empty advance must be persisted" + ); + } + + #[test] + fn ordinary_empty_range_fold_does_not_skip_the_slot_immediately_after_the_checked_range() { + initialize_test_network_spec(); + let executor = ReamExecutor::new().expect("executor should start"); + let (mut peer_manager, peer_id) = test_peer_manager_with_one_peer(); + + let root = B256::ZERO; + let frontier_id = test_frontier_id(0); + let range = Range::new(11, 10); + let mut block_cache = BlockCache::new(root, 10); + block_cache.mark_block_range_in_progress(range); + let handle = + executor.spawn(async move { StreamOutcome::Complete(Vec::::new()) }); + let mut tasks = vec![DownloadTask::new_block_range( + handle, + range, + peer_id, + Some(frontier_id), + )]; + + let mut finalized_frontier = Some(test_frontier(frontier_id, root, 10)); + let mut head_frontier = None; + let mut saw_empty_range = false; + for _ in 0..500 { + let mut frontiers = FrontierStore { + finalized: &mut finalized_frontier, + head: &mut head_frontier, + }; + poll_ready_tasks( + &mut tasks, + &mut block_cache, + &mut peer_manager, + &mut frontiers, + &HashSet::new(), + &mut saw_empty_range, + &[peer_id], + ) + .expect("poll_ready_tasks should not error"); + if tasks.is_empty() { + break; + } + std::thread::sleep(Duration::from_millis(2)); + } + + assert_eq!( + block_cache.next_start_slot(), + 20, + "a [11, 21) empty response confirms up to slot 20 inclusive -- the cursor must \ + land there, not at 21, or the next dispatch (cursor + 1) would skip slot 21 \ + without ever checking it" + ); + let confirmed = finalized_frontier + .as_ref() + .expect("frontier should still exist") + .confirmed_empty_through + .as_ref() + .expect("an empty response under a tracked frontier must fold into persisted coverage"); + assert_eq!( + confirmed.end_slot_exclusive, 21, + "persisted coverage keeps its own exclusive-end convention independent of the \ + cache's inclusive cursor" + ); + } + + #[test] + fn cooldown_is_armed_once_not_re_extended_on_every_commit() { + initialize_test_network_spec(); + let (_data_dir, beacon_chain) = test_beacon_chain(); + let executor = ReamExecutor::new().expect("executor should start"); + let (p2p_sender, _p2p_receiver) = tokio::sync::mpsc::unbounded_channel(); + let mut syncer = BlockRangeSyncer::new( + Arc::new(beacon_chain), + p2p_sender, + test_network_state(), + executor, + ); + + let observation = FrontierObservation { + anchor_root: B256::ZERO, + anchor_slot: 10, + phase: SyncPhase::Finalized, + scan_start_slot: 0, + target_slot: 100, + }; + let peer = PeerId::random(); + let record = RemoteObservation { + observation: observation.clone(), + reason: RemoteNoProgressReason::ProbeNotFound, + implicated_peers: HashSet::from([peer]), + failed_candidates: HashSet::new(), + }; + + syncer.record_remote_no_progress(SyncPhase::Finalized, record.clone(), HashSet::new()); + let first_deadline = syncer + .finalized_frontier + .as_ref() + .expect("frontier should exist") + .cooldown_until + .expect("cooldown should be armed once a peer is attempted"); + + syncer.record_remote_no_progress(SyncPhase::Finalized, record, HashSet::new()); + let second_deadline = syncer + .finalized_frontier + .as_ref() + .expect("frontier should exist") + .cooldown_until + .expect("cooldown should still be armed"); + assert_eq!( + first_deadline, second_deadline, + "an already-armed cooldown must not be re-extended by a later commit, or it would \ + never elapse if segments repeat faster than the cooldown duration" + ); + } + + #[test] + fn refresh_attempt_round_clears_failed_candidates_alongside_attempted_peers() { + let mut frontier = test_frontier(test_frontier_id(0), B256::ZERO, 10); + frontier.attempted_peers.insert(PeerId::random()); + frontier.failed_candidates.insert(FailedRecoveryCandidate { + peer_id: PeerId::random(), + candidate_root: B256::repeat_byte(1), + target_bucket: 0, + }); + let now = Instant::now(); + frontier.cooldown_until = Some(now); + + frontier.refresh_attempt_round(now + Duration::from_secs(1)); + + assert!(frontier.attempted_peers.is_empty()); + assert!( + frontier.failed_candidates.is_empty(), + "failed_candidates must clear alongside attempted_peers on cooldown expiry, or a \ + candidate could stay excluded forever even after the peer cooldown reopens" + ); + assert!(frontier.cooldown_until.is_none()); } } diff --git a/crates/networking/syncer/src/block_range/peer_manager.rs b/crates/networking/syncer/src/block_range/peer_manager.rs index f2f0a209a..c404ed692 100644 --- a/crates/networking/syncer/src/block_range/peer_manager.rs +++ b/crates/networking/syncer/src/block_range/peer_manager.rs @@ -1,14 +1,54 @@ use std::{ - collections::{HashMap, hash_map::Entry}, + collections::{HashMap, HashSet, hash_map::Entry}, sync::Arc, - time::Instant, + time::{Duration, Instant}, }; use libp2p::PeerId; use ream_consensus_misc::constants::beacon::SLOTS_PER_EPOCH; use ream_p2p::network::beacon::{network_state::NetworkState, peer::CachedPeer}; +use ream_req_resp::beacon::messages::status::Status; use tracing::warn; +/// How long a peer stays banned before it becomes eligible to rejoin the peer set. +const BAN_DURATION: Duration = Duration::from_secs(300); + +pub const MIN_SYNC_PEERS: usize = 3; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TargetSelection { + Ready { + target_slot: u64, + eligible_peers: Vec, + }, + NoQuorum, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TargetQualification { + FinalizedEpoch(u64), + HeadEpoch(u64), +} + +/// Why a peer was banned. Still just bans outright either way, but structured (instead of a +/// free-text string) so a future scoring system can weigh severities without touching call sites. +#[derive(Debug, Clone)] +pub enum BanReason { + /// Disconnect, timeout, decode error, or a response that didn't fit the cache. + ProtocolError(String), + /// Failed KZG or inclusion proof verification. + InvalidProof(String), +} + +impl std::fmt::Display for BanReason { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + BanReason::ProtocolError(detail) => write!(f, "protocol error: {detail}"), + BanReason::InvalidProof(detail) => write!(f, "invalid proof: {detail}"), + } + } +} + #[derive(Debug, Clone)] pub enum PeerStatus { Idle, @@ -19,13 +59,15 @@ pub enum PeerStatus { pub struct PeerInfo { pub peer: CachedPeer, pub peer_status: PeerStatus, + pub processed_blocks: u64, + pub sync_requests_started: u64, } pub struct PeerManager { network_state: Arc, peers: HashMap, banned_peers: HashMap, - ban_reasons: HashMap, + ban_reasons: HashMap, } impl PeerManager { @@ -39,6 +81,12 @@ impl PeerManager { } pub fn update_peer_set(&mut self) { + let now = Instant::now(); + self.banned_peers + .retain(|_, banned_at| now.duration_since(*banned_at) < BAN_DURATION); + self.ban_reasons + .retain(|peer_id, _| self.banned_peers.contains_key(peer_id)); + let connected_peers = self.network_state.connected_peers(); for peer in &connected_peers { if self.banned_peers.contains_key(&peer.peer_id) { @@ -53,6 +101,8 @@ impl PeerManager { entry.insert(PeerInfo { peer: peer.clone(), peer_status: PeerStatus::Idle, + processed_blocks: 0, + sync_requests_started: 0, }); } } @@ -63,8 +113,7 @@ impl PeerManager { .retain(|peer_id, _| connected_peers.iter().any(|peer| peer.peer_id == *peer_id)); } - /// Bans a peer - pub fn ban_peer(&mut self, peer_id: &PeerId, reason: String) { + pub fn ban_peer(&mut self, peer_id: &PeerId, reason: BanReason) { self.ban_reasons.insert(*peer_id, reason); if let Some(peer_info) = self.peers.remove(peer_id) { self.banned_peers @@ -74,14 +123,48 @@ impl PeerManager { } } + fn reserve(&mut self, peer_id: &PeerId) -> Option { + let peer_info = self.peers.get_mut(peer_id)?; + if !matches!(peer_info.peer_status, PeerStatus::Idle) { + return None; + } + peer_info.peer_status = PeerStatus::Downloading; + peer_info.sync_requests_started += 1; + Some(peer_info.peer.clone()) + } + /// Fetches an idle peer from the peer set. /// /// Will set the peer status to `Downloading` if an idle peer is found. pub fn fetch_idle_peer(&mut self) -> Option { - for peer_info in self.peers.values_mut() { - if let PeerStatus::Idle = peer_info.peer_status { - peer_info.peer_status = PeerStatus::Downloading; - return Some(peer_info.peer.clone()); + let idle_peer_id = self + .peers + .iter() + .find(|(_, peer_info)| matches!(peer_info.peer_status, PeerStatus::Idle)) + .map(|(peer_id, _)| *peer_id)?; + self.reserve(&idle_peer_id) + } + + pub fn fetch_idle_peer_from(&mut self, eligible: &[PeerId]) -> Option { + for peer_id in eligible { + if let Some(peer) = self.reserve(peer_id) { + return Some(peer); + } + } + None + } + + pub fn fetch_idle_peer_from_excluding( + &mut self, + eligible: &[PeerId], + excluded: &HashSet, + ) -> Option { + for peer_id in eligible { + if excluded.contains(peer_id) { + continue; + } + if let Some(peer) = self.reserve(peer_id) { + return Some(peer); } } None @@ -109,20 +192,484 @@ impl PeerManager { } } - pub fn finalized_slot(&self) -> Option { - let mut frequencies = HashMap::new(); + pub fn record_processed_blocks(&mut self, peer_id: &PeerId, count: u64) { + if let Some(peer_info) = self.peers.get_mut(peer_id) { + peer_info.processed_blocks += count; + } + } - for peer in self.peers.values() { - if let Some(status) = &peer.peer.status { - *frequencies - .entry(status.finalized_epoch * SLOTS_PER_EPOCH) - .or_insert(0) += 1; + pub fn best_finalized(&self, our_finalized_epoch: u64) -> TargetSelection { + let mut votes: HashMap = HashMap::new(); + let mut candidates: Vec<(PeerId, u64, u64)> = Vec::new(); + + for (peer_id, peer_info) in &self.peers { + let Some(status) = &peer_info.peer.status else { + continue; + }; + if status.finalized_epoch < our_finalized_epoch { + continue; } + *votes.entry(status.finalized_epoch).or_insert(0) += 1; + candidates.push((*peer_id, status.finalized_epoch, status.head_slot)); } - frequencies + let Some(winner_epoch) = votes + .iter() + .max_by(|(epoch_a, votes_a), (epoch_b, votes_b)| { + votes_a.cmp(votes_b).then(epoch_a.cmp(epoch_b)) + }) + .map(|(&epoch, _)| epoch) + else { + return TargetSelection::NoQuorum; + }; + + let mut eligible: Vec<(PeerId, u64, u64)> = candidates .into_iter() - .max_by_key(|&(_, count)| count) - .map(|(slot, _)| slot) + .filter(|(_, epoch, _)| *epoch >= winner_epoch) + .collect(); + eligible.sort_by(|a, b| b.1.cmp(&a.1).then(b.2.cmp(&a.2))); + + TargetSelection::Ready { + target_slot: winner_epoch * SLOTS_PER_EPOCH, + eligible_peers: eligible.into_iter().map(|(id, ..)| id).collect(), + } + } + + pub fn best_non_finalized(&self, min_peers: usize, our_head_epoch: u64) -> TargetSelection { + let our_head_slot = our_head_epoch * SLOTS_PER_EPOCH; + let mut epoch_votes: HashMap = HashMap::new(); + let mut candidates: Vec<(PeerId, u64, u64)> = Vec::new(); + + for (peer_id, peer_info) in &self.peers { + let Some(status) = &peer_info.peer.status else { + continue; + }; + if status.head_slot <= our_head_slot { + continue; + } + let epoch = status.head_slot / SLOTS_PER_EPOCH; + *epoch_votes.entry(epoch).or_insert(0) += 1; + candidates.push((*peer_id, epoch, status.head_slot)); + } + + let Some(target_epoch) = epoch_votes + .iter() + .filter(|&(_, &votes)| votes >= min_peers) + .map(|(&epoch, _)| epoch) + .max() + else { + return TargetSelection::NoQuorum; + }; + + let eligible: Vec<(PeerId, u64)> = candidates + .into_iter() + .filter(|(_, epoch, _)| *epoch >= target_epoch) + .map(|(id, _, head_slot)| (id, head_slot)) + .collect(); + + let target_slot = eligible + .iter() + .filter(|(_, head_slot)| head_slot / SLOTS_PER_EPOCH == target_epoch) + .map(|(_, head_slot)| *head_slot) + .max() + .unwrap_or(target_epoch * SLOTS_PER_EPOCH); + + let mut eligible = eligible; + eligible.sort_by_key(|&(_, head_slot)| std::cmp::Reverse(head_slot)); + + TargetSelection::Ready { + target_slot, + eligible_peers: eligible.into_iter().map(|(id, _)| id).collect(), + } + } + + pub fn status_of(&self, peer_id: &PeerId) -> Option { + self.peers + .get(peer_id) + .and_then(|info| info.peer.status.clone()) + } + + pub fn exact_finalized_epoch_peers(&self, epoch: u64) -> Vec { + self.peers + .iter() + .filter(|(_, info)| { + info.peer + .status + .as_ref() + .is_some_and(|status| status.finalized_epoch == epoch) + }) + .map(|(peer_id, _)| *peer_id) + .collect() + } + + pub fn peers_satisfying(&self, qualification: TargetQualification) -> Vec { + self.peers + .iter() + .filter(|(_, info)| { + let Some(status) = &info.peer.status else { + return false; + }; + match qualification { + TargetQualification::FinalizedEpoch(epoch) => status.finalized_epoch >= epoch, + TargetQualification::HeadEpoch(epoch) => { + status.head_slot / SLOTS_PER_EPOCH >= epoch + } + } + }) + .map(|(peer_id, _)| *peer_id) + .collect() + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use discv5::{Enr, enr::CombinedKey}; + use parking_lot::RwLock; + use ream_peer::{ConnectionState, Direction}; + use ream_req_resp::beacon::messages::{meta_data::GetMetaDataV3, status::Status}; + + use super::*; + + fn test_network_state() -> Arc { + let enr_key = CombinedKey::generate_secp256k1(); + Arc::new(NetworkState { + local_enr: RwLock::new(Enr::builder().build(&enr_key).expect("valid enr")), + peer_table: RwLock::new(HashMap::new()), + meta_data: RwLock::new(GetMetaDataV3::default()), + status: RwLock::new(Status::default()), + data_dir: PathBuf::new(), + }) + } + + fn test_peer(status: Status) -> CachedPeer { + let mut peer = CachedPeer::new( + PeerId::random(), + None, + ConnectionState::Connected, + Direction::Outbound, + None, + ); + peer.status = Some(status); + peer + } + + fn insert_idle(peer_manager: &mut PeerManager, peer: CachedPeer) { + peer_manager.peers.insert( + peer.peer_id, + PeerInfo { + peer, + peer_status: PeerStatus::Idle, + processed_blocks: 0, + sync_requests_started: 0, + }, + ); + } + + #[test] + fn best_finalized_excludes_peers_behind_our_epoch() { + let mut peer_manager = PeerManager::new(test_network_state()); + insert_idle( + &mut peer_manager, + test_peer(Status { + finalized_epoch: 5, + ..Default::default() + }), + ); + + assert_eq!( + peer_manager.best_finalized(10), + TargetSelection::NoQuorum, + "the only peer is behind our own finalized epoch, so there's no target to chase" + ); + } + + #[test] + fn best_finalized_is_plurality_not_majority_with_tie_break_by_higher_epoch() { + let mut peer_manager = PeerManager::new(test_network_state()); + for _ in 0..2 { + insert_idle( + &mut peer_manager, + test_peer(Status { + finalized_epoch: 10, + ..Default::default() + }), + ); + } + insert_idle( + &mut peer_manager, + test_peer(Status { + finalized_epoch: 12, + ..Default::default() + }), + ); + + let TargetSelection::Ready { + target_slot, + eligible_peers, + } = peer_manager.best_finalized(0) + else { + panic!("expected a target"); + }; + assert_eq!(target_slot, 10 * SLOTS_PER_EPOCH); + assert_eq!(eligible_peers.len(), 3); + } + + #[test] + fn best_non_finalized_uses_a_threshold_not_a_plurality_vote() { + let mut peer_manager = PeerManager::new(test_network_state()); + for _ in 0..3 { + insert_idle( + &mut peer_manager, + test_peer(Status { + head_slot: 20 * SLOTS_PER_EPOCH, + ..Default::default() + }), + ); + } + insert_idle( + &mut peer_manager, + test_peer(Status { + head_slot: 21 * SLOTS_PER_EPOCH, + ..Default::default() + }), + ); + + let TargetSelection::Ready { target_slot, .. } = peer_manager.best_non_finalized(1, 0) + else { + panic!("expected a target"); + }; + assert_eq!(target_slot, 21 * SLOTS_PER_EPOCH); + } + + #[test] + fn best_non_finalized_returns_no_quorum_below_min_peers_even_with_peers_ahead() { + let mut peer_manager = PeerManager::new(test_network_state()); + insert_idle( + &mut peer_manager, + test_peer(Status { + head_slot: 20 * SLOTS_PER_EPOCH, + ..Default::default() + }), + ); + + assert_eq!( + peer_manager.best_non_finalized(3, 0), + TargetSelection::NoQuorum, + "one peer is ahead of us, but not enough to clear min_peers=3" + ); + } + + #[test] + fn best_non_finalized_refines_target_to_highest_actual_head_slot_in_winning_epoch() { + let mut peer_manager = PeerManager::new(test_network_state()); + insert_idle( + &mut peer_manager, + test_peer(Status { + head_slot: 20 * SLOTS_PER_EPOCH + 3, + ..Default::default() + }), + ); + insert_idle( + &mut peer_manager, + test_peer(Status { + head_slot: 20 * SLOTS_PER_EPOCH + 9, + ..Default::default() + }), + ); + + let TargetSelection::Ready { target_slot, .. } = peer_manager.best_non_finalized(1, 0) + else { + panic!("expected a target"); + }; + assert_eq!(target_slot, 20 * SLOTS_PER_EPOCH + 9); + } + + #[test] + fn fetch_idle_peer_from_returns_first_idle_in_given_order() { + let mut peer_manager = PeerManager::new(test_network_state()); + let first = test_peer(Status::default()); + let second = test_peer(Status::default()); + let first_id = first.peer_id; + let second_id = second.peer_id; + insert_idle(&mut peer_manager, first); + insert_idle(&mut peer_manager, second); + + let peer = peer_manager + .fetch_idle_peer_from(&[second_id, first_id]) + .expect("a peer should be found"); + assert_eq!(peer.peer_id, second_id); + + assert!( + peer_manager + .fetch_idle_peer_from(&[PeerId::random()]) + .is_none() + ); + } + + #[test] + fn fetch_idle_peer_from_excluding_skips_excluded_peers() { + let mut peer_manager = PeerManager::new(test_network_state()); + let peer = test_peer(Status::default()); + let peer_id = peer.peer_id; + insert_idle(&mut peer_manager, peer); + + let mut excluded = HashSet::new(); + excluded.insert(peer_id); + assert!( + peer_manager + .fetch_idle_peer_from_excluding(&[peer_id], &excluded) + .is_none() + ); + } + + #[test] + fn peers_satisfying_distinguishes_finalized_and_head_qualification() { + let mut peer_manager = PeerManager::new(test_network_state()); + let high_head_low_finalized = test_peer(Status { + head_slot: 1_000, + finalized_epoch: 1, + ..Default::default() + }); + let id = high_head_low_finalized.peer_id; + insert_idle(&mut peer_manager, high_head_low_finalized); + + assert!( + peer_manager + .peers_satisfying(TargetQualification::FinalizedEpoch(10)) + .is_empty() + ); + assert_eq!( + peer_manager.peers_satisfying(TargetQualification::HeadEpoch(0)), + vec![id] + ); + } + + #[test] + fn bans_expire_after_ban_duration() { + let mut peer_manager = PeerManager::new(test_network_state()); + let peer = test_peer(Status::default()); + let peer_id = peer.peer_id; + insert_idle(&mut peer_manager, peer); + + peer_manager.ban_peer(&peer_id, BanReason::ProtocolError("test".to_string())); + assert!(peer_manager.banned_peers.contains_key(&peer_id)); + + // Simulate the ban having happened BAN_DURATION ago. + peer_manager.banned_peers.insert( + peer_id, + Instant::now() - BAN_DURATION - Duration::from_secs(1), + ); + + peer_manager.update_peer_set(); + assert!(!peer_manager.banned_peers.contains_key(&peer_id)); + } + + #[test] + fn mark_peer_as_idle_recovers_a_stuck_downloading_peer() { + let mut peer_manager = PeerManager::new(test_network_state()); + let peer = test_peer(Status::default()); + let peer_id = peer.peer_id; + peer_manager.peers.insert( + peer_id, + PeerInfo { + peer, + peer_status: PeerStatus::Downloading, + processed_blocks: 0, + sync_requests_started: 0, + }, + ); + + peer_manager.mark_peer_as_idle(&peer_id); + + assert!(matches!( + peer_manager + .peers + .get(&peer_id) + .expect("peer exists") + .peer_status, + PeerStatus::Idle + )); + } + + #[test] + fn reserving_a_peer_credits_a_started_sync_request_every_time_including_after_reuse() { + let mut peer_manager = PeerManager::new(test_network_state()); + let peer = test_peer(Status::default()); + let peer_id = peer.peer_id; + insert_idle(&mut peer_manager, peer); + + peer_manager + .fetch_idle_peer_from(&[peer_id]) + .expect("peer is idle"); + assert_eq!( + peer_manager + .peers + .get(&peer_id) + .expect("peer exists") + .sync_requests_started, + 1 + ); + + assert!(peer_manager.fetch_idle_peer_from(&[peer_id]).is_none()); + assert_eq!( + peer_manager + .peers + .get(&peer_id) + .expect("peer exists") + .sync_requests_started, + 1 + ); + + peer_manager.mark_peer_as_idle(&peer_id); + peer_manager + .fetch_idle_peer_from_excluding(&[peer_id], &HashSet::new()) + .expect("peer is idle again"); + assert_eq!( + peer_manager + .peers + .get(&peer_id) + .expect("peer exists") + .sync_requests_started, + 2 + ); + } + + #[test] + fn record_processed_blocks_accumulates_and_ignores_unknown_peers() { + let mut peer_manager = PeerManager::new(test_network_state()); + let peer = test_peer(Status::default()); + let peer_id = peer.peer_id; + insert_idle(&mut peer_manager, peer); + + peer_manager.record_processed_blocks(&peer_id, 3); + peer_manager.record_processed_blocks(&peer_id, 2); + assert_eq!( + peer_manager + .peers + .get(&peer_id) + .expect("peer exists") + .processed_blocks, + 5 + ); + + peer_manager.record_processed_blocks(&PeerId::random(), 1); + } + + #[test] + fn peers_satisfying_head_epoch_includes_a_peer_below_the_refined_max_slot() { + let mut peer_manager = PeerManager::new(test_network_state()); + let below_refined_max = test_peer(Status { + head_slot: 20 * SLOTS_PER_EPOCH + 3, + ..Default::default() + }); + let id = below_refined_max.peer_id; + insert_idle(&mut peer_manager, below_refined_max); + + assert_eq!( + peer_manager.peers_satisfying(TargetQualification::HeadEpoch(20)), + vec![id] + ); } } diff --git a/crates/networking/syncer/src/block_range/peer_range_downloader.rs b/crates/networking/syncer/src/block_range/peer_range_downloader.rs index 9ea0976c2..c8c773de0 100644 --- a/crates/networking/syncer/src/block_range/peer_range_downloader.rs +++ b/crates/networking/syncer/src/block_range/peer_range_downloader.rs @@ -1,21 +1,26 @@ use alloy_primitives::B256; -use anyhow::bail; use libp2p::PeerId; use ream_consensus_beacon::{ blob_sidecar::{BlobIdentifier, BlobSidecar}, + data_column_sidecar::DataColumnSidecar, electra::beacon_block::SignedBeaconBlock, }; use ream_executor::ReamExecutor; -use ream_p2p::network::beacon::channel::{P2PCallbackResponse, P2PMessage, P2PRequest}; -use ream_req_resp::beacon::messages::BeaconResponseMessage; -use ssz::Encode; +use ream_p2p::network::beacon::channel::{ + P2PCallbackError, P2PCallbackResponse, P2PMessage, P2PRequest, +}; +use ream_req_resp::{ + beacon::messages::{BeaconResponseMessage, data_column_sidecars::DataColumnsByRootIdentifier}, + error::ReqRespError, + inbound_protocol::ResponseCode, +}; use tokio::{ sync::mpsc::{self, UnboundedSender}, task::JoinHandle, }; use tracing::info; -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct Range { pub start_slot: u64, pub count: u64, @@ -27,6 +32,70 @@ impl Range { } } +#[derive(Debug)] +pub enum DownloadFailure { + Transport(String), + InvalidData(String), + RemoteError { code: ResponseCode, message: String }, +} + +pub enum StreamOutcome { + Complete(Vec), + Failed(DownloadFailure), +} + +fn classify_req_resp_error(err: ReqRespError) -> DownloadFailure { + match err { + ReqRespError::RemoteError { code, message } => { + DownloadFailure::RemoteError { code, message } + } + ReqRespError::InvalidData(message) => DownloadFailure::InvalidData(message), + other => DownloadFailure::Transport(format!("{other:?}")), + } +} + +async fn drain_responses( + mut rx: mpsc::Receiver>, + mut extract: impl FnMut(BeaconResponseMessage) -> Result, +) -> StreamOutcome { + let mut items = vec![]; + + while let Some(response) = rx.recv().await { + match response { + Ok(P2PCallbackResponse::ResponseMessage(message)) => { + match extract(message.as_ref().clone()) { + Ok(item) => items.push(item), + Err(err) => return StreamOutcome::Failed(err), + } + } + Ok(P2PCallbackResponse::EndOfStream) => { + info!("End of request stream received."); + return StreamOutcome::Complete(items); + } + Ok(P2PCallbackResponse::Disconnected) => { + return StreamOutcome::Failed(DownloadFailure::Transport( + "peer disconnected while receiving response".to_string(), + )); + } + Ok(P2PCallbackResponse::Timeout) => { + return StreamOutcome::Failed(DownloadFailure::Transport( + "request timed out".to_string(), + )); + } + Err(P2PCallbackError::ReqResp(err)) => { + return StreamOutcome::Failed(classify_req_resp_error(err)); + } + Err(P2PCallbackError::Other(err)) => { + return StreamOutcome::Failed(DownloadFailure::Transport(format!("{err:?}"))); + } + } + } + + StreamOutcome::Failed(DownloadFailure::Transport( + "channel closed before EndOfStream".to_string(), + )) +} + pub struct PeerRangeDownloader; impl PeerRangeDownloader { @@ -35,10 +104,9 @@ impl PeerRangeDownloader { p2p_sender: UnboundedSender, executor: ReamExecutor, range: Range, - ) -> JoinHandle>>> { + ) -> JoinHandle>> { executor.spawn(async move { - let mut beacon_blocks = vec![]; - let (callback, mut rx) = mpsc::channel(100); + let (callback, rx) = mpsc::channel(100); p2p_sender .send(P2PMessage::Request(P2PRequest::BlockRange { peer_id, @@ -48,37 +116,13 @@ impl PeerRangeDownloader { })) .expect("Failed to send block range request"); - while let Some(response) = rx.recv().await { - match response { - Ok(P2PCallbackResponse::ResponseMessage(message)) => { - if let BeaconResponseMessage::BeaconBlocksByRange(blocks) = - message.as_ref().clone() - { - info!( - "Received block response with slot {} length {}", - blocks.message.slot, - blocks.as_ssz_bytes().len() - ); - beacon_blocks.push(blocks); - } - } - Ok(P2PCallbackResponse::EndOfStream) => { - info!("End of block range request stream received."); - break; - } - Ok(P2PCallbackResponse::Disconnected) => { - bail!("Peer disconnected while receiving block range."); - } - Ok(P2PCallbackResponse::Timeout) => { - bail!("Block range request timed out."); - } - Err(err) => { - info!("Error receiving BeaconBlocks from block range request: {err:?}"); - } - } - } - - Ok(beacon_blocks) + drain_responses(rx, |message| match message { + BeaconResponseMessage::BeaconBlocksByRange(block) => Ok(block), + other => Err(DownloadFailure::InvalidData(format!( + "unexpected response variant for a BlockRange request: {other:?}" + ))), + }) + .await }) } } @@ -91,49 +135,24 @@ impl PeerRootsDownloader { p2p_sender: UnboundedSender, executor: ReamExecutor, roots: Vec, - ) -> JoinHandle>>> { + ) -> JoinHandle>> { executor.spawn(async move { - let mut beacon_blocks = vec![]; - let (callback, mut rx) = mpsc::channel(100); + let (callback, rx) = mpsc::channel(100); p2p_sender .send(P2PMessage::Request(P2PRequest::BlockRoots { peer_id, - roots: roots.to_vec(), + roots, callback, })) .expect("Failed to send block roots request"); - while let Some(response) = rx.recv().await { - match response { - Ok(P2PCallbackResponse::ResponseMessage(message)) => { - if let BeaconResponseMessage::BeaconBlocksByRoot(blocks) = - message.as_ref().clone() - { - info!( - "Received block response with slot {} length {}", - blocks.message.slot, - blocks.as_ssz_bytes().len() - ); - beacon_blocks.push(blocks); - } - } - Ok(P2PCallbackResponse::EndOfStream) => { - info!("End of block roots request stream received."); - break; - } - Ok(P2PCallbackResponse::Disconnected) => { - bail!("Peer disconnected while receiving block roots."); - } - Ok(P2PCallbackResponse::Timeout) => { - bail!("Block roots request timed out."); - } - Err(err) => { - info!("Error receiving BeaconBlocks from block roots request: {err:?}"); - } - } - } - - Ok(beacon_blocks) + drain_responses(rx, |message| match message { + BeaconResponseMessage::BeaconBlocksByRoot(block) => Ok(block), + other => Err(DownloadFailure::InvalidData(format!( + "unexpected response variant for a BlockRoots request: {other:?}" + ))), + }) + .await }) } } @@ -146,49 +165,245 @@ impl PeerBlobIdentifierDownloader { p2p_sender: UnboundedSender, executor: ReamExecutor, blob_identifiers: Vec, - ) -> JoinHandle>>> { + ) -> JoinHandle>> { executor.spawn(async move { - let mut blob_sidecars = vec![]; - let (callback, mut rx) = mpsc::channel(100); + let (callback, rx) = mpsc::channel(100); p2p_sender .send(P2PMessage::Request(P2PRequest::BlobIdentifiers { peer_id, - blob_identifiers: blob_identifiers.to_vec(), + blob_identifiers, callback, })) .expect("Failed to send blob identifiers request"); - while let Some(response) = rx.recv().await { - match response { - Ok(P2PCallbackResponse::ResponseMessage(message)) => { - if let BeaconResponseMessage::BlobSidecarsByRoot(blob_sidecar) = - message.as_ref().clone() - { - info!( - "Received blob sidecar response with index {} length {}", - blob_sidecar.index, - blob_sidecar.as_ssz_bytes().len() - ); - blob_sidecars.push(blob_sidecar); - } - } - Ok(P2PCallbackResponse::EndOfStream) => { - info!("End of blob roots request stream received."); - break; - } - Ok(P2PCallbackResponse::Disconnected) => { - bail!("Peer disconnected while receiving blob sidecars."); - } - Ok(P2PCallbackResponse::Timeout) => { - bail!("Blob identifiers request timed out."); - } - Err(err) => { - info!("Error receiving blobs from blob roots request: {err:?}"); - } - } - } + drain_responses(rx, |message| match message { + BeaconResponseMessage::BlobSidecarsByRoot(blob_sidecar) => Ok(blob_sidecar), + other => Err(DownloadFailure::InvalidData(format!( + "unexpected response variant for a BlobIdentifiers request: {other:?}" + ))), + }) + .await + }) + } +} + +pub struct PeerDataColumnRangeDownloader; + +impl PeerDataColumnRangeDownloader { + pub fn start( + peer_id: PeerId, + p2p_sender: UnboundedSender, + executor: ReamExecutor, + range: Range, + columns: Vec, + ) -> JoinHandle>> { + executor.spawn(async move { + let (callback, rx) = mpsc::channel(100); + p2p_sender + .send(P2PMessage::Request(P2PRequest::DataColumnRange { + peer_id, + start: range.start_slot, + count: range.count, + columns, + callback, + })) + .expect("Failed to send data column range request"); + + drain_responses(rx, |message| match message { + BeaconResponseMessage::DataColumnSidecarsByRange(column) => Ok(column), + other => Err(DownloadFailure::InvalidData(format!( + "unexpected response variant for a DataColumnRange request: {other:?}" + ))), + }) + .await + }) + } +} + +pub struct PeerDataColumnIdentifierDownloader; + +impl PeerDataColumnIdentifierDownloader { + pub fn start( + peer_id: PeerId, + p2p_sender: UnboundedSender, + executor: ReamExecutor, + identifiers: Vec, + ) -> JoinHandle>> { + executor.spawn(async move { + let (callback, rx) = mpsc::channel(100); + p2p_sender + .send(P2PMessage::Request(P2PRequest::DataColumnIdentifiers { + peer_id, + column_identifiers: identifiers, + callback, + })) + .expect("Failed to send data column identifiers request"); + + drain_responses(rx, |message| match message { + BeaconResponseMessage::DataColumnSidecarsByRoot(column) => Ok(column), + other => Err(DownloadFailure::InvalidData(format!( + "unexpected response variant for a DataColumnIdentifiers request: {other:?}" + ))), + }) + .await + }) + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use super::*; + + fn test_block() -> SignedBeaconBlock { + SignedBeaconBlock { + message: Default::default(), + signature: Default::default(), + } + } + + #[tokio::test] + async fn drain_responses_rejects_an_unexpected_variant_as_invalid_data() { + let (tx, rx) = mpsc::channel(10); + tx.send(Ok(P2PCallbackResponse::ResponseMessage(Arc::new( + BeaconResponseMessage::BeaconBlocksByRange(test_block()), + )))) + .await + .expect("send should succeed"); + tx.send(Ok(P2PCallbackResponse::ResponseMessage(Arc::new( + BeaconResponseMessage::BeaconBlocksByRoot(test_block()), + )))) + .await + .expect("send should succeed"); + + let outcome = drain_responses(rx, |message| match message { + BeaconResponseMessage::BeaconBlocksByRange(block) => Ok(block), + other => Err(DownloadFailure::InvalidData(format!( + "unexpected: {other:?}" + ))), + }) + .await; - Ok(blob_sidecars) + assert!(matches!( + outcome, + StreamOutcome::Failed(DownloadFailure::InvalidData(_)) + )); + } + + #[tokio::test] + async fn drain_responses_completes_on_clean_end_of_stream() { + let (tx, rx) = mpsc::channel(10); + tx.send(Ok(P2PCallbackResponse::ResponseMessage(Arc::new( + BeaconResponseMessage::BeaconBlocksByRange(test_block()), + )))) + .await + .expect("send should succeed"); + tx.send(Ok(P2PCallbackResponse::EndOfStream)) + .await + .expect("send should succeed"); + + let outcome = drain_responses(rx, |message| match message { + BeaconResponseMessage::BeaconBlocksByRange(block) => Ok(block), + other => Err(DownloadFailure::InvalidData(format!( + "unexpected: {other:?}" + ))), + }) + .await; + + let StreamOutcome::Complete(items) = outcome else { + panic!("expected a complete outcome"); + }; + assert_eq!(items.len(), 1); + } + + #[tokio::test] + async fn drain_responses_treats_channel_close_without_end_of_stream_as_transport_failure() { + let (tx, rx) = mpsc::channel(10); + tx.send(Ok(P2PCallbackResponse::ResponseMessage(Arc::new( + BeaconResponseMessage::BeaconBlocksByRange(test_block()), + )))) + .await + .expect("send should succeed"); + drop(tx); + + let outcome = drain_responses(rx, |message| match message { + BeaconResponseMessage::BeaconBlocksByRange(block) => Ok(block), + other => Err(DownloadFailure::InvalidData(format!( + "unexpected: {other:?}" + ))), + }) + .await; + + assert!(matches!( + outcome, + StreamOutcome::Failed(DownloadFailure::Transport(_)) + )); + } + + #[tokio::test] + async fn drain_responses_fails_on_disconnect_without_banning() { + let (tx, rx) = mpsc::channel(10); + tx.send(Ok(P2PCallbackResponse::Disconnected)) + .await + .expect("send should succeed"); + + let outcome = drain_responses(rx, |message| match message { + BeaconResponseMessage::BeaconBlocksByRange(block) => Ok(block), + other => Err(DownloadFailure::InvalidData(format!( + "unexpected: {other:?}" + ))), + }) + .await; + + assert!(matches!( + outcome, + StreamOutcome::Failed(DownloadFailure::Transport(_)) + )); + } + + #[tokio::test] + async fn drain_responses_fails_on_timeout_without_banning() { + let (tx, rx) = mpsc::channel(10); + tx.send(Ok(P2PCallbackResponse::Timeout)) + .await + .expect("send should succeed"); + + let outcome = drain_responses(rx, |message| match message { + BeaconResponseMessage::BeaconBlocksByRange(block) => Ok(block), + other => Err(DownloadFailure::InvalidData(format!( + "unexpected: {other:?}" + ))), }) + .await; + + assert!(matches!( + outcome, + StreamOutcome::Failed(DownloadFailure::Transport(_)) + )); + } + + #[tokio::test] + async fn drain_responses_classifies_remote_error_separately_from_invalid_data() { + let (tx, rx) = mpsc::channel(10); + tx.send(Err(P2PCallbackError::ReqResp(ReqRespError::RemoteError { + code: ResponseCode::ResourceUnavailable, + message: "no data for this range".to_string(), + }))) + .await + .expect("send should succeed"); + + let outcome = drain_responses(rx, |message| match message { + BeaconResponseMessage::BeaconBlocksByRange(block) => Ok(block), + other => Err(DownloadFailure::InvalidData(format!( + "unexpected: {other:?}" + ))), + }) + .await; + + assert!(matches!( + outcome, + StreamOutcome::Failed(DownloadFailure::RemoteError { .. }) + )); } } diff --git a/crates/networking/syncer/src/block_range/recovery.rs b/crates/networking/syncer/src/block_range/recovery.rs new file mode 100644 index 000000000..d6519dd0c --- /dev/null +++ b/crates/networking/syncer/src/block_range/recovery.rs @@ -0,0 +1,844 @@ +use std::{collections::HashSet, time::Instant}; + +use alloy_primitives::B256; +use libp2p::PeerId; +use ream_consensus_beacon::electra::beacon_block::SignedBeaconBlock; +use ream_consensus_misc::{constants::beacon::SLOTS_PER_EPOCH, misc::compute_start_slot_at_epoch}; +use ream_fork_choice_beacon::store::Store; +use ream_storage::tables::{field::REDBField, table::REDBTable}; +use tokio::time::Duration; +use tracing::warn; +use tree_hash::TreeHash; + +use super::{ + BanReason, FailedRecoveryCandidate, FrontierObservation, PeerManager, RemoteNoProgressReason, + SyncPhase, + block_cache::validate_range_chain, + peer_range_downloader::{PeerRangeDownloader, PeerRootsDownloader, Range, StreamOutcome}, + target_bucket, +}; +use crate::block_range::BlockRangeSyncer; + +pub(super) const MAX_RECOVERY_PEERS_PER_ROUND: usize = 3; +pub(super) const MAX_PROBE_REQUESTS_PER_ROUND: usize = 10; +pub(super) const MAX_TOTAL_ANCESTOR_REQUESTS_PER_ROUND: u64 = 128; +pub(super) const MAX_RECOVERY_SEED_BLOCKS: usize = 48; +pub(super) const RECOVERY_ROUND_TIMEOUT: Duration = Duration::from_secs(60); + +struct RecoveryBudget { + deadline: Instant, + requests_used: u64, +} + +impl RecoveryBudget { + fn new() -> Self { + Self { + deadline: Instant::now() + RECOVERY_ROUND_TIMEOUT, + requests_used: 0, + } + } + + fn has_capacity(&self) -> bool { + Instant::now() < self.deadline && self.requests_used < MAX_TOTAL_ANCESTOR_REQUESTS_PER_ROUND + } + + fn consume(&mut self) { + self.requests_used += 1; + } +} + +pub(super) struct CoverageAdvance { + pub parent_root: B256, + pub parent_slot: u64, + pub covered_through_slot: u64, + pub confirming_peers: HashSet, + pub proven_empty: bool, +} + +pub(super) struct RecoverySeed { + pub(super) ancestor_root: B256, + pub(super) ancestor_slot: u64, + pub(super) forward_blocks: Vec, + pub(super) target_slot: u64, + pub(super) source_peer: PeerId, +} + +pub(super) enum RecoveryOutcome { + AdvancedCoverage(CoverageAdvance), + Seeded(RecoverySeed), + NoProgress { + reason: RemoteNoProgressReason, + implicated_peers: HashSet, + failed_candidates: HashSet, + }, +} + +fn no_progress( + reason: RemoteNoProgressReason, + implicated_peers: HashSet, +) -> anyhow::Result { + Ok(RecoveryOutcome::NoProgress { + reason, + implicated_peers, + failed_candidates: HashSet::new(), + }) +} + +fn budget_exhausted(implicated_peers: HashSet) -> anyhow::Result { + no_progress( + RemoteNoProgressReason::RecoveryBudgetExhausted, + implicated_peers, + ) +} + +fn is_processable_connection_point( + store: &Store, + parent_root: B256, + first_child_slot: u64, +) -> anyhow::Result { + if store.db.block_provider().get(parent_root)?.is_none() { + return Ok(false); + } + if store.db.state_provider().get(parent_root)?.is_none() { + return Ok(false); + } + let finalized_checkpoint = store.db.finalized_checkpoint_provider().get()?; + let finalized_slot = compute_start_slot_at_epoch(finalized_checkpoint.epoch); + if first_child_slot <= finalized_slot { + return Ok(false); + } + let checkpoint_block = store.get_checkpoint_block(parent_root, finalized_checkpoint.epoch)?; + Ok(checkpoint_block == finalized_checkpoint.root) +} + +enum ProbeStep { + Empty, + Found(Vec), + Failed, +} + +fn reserve_round_robin( + peer_manager: &mut PeerManager, + peers: &[PeerId], + excluded: &HashSet, + start_index: usize, +) -> Option<(PeerId, usize)> { + if peers.is_empty() { + return None; + } + for offset in 0..peers.len() { + let candidate = peers[(start_index + offset) % peers.len()]; + if excluded.contains(&candidate) { + continue; + } + if let Some(peer) = + peer_manager.fetch_idle_peer_from_excluding(&[candidate], &HashSet::new()) + { + return Some((peer.peer_id, offset + 1)); + } + } + None +} + +impl BlockRangeSyncer { + pub(super) async fn run_recovery( + &mut self, + observation: &FrontierObservation, + candidate_peers: &[PeerId], + excluded: &HashSet, + ) -> anyhow::Result { + let mut budget = RecoveryBudget::new(); + let tier = self + .frontier_for(observation.phase) + .as_ref() + .map(|frontier| frontier.tier()) + .unwrap_or(0); + + let baseline_end = self + .frontier_for(observation.phase) + .as_ref() + .and_then(|frontier| frontier.confirmed_empty_through.as_ref()) + .map(|coverage| coverage.end_slot_exclusive) + .unwrap_or(observation.anchor_slot + 1); + + let probe_peers: Vec = candidate_peers + .iter() + .filter(|peer_id| !excluded.contains(peer_id)) + .take(MAX_RECOVERY_PEERS_PER_ROUND) + .copied() + .collect(); + + if probe_peers.is_empty() { + return no_progress(RemoteNoProgressReason::ProbeNotFound, HashSet::new()); + } + + let mut implicated_peers: HashSet = HashSet::new(); + let mut cursor = baseline_end; + let mut confirmed_end: Option<(u64, HashSet)> = None; + let mut found: Option<(Vec, PeerId)> = None; + let mut failed_this_round: HashSet = HashSet::new(); + let mut next_peer_index = 0usize; + + for _ in 0..MAX_PROBE_REQUESTS_PER_ROUND { + if !budget.has_capacity() || cursor > observation.target_slot { + break; + } + let Some((peer_id, advance_by)) = reserve_round_robin( + &mut self.peer_manager, + &probe_peers, + &failed_this_round, + next_peer_index, + ) else { + break; + }; + next_peer_index = (next_peer_index + advance_by) % probe_peers.len(); + let window = SLOTS_PER_EPOCH.min(observation.target_slot.saturating_add(1) - cursor); + let range = Range::new(cursor, window); + budget.consume(); + + match self + .probe_range(peer_id, range, &mut implicated_peers) + .await + { + ProbeStep::Empty => { + match &mut confirmed_end { + Some((end, peers)) => { + *end = cursor + window; + peers.insert(peer_id); + } + None => confirmed_end = Some((cursor + window, HashSet::from([peer_id]))), + } + cursor += window; + } + ProbeStep::Found(blocks) => { + found = Some((blocks, peer_id)); + break; + } + ProbeStep::Failed => { + failed_this_round.insert(peer_id); + } + } + } + + if let Some((blocks, peer_id)) = found { + return self + .resolve_candidate( + observation, + blocks, + HashSet::from([peer_id]), + &probe_peers, + &mut budget, + implicated_peers, + ) + .await; + } + + if let Some((end, peers)) = confirmed_end + && end > baseline_end + { + return Ok(RecoveryOutcome::AdvancedCoverage(CoverageAdvance { + parent_root: observation.anchor_root, + parent_slot: observation.anchor_slot, + covered_through_slot: end - 1, + proven_empty: true, + confirming_peers: peers, + })); + } + + if tier < 3 { + return no_progress(RemoteNoProgressReason::ProbeNotFound, implicated_peers); + } + + self.run_fallback( + observation, + candidate_peers, + excluded, + &mut budget, + implicated_peers, + ) + .await + } + + async fn probe_range( + &mut self, + peer_id: PeerId, + range: Range, + implicated_peers: &mut HashSet, + ) -> ProbeStep { + let handle = PeerRangeDownloader::start( + peer_id, + self.p2p_sender.clone(), + self.executor.clone(), + range, + ); + let result = handle.await; + let idle = IdleOnDrop::new(&mut self.peer_manager, peer_id); + let outcome = match result { + Ok(Ok(outcome)) => outcome, + Ok(Err(err)) => { + warn!("Recovery probe task failed: {err}"); + implicated_peers.insert(peer_id); + return ProbeStep::Failed; + } + Err(err) => { + warn!("Recovery probe task panicked: {err}"); + implicated_peers.insert(peer_id); + return ProbeStep::Failed; + } + }; + drop(idle); + + match outcome { + StreamOutcome::Complete(blocks) if blocks.is_empty() => { + implicated_peers.insert(peer_id); + ProbeStep::Empty + } + StreamOutcome::Complete(blocks) => { + let range_end = range.start_slot + range.count; + let out_of_range = blocks.iter().any(|block| { + block.message.slot < range.start_slot || block.message.slot >= range_end + }); + if out_of_range || validate_range_chain(&blocks).is_err() { + self.peer_manager.ban_peer( + &peer_id, + BanReason::ProtocolError( + "block(s) outside the requested range or not a valid chain".to_string(), + ), + ); + return ProbeStep::Failed; + } + ProbeStep::Found(blocks) + } + StreamOutcome::Failed(_) => { + implicated_peers.insert(peer_id); + ProbeStep::Failed + } + } + } + + async fn run_fallback( + &mut self, + observation: &FrontierObservation, + candidate_peers: &[PeerId], + excluded: &HashSet, + budget: &mut RecoveryBudget, + implicated_peers: HashSet, + ) -> anyhow::Result { + if !budget.has_capacity() { + return budget_exhausted(implicated_peers); + } + match observation.phase { + SyncPhase::Head => { + let eligible: Vec = candidate_peers + .iter() + .filter(|peer_id| !excluded.contains(peer_id)) + .copied() + .collect(); + let Some(peer) = self + .peer_manager + .fetch_idle_peer_from_excluding(&eligible, &HashSet::new()) + else { + return no_progress(RemoteNoProgressReason::AncestorNotFound, implicated_peers); + }; + let peer_id = peer.peer_id; + let end_exclusive = observation.target_slot.saturating_add(1); + let window = SLOTS_PER_EPOCH.min(end_exclusive); + let start = end_exclusive.saturating_sub(window); + let range = Range::new(start, window.max(1)); + let mut implicated = implicated_peers; + budget.consume(); + match self.probe_range(peer_id, range, &mut implicated).await { + ProbeStep::Found(blocks) => { + self.resolve_candidate( + observation, + blocks, + HashSet::from([peer_id]), + &eligible, + budget, + implicated, + ) + .await + } + _ => no_progress(RemoteNoProgressReason::AncestorNotFound, implicated), + } + } + SyncPhase::Finalized => { + let epoch = observation.target_slot / SLOTS_PER_EPOCH; + let bucket = target_bucket(observation.target_slot); + let previously_failed = self + .frontier_for(observation.phase) + .as_ref() + .map(|frontier| frontier.failed_candidates.clone()) + .unwrap_or_default(); + + let exact_epoch_peers = self.peer_manager.exact_finalized_epoch_peers(epoch); + let ordered_peers = self.group_by_finalized_root_agreement(&exact_epoch_peers); + + let mut implicated = implicated_peers; + let mut failed_candidates = HashSet::new(); + + for peer_id in &ordered_peers { + let peer_id = *peer_id; + if !budget.has_capacity() { + break; + } + if excluded.contains(&peer_id) { + continue; + } + let Some(status) = self.peer_manager.status_of(&peer_id) else { + continue; + }; + let candidate = FailedRecoveryCandidate { + peer_id, + candidate_root: status.finalized_root, + target_bucket: bucket, + }; + if previously_failed.contains(&candidate) + || failed_candidates.contains(&candidate) + { + continue; + } + let Some(reserved) = self + .peer_manager + .fetch_idle_peer_from_excluding(&[peer_id], &HashSet::new()) + else { + continue; + }; + debug_assert_eq!(reserved.peer_id, peer_id); + + budget.consume(); + let Some(block) = self + .fetch_single_root(peer_id, status.finalized_root, &mut implicated) + .await + else { + failed_candidates.insert(candidate); + continue; + }; + + let required_backtrack_floor = block + .message + .slot + .saturating_sub(MAX_TOTAL_ANCESTOR_REQUESTS_PER_ROUND); + if status.earliest_available_slot > required_backtrack_floor { + implicated.insert(peer_id); + failed_candidates.insert(candidate); + continue; + } + + match self + .resolve_candidate( + observation, + vec![block], + HashSet::from([peer_id]), + &ordered_peers, + budget, + implicated, + ) + .await? + { + RecoveryOutcome::NoProgress { + reason, + implicated_peers: inner_implicated, + failed_candidates: mut inner_failed, + } => { + inner_failed.insert(candidate); + inner_failed.extend(failed_candidates.iter().copied()); + if reason == RemoteNoProgressReason::RecoveryBudgetExhausted { + return Ok(RecoveryOutcome::NoProgress { + reason, + implicated_peers: inner_implicated, + failed_candidates: inner_failed, + }); + } + implicated = inner_implicated; + failed_candidates = inner_failed; + } + other => return Ok(other), + } + } + + let reason = if budget.has_capacity() { + RemoteNoProgressReason::AncestorNotFound + } else { + RemoteNoProgressReason::RecoveryBudgetExhausted + }; + Ok(RecoveryOutcome::NoProgress { + reason, + implicated_peers: implicated, + failed_candidates, + }) + } + } + } + + fn group_by_finalized_root_agreement(&self, peers: &[PeerId]) -> Vec { + let mut by_root: std::collections::BTreeMap> = + std::collections::BTreeMap::new(); + for &peer_id in peers { + let Some(status) = self.peer_manager.status_of(&peer_id) else { + continue; + }; + by_root + .entry(status.finalized_root) + .or_default() + .push(peer_id); + } + for group in by_root.values_mut() { + group.sort(); + } + let mut groups: Vec<(B256, Vec)> = by_root.into_iter().collect(); + groups.sort_by(|(root_a, group_a), (root_b, group_b)| { + group_b.len().cmp(&group_a.len()).then(root_a.cmp(root_b)) + }); + groups.into_iter().flat_map(|(_, group)| group).collect() + } + + async fn fetch_single_root( + &mut self, + peer_id: PeerId, + root: B256, + implicated_peers: &mut HashSet, + ) -> Option { + let handle = PeerRootsDownloader::start( + peer_id, + self.p2p_sender.clone(), + self.executor.clone(), + vec![root], + ); + let result = handle.await; + let idle = IdleOnDrop::new(&mut self.peer_manager, peer_id); + let outcome = match result { + Ok(Ok(outcome)) => outcome, + _ => { + drop(idle); + implicated_peers.insert(peer_id); + return None; + } + }; + drop(idle); + + match outcome { + StreamOutcome::Complete(blocks) if blocks.len() == 1 => { + let block = &blocks[0]; + if block.message.tree_hash_root() == root { + Some(blocks.into_iter().next().expect("checked len == 1 above")) + } else { + self.peer_manager.ban_peer( + &peer_id, + BanReason::ProtocolError("returned an unrequested root".to_string()), + ); + None + } + } + StreamOutcome::Complete(_) => { + implicated_peers.insert(peer_id); + None + } + StreamOutcome::Failed(_) => { + implicated_peers.insert(peer_id); + None + } + } + } + + async fn resolve_candidate( + &mut self, + observation: &FrontierObservation, + blocks: Vec, + confirming_peers: HashSet, + fallback_peers: &[PeerId], + budget: &mut RecoveryBudget, + implicated_peers: HashSet, + ) -> anyhow::Result { + let mut blocks = blocks; + blocks.sort_by_key(|block| block.message.slot); + + let primary_peer = *confirming_peers + .iter() + .next() + .expect("resolve_candidate is always called with a non-empty confirming_peers set"); + + let leading_parent = blocks[0].message.parent_root; + let leading_slot = blocks[0].message.slot; + + let connected_at = { + let store = self.beacon_chain.store.lock().await; + if is_processable_connection_point(&store, leading_parent, leading_slot)? { + store + .db + .block_provider() + .get(leading_parent)? + .map(|block| block.message.slot) + } else { + None + } + }; + + if let Some(leading_parent_slot) = connected_at { + return self + .trim_and_seed( + observation, + (leading_parent, leading_parent_slot), + blocks, + confirming_peers, + implicated_peers, + primary_peer, + ) + .await; + } + + if !budget.has_capacity() { + return budget_exhausted(implicated_peers); + } + + let finalized_epoch_start = { + let store = self.beacon_chain.store.lock().await; + compute_start_slot_at_epoch(store.db.finalized_checkpoint_provider().get()?.epoch) + }; + + let mut chain: Vec = blocks; + let mut visited: HashSet = HashSet::from([leading_parent]); + let mut cursor_root = leading_parent; + let mut cursor_slot = leading_slot; + let mut implicated_peers = implicated_peers; + let ancestor_walk_peers: Vec = confirming_peers + .iter() + .copied() + .chain(fallback_peers.iter().copied()) + .collect::>() + .into_iter() + .collect(); + let mut next_peer_index = 0usize; + let mut hop_failed: HashSet = HashSet::new(); + + while budget.has_capacity() && cursor_slot > finalized_epoch_start { + let Some((peer_id, advance_by)) = reserve_round_robin( + &mut self.peer_manager, + &ancestor_walk_peers, + &hop_failed, + next_peer_index, + ) else { + return no_progress(RemoteNoProgressReason::AncestorNotFound, implicated_peers); + }; + next_peer_index = (next_peer_index + advance_by) % ancestor_walk_peers.len(); + budget.consume(); + let Some(cursor_block) = self + .fetch_single_root(peer_id, cursor_root, &mut implicated_peers) + .await + else { + hop_failed.insert(peer_id); + continue; + }; + hop_failed.clear(); + if cursor_block.message.slot >= cursor_slot { + self.peer_manager.ban_peer( + &peer_id, + BanReason::ProtocolError( + "ancestor walk did not strictly decrease slot".to_string(), + ), + ); + return no_progress(RemoteNoProgressReason::AncestorNotFound, implicated_peers); + } + let grandparent_root = cursor_block.message.parent_root; + let first_child_slot = cursor_block.message.slot; + let grandparent_slot = { + let store = self.beacon_chain.store.lock().await; + if is_processable_connection_point(&store, grandparent_root, first_child_slot)? { + store + .db + .block_provider() + .get(grandparent_root)? + .map(|block| block.message.slot) + } else { + None + } + }; + chain.insert(0, cursor_block); + if let Some(grandparent_slot) = grandparent_slot { + return self + .trim_and_seed( + observation, + (grandparent_root, grandparent_slot), + chain, + confirming_peers, + implicated_peers, + peer_id, + ) + .await; + } + if !visited.insert(grandparent_root) { + return no_progress(RemoteNoProgressReason::AncestorNotFound, implicated_peers); + } + cursor_root = grandparent_root; + cursor_slot = first_child_slot; + } + + budget_exhausted(implicated_peers) + } + + async fn trim_and_seed( + &mut self, + observation: &FrontierObservation, + ancestor: (B256, u64), + mut chain: Vec, + confirming_peers: HashSet, + implicated_peers: HashSet, + primary_peer: PeerId, + ) -> anyhow::Result { + let (ancestor_root, ancestor_slot) = ancestor; + chain.retain(|block| { + block.message.slot > ancestor_slot && block.message.slot <= observation.target_slot + }); + + let mut new_ancestor_root = ancestor_root; + let mut new_ancestor_slot = ancestor_slot; + { + let store = self.beacon_chain.store.lock().await; + while let Some(first) = chain.first() { + let first_root = first.message.tree_hash_root(); + let first_slot = first.message.slot; + let next_child_slot = chain.get(1).map_or(first_slot, |block| block.message.slot); + if is_processable_connection_point(&store, first_root, next_child_slot)? { + new_ancestor_root = first_root; + new_ancestor_slot = first_slot; + chain.remove(0); + } else { + break; + } + } + } + + if chain.is_empty() { + if new_ancestor_slot < observation.anchor_slot { + return no_progress(RemoteNoProgressReason::NoNewDescendants, implicated_peers); + } + return Ok(RecoveryOutcome::AdvancedCoverage(CoverageAdvance { + parent_root: new_ancestor_root, + parent_slot: new_ancestor_slot, + covered_through_slot: new_ancestor_slot, + proven_empty: false, + confirming_peers, + })); + } + + if chain.len() > MAX_RECOVERY_SEED_BLOCKS { + chain.truncate(MAX_RECOVERY_SEED_BLOCKS); + } + + Ok(RecoveryOutcome::Seeded(RecoverySeed { + ancestor_root: new_ancestor_root, + ancestor_slot: new_ancestor_slot, + forward_blocks: chain, + target_slot: observation.target_slot, + source_peer: primary_peer, + })) + } +} + +struct IdleOnDrop<'a> { + peer_manager: &'a mut super::PeerManager, + peer_id: PeerId, +} + +impl<'a> IdleOnDrop<'a> { + fn new(peer_manager: &'a mut super::PeerManager, peer_id: PeerId) -> Self { + Self { + peer_manager, + peer_id, + } + } +} + +impl Drop for IdleOnDrop<'_> { + fn drop(&mut self) { + self.peer_manager.mark_peer_as_idle(&self.peer_id); + } +} + +#[cfg(test)] +mod tests { + use std::{path::PathBuf, sync::Arc}; + + use discv5::{Enr, enr::CombinedKey}; + use libp2p::PeerId; + use parking_lot::RwLock; + use ream_p2p::network::beacon::{network_state::NetworkState, peer::CachedPeer}; + use ream_peer::{ConnectionState, Direction}; + use ream_req_resp::beacon::messages::{meta_data::GetMetaDataV3, status::Status}; + + use super::*; + + fn test_network_state() -> Arc { + let enr_key = CombinedKey::generate_secp256k1(); + Arc::new(NetworkState { + local_enr: RwLock::new(Enr::builder().build(&enr_key).expect("valid enr")), + peer_table: RwLock::new(std::collections::HashMap::new()), + meta_data: RwLock::new(GetMetaDataV3::default()), + status: RwLock::new(Status::default()), + data_dir: PathBuf::new(), + }) + } + + fn idle_peer_manager_with(peer_ids: &[PeerId]) -> PeerManager { + let network_state = test_network_state(); + for &peer_id in peer_ids { + let mut peer = CachedPeer::new( + peer_id, + None, + ConnectionState::Connected, + Direction::Outbound, + None, + ); + peer.status = Some(Status::default()); + network_state.peer_table.write().insert(peer_id, peer); + } + let mut peer_manager = PeerManager::new(network_state); + peer_manager.update_peer_set(); + peer_manager + } + + #[test] + fn recovery_budget_stops_capacity_at_the_request_cap_regardless_of_time_remaining() { + let mut budget = RecoveryBudget::new(); + assert!(budget.has_capacity()); + for _ in 0..MAX_TOTAL_ANCESTOR_REQUESTS_PER_ROUND { + assert!( + budget.has_capacity(), + "must still have capacity before the cap is reached" + ); + budget.consume(); + } + assert!( + !budget.has_capacity(), + "the shared budget must run out at exactly MAX_TOTAL_ANCESTOR_REQUESTS_PER_ROUND, \ + regardless of which phase (probe, fallback, or ancestor-walk hops) consumed it" + ); + } + + #[test] + fn recovery_budget_expires_on_deadline_even_with_requests_remaining() { + let mut budget = RecoveryBudget::new(); + budget.deadline = Instant::now(); + std::thread::sleep(Duration::from_millis(1)); + assert!( + !budget.has_capacity(), + "a round past its soft deadline must refuse further dispatch even if the request \ + count is nowhere near the cap -- fallback and ancestor-walk hops must respect this \ + the same way the forward probe already does" + ); + } + + #[test] + fn reserve_round_robin_skips_excluded_and_wraps_around() { + let peer_a = PeerId::random(); + let peer_b = PeerId::random(); + let mut peer_manager = idle_peer_manager_with(&[peer_a, peer_b]); + + let peers = [peer_a, peer_b]; + let excluded = HashSet::from([peer_a]); + let (selected, advance_by) = reserve_round_robin(&mut peer_manager, &peers, &excluded, 0) + .expect("peer_b should still be selectable"); + assert_eq!(selected, peer_b); + assert_eq!(advance_by, 2); + } +}