diff --git a/Cargo.lock b/Cargo.lock index f5d03b1..36c596d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -885,6 +885,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" dependencies = [ "serde", + "zeroize", ] [[package]] @@ -1178,6 +1179,7 @@ dependencies = [ "cc", "cfg-if", "constant_time_eq", + "zeroize", ] [[package]] @@ -1424,9 +1426,9 @@ checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" [[package]] name = "commonware-broadcast" -version = "0.0.58" +version = "0.0.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a5bee66c7680e78cdc6d47ccb2d4e3c3db4f00b981f2efb6cc6a329ede1ee7d" +checksum = "cc71e670241b8b07840cbb01d0d9521ae68bc2195f620adaeb57f76c51584bd5" dependencies = [ "bytes", "commonware-codec", @@ -1443,9 +1445,9 @@ dependencies = [ [[package]] name = "commonware-codec" -version = "0.0.58" +version = "0.0.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b883694c401f119519c984524236fc883d34bc7ac6a5c623f147d6c4ff937eeb" +checksum = "881d86d9cc88835dac1774701cfc296f8485acbbeb6c50d0c04246e066f3a169" dependencies = [ "bytes", "paste", @@ -1454,9 +1456,9 @@ dependencies = [ [[package]] name = "commonware-consensus" -version = "0.0.58" +version = "0.0.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439a1a5630465b65e2e0f531a23543412b3a01f051b815faeda7b87295b89830" +checksum = "8f8ba91ccb78397634c478b1199a3d3ff20eaa5dc637a1a657450fa3eafbc0ae" dependencies = [ "bytes", "cfg-if", @@ -1480,19 +1482,22 @@ dependencies = [ [[package]] name = "commonware-cryptography" -version = "0.0.58" +version = "0.0.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b17804ba29033a971c6129ecd5c80b36ca4f97f525b6f2605a38fabbbd4fdb69" +checksum = "e97eba4685c332ae24a5842893f37c839371c33647645c314a124a7e4ee9ba60" dependencies = [ "blake3", "blst", "bytes", + "cfg-if", "commonware-codec", "commonware-utils", "ed25519-consensus", "getrandom 0.2.16", "p256", "rand 0.8.5", + "rand_chacha 0.3.1", + "rand_core 0.6.4", "rayon", "sha2 0.10.9", "thiserror 2.0.12", @@ -1501,9 +1506,9 @@ dependencies = [ [[package]] name = "commonware-macros" -version = "0.0.58" +version = "0.0.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14b2d9cb09f208592740cf5d49a6df00ebb87d709f4e68e550c8c153bfa977c" +checksum = "25989115bb59d92b1b7f79018792b31882dbe08e3ecaa6fd4ade5c4244f0d4e3" dependencies = [ "futures", "proc-macro2", @@ -1515,9 +1520,9 @@ dependencies = [ [[package]] name = "commonware-p2p" -version = "0.0.58" +version = "0.0.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abc4dc1c6997240c7322542b22e912ee39d0f7da47fb611e9be14d7fb36ca0e8" +checksum = "6e25f8905f151ae133e13866b208d485f4fb17121bf73a57f6587ffaaac2c9a0" dependencies = [ "bytes", "commonware-codec", @@ -1538,9 +1543,9 @@ dependencies = [ [[package]] name = "commonware-resolver" -version = "0.0.58" +version = "0.0.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55b6111c0a24e505a076f6f42d3cb1182b55f05160446760c5cef97c7a3c59b7" +checksum = "c778ccaea860de1fd11305cb51f2ef1de085eafd707adbf48a533afdd8742004" dependencies = [ "bimap", "bytes", @@ -1561,9 +1566,9 @@ dependencies = [ [[package]] name = "commonware-runtime" -version = "0.0.58" +version = "0.0.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5961bab61137cedb725be74aced2d25694c9d6e5be195d2ce001f8d4b191628" +checksum = "ffb9485b85b504ae6ae37c49a94050b1d3eafe222778e46d354ed0ca65259575" dependencies = [ "async-lock", "axum 0.8.4", @@ -1583,6 +1588,7 @@ dependencies = [ "rand 0.8.5", "rayon", "sha2 0.10.9", + "sysinfo", "thiserror 2.0.12", "tokio", "tracing", @@ -1592,9 +1598,9 @@ dependencies = [ [[package]] name = "commonware-storage" -version = "0.0.58" +version = "0.0.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6b74959b9e6e611fd7764858d9d571e026b454cd3fa716f33c02e61fd940cb1" +checksum = "6ac3dfa058a662ce951500abebb865608afc68bfbde206d23a591b6aa3b235f9" dependencies = [ "bytes", "commonware-codec", @@ -1614,9 +1620,9 @@ dependencies = [ [[package]] name = "commonware-stream" -version = "0.0.58" +version = "0.0.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e96fdc2c280f2e6b86f32893231b63a20c847c4350c0137c9a2172b83adfdc0" +checksum = "8d7599a9b3b015b9eebe87a4cc572c223843eb281e06262ea13c92db5b29f9a5" dependencies = [ "bytes", "chacha20poly1305", @@ -1635,9 +1641,9 @@ dependencies = [ [[package]] name = "commonware-utils" -version = "0.0.58" +version = "0.0.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "948bfc7f5f7136a6708bc0c18600456f74d9d3dc3c66e613d922427f06263129" +checksum = "5b052e7cbeed6ba05dee34b82434dd4e76ed5dd3b1c4585eafab08233e4033f9" dependencies = [ "bytes", "commonware-codec", @@ -2033,7 +2039,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ "const-oid", - "pem-rfc7468", "zeroize", ] @@ -2201,7 +2206,6 @@ dependencies = [ "curve25519-dalek-ng", "hex", "rand_core 0.6.4", - "serde", "sha2 0.9.9", "thiserror 1.0.69", "zeroize", @@ -2228,7 +2232,6 @@ dependencies = [ "ff", "generic-array", "group", - "pem-rfc7468", "pkcs8", "rand_core 0.6.4", "sec1", @@ -3494,7 +3497,7 @@ dependencies = [ "once_cell", "procfs", "rlimit", - "windows", + "windows 0.58.0", ] [[package]] @@ -3584,6 +3587,15 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21" +[[package]] +name = "ntapi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" +dependencies = [ + "winapi", +] + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -4001,15 +4013,6 @@ dependencies = [ "serde", ] -[[package]] -name = "pem-rfc7468" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" -dependencies = [ - "base64ct", -] - [[package]] name = "percent-encoding" version = "2.3.1" @@ -5505,7 +5508,6 @@ dependencies = [ "commonware-codec", "commonware-consensus", "commonware-cryptography", - "commonware-resolver", "commonware-runtime", "commonware-storage", "commonware-utils", @@ -5577,6 +5579,7 @@ dependencies = [ "commonware-codec", "commonware-consensus", "commonware-cryptography", + "commonware-resolver", "commonware-utils", "dirs 6.0.0", "ethereum_ssz", @@ -5643,6 +5646,20 @@ dependencies = [ "syn 2.0.104", ] +[[package]] +name = "sysinfo" +version = "0.33.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fc858248ea01b66f19d8e8a6d55f41deaf91e9d495246fd01368d99935c6c01" +dependencies = [ + "core-foundation-sys", + "libc", + "memchr", + "ntapi", + "rayon", + "windows 0.57.0", +] + [[package]] name = "system-configuration" version = "0.6.1" @@ -6469,6 +6486,16 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143" +dependencies = [ + "windows-core 0.57.0", + "windows-targets 0.52.6", +] + [[package]] name = "windows" version = "0.58.0" @@ -6479,6 +6506,18 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-core" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d" +dependencies = [ + "windows-implement 0.57.0", + "windows-interface 0.57.0", + "windows-result 0.1.2", + "windows-targets 0.52.6", +] + [[package]] name = "windows-core" version = "0.58.0" @@ -6505,6 +6544,17 @@ dependencies = [ "windows-strings 0.4.2", ] +[[package]] +name = "windows-implement" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.104", +] + [[package]] name = "windows-implement" version = "0.58.0" @@ -6527,6 +6577,17 @@ dependencies = [ "syn 2.0.104", ] +[[package]] +name = "windows-interface" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.104", +] + [[package]] name = "windows-interface" version = "0.58.0" @@ -6566,6 +6627,15 @@ dependencies = [ "windows-strings 0.4.2", ] +[[package]] +name = "windows-result" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-result" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index 6208be2..8c3f56d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,16 +14,16 @@ summit-syncer = {path = "syncer"} summit-finalizer = {path = "finalizer"} summit-rpc = {path = "rpc"} -commonware-consensus = "0.0.58" -commonware-cryptography = "0.0.58" -commonware-storage = "0.0.58" -commonware-runtime = "0.0.58" -commonware-codec = "0.0.58" -commonware-p2p = "0.0.58" -commonware-broadcast = "0.0.58" -commonware-utils = "0.0.58" -commonware-resolver = "0.0.58" -commonware-macros = "0.0.58" +commonware-consensus = "0.0.62" +commonware-cryptography = "0.0.62" +commonware-storage = "0.0.62" +commonware-runtime = "0.0.62" +commonware-codec = "0.0.62" +commonware-p2p = "0.0.62" +commonware-broadcast = "0.0.62" +commonware-utils = "0.0.62" +commonware-resolver = "0.0.62" +commonware-macros = "0.0.62" alloy-consensus = "1.0.12" alloy-eips = { version = "1.0.19", features = ["ssz"] } diff --git a/finalizer/Cargo.toml b/finalizer/Cargo.toml index a87d954..d843ee7 100644 --- a/finalizer/Cargo.toml +++ b/finalizer/Cargo.toml @@ -10,7 +10,6 @@ summit-syncer.workspace = true commonware-codec.workspace = true commonware-consensus.workspace = true commonware-cryptography.workspace = true -commonware-resolver.workspace = true commonware-runtime.workspace = true commonware-storage.workspace = true commonware-utils.workspace = true diff --git a/finalizer/src/actor.rs b/finalizer/src/actor.rs index 3395689..0fce48b 100644 --- a/finalizer/src/actor.rs +++ b/finalizer/src/actor.rs @@ -1,10 +1,12 @@ +use crate::db::{Config as StateConfig, FinalizerState}; +use crate::{FinalizerConfig, FinalizerMailbox, FinalizerMessage, block_fetcher::BlockFetcher}; use alloy_eips::eip4895::Withdrawal; use alloy_primitives::Address; #[cfg(debug_assertions)] use alloy_primitives::hex; use alloy_rpc_types_engine::ForkchoiceState; use commonware_codec::{DecodeExt as _, ReadExt as _}; -use commonware_cryptography::Verifier as _; +use commonware_cryptography::{Hasher, Sha256, Verifier as _}; use commonware_runtime::{Clock, Handle, Metrics, Spawner, Storage}; use commonware_storage::translator::TwoCap; use commonware_utils::{NZU64, NZUsize, hex}; @@ -23,17 +25,12 @@ use summit_types::account::{ValidatorAccount, ValidatorStatus}; use summit_types::checkpoint::Checkpoint; use summit_types::consensus_state_query::{ConsensusStateRequest, ConsensusStateResponse}; use summit_types::execution_request::ExecutionRequest; +use summit_types::registry::Registry; use summit_types::utils::{is_last_block_of_epoch, is_penultimate_block_of_epoch}; use summit_types::{Block, BlockAuxData, Digest, FinalizedHeader, PublicKey, Signature}; use summit_types::{BlockEnvelope, EngineClient, consensus_state::ConsensusState}; use tracing::{info, warn}; -use crate::db::{Config as StateConfig, FinalizerState}; -use crate::{ - FinalizerConfig, FinalizerMailbox, FinalizerMessage, block_fetcher::BlockFetcher, - registry::Registry, -}; - const WRITE_BUFFER: NonZero = NZUsize!(1024 * 1024); pub struct Finalizer< @@ -100,9 +97,7 @@ impl { pub mailbox_size: usize, diff --git a/finalizer/src/db.rs b/finalizer/src/db.rs index 7152816..8363e97 100644 --- a/finalizer/src/db.rs +++ b/finalizer/src/db.rs @@ -211,7 +211,7 @@ impl FinalizerState { // Commit all pending changes to the database pub async fn commit(&mut self) { self.store - .commit() + .commit(None) .await .expect("failed to commit to database"); } diff --git a/finalizer/src/lib.rs b/finalizer/src/lib.rs index de0632a..1c28c52 100644 --- a/finalizer/src/lib.rs +++ b/finalizer/src/lib.rs @@ -5,4 +5,3 @@ pub use config::*; pub mod actor; pub mod block_fetcher; pub mod db; -pub mod registry; diff --git a/node/src/engine.rs b/node/src/engine.rs index 7db5b41..8ed272f 100644 --- a/node/src/engine.rs +++ b/node/src/engine.rs @@ -13,9 +13,9 @@ use rand::{CryptoRng, Rng}; use std::num::NonZero; use summit_application::ApplicationConfig; use summit_finalizer::actor::Finalizer; -use summit_finalizer::registry::Registry; use summit_finalizer::{FinalizerConfig, FinalizerMailbox}; use summit_syncer::Orchestrator; +use summit_types::registry::Registry; use summit_types::{Block, Digest, EngineClient, PrivateKey, PublicKey}; use tracing::{error, warn}; @@ -117,7 +117,7 @@ impl Digest { - commonware_cryptography::sha256::hash(&PROTOCOL_VERSION.to_le_bytes()) + Sha256::hash(&PROTOCOL_VERSION.to_le_bytes()) } /// Parse a substring from a metric name using XML-like tags diff --git a/node/src/tests/checkpointing.rs b/node/src/tests/checkpointing.rs index 11a429f..bd3a4ae 100644 --- a/node/src/tests/checkpointing.rs +++ b/node/src/tests/checkpointing.rs @@ -21,8 +21,8 @@ fn test_checkpoint_created() { // and store it to disk let n = 10; let link = Link { - latency: 80.0, - jitter: 10.0, + latency: Duration::from_millis(80), + jitter: Duration::from_millis(10), success_rate: 1.0, }; // Create context @@ -173,8 +173,8 @@ fn test_previous_header_hash_matches() { // This test verifies that these hashes match. let n = 10; let link = Link { - latency: 80.0, - jitter: 10.0, + latency: Duration::from_millis(80), + jitter: Duration::from_millis(10), success_rate: 1.0, }; // Create context @@ -338,8 +338,8 @@ fn test_single_engine_with_checkpoint() { // Test that an Engine instance can be initialized with a pre-created checkpoint // and properly load the consensus state from it let link = Link { - latency: 80.0, - jitter: 10.0, + latency: Duration::from_millis(80), + jitter: Duration::from_millis(10), success_rate: 1.0, }; // Create context @@ -429,8 +429,8 @@ fn test_node_joins_later_with_checkpoint() { // it uses that checkpoint to initialize the consensus DB let n = 5; let link = Link { - latency: 80.0, - jitter: 10.0, + latency: Duration::from_millis(80), + jitter: Duration::from_millis(10), success_rate: 1.0, }; // Create context @@ -632,3 +632,215 @@ fn test_node_joins_later_with_checkpoint() { context.auditor().state() }); } + +#[test_traced("INFO")] +fn test_node_joins_later_with_checkpoint_not_in_genesis() { + // Creates a network of 5 nodes, and starts only 4 of them. + // The last node starts after the first checkpoint was created, and + // it uses that checkpoint to initialize the consensus DB + // In this test the joining node is not included in the list of peers that is passed to the engine. + let n = 5; + let link = Link { + latency: Duration::from_millis(80), + jitter: Duration::from_millis(10), + success_rate: 1.0, + }; + // Create context + let cfg = deterministic::Config::default().with_seed(0); + let executor = Runner::from(cfg); + executor.start(|context| async move { + // Create simulated network + let (network, mut oracle) = Network::new( + context.with_label("network"), + simulated::Config { + max_size: 1024 * 1024, + }, + ); + // Start network + network.start(); + // Register participants + let mut signers = Vec::new(); + let mut validators = Vec::new(); + for i in 0..n { + let signer = PrivateKey::from_seed(i as u64); + let pk = signer.public_key(); + signers.push(signer); + validators.push(pk); + } + validators.sort(); + signers.sort_by_key(|s| s.public_key()); + + // Separate initial validators from late joiner + let initial_validators = &validators[..validators.len() - 1]; + + // Register and link only initial validators + let mut registrations = common::register_validators(&mut oracle, initial_validators).await; + common::link_validators(&mut oracle, initial_validators, link.clone(), None).await; + // Create the engine clients + let genesis_hash = + from_hex_formatted(common::GENESIS_HASH).expect("failed to decode genesis hash"); + let genesis_hash: [u8; 32] = genesis_hash + .try_into() + .expect("failed to convert genesis hash"); + + let engine_client_network = MockEngineNetworkBuilder::new(genesis_hash).build(); + + // Create instances + let mut public_keys = HashSet::new(); + let mut consensus_state_queries = HashMap::new(); + + // Start all the engines, except for one + let signer_joining_later = signers.pop().unwrap(); + + for (idx, signer) in signers.into_iter().enumerate() { + // Create signer context + let public_key = signer.public_key(); + public_keys.insert(public_key.clone()); + + // Configure engine + let uid = format!("validator-{public_key}"); + let namespace = String::from("_SEISMIC_BFT"); + + let engine_client = engine_client_network.create_client(uid.clone()); + + let config = get_default_engine_config( + engine_client, + uid.clone(), + genesis_hash, + namespace, + signer, + initial_validators.to_vec(), + None, + ); + let engine = Engine::new(context.with_label(&uid), config).await; + consensus_state_queries.insert(idx, engine.finalizer_mailbox.clone()); + + // Get networking + let (pending, resolver, broadcast, backfill) = + registrations.remove(&public_key).unwrap(); + + // Start engine + engine.start(pending, resolver, broadcast, backfill); + } + + // Wait for the validators to checkpoint + let consensus_state_query = consensus_state_queries.get(&0).unwrap(); + let checkpoint = loop { + if let Some(checkpoint) = consensus_state_query.clone().get_latest_checkpoint().await { + break checkpoint; + } + context.sleep(Duration::from_secs(1)).await; + }; + + loop { + if consensus_state_query.get_latest_height().await >= 20 { + break; + } + context.sleep(Duration::from_secs(1)).await; + } + + // Now register and join the final validator to the network + let public_key = signer_joining_later.public_key(); + + // Register the late joining validator + let late_registrations = + common::register_validators(&mut oracle, &[public_key.clone()]).await; + + // Join the validator to the network + common::join_validator(&mut oracle, &public_key, initial_validators, link).await; + + // Allow p2p connections to establish before starting engine + context.sleep(Duration::from_millis(100)).await; + + public_keys.insert(public_key.clone()); + + // Configure engine + let uid = format!("validator-{public_key}"); + let namespace = String::from("_SEISMIC_BFT"); + + let engine_client = engine_client_network.create_client(uid.clone()); + + // This corresponds to snapshotting Reth + let consensus_state = ConsensusState::try_from(&checkpoint).unwrap(); + let from_block = consensus_state.latest_height + 1; + let eth_hash = consensus_state.forkchoice.head_block_hash.into(); + + engine_client.load_checkpoint(consensus_state.latest_height, eth_hash); + + let config = get_default_engine_config( + engine_client, + uid.clone(), + genesis_hash, + namespace, + signer_joining_later, + initial_validators.to_vec(), + Some(checkpoint), + ); + let engine = Engine::new(context.with_label(&uid), config).await; + + // Get networking from late registrations + let (pending, resolver, broadcast, backfill) = + late_registrations.into_iter().next().unwrap().1; + + // Start engine + engine.start(pending, resolver, broadcast, backfill); + + // Poll metrics + let stop_height = 3 * EPOCH_NUM_BLOCKS; + let mut nodes_finished = HashSet::new(); + loop { + let metrics = context.encode(); + + // Iterate over all lines + let mut success = false; + for line in metrics.lines() { + // Ensure it is a metrics line + if !line.starts_with("validator-") { + continue; + } + + // Split metric and value + let mut parts = line.split_whitespace(); + let metric = parts.next().unwrap(); + let value = parts.next().unwrap(); + + // If ends with peers_blocked, ensure it is zero + if metric.ends_with("_peers_blocked") { + let value = value.parse::().unwrap(); + assert_eq!(value, 0); + } + + if metric.ends_with("finalizer_height") { + let value = value.parse::().unwrap(); + if value == stop_height { + nodes_finished.insert(metric.to_string()); + if nodes_finished.len() as u32 == n { + success = true; + break; + } + } + } + + if nodes_finished.len() as u32 >= n { + success = true; + break; + } + } + if success { + break; + } + + // Still waiting for all validators to complete + context.sleep(Duration::from_secs(1)).await; + } + + // Check that all nodes have the same canonical chain + assert!( + engine_client_network + .verify_consensus(Some(from_block), Some(stop_height)) + .is_ok() + ); + + context.auditor().state() + }); +} diff --git a/node/src/tests/execution_requests.rs b/node/src/tests/execution_requests.rs index 2523802..b51ab9e 100644 --- a/node/src/tests/execution_requests.rs +++ b/node/src/tests/execution_requests.rs @@ -24,8 +24,8 @@ fn test_deposit_request_single() { // and withdrawal credentials were added correctly. let n = 10; let link = Link { - latency: 80.0, - jitter: 10.0, + latency: Duration::from_millis(80), + jitter: Duration::from_millis(10), success_rate: 0.98, }; // Create context @@ -195,8 +195,8 @@ fn test_deposit_request_top_up() { // validator balance is the sum of the amounts of both deposit requests. let n = 10; let link = Link { - latency: 80.0, - jitter: 10.0, + latency: Duration::from_millis(80), + jitter: Duration::from_millis(10), success_rate: 0.98, }; // Create context @@ -389,8 +389,8 @@ fn test_deposit_and_withdrawal_request_single() { // withdrawal request (execution request) that was initially added to block 7. let n = 10; let link = Link { - latency: 80.0, - jitter: 10.0, + latency: Duration::from_millis(80), + jitter: Duration::from_millis(10), success_rate: 0.98, }; // Create context @@ -595,8 +595,8 @@ fn test_partial_withdrawal_balance_below_minimum_stake() { // is no balance left. let n = 10; let link = Link { - latency: 80.0, - jitter: 10.0, + latency: Duration::from_millis(80), + jitter: Duration::from_millis(10), success_rate: 0.98, }; // Create context @@ -811,8 +811,8 @@ fn test_deposit_less_than_min_stake_and_withdrawal() { // The balance should still increase and the withdrawal should work as well. let n = 10; let link = Link { - latency: 80.0, - jitter: 10.0, + latency: Duration::from_millis(80), + jitter: Duration::from_millis(10), success_rate: 0.98, }; // Create context @@ -1024,8 +1024,8 @@ fn test_deposit_and_withdrawal_request_multiple() { // (from different public keys). let n = 10; let link = Link { - latency: 80.0, - jitter: 10.0, + latency: Duration::from_millis(80), + jitter: Duration::from_millis(10), success_rate: 0.98, }; // Create context @@ -1257,8 +1257,8 @@ fn test_deposit_request_invalid_signature() { // verifies that the request is rejected. let n = 10; let link = Link { - latency: 80.0, - jitter: 10.0, + latency: Duration::from_millis(80), + jitter: Duration::from_millis(10), success_rate: 0.98, }; // Create context diff --git a/node/src/tests/mod.rs b/node/src/tests/mod.rs index 0eb18f4..3b831f0 100644 --- a/node/src/tests/mod.rs +++ b/node/src/tests/mod.rs @@ -5,12 +5,13 @@ mod syncer; use crate::test_harness::common::run_until_height; use commonware_macros::test_traced; use commonware_p2p::simulated::Link; +use std::time::Duration; #[test_traced("INFO")] fn test_20_blocks_and_verify() { let link = Link { - latency: 80.0, - jitter: 10.0, + latency: Duration::from_millis(80), + jitter: Duration::from_millis(10), success_rate: 0.98, }; run_until_height(10, 0, link.clone(), 20, true); diff --git a/node/src/tests/syncer.rs b/node/src/tests/syncer.rs index cad61f4..83bb0e4 100644 --- a/node/src/tests/syncer.rs +++ b/node/src/tests/syncer.rs @@ -20,8 +20,8 @@ fn test_node_joins_later_no_checkpoint() { // in the syncer works. let n = 5; let link = Link { - latency: 80.0, - jitter: 10.0, + latency: Duration::from_millis(80), + jitter: Duration::from_millis(10), success_rate: 1.0, }; // Create context @@ -209,3 +209,204 @@ fn test_node_joins_later_no_checkpoint() { context.auditor().state() }); } + +#[test_traced("INFO")] +fn test_node_joins_later_no_checkpoint_not_in_genesis() { + // Creates a network of 5 nodes, and starts only 4 of them. + // The last node starts after 10 blocks, to ensure that the block backfilling + // in the syncer works. + // In this test the joining node is not included in the list of peers that is passed to the engine. + let n = 5; + let link = Link { + latency: Duration::from_millis(80), + jitter: Duration::from_millis(10), + success_rate: 1.0, + }; + // Create context + let cfg = deterministic::Config::default().with_seed(0); + let executor = Runner::from(cfg); + executor.start(|context| async move { + // Create simulated network + let (network, mut oracle) = Network::new( + context.with_label("network"), + simulated::Config { + max_size: 1024 * 1024, + }, + ); + // Start network + network.start(); + // Register participants + let mut signers = Vec::new(); + let mut validators = Vec::new(); + for i in 0..n { + let signer = PrivateKey::from_seed(i as u64); + let pk = signer.public_key(); + signers.push(signer); + validators.push(pk); + } + validators.sort(); + signers.sort_by_key(|s| s.public_key()); + + // Separate initial validators from late joiner + let initial_validators = &validators[..validators.len() - 1]; + + // Register and link only initial validators + let mut registrations = common::register_validators(&mut oracle, initial_validators).await; + common::link_validators(&mut oracle, initial_validators, link.clone(), None).await; + // Create the engine clients + let genesis_hash = + from_hex_formatted(common::GENESIS_HASH).expect("failed to decode genesis hash"); + let genesis_hash: [u8; 32] = genesis_hash + .try_into() + .expect("failed to convert genesis hash"); + + let engine_client_network = MockEngineNetworkBuilder::new(genesis_hash).build(); + + // Create instances + let mut public_keys = HashSet::new(); + let mut consensus_state_queries = HashMap::new(); + + // Start all the engines, except for one + let signer_joining_later = signers.pop().unwrap(); + + for (idx, signer) in signers.into_iter().enumerate() { + // Create signer context + let public_key = signer.public_key(); + public_keys.insert(public_key.clone()); + + // Configure engine + let uid = format!("validator-{public_key}"); + let namespace = String::from("_SEISMIC_BFT"); + + let engine_client = engine_client_network.create_client(uid.clone()); + + let config = get_default_engine_config( + engine_client, + uid.clone(), + genesis_hash, + namespace, + signer, + initial_validators.to_vec(), + None, + ); + let engine = Engine::new(context.with_label(&uid), config).await; + consensus_state_queries.insert(idx, engine.finalizer_mailbox.clone()); + + // Get networking + let (pending, resolver, broadcast, backfill) = + registrations.remove(&public_key).unwrap(); + + // Start engine + engine.start(pending, resolver, broadcast, backfill); + } + + // Wait for the validators to checkpoint + let consensus_state_query = consensus_state_queries.get(&0).unwrap(); + let _checkpoint = loop { + if let Some(checkpoint) = consensus_state_query.clone().get_latest_checkpoint().await { + break checkpoint; + } + context.sleep(Duration::from_secs(1)).await; + }; + + // Now register and join the final validator to the network + let public_key = signer_joining_later.public_key(); + + // Register the late joining validator + let late_registrations = + common::register_validators(&mut oracle, &[public_key.clone()]).await; + + // Join the validator to the network + common::join_validator(&mut oracle, &public_key, initial_validators, link).await; + + // Allow p2p connections to establish before starting engine + context.sleep(Duration::from_millis(100)).await; + + public_keys.insert(public_key.clone()); + + // Configure engine + let uid = format!("validator-{public_key}"); + let namespace = String::from("_SEISMIC_BFT"); + + let engine_client = engine_client_network.create_client(uid.clone()); + + // Joining node uses initial_validators for syncer verification + // since historical blocks were finalized by only those 4 validators + let config = get_default_engine_config( + engine_client, + uid.clone(), + genesis_hash, + namespace, + signer_joining_later, + initial_validators.to_vec(), + None, + ); + let engine = Engine::new(context.with_label(&uid), config).await; + + // Get networking from late registrations + let (pending, resolver, broadcast, backfill) = + late_registrations.into_iter().next().unwrap().1; + + // Start engine + engine.start(pending, resolver, broadcast, backfill); + + // Poll metrics + let stop_height = 2 * EPOCH_NUM_BLOCKS; + let mut nodes_finished = HashSet::new(); + loop { + let metrics = context.encode(); + + // Iterate over all lines + let mut success = false; + for line in metrics.lines() { + // Ensure it is a metrics line + if !line.starts_with("validator-") { + continue; + } + + // Split metric and value + let mut parts = line.split_whitespace(); + let metric = parts.next().unwrap(); + let value = parts.next().unwrap(); + + // If ends with peers_blocked, ensure it is zero + if metric.ends_with("_peers_blocked") { + let value = value.parse::().unwrap(); + println!("{} -> {}", metric, value); + assert_eq!(value, 0); + } + + if metric.ends_with("finalizer_height") { + let value = value.parse::().unwrap(); + if value == stop_height { + nodes_finished.insert(metric.to_string()); + if nodes_finished.len() as u32 == n { + success = true; + break; + } + } + } + + if nodes_finished.len() as u32 >= n { + success = true; + break; + } + } + if success { + break; + } + + // Still waiting for all validators to complete + context.sleep(Duration::from_secs(1)).await; + } + + // Check that all nodes have the same canonical chain + assert!( + engine_client_network + .verify_consensus(None, Some(stop_height)) + .is_ok() + ); + + context.auditor().state() + }); +} diff --git a/syncer/src/actor.rs b/syncer/src/actor.rs index 22c1f2a..874af84 100644 --- a/syncer/src/actor.rs +++ b/syncer/src/actor.rs @@ -2,15 +2,14 @@ use std::{collections::BTreeSet, num::NonZero, time::Duration}; use crate::{ Orchestration, Orchestrator, - coordinator::Coordinator, handler::Handler, ingress::{Mailbox, Message}, key::{MultiIndex, Value}, }; use commonware_broadcast::{Broadcaster as _, buffered}; use commonware_codec::{DecodeExt as _, Encode as _}; -use commonware_consensus::Viewable as _; use commonware_consensus::simplex::types::Finalization; +use commonware_consensus::{Supervisor, Viewable as _}; use commonware_macros::select; use commonware_p2p::{Receiver, Recipients, Sender, utils::requester}; use commonware_resolver::{Resolver as _, p2p}; @@ -28,6 +27,7 @@ use governor::Quota; #[cfg(feature = "prom")] use metrics::histogram; use rand::Rng; +use summit_types::registry::Registry; use summit_types::{Block, Digest, Finalized, Notarized, PublicKey, Signature}; use tracing::{debug, warn}; @@ -57,7 +57,7 @@ pub struct Actor, orchestrator_mailbox: mpsc::Receiver, public_key: PublicKey, - participants: Vec, + registry: Registry, mailbox_size: usize, backfill_quota: Quota, activity_timeout: u64, @@ -156,7 +156,7 @@ impl Acto finalized: finalized_archive, blocks: block_archive, public_key: config.public_key, - participants: config.participants, + registry: config.registry, mailbox_size: config.mailbox_size, backfill_quota: config.backfill_quota, activity_timeout: config.activity_timeout, @@ -189,14 +189,13 @@ impl Acto ), mut tx_finalizer: mpsc::Sender<()>, ) { - let coordinator = Coordinator::new(self.participants.clone()); let (handler_sender, mut handler_receiver) = mpsc::channel(self.mailbox_size); let handler = Handler::new(handler_sender); let (resolver_engine, mut resolver) = p2p::Engine::new( self.context.with_label("resolver"), p2p::Config { - coordinator, + coordinator: self.registry.clone(), consumer: handler.clone(), producer: handler, mailbox_size: self.mailbox_size, @@ -594,7 +593,8 @@ impl Acto continue; }; - if !notarization.proof.verify(self.namespace.as_bytes(), &self.participants) { + let participants = self.registry.participants(view).expect("registry cannot be empty"); + if !notarization.proof.verify(self.namespace.as_bytes(), participants) { let _ = response.send(false); continue; } @@ -628,7 +628,9 @@ impl Acto let _ = response.send(false); continue; }; - if !finalization.proof.verify(self.namespace.as_bytes(), &self.participants) { + let view = finalization.proof.proposal.view; + let participants = self.registry.participants(view).expect("registry cannot be empty"); + if !finalization.proof.verify(self.namespace.as_bytes(), participants) { let _ = response.send(false); continue; } diff --git a/syncer/src/lib.rs b/syncer/src/lib.rs index c92f6ef..f639d89 100644 --- a/syncer/src/lib.rs +++ b/syncer/src/lib.rs @@ -4,6 +4,8 @@ pub mod ingress; use commonware_runtime::buffer::PoolRef; pub use ingress::*; use summit_types::PublicKey; +use summit_types::registry::Registry; + pub mod coordinator; pub mod handler; pub mod key; @@ -14,7 +16,7 @@ pub struct Config { pub public_key: PublicKey, - pub participants: Vec, + pub registry: Registry, /// Number of messages from consensus to hold in our backlog /// before blocking. diff --git a/types/Cargo.toml b/types/Cargo.toml index fb45ec5..694af64 100644 --- a/types/Cargo.toml +++ b/types/Cargo.toml @@ -8,6 +8,7 @@ commonware-cryptography.workspace = true commonware-consensus.workspace = true commonware-codec.workspace = true commonware-utils.workspace = true +commonware-resolver.workspace = true alloy-rpc-types-engine.workspace = true alloy-eips.workspace = true diff --git a/types/src/lib.rs b/types/src/lib.rs index 7402c83..fa76a25 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -7,8 +7,10 @@ pub mod engine_client; pub mod execution_request; pub mod genesis; pub mod header; +pub mod registry; pub mod utils; pub mod withdrawal; + use alloy_rpc_types_engine::ForkchoiceState; pub use block::*; pub use engine_client::*; diff --git a/finalizer/src/registry.rs b/types/src/registry.rs similarity index 94% rename from finalizer/src/registry.rs rename to types/src/registry.rs index 801d3fd..093eeae 100644 --- a/finalizer/src/registry.rs +++ b/types/src/registry.rs @@ -1,8 +1,8 @@ +use crate::PublicKey; use commonware_consensus::{Supervisor as Su, simplex::types::View}; use commonware_resolver::p2p; use std::collections::BTreeMap; use std::sync::{Arc, RwLock}; -use summit_types::PublicKey; use tracing::warn; #[derive(Default, Clone, Debug)] @@ -175,7 +175,7 @@ mod tests { fn create_test_pubkeys(count: usize) -> Vec { (0..count) .map(|i| { - let private_key = summit_types::PrivateKey::from_seed(i as u64); + let private_key = crate::PrivateKey::from_seed(i as u64); private_key.public_key() }) .collect() @@ -208,7 +208,7 @@ mod tests { #[test] fn test_update_registry_add_participant() { let registry = create_test_registry(2); - let new_participant = summit_types::PrivateKey::from_seed(99).public_key(); + let new_participant = crate::PrivateKey::from_seed(99).public_key(); // Add participant to view 1 registry.update_registry(1, vec![new_participant.clone()], vec![]); @@ -263,7 +263,7 @@ mod tests { #[test] fn test_update_registry_remove_nonexistent_participant() { let registry = create_test_registry(2); - let nonexistent_participant = summit_types::PrivateKey::from_seed(999).public_key(); + let nonexistent_participant = crate::PrivateKey::from_seed(999).public_key(); // Try to remove non-existent participant - should log warning but not fail registry.update_registry(1, vec![], vec![nonexistent_participant]); @@ -307,7 +307,7 @@ mod tests { assert_eq!(registry.participants(0).unwrap(), original_participants); // Add participant to create view 3 - let new_participant = summit_types::PrivateKey::from_seed(100).public_key(); + let new_participant = crate::PrivateKey::from_seed(100).public_key(); registry.update_registry(3, vec![new_participant.clone()], vec![]); // Views 0, 1, 2 should still use original participants (largest view <= requested) @@ -337,7 +337,7 @@ mod tests { } // Test non-existing participant - let non_participant = summit_types::PrivateKey::from_seed(999).public_key(); + let non_participant = crate::PrivateKey::from_seed(999).public_key(); assert_eq!(registry.is_participant(0, &non_participant), None); // Test with view that uses latest available participants @@ -373,7 +373,7 @@ mod tests { assert_eq!(registry.peer_set_id(), 0); // Add participant to create view 1 - let new_participant = summit_types::PrivateKey::from_seed(100).public_key(); + let new_participant = crate::PrivateKey::from_seed(100).public_key(); registry.update_registry(1, vec![new_participant], vec![]); // Peer set ID should now be 1 @@ -389,7 +389,7 @@ mod tests { assert_eq!(initial_peers.len(), 2); // Add participant - let new_participant = summit_types::PrivateKey::from_seed(100).public_key(); + let new_participant = crate::PrivateKey::from_seed(100).public_key(); registry.update_registry(1, vec![new_participant.clone()], vec![]); // Peers should now reflect the latest view @@ -404,8 +404,8 @@ mod tests { let registry = create_test_registry(2); // Add participants to different views - let participant_a = summit_types::PrivateKey::from_seed(100).public_key(); - let participant_b = summit_types::PrivateKey::from_seed(101).public_key(); + let participant_a = crate::PrivateKey::from_seed(100).public_key(); + let participant_b = crate::PrivateKey::from_seed(101).public_key(); registry.update_registry(3, vec![participant_a.clone()], vec![]); registry.update_registry(7, vec![participant_b.clone()], vec![]); @@ -431,7 +431,7 @@ mod tests { let original_participants = registry.participants(0).unwrap().clone(); // Add participant to view 1 - let new_participant = summit_types::PrivateKey::from_seed(100).public_key(); + let new_participant = crate::PrivateKey::from_seed(100).public_key(); registry.update_registry(1, vec![new_participant.clone()], vec![]); // Original view should remain unchanged @@ -517,8 +517,8 @@ mod tests { let registry = create_test_registry(2); // Add participants to sparse views: 0, 3, 7 - let participant_a = summit_types::PrivateKey::from_seed(100).public_key(); - let participant_b = summit_types::PrivateKey::from_seed(101).public_key(); + let participant_a = crate::PrivateKey::from_seed(100).public_key(); + let participant_b = crate::PrivateKey::from_seed(101).public_key(); registry.update_registry(3, vec![participant_a.clone()], vec![]); registry.update_registry(7, vec![participant_b.clone()], vec![]); @@ -555,7 +555,7 @@ mod tests { let registry = create_test_registry(4); // Add participant at view 2 - let new_participant = summit_types::PrivateKey::from_seed(100).public_key(); + let new_participant = crate::PrivateKey::from_seed(100).public_key(); registry.update_registry(2, vec![new_participant.clone()], vec![]); // Leader for view 0-1 should use 4-participant set from view 0 @@ -590,7 +590,7 @@ mod tests { let original_participants = registry.participants(0).unwrap().clone(); // Add participant at view 3 - let new_participant = summit_types::PrivateKey::from_seed(100).public_key(); + let new_participant = crate::PrivateKey::from_seed(100).public_key(); registry.update_registry(3, vec![new_participant.clone()], vec![]); // Original participants should be found in all views @@ -631,8 +631,8 @@ mod tests { assert_eq!(registry.peer_set_id(), 0); // Add participants to different views - let participant_a = summit_types::PrivateKey::from_seed(100).public_key(); - let participant_b = summit_types::PrivateKey::from_seed(101).public_key(); + let participant_a = crate::PrivateKey::from_seed(100).public_key(); + let participant_b = crate::PrivateKey::from_seed(101).public_key(); registry.update_registry(5, vec![participant_a], vec![]); assert_eq!(registry.peer_set_id(), 5); @@ -689,8 +689,8 @@ mod tests { let original_participants = registry.participants(0).unwrap().clone(); // Create new participants to add and remove existing ones - let new_participant_a = summit_types::PrivateKey::from_seed(200).public_key(); - let new_participant_b = summit_types::PrivateKey::from_seed(201).public_key(); + let new_participant_a = crate::PrivateKey::from_seed(200).public_key(); + let new_participant_b = crate::PrivateKey::from_seed(201).public_key(); let participant_to_remove = original_participants[0].clone(); // Add two participants and remove one in a single operation