From 1770660f0c00e473e6327d4efe9157ab0f1eee05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Fri, 12 Jun 2026 12:38:18 -0300 Subject: [PATCH 1/2] simulators/lean: decode devnet5 blocks with the merged-proof shape Devnet5 replaced the per-attestation signature list on the signed-block envelope with a single merged multi-message aggregate proof, matching leanSpec's SignedBlock { block, proof: MultiMessageAggregate }. The simulator still decoded every signed block with the devnet4 shape SignedBlock { block, signature: BlockSignatures }, so reqresp blocks_by_root tests failed against devnet5 clients with OffsetIntoFixedPortion(4): the merged-proof container opens with offset 4, which lands inside the devnet4 signatures container's fixed portion. The rpc-compat finalized-block decoder had the same mismatch. Keep both envelope shapes as wire-only types and select between them from the devnet under test, mirroring how the state decoder already splits devnet formats. Scenario code now works with the inner block only: decoding validates the envelope structurally, then returns the block, since no test inspects signature or proof contents. --- simulators/lean/src/scenarios/gossip.rs | 20 ++--- simulators/lean/src/scenarios/reqresp.rs | 44 +++++------ simulators/lean/src/scenarios/rpc_compat.rs | 56 +++++++++---- simulators/lean/src/scenarios/validation.rs | 16 ++-- simulators/lean/src/utils/libp2p_mock.rs | 88 ++++++++++++++++----- simulators/lean/src/utils/util.rs | 8 ++ 6 files changed, 159 insertions(+), 73 deletions(-) diff --git a/simulators/lean/src/scenarios/gossip.rs b/simulators/lean/src/scenarios/gossip.rs index 8c6dad299a..a2dcdcf996 100644 --- a/simulators/lean/src/scenarios/gossip.rs +++ b/simulators/lean/src/scenarios/gossip.rs @@ -1,6 +1,6 @@ use crate::utils::libp2p_mock::{ - decode_request, encode_gossip_data, extract_ip_port, lean_block_topic, replace_multiaddr_ip, - LeanSignedBlock, MockBehaviourEvent, MockNode, Status, RESPONSE_CODE_SUCCESS, + decode_request, encode_gossip_block, extract_ip_port, lean_block_topic, replace_multiaddr_ip, + LeanBlock, MockBehaviourEvent, MockNode, Status, RESPONSE_CODE_SUCCESS, }; use crate::utils::util::{ expect_single_client, lean_clients, lean_environment, lean_single_client_runtime_setup, @@ -222,8 +222,8 @@ dyn_async! { mock.subscribe(&wrong_topic) .expect("mock should subscribe to wrong-fork topic"); - let block = LeanSignedBlock::build_minimal(1, 0, B256::ZERO, B256::ZERO); - let block_bytes = encode_gossip_data(&block); + let block = LeanBlock::build_minimal(1, 0, B256::ZERO, B256::ZERO); + let block_bytes = encode_gossip_block(&block); if let Err(e) = mock.publish(wrong_topic, block_bytes) { // If there are no peers subscribed to the wrong topic, the publish // will fail. This is expected because the client only subscribes to @@ -274,15 +274,15 @@ dyn_async! { let block_topic = lean_block_topic(&fork_digest); - let parent_block = LeanSignedBlock::build_minimal(1, 0, B256::ZERO, B256::ZERO); + let parent_block = LeanBlock::build_minimal(1, 0, B256::ZERO, B256::ZERO); let parent_root = B256::from_slice(&[0xca; 32]); - let child_block = LeanSignedBlock::build_minimal(2, 0, parent_root, B256::ZERO); + let child_block = LeanBlock::build_minimal(2, 0, parent_root, B256::ZERO); let fork_choice_before = load_fork_choice_response(&client).await; let block_count_before = fork_choice_before.nodes.len(); - let child_bytes = encode_gossip_data(&child_block); + let child_bytes = encode_gossip_block(&child_block); mock.publish(block_topic.clone(), child_bytes) .expect("should publish orphan child block"); @@ -295,7 +295,7 @@ dyn_async! { "client should not process orphan block" ); - let parent_bytes = encode_gossip_data(&parent_block); + let parent_bytes = encode_gossip_block(&parent_block); mock.publish(block_topic, parent_bytes) .expect("should publish parent block"); @@ -319,8 +319,8 @@ dyn_async! { mock.process_events_for(Duration::from_secs(3)).await; let block_topic = lean_block_topic(&fork_digest); - let invalid_block = LeanSignedBlock::build_minimal(1, 9999, B256::ZERO, B256::ZERO); - let block_bytes = encode_gossip_data(&invalid_block); + let invalid_block = LeanBlock::build_minimal(1, 9999, B256::ZERO, B256::ZERO); + let block_bytes = encode_gossip_block(&invalid_block); let fork_choice_before = load_fork_choice_response(&client).await; let block_count_before = fork_choice_before.nodes.len(); diff --git a/simulators/lean/src/scenarios/reqresp.rs b/simulators/lean/src/scenarios/reqresp.rs index 480c13bef5..42e3ce276b 100644 --- a/simulators/lean/src/scenarios/reqresp.rs +++ b/simulators/lean/src/scenarios/reqresp.rs @@ -5,8 +5,8 @@ use crate::utils::helper::{ use crate::utils::libp2p_mock::{ client_multiaddr, compute_client_peer_id, decode_request, encode_request, encode_request_raw, extract_ip_port, replace_multiaddr_ip, BlocksByRangeV1Request, BlocksByRootV1Request, - Checkpoint, LeanSignedBlock, MockNode, Status, MAX_REQUEST_BLOCKS, - RESPONSE_CODE_INVALID_REQUEST, RESPONSE_CODE_SUCCESS, + Checkpoint, LeanBlock, MockNode, Status, MAX_REQUEST_BLOCKS, RESPONSE_CODE_INVALID_REQUEST, + RESPONSE_CODE_SUCCESS, }; use crate::utils::util::{ default_genesis_time, fork_choice_head_slot, http_client, lean_api_url, lean_clients, @@ -16,7 +16,7 @@ use crate::utils::util::{ }; use alloy_primitives::B256; use hivesim::{dyn_async, Client, Test}; -use ssz::{Decode, Encode}; +use ssz::Encode; use std::time::Duration; use tokio::time::sleep; @@ -989,11 +989,11 @@ dyn_async! { .collect::>(); assert_eq!(success_payloads.len(), 1, "client should return exactly one known block"); - let signed_block = LeanSignedBlock::from_ssz_bytes(success_payloads[0]) + let block = LeanBlock::from_signed_wire_ssz_bytes(success_payloads[0]) .expect("returned head block should decode from SSZ"); - assert_eq!(signed_block.block.slot, expected_node.slot, "returned block slot should match fork_choice head"); - assert_eq!(signed_block.block.parent_root, expected_node.parent_root, "returned block parent should match fork_choice head"); - assert_eq!(signed_block.block.proposer_index, expected_node.proposer_index, "returned block proposer should match fork_choice head"); + assert_eq!(block.slot, expected_node.slot, "returned block slot should match fork_choice head"); + assert_eq!(block.parent_root, expected_node.parent_root, "returned block parent should match fork_choice head"); + assert_eq!(block.proposer_index, expected_node.proposer_index, "returned block proposer should match fork_choice head"); } } @@ -1047,11 +1047,11 @@ dyn_async! { ); for (payload, expected_node) in success_payloads.iter().zip(known_nodes.iter()) { - let signed_block = LeanSignedBlock::from_ssz_bytes(payload) + let block = LeanBlock::from_signed_wire_ssz_bytes(payload) .expect("returned block should decode from SSZ"); - assert_eq!(signed_block.block.slot, expected_node.slot, "returned block slot should match requested fork_choice node"); - assert_eq!(signed_block.block.parent_root, expected_node.parent_root, "returned block parent should match requested fork_choice node"); - assert_eq!(signed_block.block.proposer_index, expected_node.proposer_index, "returned block proposer should match requested fork_choice node"); + assert_eq!(block.slot, expected_node.slot, "returned block slot should match requested fork_choice node"); + assert_eq!(block.parent_root, expected_node.parent_root, "returned block parent should match requested fork_choice node"); + assert_eq!(block.proposer_index, expected_node.proposer_index, "returned block proposer should match requested fork_choice node"); } } } @@ -1250,11 +1250,11 @@ dyn_async! { .collect::>(); assert_eq!(success_payloads.len(), 1, "client should return exactly one known range block"); - let signed_block = LeanSignedBlock::from_ssz_bytes(success_payloads[0]) + let block = LeanBlock::from_signed_wire_ssz_bytes(success_payloads[0]) .expect("returned range block should decode from SSZ"); - assert_eq!(signed_block.block.slot, expected_node.slot, "returned range block slot should match requested slot"); - assert_eq!(signed_block.block.parent_root, expected_node.parent_root, "returned range block parent should match fork_choice node"); - assert_eq!(signed_block.block.proposer_index, expected_node.proposer_index, "returned range block proposer should match fork_choice node"); + assert_eq!(block.slot, expected_node.slot, "returned range block slot should match requested slot"); + assert_eq!(block.parent_root, expected_node.parent_root, "returned range block parent should match fork_choice node"); + assert_eq!(block.proposer_index, expected_node.proposer_index, "returned range block proposer should match fork_choice node"); } } @@ -1309,26 +1309,26 @@ dyn_async! { let mut previous_slot = None; for payload in success_payloads { - let signed_block = LeanSignedBlock::from_ssz_bytes(payload) + let block = LeanBlock::from_signed_wire_ssz_bytes(payload) .expect("returned range block should decode from SSZ"); assert!( - signed_block.block.slot >= start_slot && signed_block.block.slot < start_slot + count, + block.slot >= start_slot && block.slot < start_slot + count, "returned range block slot should be inside requested range" ); if let Some(previous_slot) = previous_slot { assert!( - signed_block.block.slot > previous_slot, + block.slot > previous_slot, "BlocksByRange responses should be strictly increasing by slot" ); } if let Some(expected_node) = known_nodes .iter() - .find(|node| node.slot == signed_block.block.slot) + .find(|node| node.slot == block.slot) { - assert_eq!(signed_block.block.parent_root, expected_node.parent_root, "returned range block parent should match fork_choice node"); - assert_eq!(signed_block.block.proposer_index, expected_node.proposer_index, "returned range block proposer should match fork_choice node"); + assert_eq!(block.parent_root, expected_node.parent_root, "returned range block parent should match fork_choice node"); + assert_eq!(block.proposer_index, expected_node.proposer_index, "returned range block proposer should match fork_choice node"); } - previous_slot = Some(signed_block.block.slot); + previous_slot = Some(block.slot); } } } diff --git a/simulators/lean/src/scenarios/rpc_compat.rs b/simulators/lean/src/scenarios/rpc_compat.rs index 4754ab6b48..170c8bd4d3 100644 --- a/simulators/lean/src/scenarios/rpc_compat.rs +++ b/simulators/lean/src/scenarios/rpc_compat.rs @@ -18,7 +18,7 @@ use reqwest::header::CONTENT_TYPE; use ssz::Decode as SszDecode; use ssz_derive::Decode; use ssz_types::{ - typenum::{U1073741824, U262144, U4096}, + typenum::{U1073741824, U262144, U4096, U524288}, BitList, VariableList, }; use std::time::Duration; @@ -133,18 +133,36 @@ struct LeanRpcBlock { body: LeanRpcBlockBody, } +/// Devnet4 per-attestation signature payload on the signed-block envelope. #[derive(Debug, Clone, PartialEq, Eq, Decode)] struct LeanRpcBlockSignatures { attestation_signatures: VariableList, proposer_signature: LeanSignature, } +/// Devnet4 signed-block wire shape: block plus per-attestation signatures. #[derive(Debug, Clone, PartialEq, Eq, Decode)] -struct LeanRpcSignedBlock { +struct LeanRpcSignedBlockDevnet4 { block: LeanRpcBlock, signature: LeanRpcBlockSignatures, } +/// Current signed-block wire shape (devnet5 and later): block plus one merged +/// multi-message aggregate proof, matching leanSpec's +/// `SignedBlock { block, proof: MultiMessageAggregate }`. +#[derive(Debug, Clone, PartialEq, Eq, Decode)] +struct LeanRpcSignedBlock { + block: LeanRpcBlock, + proof: LeanRpcMultiMessageAggregate, +} + +/// Merged proof covering every attestation in the body plus the proposer +/// signature, matching leanSpec's `MultiMessageAggregate`. +#[derive(Debug, Clone, PartialEq, Eq, Decode)] +struct LeanRpcMultiMessageAggregate { + proof: VariableList, +} + impl From for LeanState { fn from(state: LeanStateDevnet4) -> Self { let tree_hash_root = state.tree_hash_root(); @@ -303,12 +321,22 @@ async fn load_finalized_block_bytes(client: &Client) -> Vec { .to_vec() } -fn decode_finalized_block(ssz_bytes: &[u8]) -> LeanRpcSignedBlock { - LeanRpcSignedBlock::from_ssz_bytes(ssz_bytes) - .unwrap_or_else(|err| panic!("Unable to decode SSZ finalized block: {err:?}")) +/// Decode the block carried by a finalized signed-block payload, using the +/// wire shape of the selected devnet. The signature payload is validated +/// structurally by the decode and then discarded. +fn decode_finalized_block(ssz_bytes: &[u8]) -> LeanRpcBlock { + if selected_lean_devnet().uses_merged_block_proof() { + LeanRpcSignedBlock::from_ssz_bytes(ssz_bytes) + .map(|signed| signed.block) + .unwrap_or_else(|err| panic!("Unable to decode SSZ finalized block: {err:?}")) + } else { + LeanRpcSignedBlockDevnet4::from_ssz_bytes(ssz_bytes) + .map(|signed| signed.block) + .unwrap_or_else(|err| panic!("Unable to decode SSZ finalized block: {err:?}")) + } } -async fn load_finalized_block(client: &Client) -> LeanRpcSignedBlock { +async fn load_finalized_block(client: &Client) -> LeanRpcBlock { decode_finalized_block(&load_finalized_block_bytes(client).await) } @@ -372,7 +400,7 @@ async fn wait_for_finalized_state_and_block_to_reach_observed_finalized_slot( client: &Client, ) -> ( LeanState, - LeanRpcSignedBlock, + LeanRpcBlock, ForkChoiceResponse, CheckpointResponse, ) { @@ -388,13 +416,13 @@ async fn wait_for_finalized_state_and_block_to_reach_observed_finalized_slot( let current_fork_choice = load_fork_choice_response(client).await; last_state_slot = Some(state.slot); - last_block_slot = Some(block.block.slot); + last_block_slot = Some(block.slot); last_current_fork_choice_slot = Some(current_fork_choice.finalized.slot); - let block_root = block.block.tree_hash_root(); + let block_root = block.tree_hash_root(); if state.slot >= target_finalized.slot - && block.block.slot == state.slot - && block.block.state_root == state.tree_hash_root + && block.slot == state.slot + && block.state_root == state.tree_hash_root && block_root == current_fork_choice.finalized.root { return (state, block, current_fork_choice, target_finalized); @@ -1351,14 +1379,14 @@ dyn_async! { let (state, block, fork_choice, observed_finalized) = wait_for_finalized_state_and_block_to_reach_observed_finalized_slot(&context.client_under_test).await; - let finalized_block_root = block.block.tree_hash_root(); + let finalized_block_root = block.tree_hash_root(); assert_eq!( - block.block.state_root, state.tree_hash_root, + block.state_root, state.tree_hash_root, "finalized block state_root should match the finalized state tree hash root" ); assert_eq!( - block.block.slot, state.slot, + block.slot, state.slot, "finalized block slot should match the finalized state slot" ); assert_eq!( diff --git a/simulators/lean/src/scenarios/validation.rs b/simulators/lean/src/scenarios/validation.rs index 18e02b0fa3..d97d05346d 100644 --- a/simulators/lean/src/scenarios/validation.rs +++ b/simulators/lean/src/scenarios/validation.rs @@ -1,6 +1,6 @@ use crate::utils::libp2p_mock::{ - decode_request, encode_gossip_data, extract_ip_port, lean_block_topic, replace_multiaddr_ip, - LeanSignedBlock, MockNode, Status, RESPONSE_CODE_SUCCESS, + decode_request, encode_gossip_block, extract_ip_port, lean_block_topic, replace_multiaddr_ip, + LeanBlock, MockNode, Status, RESPONSE_CODE_SUCCESS, }; use crate::utils::util::{ expect_single_client, lean_clients, lean_environment, lean_single_client_runtime_setup, @@ -121,10 +121,10 @@ dyn_async! { mock.process_events_for(Duration::from_secs(3)).await; - let invalid_block = LeanSignedBlock::build_minimal( + let invalid_block = LeanBlock::build_minimal( 1, 9999, B256::ZERO, B256::ZERO ); - let block_bytes = encode_gossip_data(&invalid_block); + let block_bytes = encode_gossip_block(&invalid_block); mock.publish(block_topic, block_bytes) .expect("should publish invalid block"); @@ -197,10 +197,10 @@ dyn_async! { mock.process_events_for(Duration::from_secs(3)).await; - let invalid_block = LeanSignedBlock::build_minimal( + let invalid_block = LeanBlock::build_minimal( 1, 0, B256::from_slice(&[0xde; 32]), B256::ZERO ); - let block_bytes = encode_gossip_data(&invalid_block); + let block_bytes = encode_gossip_block(&invalid_block); mock.publish(block_topic, block_bytes) .expect("should publish invalid block"); @@ -273,10 +273,10 @@ dyn_async! { mock.process_events_for(Duration::from_secs(3)).await; - let invalid_block = LeanSignedBlock::build_minimal( + let invalid_block = LeanBlock::build_minimal( 1, 0, B256::ZERO, B256::from_slice(&[0xbe; 32]) ); - let block_bytes = encode_gossip_data(&invalid_block); + let block_bytes = encode_gossip_block(&invalid_block); mock.publish(block_topic, block_bytes) .expect("should publish invalid block"); diff --git a/simulators/lean/src/utils/libp2p_mock.rs b/simulators/lean/src/utils/libp2p_mock.rs index 4ca33698b2..4370acdbb0 100644 --- a/simulators/lean/src/utils/libp2p_mock.rs +++ b/simulators/lean/src/utils/libp2p_mock.rs @@ -24,11 +24,13 @@ use snap::{read::FrameDecoder, write::FrameEncoder}; use ssz::Encode; use ssz_derive::{Decode as SszDecodeDerive, Encode as SszEncodeDerive}; use ssz_types::{ - typenum::{U1024, U1048576, U4096}, + typenum::{U1024, U1048576, U4096, U524288}, BitList, VariableList, }; use tokio::time::timeout; +use crate::utils::util::selected_lean_devnet; + // Protocol strings for lean reqresp pub const LEAN_STATUS_PROTOCOL: &str = "/leanconsensus/req/status/1/ssz_snappy"; pub const LEAN_BLOCKS_BY_ROOT_PROTOCOL: &str = "/leanconsensus/req/blocks_by_root/1/ssz_snappy"; @@ -186,20 +188,41 @@ pub struct LeanBlock { pub body: LeanBlockBody, } +/// Devnet4 per-attestation signature payload on the signed-block envelope. #[derive(Debug, Default, Clone, PartialEq, Eq, SszEncodeDerive, SszDecodeDerive)] pub struct LeanBlockSignatures { pub attestation_signatures: VariableList, pub proposer_signature: LeanSignature, } +/// Devnet4 signed-block wire shape: block plus per-attestation signatures. #[derive(Debug, Default, Clone, PartialEq, Eq, SszEncodeDerive, SszDecodeDerive)] -pub struct LeanSignedBlock { +pub struct LeanSignedBlockDevnet4 { pub block: LeanBlock, pub signature: LeanBlockSignatures, } -impl LeanSignedBlock { - /// Build a minimal valid-SSZ signed block for the given slot with the specified parent. +/// Merged proof covering every attestation in the body plus the proposer +/// signature over the block root. +/// +/// Matches leanSpec's `MultiMessageAggregate`: a single ByteList512KiB of +/// compact public-key-free aggregate proof bytes. +#[derive(Debug, Default, Clone, PartialEq, Eq, SszEncodeDerive, SszDecodeDerive)] +pub struct LeanMultiMessageAggregate { + pub proof: VariableList, +} + +/// Current signed-block wire shape (devnet5 and later): block plus one merged +/// multi-message aggregate proof, matching leanSpec's +/// `SignedBlock { block, proof: MultiMessageAggregate }`. +#[derive(Debug, Default, Clone, PartialEq, Eq, SszEncodeDerive, SszDecodeDerive)] +pub struct LeanSignedBlock { + pub block: LeanBlock, + pub proof: LeanMultiMessageAggregate, +} + +impl LeanBlock { + /// Build a minimal valid-SSZ block for the given slot with the specified parent. pub fn build_minimal( slot: u64, proposer_index: u64, @@ -207,26 +230,53 @@ impl LeanSignedBlock { state_root: B256, ) -> Self { Self { - block: LeanBlock { - slot, - proposer_index, - parent_root, - state_root, - body: LeanBlockBody { - attestations: VariableList::new(vec![]).expect("empty attestation list"), - }, - }, - signature: LeanBlockSignatures { - attestation_signatures: VariableList::new(vec![]).expect("empty signature list"), - proposer_signature: LeanSignature::default(), + slot, + proposer_index, + parent_root, + state_root, + body: LeanBlockBody { + attestations: VariableList::new(vec![]).expect("empty attestation list"), }, } } + + /// Decode the block carried by a signed-block payload, using the wire + /// shape of the selected devnet. + /// + /// The signature payload is validated structurally by the decode and + /// then discarded; the mock never verifies proofs. + pub fn from_signed_wire_ssz_bytes(bytes: &[u8]) -> Result { + if selected_lean_devnet().uses_merged_block_proof() { + ::from_ssz_bytes(bytes).map(|signed| signed.block) + } else { + ::from_ssz_bytes(bytes) + .map(|signed| signed.block) + } + } + + /// Encode the block into a signed-block envelope with an empty proof or + /// signature payload, using the wire shape of the selected devnet. + pub fn to_signed_wire_ssz_bytes(&self) -> Vec { + if selected_lean_devnet().uses_merged_block_proof() { + LeanSignedBlock { + block: self.clone(), + proof: LeanMultiMessageAggregate::default(), + } + .as_ssz_bytes() + } else { + LeanSignedBlockDevnet4 { + block: self.clone(), + signature: LeanBlockSignatures::default(), + } + .as_ssz_bytes() + } + } } -/// Encode data for gossipsub: snappy-compressed SSZ bytes (no varint prefix). -pub fn encode_gossip_data(item: &T) -> Vec { - let ssz_bytes = item.as_ssz_bytes(); +/// Encode a block as an unsigned gossip block message using the selected +/// devnet's wire shape: snappy-compressed SSZ bytes (no varint prefix). +pub fn encode_gossip_block(block: &LeanBlock) -> Vec { + let ssz_bytes = block.to_signed_wire_ssz_bytes(); let mut encoder = FrameEncoder::new(Vec::new()); encoder .write_all(&ssz_bytes) diff --git a/simulators/lean/src/utils/util.rs b/simulators/lean/src/utils/util.rs index caa0c8f54f..4e3874641d 100644 --- a/simulators/lean/src/utils/util.rs +++ b/simulators/lean/src/utils/util.rs @@ -61,6 +61,14 @@ impl LeanDevnet { pub(crate) fn uses_latest_leanspec_format(self) -> bool { matches!(self, Self::Devnet4 | Self::Devnet5) } + + /// Devnet5 replaced the per-attestation signature list on the signed-block + /// envelope with a single merged multi-message aggregate proof, matching + /// leanSpec's `SignedBlock { block, proof: MultiMessageAggregate }`. + /// Devnet4 still ships `SignedBlock { block, signature: BlockSignatures }`. + pub(crate) fn uses_merged_block_proof(self) -> bool { + matches!(self, Self::Devnet5) + } } impl fmt::Display for LeanDevnet { From 4ab4cfd589f1b7792dcfe5f4e73ede29ecd096f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Fri, 12 Jun 2026 16:18:10 -0300 Subject: [PATCH 2/2] simulators/lean: support leanSpec's restructured package layout in the helper leanSpec moved its packages from subspecs/types to node/spec (leanSpec PR #788 and follow-ups), so a devnet5 helper built from leanSpec main crashes on startup with ModuleNotFoundError and every helper-dependent devnet5 test fails at setup. The devnet4 helper stays pinned on the old layout, so the runner now tries the new import locations first and falls back to the old ones, both for static imports and the dynamically imported modules used by the compatibility patches. The trusted gossip attestation override additionally follows two API renames on the new layout: the voter field (validator_id -> validator_index) and the registry-bounds predicate (is_valid -> is_within_registry). --- clients/lean-spec-client/lean_spec_runtime.py | 169 ++++++++++++++---- .../lean/helper/lean_spec_client_runner.py | 164 ++++++++++++----- 2 files changed, 245 insertions(+), 88 deletions(-) diff --git a/clients/lean-spec-client/lean_spec_runtime.py b/clients/lean-spec-client/lean_spec_runtime.py index 9c31d686f1..d84f78ecb0 100644 --- a/clients/lean-spec-client/lean_spec_runtime.py +++ b/clients/lean-spec-client/lean_spec_runtime.py @@ -14,6 +14,7 @@ import shutil import sys import time +import typing import types from pathlib import Path from typing import Any, Final @@ -51,40 +52,88 @@ def decorate(func: object) -> object: install_numba_njit_stub() +# leanSpec restructured its packages over time (subspecs/types -> node/spec). +# The devnet5 helper builds from leanSpec main (new layout) while the devnet4 +# helper stays pinned on the old layout, so every import tries the new +# location first and falls back to the old ones. try: + from lean_spec.spec.forks import DEFAULT_REGISTRY +except (ImportError, ModuleNotFoundError): from lean_spec.forks import DEFAULT_REGISTRY -except ModuleNotFoundError: - DEFAULT_REGISTRY = None try: - from lean_spec.subspecs.containers import Checkpoint - from lean_spec.subspecs.containers.validator import SubnetId, ValidatorIndex + from lean_spec.spec.forks.lstar.containers import Checkpoint, SubnetId, ValidatorIndex except (ImportError, ModuleNotFoundError): try: - from lean_spec.types import Checkpoint, SubnetId, ValidatorIndex + from lean_spec.subspecs.containers import Checkpoint + from lean_spec.subspecs.containers.validator import SubnetId, ValidatorIndex except (ImportError, ModuleNotFoundError): - from lean_spec.forks.lstar.containers import Checkpoint - from lean_spec.forks.lstar.containers.validator import SubnetId, ValidatorIndex - -from lean_spec.subspecs.chain.config import ATTESTATION_COMMITTEE_COUNT -from lean_spec.subspecs.genesis.config import GenesisConfig -from lean_spec.subspecs.networking.enr import ENR, keys as enr_keys -from lean_spec.subspecs.networking.gossipsub import GossipTopic -from lean_spec.subspecs.networking.reqresp.message import Status -from lean_spec.subspecs.networking.transport.identity import IdentityKeypair -from lean_spec.subspecs.networking.transport.identity.keypair import Secp256k1PublicKey -from lean_spec.subspecs.ssz.hash import hash_tree_root -from lean_spec.subspecs.xmss import SecretKey -from lean_spec.subspecs.xmss import aggregation as xmss_aggregation_module -from lean_spec.types import Bytes32 -from lean_spec.types.collections import SSZList, SSZVector, _validate_offsets -from lean_spec.types.constants import OFFSET_BYTE_LENGTH -from lean_spec.types.container import Container -from lean_spec.types.exceptions import SSZSerializationError, SSZValueError -from lean_spec.types.uint import Uint32 + try: + from lean_spec.types import Checkpoint, SubnetId, ValidatorIndex + except (ImportError, ModuleNotFoundError): + from lean_spec.forks.lstar.containers import Checkpoint + from lean_spec.forks.lstar.containers.validator import SubnetId, ValidatorIndex + +try: + from lean_spec.node.genesis import GenesisConfig + from lean_spec.node.networking.enr import ENR, keys as enr_keys + from lean_spec.node.networking.gossipsub import GossipTopic + from lean_spec.node.networking.reqresp.message import Status + from lean_spec.node.networking.transport.identity import ( + IdentityKeypair, + Secp256k1PublicKey, + ) + from lean_spec.spec.crypto.merkleization import hash_tree_root + from lean_spec.spec.crypto.xmss import SecretKey + from lean_spec.spec.forks.lstar import aggregation as xmss_aggregation_module + from lean_spec.spec.forks.lstar.config import ATTESTATION_COMMITTEE_COUNT + from lean_spec.spec.ssz import ( + Bytes32, + Container, + SSZList, + SSZSerializationError, + SSZValueError, + SSZVector, + Uint32, + ) + from lean_spec.spec.ssz.collections import _validate_offsets + from lean_spec.spec.ssz.ssz_base import BYTES_PER_LENGTH_OFFSET as OFFSET_BYTE_LENGTH +except (ImportError, ModuleNotFoundError): + from lean_spec.subspecs.chain.config import ATTESTATION_COMMITTEE_COUNT + from lean_spec.subspecs.genesis.config import GenesisConfig + from lean_spec.subspecs.networking.enr import ENR, keys as enr_keys + from lean_spec.subspecs.networking.gossipsub import GossipTopic + from lean_spec.subspecs.networking.reqresp.message import Status + from lean_spec.subspecs.networking.transport.identity import IdentityKeypair + from lean_spec.subspecs.networking.transport.identity.keypair import Secp256k1PublicKey + from lean_spec.subspecs.ssz.hash import hash_tree_root + from lean_spec.subspecs.xmss import SecretKey + from lean_spec.subspecs.xmss import aggregation as xmss_aggregation_module + from lean_spec.types import Bytes32 + from lean_spec.types.collections import SSZList, SSZVector, _validate_offsets + from lean_spec.types.constants import OFFSET_BYTE_LENGTH + from lean_spec.types.container import Container + from lean_spec.types.exceptions import SSZSerializationError, SSZValueError + from lean_spec.types.uint import Uint32 logger = logging.getLogger("lean_spec_client_runner") + +def import_first_module(*module_names: str) -> types.ModuleType: + """Import the first available module from a list of historical locations. + + Mirrors the try/except import chains above for dynamically imported + modules: new-layout names come first, old-layout names act as fallbacks + for the pinned devnet4 helper checkout. + """ + for module_name in module_names[:-1]: + try: + return importlib.import_module(module_name) + except (ImportError, ModuleNotFoundError): + continue + return importlib.import_module(module_names[-1]) + + BOOTNODE_DIAL_TIMEOUT_SECS: Final = 10.0 STATUS_REFRESH_INTERVAL_SECS: Final = 1.0 PREPARED_ASSETS_DIR: Final = Path( @@ -271,8 +320,14 @@ def container_deserialize_without_validation( var_fields = [] bytes_read = 0 + # Older leanSpec layouts resolve SSZ field types through a private + # helper; newer ones store the SSZ type directly on the annotation. + annotation_resolver = getattr(cls, "_get_ssz_field_type", None) for field_name, field_info in cls.model_fields.items(): - field_type = cls._get_ssz_field_type(field_info.annotation) + if annotation_resolver is not None: + field_type = annotation_resolver(field_info.annotation) + else: + field_type = field_info.annotation if field_type.is_fixed_size(): size = field_type.get_byte_length() @@ -328,19 +383,40 @@ def install_snappy_compress_fallback() -> None: if _SNAPPY_COMPRESS_FALLBACK_INSTALLED: return - snappy_package_module = importlib.import_module("lean_spec.snappy") - snappy_compress_module = importlib.import_module("lean_spec.snappy.compress") - snappy_framing_module = importlib.import_module("lean_spec.snappy.framing") - reqresp_codec_module = importlib.import_module("lean_spec.subspecs.networking.reqresp.codec") - networking_service_module = importlib.import_module( - "lean_spec.subspecs.networking.service.service" + snappy_package_module = import_first_module("lean_spec.node.snappy", "lean_spec.snappy") + snappy_compress_module = import_first_module( + "lean_spec.node.snappy.compress", "lean_spec.snappy.compress" + ) + snappy_framing_module = import_first_module( + "lean_spec.node.snappy.framing", "lean_spec.snappy.framing" + ) + reqresp_codec_module = import_first_module( + "lean_spec.node.networking.reqresp.codec", + "lean_spec.subspecs.networking.reqresp.codec", + ) + networking_service_module = import_first_module( + "lean_spec.node.networking.service.service", + "lean_spec.subspecs.networking.service.service", ) + def encode_snappy_uncompressed_length(value: int) -> bytes: + # Standard unsigned LEB128, inlined because the varint helper moved + # modules (and changed name) across leanSpec layouts. + output = bytearray() + while True: + byte = value & 0x7F + value >>= 7 + if value: + output.append(byte | 0x80) + else: + output.append(byte) + return bytes(output) + def literal_only_compress(data: bytes) -> bytes: if not data: - return snappy_compress_module.encode_varint32(0) + return encode_snappy_uncompressed_length(0) - output = bytearray(snappy_compress_module.encode_varint32(len(data))) + output = bytearray(encode_snappy_uncompressed_length(len(data))) offset = 0 while offset < len(data): block_end = min(offset + snappy_compress_module.BLOCK_SIZE, len(data)) @@ -564,6 +640,21 @@ def write_validator_keys( yaml.safe_dump(manifest, manifest_file, sort_keys=False) +def genesis_validator_pubkey_keys() -> tuple[str, str]: + """Return the per-validator pubkey YAML keys the active GenesisConfig expects. + + Newer leanSpec checkouts spell the keys out in full + (attestation_public_key); older ones use the abbreviated form + (attestation_pubkey). Introspect the validator entry model so the + emitted YAML always matches the checkout that will parse it. + """ + entry_annotation = GenesisConfig.model_fields["genesis_validators"].annotation + (entry_class,) = typing.get_args(entry_annotation) + if "attestation_public_key" in entry_class.model_fields: + return ("attestation_public_key", "proposal_public_key") + return ("attestation_pubkey", "proposal_pubkey") + + def write_genesis_config( validators: list[dict[str, str]], genesis_time: int, @@ -575,10 +666,11 @@ def write_genesis_config( genesis.setdefault("NUM_VALIDATORS", len(validators)) if uses_latest_leanspec_format(): + attestation_key, proposal_key = genesis_validator_pubkey_keys() genesis["GENESIS_VALIDATORS"] = [ { - "attestation_pubkey": f"0x{validator[ATTESTATION_PUBKEY_FIELD]}", - "proposal_pubkey": f"0x{validator[PROPOSAL_PUBKEY_FIELD]}", + attestation_key: f"0x{validator[ATTESTATION_PUBKEY_FIELD]}", + proposal_key: f"0x{validator[PROPOSAL_PUBKEY_FIELD]}", } for validator in validators ] @@ -635,10 +727,11 @@ def helper_genesis_metadata() -> dict[str, object]: validators = genesis_config.get("GENESIS_VALIDATORS", []) if uses_latest_leanspec_format(): + attestation_key, proposal_key = genesis_validator_pubkey_keys() genesis_validator_entries = [ { - "attestation_public_key": validator["attestation_pubkey"], - "proposal_public_key": validator["proposal_pubkey"], + "attestation_public_key": validator[attestation_key], + "proposal_public_key": validator[proposal_key], } for validator in validators ] @@ -917,8 +1010,6 @@ def build_node_config( } if node_config_supports(node_config_type, "fork"): - if DEFAULT_REGISTRY is None: - raise RuntimeError("LeanSpec NodeConfig requires a fork, but no fork registry exists") config["fork"] = DEFAULT_REGISTRY.current config["network_name"] = gossip_network_name(fork_digest) else: diff --git a/simulators/lean/helper/lean_spec_client_runner.py b/simulators/lean/helper/lean_spec_client_runner.py index a0d5cdadb7..adbaf75535 100644 --- a/simulators/lean/helper/lean_spec_client_runner.py +++ b/simulators/lean/helper/lean_spec_client_runner.py @@ -29,6 +29,7 @@ extract_inner_block, helper_genesis_metadata, identity_keypair_from_private_key_hex, + import_first_module, install_fast_ssz_deserialization, install_low_s_identity_signature_compatibility, install_setup_prover_compatibility, @@ -48,30 +49,58 @@ uses_latest_leanspec_format, ) -from lean_spec.subspecs.api import ApiServer, ApiServerConfig -from lean_spec.subspecs.metrics import registry as metrics -from lean_spec.subspecs.networking.client import LiveNetworkEventSource -from lean_spec.subspecs.networking.service.events import GossipBlockEvent -from lean_spec.subspecs.networking.transport.identity import IdentityKeypair -from lean_spec.subspecs.networking.transport.quic.connection import ( - QuicConnectionManager, -) -from lean_spec.subspecs.node import Node, NodeConfig +# leanSpec restructured its packages over time (subspecs/types -> node/spec). +# The devnet5 helper builds from leanSpec main (new layout) while the devnet4 +# helper stays pinned on the old layout, so every import tries the new +# location first and falls back to the old ones. try: - from lean_spec.subspecs.xmss.aggregation import ( - AggregatedSignatureProof as TypeOneSignatureProof, + from lean_spec.node.api import ApiServer, ApiServerConfig + from lean_spec.node.metrics import registry as metrics + from lean_spec.node.networking.client import LiveNetworkEventSource + from lean_spec.node.networking.service.events import GossipBlockEvent + from lean_spec.node.networking.transport.identity import IdentityKeypair + from lean_spec.node.networking.transport.quic.connection import ( + QuicConnectionManager, ) -except ImportError: - from lean_spec.subspecs.xmss.aggregation import TypeOneMultiSignature as TypeOneSignatureProof -from lean_spec.subspecs.ssz.hash import hash_tree_root -from lean_spec.subspecs.validator import ValidatorRegistry -from lean_spec.subspecs.validator.registry import ValidatorEntry -from lean_spec.types import Bytes32 -try: - from lean_spec.types.participation import AggregationBits, ValidatorIndices -except ImportError: - from lean_spec.types import AggregationBits, ValidatorIndices -from lean_spec.types.uint import Uint64 + from lean_spec.node.node import Node, NodeConfig + from lean_spec.node.validator import ValidatorRegistry + from lean_spec.node.validator.registry import ValidatorEntry + from lean_spec.spec.crypto.merkleization import hash_tree_root + from lean_spec.spec.forks.lstar.containers import ( + AggregationBits, + SingleMessageAggregate as TypeOneSignatureProof, + ValidatorIndices, + ) + from lean_spec.spec.ssz import Bytes32, Uint64 +except (ImportError, ModuleNotFoundError): + from lean_spec.subspecs.api import ApiServer, ApiServerConfig + from lean_spec.subspecs.metrics import registry as metrics + from lean_spec.subspecs.networking.client import LiveNetworkEventSource + from lean_spec.subspecs.networking.service.events import GossipBlockEvent + from lean_spec.subspecs.networking.transport.identity import IdentityKeypair + from lean_spec.subspecs.networking.transport.quic.connection import ( + QuicConnectionManager, + ) + from lean_spec.subspecs.node import Node, NodeConfig + + try: + from lean_spec.subspecs.xmss.aggregation import ( + AggregatedSignatureProof as TypeOneSignatureProof, + ) + except ImportError: + from lean_spec.subspecs.xmss.aggregation import ( + TypeOneMultiSignature as TypeOneSignatureProof, + ) + from lean_spec.subspecs.ssz.hash import hash_tree_root + from lean_spec.subspecs.validator import ValidatorRegistry + from lean_spec.subspecs.validator.registry import ValidatorEntry + from lean_spec.types import Bytes32 + + try: + from lean_spec.types.participation import AggregationBits, ValidatorIndices + except ImportError: + from lean_spec.types import AggregationBits, ValidatorIndices + from lean_spec.types.uint import Uint64 DEFAULT_GOSSIP_FORK_DIGEST: Final = "devnet0" DEFAULT_LISTEN_PORT: Final = 9001 @@ -116,8 +145,9 @@ def install_reqresp_block_cache_tracking() -> None: if _REQRESP_BLOCK_CACHE_TRACKING_INSTALLED: return - reqresp_client_module = importlib.import_module( - "lean_spec.subspecs.networking.client.reqresp_client" + reqresp_client_module = import_first_module( + "lean_spec.node.networking.client.reqresp_client", + "lean_spec.subspecs.networking.client.reqresp_client", ) reqresp_client_cls = reqresp_client_module.ReqRespClient original_request_blocks_by_root = reqresp_client_cls.request_blocks_by_root @@ -144,8 +174,9 @@ def install_network_service_gossip_tracking() -> None: if _NETWORK_SERVICE_GOSSIP_TRACKING_INSTALLED: return - networking_service_module = importlib.import_module( - "lean_spec.subspecs.networking.service.service" + networking_service_module = import_first_module( + "lean_spec.node.networking.service.service", + "lean_spec.subspecs.networking.service.service", ) network_service_cls = networking_service_module.NetworkService original_handle_event = network_service_cls._handle_event @@ -289,11 +320,13 @@ def install_identify_protocol_compatibility() -> None: if _IDENTIFY_PROTOCOL_COMPAT_INSTALLED: return - live_module = importlib.import_module( - "lean_spec.subspecs.networking.client.event_source.live" + live_module = import_first_module( + "lean_spec.node.networking.client.event_source.live", + "lean_spec.subspecs.networking.client.event_source.live", ) - protocol_module = importlib.import_module( - "lean_spec.subspecs.networking.client.event_source.protocol" + protocol_module = import_first_module( + "lean_spec.node.networking.client.event_source.protocol", + "lean_spec.subspecs.networking.client.event_source.protocol", ) extra_protocols = { @@ -370,18 +403,25 @@ def install_child_only_aggregation_compatibility() -> None: if _CHILD_ONLY_AGGREGATION_COMPAT_INSTALLED: return - original_to_aggregation_bits = ValidatorIndices.to_aggregation_bits - original_aggregate = TypeOneSignatureProof.aggregate - aggregate_parameters = inspect.signature(original_aggregate).parameters + # Both patches work around old-layout quirks. Newer leanSpec checkouts + # fixed the underlying behavior and dropped or reshaped these methods, + # so each patch only installs when its target still exists. + original_to_aggregation_bits = getattr(ValidatorIndices, "to_aggregation_bits", None) + original_aggregate = getattr(TypeOneSignatureProof, "aggregate", None) + aggregate_parameters = ( + inspect.signature(original_aggregate).parameters if original_aggregate else {} + ) - def to_aggregation_bits_allowing_empty( - self: ValidatorIndices, - ) -> AggregationBits: - if not self.data: - return AggregationBits(data=[]) - return original_to_aggregation_bits(self) + if original_to_aggregation_bits is not None: - ValidatorIndices.to_aggregation_bits = to_aggregation_bits_allowing_empty + def to_aggregation_bits_allowing_empty( + self: ValidatorIndices, + ) -> AggregationBits: + if not self.data: + return AggregationBits(data=[]) + return original_to_aggregation_bits(self) + + ValidatorIndices.to_aggregation_bits = to_aggregation_bits_allowing_empty if {"children", "raw_xmss", "xmss_participants"}.issubset(aggregate_parameters): @@ -421,8 +461,12 @@ def install_trusted_gossip_attestation_compatibility() -> None: return try: - from lean_spec.forks.lstar.spec import LstarSpec - from lean_spec.forks.lstar.store import AttestationSignatureEntry + try: + from lean_spec.spec.forks.lstar.containers import AttestationSignatureEntry + from lean_spec.spec.forks.lstar.spec import LstarSpec + except (ImportError, ModuleNotFoundError): + from lean_spec.forks.lstar.spec import LstarSpec + from lean_spec.forks.lstar.store import AttestationSignatureEntry except ImportError as err: logger.debug("LeanSpec trusted attestation compatibility unavailable: %s", err) return @@ -433,7 +477,12 @@ def on_gossip_attestation_without_signature_verification( signed_attestation: Any, is_aggregator: bool = False, ) -> Any: - validator_id = signed_attestation.validator_id + # The voter field and the registry-bounds predicate were renamed on + # newer leanSpec layouts (validator_id -> validator_index, is_valid -> + # is_within_registry); resolve whichever spelling the checkout has. + validator_index = getattr(signed_attestation, "validator_index", None) + if validator_index is None: + validator_index = signed_attestation.validator_id attestation_data = signed_attestation.data signature = signed_attestation.signature @@ -444,8 +493,12 @@ def on_gossip_attestation_without_signature_verification( f"No state available to validate attestation for target block " f"{attestation_data.target.root.hex()}" ) - assert validator_id.is_valid(Uint64(len(key_state.validators))), ( - f"Validator {validator_id} not found in state {attestation_data.target.root.hex()}" + in_registry = getattr(validator_index, "is_within_registry", None) + if in_registry is None: + in_registry = validator_index.is_valid + assert in_registry(Uint64(len(key_state.validators))), ( + f"Validator {validator_index} not found in state " + f"{attestation_data.target.root.hex()}" ) if not is_aggregator: @@ -453,7 +506,7 @@ def on_gossip_attestation_without_signature_verification( new_committee_sigs = {k: set(v) for k, v in store.attestation_signatures.items()} new_committee_sigs.setdefault(attestation_data, set()).add( - AttestationSignatureEntry(validator_id, signature) + AttestationSignatureEntry(validator_index, signature) ) return store.model_copy(update={"attestation_signatures": new_committee_sigs}) @@ -469,13 +522,19 @@ def install_checkpoint_finalization_compatibility() -> None: return try: - from lean_spec.forks.lstar.spec import LstarSpec + try: + from lean_spec.spec.forks.lstar.spec import LstarSpec + except (ImportError, ModuleNotFoundError): + from lean_spec.forks.lstar.spec import LstarSpec except ImportError as err: logger.debug("LeanSpec checkpoint finalization compatibility unavailable: %s", err) return try: - from lean_spec.subspecs.validator.service import ValidatorService + try: + from lean_spec.node.validator.service import ValidatorService + except (ImportError, ModuleNotFoundError): + from lean_spec.subspecs.validator.service import ValidatorService except ImportError: ValidatorService = None @@ -882,7 +941,14 @@ async def cache_and_publish_block(signed_block: object) -> None: node.validator_service.on_block = cache_and_publish_block event_source.set_block_lookup(lookup_reqresp_block) - event_source.set_block_by_slot_lookup(lookup_reqresp_block_by_slot) + # Newer leanSpec checkouts dropped the by-slot convenience setter but + # kept the handler field it assigned, so fall back to wiring the field + # directly when the setter is gone. + set_block_by_slot_lookup = getattr(event_source, "set_block_by_slot_lookup", None) + if set_block_by_slot_lookup is not None: + set_block_by_slot_lookup(lookup_reqresp_block_by_slot) + else: + event_source._reqresp_handler.block_by_slot_lookup = lookup_reqresp_block_by_slot event_source.set_current_slot_lookup(current_known_slot) listener_task = await start_listener_and_gossipsub(event_source, listen_addr())