diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index c4dee67..89527ce 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -101,9 +101,10 @@ jobs: # so a suite leanSpec renames or drops breaks the job here rather than quietly halving # the vector count downstream. # - # The state-transition tree is taken whole rather than suite by suite: its harness - # already names all fifteen suites and fails on any that matches no file, so listing - # them again here would only put the same list in two places. + # The state-transition and fork-choice trees are taken whole rather than suite by + # suite: each harness already names every suite it consumes and fails on any that + # matches no file, so listing them again here would only put the same list in two + # places. - name: Extract consumed fixture suites run: | mkdir -p fixtures-prod @@ -112,7 +113,8 @@ jobs: '*/ssz/*/ssz/test_xmss_containers/*.json' \ '*/justifiability/*/state_transition/test_justifiability/*.json' \ '*/slot_clock/*/chain/test_slot_clock/*.json' \ - '*/state_transition/*/state_transition/*/*.json' + '*/state_transition/*/state_transition/*/*.json' \ + '*/fork_choice/*/fork_choice/*/*.json' # The whole workspace, so a crate that starts consuming fixtures is picked up without # editing this job. Every fixture test skips when VERITY_FIXTURES is unset, which is diff --git a/crates/verity-chain/Cargo.toml b/crates/verity-chain/Cargo.toml index 339bf54..b620df8 100644 --- a/crates/verity-chain/Cargo.toml +++ b/crates/verity-chain/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "verity-chain" -description = "Consensus decisions over the container types: the state transition, justification candidacy, and the slot clock." +description = "Consensus decisions over the container types: the state transition, fork choice, justification candidacy, and the slot clock." version.workspace = true edition.workspace = true rust-version.workspace = true diff --git a/crates/verity-chain/src/error.rs b/crates/verity-chain/src/error.rs index b9f8421..1e5c263 100644 --- a/crates/verity-chain/src/error.rs +++ b/crates/verity-chain/src/error.rs @@ -6,9 +6,10 @@ //! leanSpec enum it mirrors so the two stay greppable against each other. //! //! Only the reasons Verity can currently produce are defined. leanSpec's enum has 36; the -//! rest belong to fork choice and gossip validation, and land with them. An unmodelled -//! reason is not silently tolerated: [`RejectionReason::as_str`] is what the fixture suites -//! compare against, so a vector expecting a reason this enum lacks fails the run. +//! four still absent are raised from paths this workspace has not reached — proposer-index +//! range checking, block-proof verification, and wire decoding. An unmodelled reason is not +//! silently tolerated: [`RejectionReason::as_str`] is what the fixture suites compare +//! against, so a vector expecting a reason this enum lacks fails the run. //! //! One variant is here ahead of the code that leanSpec raises it from. //! [`RejectionReason::BlockSlotGapTooLarge`] guards the transition's empty-slot walk, which @@ -54,6 +55,44 @@ pub enum RejectionReason { JustifiedSlotOutOfRange, /// A tracked justification root is the zero hash, which marks a slot with no block. ZeroHashJustificationRoot, + /// The anchor block does not commit to the anchor state it was handed with. + AnchorStateRootMismatch, + /// The block's parent has no state in the store, so the transition has nothing to start from. + UnknownParentBlock, + /// The block's slot runs past the horizon the store's own clock admits. + BlockTooFarInFuture, + /// The block repeats one attestation data entry, which the wire format forbids. + DuplicateAttestationData, + /// The vote names a source block the store has never seen. + UnknownSourceBlock, + /// The vote names a target block the store has never seen. + UnknownTargetBlock, + /// The vote names a head block the store has never seen. + UnknownHeadBlock, + /// The vote's source sits later than its target, which history forbids. + SourceAfterTarget, + /// The vote's head sits earlier than its target, which history forbids. + HeadOlderThanTarget, + /// The vote's source checkpoint slot disagrees with the slot of the block it names. + SourceSlotMismatch, + /// The vote's target checkpoint slot disagrees with the slot of the block it names. + TargetSlotMismatch, + /// The vote's head checkpoint slot disagrees with the slot of the block it names. + HeadSlotMismatch, + /// The vote's source does not lie on the target's chain of ancestors. + SourceNotAncestorOfTarget, + /// The vote's target does not lie on the head's chain of ancestors. + TargetNotAncestorOfHead, + /// The vote's head does not descend from the finalized block, so it can carry no weight. + HeadNotDescendantOfFinalized, + /// The vote's slot has not started locally yet, beyond the clock-skew margin. + AttestationTooFarInFuture, + /// The vote claims a head from a slot the vote itself precedes. + AttestationSlotBeforeHead, + /// The vote names a validator the target block's post-state registry does not hold. + ValidatorNotInState, + /// Signature verification failed. + InvalidSignature, } impl RejectionReason { @@ -75,6 +114,25 @@ impl RejectionReason { Self::JustificationVotesLengthMismatch => "JUSTIFICATION_VOTES_LENGTH_MISMATCH", Self::JustifiedSlotOutOfRange => "JUSTIFIED_SLOT_OUT_OF_RANGE", Self::ZeroHashJustificationRoot => "ZERO_HASH_JUSTIFICATION_ROOT", + Self::AnchorStateRootMismatch => "ANCHOR_STATE_ROOT_MISMATCH", + Self::UnknownParentBlock => "UNKNOWN_PARENT_BLOCK", + Self::BlockTooFarInFuture => "BLOCK_TOO_FAR_IN_FUTURE", + Self::DuplicateAttestationData => "DUPLICATE_ATTESTATION_DATA", + Self::UnknownSourceBlock => "UNKNOWN_SOURCE_BLOCK", + Self::UnknownTargetBlock => "UNKNOWN_TARGET_BLOCK", + Self::UnknownHeadBlock => "UNKNOWN_HEAD_BLOCK", + Self::SourceAfterTarget => "SOURCE_AFTER_TARGET", + Self::HeadOlderThanTarget => "HEAD_OLDER_THAN_TARGET", + Self::SourceSlotMismatch => "SOURCE_SLOT_MISMATCH", + Self::TargetSlotMismatch => "TARGET_SLOT_MISMATCH", + Self::HeadSlotMismatch => "HEAD_SLOT_MISMATCH", + Self::SourceNotAncestorOfTarget => "SOURCE_NOT_ANCESTOR_OF_TARGET", + Self::TargetNotAncestorOfHead => "TARGET_NOT_ANCESTOR_OF_HEAD", + Self::HeadNotDescendantOfFinalized => "HEAD_NOT_DESCENDANT_OF_FINALIZED", + Self::AttestationTooFarInFuture => "ATTESTATION_TOO_FAR_IN_FUTURE", + Self::AttestationSlotBeforeHead => "ATTESTATION_SLOT_BEFORE_HEAD", + Self::ValidatorNotInState => "VALIDATOR_NOT_IN_STATE", + Self::InvalidSignature => "INVALID_SIGNATURE", } } } @@ -107,6 +165,25 @@ mod tests { RejectionReason::JustificationVotesLengthMismatch, RejectionReason::JustifiedSlotOutOfRange, RejectionReason::ZeroHashJustificationRoot, + RejectionReason::AnchorStateRootMismatch, + RejectionReason::UnknownParentBlock, + RejectionReason::BlockTooFarInFuture, + RejectionReason::DuplicateAttestationData, + RejectionReason::UnknownSourceBlock, + RejectionReason::UnknownTargetBlock, + RejectionReason::UnknownHeadBlock, + RejectionReason::SourceAfterTarget, + RejectionReason::HeadOlderThanTarget, + RejectionReason::SourceSlotMismatch, + RejectionReason::TargetSlotMismatch, + RejectionReason::HeadSlotMismatch, + RejectionReason::SourceNotAncestorOfTarget, + RejectionReason::TargetNotAncestorOfHead, + RejectionReason::HeadNotDescendantOfFinalized, + RejectionReason::AttestationTooFarInFuture, + RejectionReason::AttestationSlotBeforeHead, + RejectionReason::ValidatorNotInState, + RejectionReason::InvalidSignature, ]; #[test] diff --git a/crates/verity-chain/src/fork_choice/attestation.rs b/crates/verity-chain/src/fork_choice/attestation.rs new file mode 100644 index 0000000..547a993 --- /dev/null +++ b/crates/verity-chain/src/fork_choice/attestation.rs @@ -0,0 +1,226 @@ +//! Admitting gossiped votes into the store's pools. +//! +//! leanSpec's `on_gossip_attestation` and `on_gossip_aggregated_attestation` each do three +//! things in a row: validate the vote, verify its signature, then record it. Verity splits +//! the signature verification out. It is the one step that needs a cryptographic library, +//! and this crate has none by design (see the crate docs); the composed entry points land +//! with `verity-crypto`, which supplies the missing middle. +//! +//! What that leaves here is every check that is a decision about the *vote* rather than +//! about the bytes signing it — which is all but two of the rejections leanSpec's gossip +//! path can produce. +//! +//! Transcribed from leanSpec `src/lean_spec/spec/forks/lstar/fork_choice.py`, read at commit +//! `0588c2d215a955a516378677a92db2a5666802f3`. + +use verity_types::config::{GOSSIP_DISPARITY_INTERVALS, INTERVALS_PER_SLOT}; +use verity_types::{AttestationData, Checkpoint, SignedAggregatedAttestation, ValidatorIndex}; + +use crate::error::RejectionReason; +use crate::fork_choice::store::{AttestationSignature, AttestationSignatureEntry, Store}; +use crate::fork_choice::weights::participants; + +/// Whether a vote is admissible against the store's current view. +/// +/// The vote must name blocks the store knows, order them the way history allows, agree with +/// those blocks' actual slots, lie on one chain, and belong to a slot that has already +/// started locally. The head must also descend from the finalized block: fork choice only +/// ever walks down from there, so an orphaned head could never carry weight, and admitting +/// one would let a stale vote re-enter after pruning dropped it. +/// +/// # Errors +/// +/// One of the ten gossip-validation [`RejectionReason`]s, named on each check below. +pub fn validate_attestation(store: &Store, data: &AttestationData) -> Result<(), RejectionReason> { + validate_availability(store, data)?; + validate_topology(store, data)?; + validate_ancestry(store, data)?; + validate_timing(store, data) +} + +/// Every block the vote names must already be in the local view. +fn validate_availability(store: &Store, data: &AttestationData) -> Result<(), RejectionReason> { + if !store.blocks.contains_key(&data.source.root) { + return Err(RejectionReason::UnknownSourceBlock); + } + if !store.blocks.contains_key(&data.target.root) { + return Err(RejectionReason::UnknownTargetBlock); + } + if !store.blocks.contains_key(&data.head.root) { + return Err(RejectionReason::UnknownHeadBlock); + } + Ok(()) +} + +/// History is linear: source at or before target, target at or before head — and each +/// checkpoint's slot must be the slot of the block it names. +fn validate_topology(store: &Store, data: &AttestationData) -> Result<(), RejectionReason> { + if data.source.slot.0 > data.target.slot.0 { + return Err(RejectionReason::SourceAfterTarget); + } + if data.head.slot.0 < data.target.slot.0 { + return Err(RejectionReason::HeadOlderThanTarget); + } + + let checks = [ + (data.source, RejectionReason::SourceSlotMismatch), + (data.target, RejectionReason::TargetSlotMismatch), + (data.head, RejectionReason::HeadSlotMismatch), + ]; + for (checkpoint, reason) in checks { + // Availability ran first, so every root here resolves. + if store.blocks.get(&checkpoint.root).map(|block| block.slot) != Some(checkpoint.slot) { + return Err(reason); + } + } + Ok(()) +} + +/// The three checkpoints must lie on one chain, rooted under the finalized block. +/// +/// Weight accrues to every ancestor of the attested head, so a head on a sibling branch +/// would steer that weight onto a chain the vote never meant to support. +fn validate_ancestry(store: &Store, data: &AttestationData) -> Result<(), RejectionReason> { + if !store.is_ancestor(data.source, data.target) { + return Err(RejectionReason::SourceNotAncestorOfTarget); + } + if !store.is_ancestor(data.target, data.head) { + return Err(RejectionReason::TargetNotAncestorOfHead); + } + if !store.is_ancestor(store.latest_finalized, data.head) { + return Err(RejectionReason::HeadNotDescendantOfFinalized); + } + Ok(()) +} + +/// A vote cannot predate the head it claims, nor arrive before its own slot has started. +/// +/// The clock-skew margin is one interval, not a whole slot. With five intervals per slot, +/// slot 10 begins at interval 50: interval 49 is admitted as skew, interval 45 would admit a +/// vote a full slot early and let an adversary pre-publish next-slot aggregates. +/// +/// The comparison stays in slot units. Multiplying a near-`u64::MAX` wire slot up into +/// intervals would overflow before it could be rejected. +fn validate_timing(store: &Store, data: &AttestationData) -> Result<(), RejectionReason> { + if data.slot.0 < data.head.slot.0 { + return Err(RejectionReason::AttestationSlotBeforeHead); + } + + let admission_horizon = store.time.0.saturating_add(GOSSIP_DISPARITY_INTERVALS); + if data.slot.0 > admission_horizon / INTERVALS_PER_SLOT { + return Err(RejectionReason::AttestationTooFarInFuture); + } + Ok(()) +} + +/// Whether a gossiped vote is admissible *and* names a validator the target block knew. +/// +/// This is the admission decision every node makes, aggregator or not. leanSpec reaches the +/// registry check on the way to resolving the signer's public key, so a node that only +/// validates and relays applies it too. +/// +/// # Errors +/// +/// Any [`RejectionReason`] from [`validate_attestation`], or +/// [`RejectionReason::ValidatorNotInState`] when the signer is outside the target's registry. +pub fn validate_attestation_signer( + store: &Store, + validator_index: ValidatorIndex, + data: &AttestationData, +) -> Result<(), RejectionReason> { + validate_attestation(store, data)?; + validate_signers(store, data.target, [validator_index]) +} + +/// Whether the target block's post-state registry holds every named validator. +/// +/// The registry is read from the target's post-state rather than the head's: that is the +/// state a verifier would resolve the signers' keys from, so an index outside it names a +/// validator the vote's own target never knew. +/// +/// # Errors +/// +/// [`RejectionReason::ValidatorNotInState`] for the first index outside the registry. +fn validate_signers( + store: &Store, + target: Checkpoint, + signers: impl IntoIterator, +) -> Result<(), RejectionReason> { + // Validation ran first, so the target's post-state is present whenever this is reached. + let registry_size = store + .states + .get(&target.root) + .map_or(0, |state| state.validators.len() as u64); + + for validator_index in signers { + if validator_index.0 >= registry_size { + return Err(RejectionReason::ValidatorNotInState); + } + } + Ok(()) +} + +/// Records one validator's signature in the aggregator's pool. +/// +/// **The caller must have verified `signature` against the validator's key first.** This +/// crate cannot: see the module docs. Every other admission check leanSpec performs on the +/// gossip path runs here, in leanSpec's order, before anything is written. +/// +/// A node that does not aggregate has no reason to call this. leanSpec validates and relays +/// such a vote without keeping it, which is [`validate_attestation`] on its own. +/// +/// # Errors +/// +/// Any [`RejectionReason`] from [`validate_attestation`], or +/// [`RejectionReason::ValidatorNotInState`] when the signer is outside the target's registry. +/// The store is left untouched on every one of them. +pub fn record_attestation_signature( + store: &mut Store, + validator_index: ValidatorIndex, + data: AttestationData, + signature: AttestationSignature, +) -> Result<(), RejectionReason> { + validate_attestation_signer(store, validator_index, &data)?; + + store + .attestation_signatures + .entry(data) + .or_default() + .insert(AttestationSignatureEntry { + validator_index, + signature, + }); + Ok(()) +} + +/// Records a gossiped aggregate proof in the pending pool. +/// +/// **The caller must have verified the proof against its participants' keys first**, for the +/// same reason as above. The proof carries no weight until an acceptance tick promotes it — +/// see [`super::timeline::accept_new_attestations`]. +/// +/// # Errors +/// +/// Any [`RejectionReason`] from [`validate_attestation`], +/// [`RejectionReason::EmptyAggregationBits`] when the proof names nobody, or +/// [`RejectionReason::ValidatorNotInState`] when a participant is outside the target's +/// registry. The store is left untouched on every one of them. +pub fn record_aggregated_payload( + store: &mut Store, + attestation: &SignedAggregatedAttestation, +) -> Result<(), RejectionReason> { + validate_attestation(store, &attestation.data)?; + + let signers: Vec = participants(&attestation.proof.participants).collect(); + if signers.is_empty() { + return Err(RejectionReason::EmptyAggregationBits); + } + validate_signers(store, attestation.data.target, signers)?; + + store + .latest_new_aggregated_payloads + .entry(attestation.data) + .or_default() + .insert(attestation.proof.clone()); + Ok(()) +} diff --git a/crates/verity-chain/src/fork_choice/block.rs b/crates/verity-chain/src/fork_choice/block.rs new file mode 100644 index 0000000..9c7e767 --- /dev/null +++ b/crates/verity-chain/src/fork_choice/block.rs @@ -0,0 +1,149 @@ +//! Importing a block into the store, and the head update that follows it. +//! +//! Signature verification is not here. leanSpec's `on_block` verifies the block proof +//! between its wire checks and the state transition; this crate has no cryptographic +//! dependency (see the crate docs), so the caller verifies before calling. Everything else +//! leanSpec does in that function runs below, in its order. +//! +//! Transcribed from leanSpec `src/lean_spec/spec/forks/lstar/fork_choice.py`, read at commit +//! `0588c2d215a955a516378677a92db2a5666802f3`. + +use std::collections::HashSet; + +use verity_types::config::HISTORICAL_ROOTS_LIMIT; +use verity_types::{AttestationData, Block, Checkpoint}; + +use crate::error::RejectionReason; +use crate::fork_choice::prune::prune_stale_attestation_data; +use crate::fork_choice::store::Store; +use crate::fork_choice::weights::{latest_votes, lmd_ghost_head}; +use crate::justification::advance_checkpoint; +use crate::merkle::hash_tree_root; +use crate::state_transition::state_transition; + +/// Imports `block` and recomputes the head. +/// +/// **The caller must have verified the block's proof first.** This crate cannot: see the +/// module docs. +/// +/// A block already in the store is accepted as a no-op — re-importing it could change +/// nothing, and treating a duplicate as an error would make gossip's own redundancy look +/// like a fault. +/// +/// # Errors +/// +/// - [`RejectionReason::UnknownParentBlock`] when the parent has no state here, which means +/// the chain below this block still has to be synced. +/// - [`RejectionReason::BlockSlotGapTooLarge`] when the block runs so far beyond its parent +/// that the transition's empty-slot walk would be unbounded. +/// - [`RejectionReason::BlockTooFarInFuture`] when the block's slot is past the horizon the +/// store's own clock admits. +/// - [`RejectionReason::DuplicateAttestationData`] when the body repeats one vote. +/// - Any [`RejectionReason`] the state transition itself produces. +/// +/// The store is left untouched on every one of them: nothing is written until the +/// transition has returned a post-state. +pub fn on_block(store: &mut Store, block: &Block) -> Result<(), RejectionReason> { + let block_root = hash_tree_root(block); + if store.blocks.contains_key(&block_root) { + return Ok(()); + } + + let post_state = { + let parent_state = store + .states + .get(&block.parent_root) + .ok_or(RejectionReason::UnknownParentBlock)?; + + if block.slot.0.saturating_sub(parent_state.slot.0) > HISTORICAL_ROOTS_LIMIT as u64 { + return Err(RejectionReason::BlockSlotGapTooLarge); + } + if block.slot.0 > store.current_slot().0.saturating_add(1) { + return Err(RejectionReason::BlockTooFarInFuture); + } + reject_duplicate_attestation_data(block)?; + + state_transition(parent_state, block)? + }; + + let previous_finalized_slot = store.latest_finalized.slot; + + store.latest_justified = + advance_checkpoint(store.latest_justified, post_state.latest_justified); + seed_block_votes(store, block); + store.blocks.insert(block_root, block.clone()); + store.states.insert(block_root, post_state); + + update_head(store); + + if store.latest_finalized.slot.0 > previous_finalized_slot.0 { + prune_stale_attestation_data(store); + } + Ok(()) +} + +/// Rejects a body that files the same vote twice. +/// +/// Collapsing the aggregates to their distinct data exposes any repeat: fewer distinct +/// entries than aggregates means one was sent twice. This is the wire-level prohibition +/// only — the transition separately bounds how many distinct entries a body may carry. +fn reject_duplicate_attestation_data(block: &Block) -> Result<(), RejectionReason> { + let attestations = &block.body.attestations; + let distinct: HashSet<&AttestationData> = attestations + .iter() + .map(|attestation| &attestation.data) + .collect(); + + if distinct.len() == attestations.len() { + Ok(()) + } else { + Err(RejectionReason::DuplicateAttestationData) + } +} + +/// Files each vote the block carries into the counted pool, with no proof behind it. +/// +/// A block's merged proof is never split back into per-vote proofs, so the entries start +/// empty and the votes add no head weight of their own. The per-vote proofs arrive on the +/// gossip path instead, which defers a block-carried vote's weight by up to one slot. +/// +/// An existing entry keeps the proofs it already has: the same vote reaching the store twice, +/// once by gossip and once inside a block, must not lose the proof that gave it weight. +fn seed_block_votes(store: &mut Store, block: &Block) { + for attestation in block.body.attestations.iter() { + store + .latest_known_aggregated_payloads + .entry(attestation.data) + .or_default(); + } +} + +/// Recomputes the head, and with it the finalized checkpoint the head's own state names. +/// +/// The walk starts at the justified root and descends to the heaviest leaf, so the head is +/// always a descendant of that root. +/// +/// The finalized checkpoint is then re-derived by climbing from the new head to its ancestor +/// at the slot the head's post-state finalized. Re-deriving it from the head is what makes +/// pruning sound: a finalized checkpoint that drifted off the head's chain would prune votes +/// that still matter. Where that ancestor cannot be resolved — a checkpoint-sync anchor +/// stores no block below itself — the trusted checkpoint stays rather than an unresolved +/// root being published. +pub fn update_head(store: &mut Store) { + let votes = latest_votes( + &store.latest_known_aggregated_payloads, + store.latest_finalized.slot, + ); + store.head = lmd_ghost_head(store, store.latest_justified.root, &votes, None); + + let Some(head_state) = store.states.get(&store.head) else { + return; + }; + let finalized_slot = head_state.latest_finalized.slot; + if let Some(root) = store.ancestor_at_slot(store.head, finalized_slot) { + store.latest_finalized = Checkpoint { + root, + slot: finalized_slot, + }; + } +} diff --git a/crates/verity-chain/src/fork_choice/duties.rs b/crates/verity-chain/src/fork_choice/duties.rs new file mode 100644 index 0000000..6a1b615 --- /dev/null +++ b/crates/verity-chain/src/fork_choice/duties.rs @@ -0,0 +1,78 @@ +//! What a validator should vote for, given the store's view. +//! +//! Only the target selection lives here. Producing and signing the attestation itself needs +//! a key and a signature library, neither of which this crate has (see the crate docs). +//! +//! Transcribed from leanSpec `src/lean_spec/spec/forks/lstar/validator_duties.py`, read at +//! commit `0588c2d215a955a516378677a92db2a5666802f3`. + +use verity_types::config::JUSTIFICATION_LOOKBACK_SLOTS; +use verity_types::{Bytes32, Checkpoint, Slot}; + +use crate::fork_choice::store::Store; +use crate::justification::is_justifiable_after; + +/// The checkpoint a validator should name as its attestation target. +/// +/// The walk starts at the head and steps back, balancing two pulls. Advancing the head is +/// what moves the chain forward; staying at or behind the safe target is what keeps the vote +/// from backing something that can still disappear. The first loop gives the head at most +/// [`JUSTIFICATION_LOOKBACK_SLOTS`] steps back toward that bound, and the second keeps +/// stepping until the slot is one that may actually be justified. +/// +/// Neither walk crosses the finalized boundary. When the safe target has fallen behind +/// finalization, the finalized slot becomes the lower bound instead, so target selection +/// never inspects a slot below it. +#[must_use = "this chooses the target; casting the vote is the validator client's job"] +pub fn attestation_target(store: &Store) -> Checkpoint { + let finalized_slot = store.latest_finalized.slot; + let safe_target_slot = slot_of(store, store.safe_target).unwrap_or(finalized_slot); + let lower_bound_slot = Slot(safe_target_slot.0.max(finalized_slot.0)); + + let mut target_root = store.head; + for _ in 0..JUSTIFICATION_LOOKBACK_SLOTS { + let Some(slot) = slot_of(store, target_root) else { + break; + }; + if slot.0 <= lower_bound_slot.0 { + break; + } + let parent = parent_of(store, target_root); + if parent == target_root { + break; + } + target_root = parent; + } + + while let Some(slot) = slot_of(store, target_root) { + if slot.0 <= finalized_slot.0 || is_justifiable_after(slot, finalized_slot) { + break; + } + let parent = parent_of(store, target_root); + if parent == target_root { + break; + } + target_root = parent; + } + + Checkpoint { + root: target_root, + slot: slot_of(store, target_root).unwrap_or(finalized_slot), + } +} + +/// The slot of a known block, or `None` where the root is not in the local view. +fn slot_of(store: &Store, root: Bytes32) -> Option { + store.blocks.get(&root).map(|block| block.slot) +} + +/// The parent of a known block, or the root itself where the walk has left the known tree. +/// +/// Both walks above treat an unchanged root as the end of the chain, which is what stops +/// them from climbing into an unknown branch or circling a block that names itself. +fn parent_of(store: &Store, root: Bytes32) -> Bytes32 { + store + .blocks + .get(&root) + .map_or(root, |block| block.parent_root) +} diff --git a/crates/verity-chain/src/fork_choice/mod.rs b/crates/verity-chain/src/fork_choice/mod.rs new file mode 100644 index 0000000..60709e4 --- /dev/null +++ b/crates/verity-chain/src/fork_choice/mod.rs @@ -0,0 +1,195 @@ +//! Fork choice: the store, and the decisions that move its head. +//! +//! # What the store is +//! +//! The state transition answers "is this block valid, and what does it produce". Fork choice +//! answers "which of the valid chains is the one". It needs memory the transition does not — +//! every block above finalization, their post-states, and the votes cast over them — and +//! that memory is the [`Store`]. +//! +//! # Why the store is mutated in place +//! +//! Everything else in this crate is a pure function returning a fresh value, and the store +//! deliberately is not. It is a long-lived aggregate with one writer, not a value passed +//! between them, and it holds a [`verity_types::State`] per unfinalized block; copying it per +//! imported block would cost `O(chain)` per block and buy nothing. What the copy guaranteed +//! is kept as a contract instead: **an entry point that returns `Err` leaves the store +//! exactly as it found it**. +//! +//! # Where the cryptography went +//! +//! leanSpec verifies signatures inside three of these operations. This crate has no +//! cryptographic dependency (see the crate docs), so each of the three is split at exactly +//! that point and the caller verifies in between: +//! +//! | leanSpec | here | the caller supplies | +//! |---|---|---| +//! | `on_block` | [`on_block`] | the block proof check | +//! | `on_gossip_attestation` | [`validate_attestation_signer`] + [`record_attestation_signature`] | the XMSS verify | +//! | `on_gossip_aggregated_attestation` | [`record_aggregated_payload`] | the aggregate verify | +//! +//! The aggregator duty at interval 2 is absent for the same reason — see [`timeline`]. + +pub mod attestation; +pub mod block; +pub mod duties; +pub mod prune; +pub mod store; +pub mod timeline; +pub mod weights; + +pub use attestation::{ + record_aggregated_payload, record_attestation_signature, validate_attestation, + validate_attestation_signer, +}; +pub use block::{on_block, update_head}; +pub use duties::attestation_target; +pub use prune::prune_stale_attestation_data; +pub use store::{AttestationSignature, AttestationSignatureEntry, Store}; +pub use timeline::{accept_new_attestations, on_tick, update_safe_target}; +pub use weights::{block_weights, latest_votes, lmd_ghost_head, participants}; + +#[cfg(test)] +pub(crate) mod testing { + //! Store builders shared by the unit tests of this module's submodules. + + use verity_types::{Block, Slot, State, ValidatorIndex}; + + use crate::merkle::hash_tree_root; + use crate::slot_clock::intervals_at_slot_start; + use crate::state_transition::testing::{empty_block_at, genesis_with}; + use crate::state_transition::{process_block, process_slots}; + + use super::{Store, on_block}; + + /// A store anchored on a genesis with `count` validators, its clock at slot 0. + pub(crate) fn anchored_on_genesis(count: u64) -> (Store, State) { + let genesis = genesis_with(count); + let mut anchor = empty_block_at(&genesis, 0); + anchor.state_root = hash_tree_root(&genesis); + anchor.parent_root = genesis.latest_block_header.parent_root; + + let store = Store::new(&genesis, &anchor, Some(ValidatorIndex(0))) + .expect("the anchor commits to its own state"); + (store, genesis) + } + + /// An empty block at `slot`, self-consistent against the state it builds on. + pub(crate) fn block_on(state: &State, slot: u64) -> Block { + let advanced = process_slots(state, Slot(slot)).expect("the slot is ahead of the state"); + let mut block = empty_block_at(&advanced, slot); + block.state_root = hash_tree_root( + &process_block(&advanced, &block).expect("an empty block on its own parent"), + ); + block + } + + /// Imports `slot`'s empty block, ticking the clock to admit it first. + pub(crate) fn import_at(store: &mut Store, state: &State, slot: u64) -> Block { + let block = block_on(state, slot); + store.time = intervals_at_slot_start(block.slot); + on_block(store, &block).expect("a self-consistent block on a known parent"); + block + } +} + +#[cfg(test)] +mod tests { + use verity_types::{Checkpoint, Interval, Slot}; + + use crate::error::RejectionReason; + use crate::merkle::hash_tree_root; + + use super::testing::{anchored_on_genesis, block_on, import_at}; + use super::{Store, on_block}; + + #[test] + fn should_reject_an_anchor_whose_block_does_not_commit_to_its_state() { + let (_, genesis) = anchored_on_genesis(4); + let mut anchor = crate::state_transition::testing::empty_block_at(&genesis, 0); + anchor.state_root = [7u8; 32]; + + assert_eq!( + Store::new(&genesis, &anchor, None).unwrap_err(), + RejectionReason::AnchorStateRootMismatch + ); + } + + #[test] + fn should_start_the_clock_at_the_anchor_slot() { + let (store, _) = anchored_on_genesis(4); + assert_eq!(store.time, Interval(0)); + assert_eq!(store.head, store.latest_finalized.root); + assert_eq!(store.latest_justified, store.latest_finalized); + } + + #[test] + fn should_leave_the_store_untouched_when_a_block_is_rejected() { + let (mut store, genesis) = anchored_on_genesis(4); + let before = store.clone(); + + // The clock still sits at slot 0, so a block two slots ahead is past the horizon. + let block = block_on(&genesis, 2); + assert_eq!( + on_block(&mut store, &block), + Err(RejectionReason::BlockTooFarInFuture) + ); + assert_eq!(store, before); + } + + #[test] + fn should_leave_the_store_untouched_when_the_parent_is_unknown() { + let (mut store, genesis) = anchored_on_genesis(4); + let before = store.clone(); + + let mut block = block_on(&genesis, 1); + block.parent_root = [9u8; 32]; + store.time = Interval(5); + assert_eq!( + on_block(&mut store, &block), + Err(RejectionReason::UnknownParentBlock) + ); + assert_eq!(store.blocks, before.blocks); + assert_eq!(store.states, before.states); + assert_eq!(store.latest_justified, before.latest_justified); + } + + #[test] + fn should_accept_a_block_already_in_the_store_without_changing_it() { + let (mut store, genesis) = anchored_on_genesis(4); + let block = import_at(&mut store, &genesis, 1); + let after_first = store.clone(); + + assert_eq!(on_block(&mut store, &block), Ok(())); + assert_eq!(store, after_first); + } + + #[test] + fn should_follow_the_chain_when_no_vote_has_any_weight() { + let (mut store, genesis) = anchored_on_genesis(4); + let first = import_at(&mut store, &genesis, 1); + + assert_eq!(store.head, hash_tree_root(&first)); + assert_eq!(store.blocks.len(), 2); + } + + #[test] + fn should_keep_the_justified_checkpoint_when_a_candidate_ties_on_slot() { + let (mut store, _) = anchored_on_genesis(4); + let original = store.latest_justified; + let tie = Checkpoint { + root: [1u8; 32], + slot: original.slot, + }; + + store.latest_justified = crate::justification::advance_checkpoint(original, tie); + assert_eq!(store.latest_justified, original); + } + + #[test] + fn should_report_the_slot_its_interval_clock_sits_in() { + let (mut store, _) = anchored_on_genesis(4); + store.time = Interval(12); + assert_eq!(store.current_slot(), Slot(2)); + } +} diff --git a/crates/verity-chain/src/fork_choice/prune.rs b/crates/verity-chain/src/fork_choice/prune.rs new file mode 100644 index 0000000..3da306a --- /dev/null +++ b/crates/verity-chain/src/fork_choice/prune.rs @@ -0,0 +1,47 @@ +//! Dropping the votes finalization has put out of reach. +//! +//! Transcribed from leanSpec `src/lean_spec/spec/forks/lstar/fork_choice.py`, read at commit +//! `0588c2d215a955a516378677a92db2a5666802f3`. + +use std::collections::HashSet; + +use verity_types::AttestationData; + +use crate::fork_choice::store::Store; + +/// Drops every vote whose head can no longer influence fork choice. +/// +/// A vote is out of reach when its head sits at or below the finalized slot, or when that +/// head is on a branch the finalized block orphaned. Neither head lies under the finalized +/// block, and fork choice only ever descends from there, so neither can credit a block the +/// walk reaches — dropping them cannot change the chosen chain. +/// +/// This is sound only because [`super::block::update_head`] re-derives the finalized +/// checkpoint from the head. Pruning against a checkpoint that had drifted onto a different +/// branch would discard votes that still matter. +/// +/// All three pools share the vote as their key, so one staleness test filters them together. +pub fn prune_stale_attestation_data(store: &mut Store) { + let finalized = store.latest_finalized; + + let survivors: HashSet = store + .attestation_signatures + .keys() + .chain(store.latest_new_aggregated_payloads.keys()) + .chain(store.latest_known_aggregated_payloads.keys()) + .filter(|data| { + data.head.slot.0 > finalized.slot.0 && store.is_ancestor(finalized, data.head) + }) + .copied() + .collect(); + + store + .attestation_signatures + .retain(|data, _| survivors.contains(data)); + store + .latest_new_aggregated_payloads + .retain(|data, _| survivors.contains(data)); + store + .latest_known_aggregated_payloads + .retain(|data, _| survivors.contains(data)); +} diff --git a/crates/verity-chain/src/fork_choice/store.rs b/crates/verity-chain/src/fork_choice/store.rs new file mode 100644 index 0000000..9838b05 --- /dev/null +++ b/crates/verity-chain/src/fork_choice/store.rs @@ -0,0 +1,166 @@ +//! The fork-choice store: the node's local view of the chain and the votes over it. +//! +//! leanSpec models the store as an immutable value and returns a fresh copy from every +//! operation. Verity does not. The store owns two maps that grow with the chain — every +//! block and every post-state above finalization — and a [`State`] is the largest value in +//! the system. Copying both per imported block would cost `O(chain)` per block for no gain: +//! `docs/src/reference/architecture.md` gives this aggregate a single writer, so no second +//! holder exists to observe an older copy. +//! +//! What the copy bought is kept as an invariant instead: **every fallible check runs before +//! any field is touched**, so an operation that returns `Err` leaves the store byte-for-byte +//! as it was. Each `&mut` entry point states this in its own contract and is tested for it. +//! +//! Transcribed from leanSpec `src/lean_spec/spec/forks/lstar/containers/store.py` and +//! `fork_choice.py`, read at commit `0588c2d215a955a516378677a92db2a5666802f3`. + +use std::cmp::Ordering; +use std::collections::{HashMap, HashSet}; + +use verity_types::{ + AttestationData, Block, Bytes32, Checkpoint, GenesisConfig, Interval, SingleMessageAggregate, + State, ValidatorIndex, +}; + +use crate::error::RejectionReason; +use crate::merkle::hash_tree_root; +use crate::slot_clock::intervals_at_slot_start; + +/// One validator's raw signature over a vote, carried without being interpreted. +/// +/// The bytes are an XMSS signature container produced by the signature library. This crate +/// takes no cryptographic dependency (see the crate docs), and never needs one for these: +/// the store only holds a signature until an aggregator folds it into a proof, and fork +/// choice weighs votes by *who* signed, never by the signature itself. Verifying the bytes +/// belongs to the caller, above this crate — see [`super::attestation`]. +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct AttestationSignature(pub Vec); + +/// A signature in the aggregator's pool, paired with the validator that produced it. +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct AttestationSignatureEntry { + /// The validator that signed. + pub validator_index: ValidatorIndex, + /// Its uninterpreted signature over the vote this entry is filed under. + pub signature: AttestationSignature, +} + +/// The node's fork-choice view: known blocks and states, the vote pools, and the checkpoints. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Store { + /// Intervals elapsed since genesis, as the node's clock has ticked them. + pub time: Interval, + /// Chain configuration, carried from the anchor state. + pub config: GenesisConfig, + /// The block fork choice currently selects. + pub head: Bytes32, + /// The deepest block a supermajority of this slot's voters already back. + pub safe_target: Bytes32, + /// The highest justified checkpoint the store has observed. + pub latest_justified: Checkpoint, + /// The highest finalized checkpoint the store has observed. + pub latest_finalized: Checkpoint, + /// Every known block, keyed by its root. + pub blocks: HashMap, + /// The post-state of every known block, keyed by the same root. + pub states: HashMap, + /// The validator this node attests for, when it runs one. + pub validator_index: Option, + /// Per-validator signatures an aggregator has collected, grouped by the vote they sign. + pub attestation_signatures: HashMap>, + /// Proofs gathered this slot. They carry no weight until an acceptance tick promotes them. + pub latest_new_aggregated_payloads: HashMap>, + /// Proofs that count toward fork-choice weight. + pub latest_known_aggregated_payloads: HashMap>, +} + +impl Store { + /// Builds a store anchored on a trusted block and its post-state. + /// + /// The anchor is treated as both justified and finalized: it is the deepest point the + /// node will ever reconsider. Time starts at the anchor slot's first interval, so a node + /// resuming from a checkpoint does not replay the intervals before it. + /// + /// # Errors + /// + /// [`RejectionReason::AnchorStateRootMismatch`] when the block does not commit to the + /// state it was handed with. The pair would otherwise seed a store whose first state + /// transition compares against a root nothing produced. + #[must_use = "this builds the store; it neither registers nor starts anything"] + pub fn new( + state: &State, + anchor_block: &Block, + validator_index: Option, + ) -> Result { + if anchor_block.state_root != hash_tree_root(state) { + return Err(RejectionReason::AnchorStateRootMismatch); + } + + let anchor_root = hash_tree_root(anchor_block); + let anchor_checkpoint = Checkpoint { + root: anchor_root, + slot: anchor_block.slot, + }; + + Ok(Self { + time: intervals_at_slot_start(anchor_block.slot), + config: state.config, + head: anchor_root, + safe_target: anchor_root, + latest_justified: anchor_checkpoint, + latest_finalized: anchor_checkpoint, + blocks: HashMap::from([(anchor_root, anchor_block.clone())]), + states: HashMap::from([(anchor_root, state.clone())]), + validator_index, + attestation_signatures: HashMap::new(), + latest_new_aggregated_payloads: HashMap::new(), + latest_known_aggregated_payloads: HashMap::new(), + }) + } + + /// The slot the store's interval clock currently sits in. + #[must_use = "this reads the clock; ticking it is `on_tick`'s job"] + pub const fn current_slot(&self) -> verity_types::Slot { + verity_types::Slot(self.time.0 / verity_types::config::INTERVALS_PER_SLOT) + } + + /// Whether one checkpoint lies on the other's chain of ancestors. + /// + /// The walk climbs parent links from `descendant` and stops the moment it leaves the + /// known tree, so an unknown branch answers `false` rather than looping. Landing above + /// the ancestor's slot without hitting it means that slot held no block on this chain, + /// which puts the ancestor off it. + #[must_use = "this answers the question; it neither records nor enforces the relation"] + pub fn is_ancestor(&self, ancestor: Checkpoint, descendant: Checkpoint) -> bool { + if ancestor.slot > descendant.slot { + return false; + } + + let mut current_root = descendant.root; + while let Some(current_block) = self.blocks.get(¤t_root) { + match current_block.slot.cmp(&ancestor.slot) { + Ordering::Equal => return current_root == ancestor.root, + Ordering::Less => return false, + Ordering::Greater => current_root = current_block.parent_root, + } + } + false + } + + /// The ancestor of `root` at `slot`, when the chain above it is fully known. + /// + /// Returns `None` where the walk leaves the known tree, or where the chain skips the + /// slot entirely — a checkpoint-sync anchor leaves exactly that hole below itself. + #[must_use = "this locates the ancestor; it does not move any checkpoint onto it"] + pub fn ancestor_at_slot(&self, root: Bytes32, slot: verity_types::Slot) -> Option { + let mut current_root = root; + loop { + let current_block = self.blocks.get(¤t_root)?; + match current_block.slot.cmp(&slot) { + Ordering::Equal => return Some(current_root), + Ordering::Less => return None, + Ordering::Greater => current_root = current_block.parent_root, + } + } + } +} diff --git a/crates/verity-chain/src/fork_choice/timeline.rs b/crates/verity-chain/src/fork_choice/timeline.rs new file mode 100644 index 0000000..f4f8132 --- /dev/null +++ b/crates/verity-chain/src/fork_choice/timeline.rs @@ -0,0 +1,97 @@ +//! The interval clock: what the store does at each fixed point inside a slot. +//! +//! A slot is five intervals, and consensus work is pinned to positions within it rather than +//! to arrival times, so every node acts on the same schedule relative to a block landing. +//! +//! One of leanSpec's five actions is absent. At interval 2 an aggregator folds the slot's +//! pooled signatures into proofs, which is a cryptographic operation and belongs to +//! `verity-crypto` (see the crate docs); the composed tick lands there. The four actions +//! below are decisions over the store alone, and they are complete. +//! +//! Transcribed from leanSpec `src/lean_spec/spec/forks/lstar/timeline.py` and +//! `fork_choice.py`, read at commit `0588c2d215a955a516378677a92db2a5666802f3`. + +use verity_types::Interval; +use verity_types::config::INTERVALS_PER_SLOT; + +use crate::fork_choice::block::update_head; +use crate::fork_choice::store::Store; +use crate::fork_choice::weights::{latest_votes, lmd_ghost_head}; + +/// Advances the store's clock to `target`, running every interval it passes through. +/// +/// Stepping one interval at a time is what keeps an action from being skipped when a node +/// falls behind and catches up across several intervals at once. +/// +/// `has_proposal` says the node expects this slot's block to have landed; it is signalled +/// only on the final step, since it describes the interval being arrived at rather than the +/// ones passed on the way. A `target` at or behind the current time does nothing. +pub fn on_tick(store: &mut Store, target: Interval, has_proposal: bool) { + while store.time.0 < target.0 { + let next = Interval(store.time.0 + 1); + tick_interval(store, has_proposal && next.0 == target.0); + } +} + +/// Advances one interval and runs whatever that position in the slot calls for. +/// +/// - Interval 0 — ingest the slot's pending votes, once the proposal has landed. +/// - Interval 3 — advance the safe target, after aggregates for this slot exist. +/// - Interval 4 — ingest the votes that accumulated through the rest of the slot. +/// +/// Interval 1 has no action, and interval 2 is the aggregator's, which is not this crate's +/// (see the module docs). +fn tick_interval(store: &mut Store, has_proposal: bool) { + store.time = Interval(store.time.0 + 1); + + match store.time.0 % INTERVALS_PER_SLOT { + 0 if has_proposal => accept_new_attestations(store), + 3 => update_safe_target(store), + 4 => accept_new_attestations(store), + _ => {} + } +} + +/// Promotes the pending proofs into the counted pool and recomputes the head. +/// +/// Proofs gathered during a slot carry no weight while they sit pending. This is the point +/// at which they begin to count, and the pending pool is emptied behind them. +pub fn accept_new_attestations(store: &mut Store) { + let pending = core::mem::take(&mut store.latest_new_aggregated_payloads); + for (data, proofs) in pending { + store + .latest_known_aggregated_payloads + .entry(data) + .or_default() + .extend(proofs); + } + update_head(store); +} + +/// Advances the safe target: the deepest block a supermajority of this slot's voters back. +/// +/// This is the block a validator can attest to without risking that it later disappears, so +/// the threshold is a strict supermajority and is rounded up — 100 validators need 67, not +/// 66. Children below it are pruned before the walk, which is why the target can stop +/// shallower than the head. +/// +/// It is weighed from the *pending* pool, not the counted one: the safe target is about what +/// this slot's voters are doing now, and the counted pool is last slot's picture. +pub fn update_safe_target(store: &mut Store) { + let validator_count = store + .states + .get(&store.head) + .map_or(0, |state| state.validators.len() as u64); + let min_target_score = (validator_count * 2).div_ceil(3); + + let votes = latest_votes( + &store.latest_new_aggregated_payloads, + store.latest_finalized.slot, + ); + store.safe_target = lmd_ghost_head( + store, + store.latest_justified.root, + &votes, + Some(min_target_score), + ); +} diff --git a/crates/verity-chain/src/fork_choice/weights.rs b/crates/verity-chain/src/fork_choice/weights.rs new file mode 100644 index 0000000..44a8dac --- /dev/null +++ b/crates/verity-chain/src/fork_choice/weights.rs @@ -0,0 +1,152 @@ +//! Turning a pool of votes into a head: the LMD view, the weights, and the GHOST walk. +//! +//! Nothing here reads or writes the store's pools. Each function takes the pool it should +//! reason about, which is what lets the same code weigh the counted pool for the head and +//! the pending pool for the safe target. +//! +//! Transcribed from leanSpec `src/lean_spec/spec/forks/lstar/fork_choice.py`, read at commit +//! `0588c2d215a955a516378677a92db2a5666802f3`. + +use std::collections::HashMap; +use std::collections::HashSet; + +use verity_types::{ + AggregationBits, AttestationData, Bytes32, SingleMessageAggregate, Slot, ValidatorIndex, +}; + +use crate::fork_choice::store::Store; +use crate::merkle::hash_tree_root; + +/// The pools the store keys by vote: proofs filed under the attestation data they cover. +pub type AggregatedPayloads = HashMap>; + +/// Each validator mapped to the latest vote it cast — the LMD view fork choice runs on. +pub type LatestVotes = HashMap; + +/// The validator indices a bitfield names. +#[must_use = "this reads the bitfield; it does not modify it"] +pub fn participants(bits: &AggregationBits) -> impl Iterator + '_ { + (0..bits.len()) + .filter(|index| bits.get(*index).unwrap_or(false)) + .map(|index| ValidatorIndex(index as u64)) +} + +/// Reduces a pool of proofs to each validator's latest still-relevant vote. +/// +/// Votes are visited newest-first, so the first one seen for a validator is the one that +/// counts. An equivocator casting two votes in one slot is settled by the larger canonical +/// attestation-data root, the same tiebreak the block walk applies to block roots — which is +/// what makes the result independent of arrival order. +/// +/// A vote whose head sits at or below the finalized slot can credit no block the walk +/// reaches, so it is skipped here and callers need not pre-filter their pool. +#[must_use = "this derives the LMD view; the pool it reads is left untouched"] +pub fn latest_votes(payloads: &AggregatedPayloads, latest_finalized_slot: Slot) -> LatestVotes { + let mut by_precedence: Vec<(&AttestationData, &HashSet)> = + payloads.iter().collect(); + // `hash_tree_root` runs once per distinct vote here, never once per validator below. + by_precedence + .sort_unstable_by_key(|(data, _)| core::cmp::Reverse((data.slot, hash_tree_root(*data)))); + + let mut latest = LatestVotes::new(); + for (data, proofs) in by_precedence { + if data.head.slot.0 <= latest_finalized_slot.0 { + continue; + } + // Every proof filed here covers this same vote, so which one the loop visits cannot + // change what gets recorded. Set order is therefore non-consensus. + for proof in proofs { + for validator_index in participants(&proof.participants) { + latest.entry(validator_index).or_insert(*data); + } + } + } + latest +} + +/// Tallies how many of those latest votes credit each block. +/// +/// A vote credits its head and every ancestor above `start_slot`. The climb stops at that +/// slot, or where the chain leaves the known tree. +#[must_use = "this returns the tally; it stores nothing on the store it reads"] +pub fn ancestor_weights( + store: &Store, + attestations: &LatestVotes, + start_slot: Slot, +) -> HashMap { + let mut weights: HashMap = HashMap::new(); + + for data in attestations.values() { + let mut current_root = data.head.root; + while let Some(current_block) = store.blocks.get(¤t_root) { + if current_block.slot.0 <= start_slot.0 { + break; + } + *weights.entry(current_root).or_default() += 1; + current_root = current_block.parent_root; + } + } + weights +} + +/// Weighs every block by the latest votes landing on it or on its descendants. +/// +/// The anchor is the finalized slot: fork choice never reconsiders anything at or below it, +/// so nothing there is weighed. +#[must_use = "this returns the weights; it caches nothing on the store"] +pub fn block_weights(store: &Store) -> HashMap { + let votes = latest_votes( + &store.latest_known_aggregated_payloads, + store.latest_finalized.slot, + ); + ancestor_weights(store, &votes, store.latest_finalized.slot) +} + +/// Walks the block tree by the LMD-GHOST rule and returns the leaf it lands on. +/// +/// From `start_root`, each step takes the heaviest child, breaking an equal-weight tie toward +/// the lexicographically larger root. `min_score` prunes children below a threshold before +/// the walk, which is how the safe target stops shallower than the head. +/// +/// An unknown `start_root` has no children and is returned unchanged; the store's own +/// invariant is that the justified root is always present. +#[must_use = "this selects the head; setting it on the store is `update_head`'s job"] +pub fn lmd_ghost_head( + store: &Store, + start_root: Bytes32, + attestations: &LatestVotes, + min_score: Option, +) -> Bytes32 { + let start_slot = store + .blocks + .get(&start_root) + .map_or(Slot(0), |block| block.slot); + let weights = ancestor_weights(store, attestations, start_slot); + let weight_of = |root: &Bytes32| weights.get(root).copied().unwrap_or_default(); + + let mut children: HashMap> = HashMap::new(); + for (root, block) in &store.blocks { + // An anchor block naming itself as parent would make the walk below loop forever. + // Nothing imported can reach that shape — `on_block` requires a known parent state — + // but a caller-supplied anchor is not checked anywhere else. + if block.parent_root == *root { + continue; + } + if min_score.is_some_and(|threshold| weight_of(root) < threshold) { + continue; + } + children.entry(block.parent_root).or_default().push(*root); + } + + let mut head = start_root; + while let Some(candidates) = children.get(&head) { + let Some(best) = candidates + .iter() + .max_by_key(|child| (weight_of(child), **child)) + else { + break; + }; + head = *best; + } + head +} diff --git a/crates/verity-chain/src/lib.rs b/crates/verity-chain/src/lib.rs index eed2c36..a82f722 100644 --- a/crates/verity-chain/src/lib.rs +++ b/crates/verity-chain/src/lib.rs @@ -7,11 +7,13 @@ //! every other crate depends on. //! //! Nothing here reads a clock, a socket, or a database. `slot_clock` takes the instant it -//! should reason about as an argument, and `state_transition` takes the block, for exactly -//! that reason. Signature verification happens before the transition is called, so this -//! crate carries no cryptographic dependency either. +//! should reason about as an argument, `state_transition` takes the block, and `fork_choice` +//! takes the interval to advance to, for exactly that reason. Signature verification happens +//! before any of them is called, so this crate carries no cryptographic dependency either — +//! see `fork_choice` for where leanSpec's three verifying entry points are split apart. pub mod error; +pub mod fork_choice; pub mod justification; pub mod merkle; pub mod proposer; @@ -19,6 +21,12 @@ pub mod slot_clock; pub mod state_transition; pub use error::RejectionReason; +pub use fork_choice::{ + AttestationSignature, AttestationSignatureEntry, Store, accept_new_attestations, + attestation_target, block_weights, on_block, on_tick, prune_stale_attestation_data, + record_aggregated_payload, record_attestation_signature, update_head, update_safe_target, + validate_attestation, validate_attestation_signer, +}; pub use justification::{ IMMEDIATE_JUSTIFICATION_WINDOW, advance_checkpoint, extend_justified_slots_to, is_justifiable_after, is_slot_justified, justified_index_after, diff --git a/crates/verity-chain/tests/common/mod.rs b/crates/verity-chain/tests/common/mod.rs index ba29ade..7df3dff 100644 --- a/crates/verity-chain/tests/common/mod.rs +++ b/crates/verity-chain/tests/common/mod.rs @@ -1,12 +1,29 @@ //! Shared plumbing for the leanSpec fixture suites. //! -//! Both suites are gated on `VERITY_FIXTURES` pointing at an extracted `fixtures-prod-scheme` +//! Every suite is gated on `VERITY_FIXTURES` pointing at an extracted `fixtures-prod-scheme` //! tree. The fast `cargo test` gate leaves it unset and the tests return; CI's fixtures job //! always sets it, and each suite fails if no case matched. +//! +//! The JSON containers below are mirrored rather than derived on the `verity-types` shapes. +//! Consensus values travel as SSZ, never as JSON; the `{"data": [...]}` wrappers and +//! camelCase names are a test-generator convention, not part of any container's shape. +//! `deny_unknown_fields` is what keeps them honest — a field leanSpec adds fails the run +//! instead of being silently skipped. + +// Each harness is its own test binary and compiles this module separately, so every one of +// them leaves some of it unused. The alternative is a per-item allow on almost every item. +#![allow(dead_code)] use std::fs; use std::path::{Path, PathBuf}; +use serde::Deserialize; +use verity_types::{ + AggregatedAttestation, AggregatedAttestations, AggregationBits, AttestationData, Block, + BlockBody, BlockHeader, Bytes32, Bytes52, Checkpoint, GenesisConfig, HistoricalBlockHashes, + JustificationValidators, JustifiedSlots, Slot, State, Validator, ValidatorIndex, Validators, +}; + /// The extracted fixture tree, when the environment points at one. pub fn fixtures_dir() -> Option { std::env::var_os("VERITY_FIXTURES").map(PathBuf::from) @@ -53,3 +70,265 @@ pub fn read_cases(paths: &[PathBuf]) -> Vec<(Str } cases } + +// --------------------------------------------------------------------------------------- +// Fixture shapes +// --------------------------------------------------------------------------------------- + +/// leanSpec wraps every SSZ collection in a `data` key. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DataList { + pub data: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct StateJson { + pub config: GenesisConfigJson, + pub slot: u64, + pub latest_block_header: BlockHeaderJson, + pub latest_justified: CheckpointJson, + pub latest_finalized: CheckpointJson, + pub historical_block_hashes: DataList, + pub justified_slots: DataList, + pub validators: DataList, + pub justifications_roots: DataList, + pub justifications_validators: DataList, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct GenesisConfigJson { + pub genesis_time: u64, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct BlockHeaderJson { + pub slot: u64, + pub proposer_index: u64, + pub parent_root: String, + pub state_root: String, + pub body_root: String, +} + +#[derive(Debug, Clone, Default, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CheckpointJson { + pub root: String, + pub slot: u64, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ValidatorJson { + pub attestation_public_key: String, + pub proposal_public_key: String, + pub index: u64, +} + +/// A block as a step or a case carries it. +/// +/// `blockRootLabel` is the generator's symbolic name for the block — `"block_2b"` — and is +/// what the fork-choice checks refer to instead of a root. It is absent from the +/// state-transition vectors, hence the default. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct BlockJson { + pub slot: u64, + pub proposer_index: u64, + pub parent_root: String, + pub state_root: String, + pub body: BlockBodyJson, + #[serde(default)] + pub block_root_label: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct BlockBodyJson { + pub attestations: DataList, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct AggregatedAttestationJson { + pub aggregation_bits: DataList, + pub data: AttestationDataJson, +} + +#[derive(Debug, Clone, Default, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AttestationDataJson { + pub slot: u64, + pub head: CheckpointJson, + pub target: CheckpointJson, + pub source: CheckpointJson, +} + +impl StateJson { + pub fn build(&self) -> Result { + Ok(State { + config: GenesisConfig { + genesis_time: self.config.genesis_time, + }, + slot: Slot(self.slot), + latest_block_header: self.latest_block_header.build()?, + latest_justified: self.latest_justified.build()?, + latest_finalized: self.latest_finalized.build()?, + historical_block_hashes: roots(&self.historical_block_hashes)?, + justified_slots: bitlist::(&self.justified_slots.data)?, + validators: validators(&self.validators)?, + justifications_roots: roots(&self.justifications_roots)?, + justifications_validators: bitlist::( + &self.justifications_validators.data, + )?, + }) + } +} + +impl BlockHeaderJson { + pub fn build(&self) -> Result { + Ok(BlockHeader { + slot: Slot(self.slot), + proposer_index: ValidatorIndex(self.proposer_index), + parent_root: bytes32(&self.parent_root)?, + state_root: bytes32(&self.state_root)?, + body_root: bytes32(&self.body_root)?, + }) + } +} + +impl CheckpointJson { + pub fn build(&self) -> Result { + Ok(Checkpoint { + root: bytes32(&self.root)?, + slot: Slot(self.slot), + }) + } +} + +impl BlockJson { + pub fn build(&self) -> Result { + let mut attestations = AggregatedAttestations::default(); + for attestation in &self.body.attestations.data { + attestations + .push(attestation.build()?) + .map_err(|error| format!("attestations: {error:?}"))?; + } + Ok(Block { + slot: Slot(self.slot), + proposer_index: ValidatorIndex(self.proposer_index), + parent_root: bytes32(&self.parent_root)?, + state_root: bytes32(&self.state_root)?, + body: BlockBody { attestations }, + }) + } +} + +impl AggregatedAttestationJson { + pub fn build(&self) -> Result { + Ok(AggregatedAttestation { + aggregation_bits: bitlist::(&self.aggregation_bits.data)?, + data: self.data.build()?, + }) + } +} + +impl AttestationDataJson { + pub fn build(&self) -> Result { + Ok(AttestationData { + slot: Slot(self.slot), + head: self.head.build()?, + target: self.target.build()?, + source: self.source.build()?, + }) + } +} + +// --------------------------------------------------------------------------------------- +// Primitive conversions +// --------------------------------------------------------------------------------------- + +/// Records a mismatch when a case asserts a value and it disagrees. +pub fn compare( + failures: &mut Vec, + name: &str, + expected: Option, + actual: Option, +) { + let (Some(expected), Some(actual)) = (expected, actual) else { + return; + }; + if expected != actual { + failures.push(format!("{name}: got {actual:?}, expected {expected:?}")); + } +} + +pub fn flags(length: usize, read: impl Fn(usize) -> Option) -> Vec { + (0..length) + .map(|index| read(index).unwrap_or(false)) + .collect() +} + +pub fn hex(bytes: &[u8]) -> String { + let mut out = String::with_capacity(2 + bytes.len() * 2); + out.push_str("0x"); + for byte in bytes { + out.push_str(&format!("{byte:02x}")); + } + out +} + +pub fn unhex(text: &str) -> Result, String> { + let body = text + .strip_prefix("0x") + .ok_or_else(|| format!("{text}: missing 0x prefix"))?; + (0..body.len()) + .step_by(2) + .map(|index| { + u8::from_str_radix(&body[index..index + 2], 16) + .map_err(|error| format!("{text}: {error}")) + }) + .collect() +} + +pub fn bytes32(text: &str) -> Result { + unhex(text)? + .try_into() + .map_err(|_| format!("{text}: not 32 bytes")) +} + +pub fn bytes52(text: &str) -> Result { + unhex(text)? + .try_into() + .map_err(|_| format!("{text}: not 52 bytes")) +} + +pub fn roots(list: &DataList) -> Result { + let mut out = HistoricalBlockHashes::default(); + for text in &list.data { + out.push(bytes32(text)?) + .map_err(|error| format!("roots: {error:?}"))?; + } + Ok(out) +} + +pub fn validators(list: &DataList) -> Result { + let mut out = Validators::default(); + for entry in &list.data { + out.push(Validator { + attestation_public_key: bytes52(&entry.attestation_public_key)?, + proposal_public_key: bytes52(&entry.proposal_public_key)?, + index: ValidatorIndex(entry.index), + }) + .map_err(|error| format!("validators: {error:?}"))?; + } + Ok(out) +} + +pub fn bitlist>>(bits: &[bool]) -> Result { + T::try_from(bits.to_vec()) + .map_err(|_| format!("bitlist of {} bits exceeds its limit", bits.len())) +} diff --git a/crates/verity-chain/tests/fork_choice/checks.rs b/crates/verity-chain/tests/fork_choice/checks.rs new file mode 100644 index 0000000..add23bc --- /dev/null +++ b/crates/verity-chain/tests/fork_choice/checks.rs @@ -0,0 +1,475 @@ +//! The generator's own per-step assertions, on top of the snapshot. + +use std::collections::{HashMap, HashSet}; + +use crate::common::compare; +use crate::fork_choice::labels::Labels; +use crate::fork_choice::shapes::Step; +use serde::Deserialize; +use verity_chain::fork_choice::duties::attestation_target; +use verity_chain::{Store, block_weights, hash_tree_root}; +use verity_types::{AttestationData, Bytes32, SingleMessageAggregate, ValidatorIndex}; + +// --------------------------------------------------------------------------------------- +// The `checks` block +// --------------------------------------------------------------------------------------- + +/// The generator's own per-step assertions, on top of the snapshot. +#[derive(Debug, Default, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields, default)] +pub struct ChecksJson { + pub time: Option, + pub head_slot: Option, + pub head_root_label: Option, + pub safe_target_slot: Option, + pub safe_target_root_label: Option, + pub latest_justified_slot: Option, + pub latest_justified_root_label: Option, + pub latest_finalized_slot: Option, + pub latest_finalized_root_label: Option, + pub attestation_target_slot: Option, + pub attestation_target_root_label: Option, + pub attestation_signature_target_slots: Option>, + pub latest_new_aggregated_target_slots: Option>, + pub latest_known_aggregated_target_slots: Option>, + pub new_pool_proof_participants: Option>>, + pub block_attestation_count: Option, + pub block_attestations: Option>, + pub attestation_checks: Option>, + pub labels_in_store: Option>, + pub lexicographic_head_among: Option>, + pub canonical_equivocation_head_among: Option>, + pub filled_block_root_label: Option, + pub reorg_depth: Option, +} + +#[derive(Debug, Default, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields, default)] +pub struct BlockAttestationCheck { + pub participants: Vec, + pub attestation_slot: Option, + pub target_slot: Option, +} + +#[derive(Debug, Default, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields, default)] +pub struct AttestationCheck { + pub validator: u64, + pub attestation_slot: Option, + pub head_slot: Option, + pub source_slot: Option, + pub source_root_label: Option, + pub target_slot: Option, + pub location: String, +} + +impl ChecksJson { + pub fn check( + &self, + failures: &mut Vec, + store: &Store, + step: &Step, + labels: &Labels, + previous_head: Bytes32, + ) -> Result<(), String> { + compare(failures, "time", self.time, Some(store.time.0)); + self.check_checkpoints(failures, store, labels)?; + self.check_target(failures, store, labels)?; + self.check_pools(failures, store); + self.check_block_body(failures, step, labels)?; + self.check_ties(failures, store, labels)?; + + if let Some(expected) = self.reorg_depth { + compare( + failures, + "reorgDepth", + Some(expected), + Some(reorg_depth(store, previous_head)), + ); + } + if let Some(names) = &self.labels_in_store { + for name in names { + let root = labels.root(name)?; + if !store.blocks.contains_key(&root) { + failures.push(format!("labelsInStore: {name} is not in the store")); + } + } + } + Ok(()) + } + + /// Head, safe target, and the two checkpoints, by slot and by symbolic name. + fn check_checkpoints( + &self, + failures: &mut Vec, + store: &Store, + labels: &Labels, + ) -> Result<(), String> { + let slot_of = |root: Bytes32| store.blocks.get(&root).map(|block| block.slot.0); + + compare(failures, "headSlot", self.head_slot, slot_of(store.head)); + compare( + failures, + "safeTargetSlot", + self.safe_target_slot, + slot_of(store.safe_target), + ); + compare( + failures, + "latestJustifiedSlot", + self.latest_justified_slot, + Some(store.latest_justified.slot.0), + ); + compare( + failures, + "latestFinalizedSlot", + self.latest_finalized_slot, + Some(store.latest_finalized.slot.0), + ); + + let named = [ + ("headRootLabel", &self.head_root_label, store.head), + ( + "safeTargetRootLabel", + &self.safe_target_root_label, + store.safe_target, + ), + ( + "latestJustifiedRootLabel", + &self.latest_justified_root_label, + store.latest_justified.root, + ), + ( + "latestFinalizedRootLabel", + &self.latest_finalized_root_label, + store.latest_finalized.root, + ), + ]; + for (name, expected, actual) in named { + let Some(label) = expected else { continue }; + labels.check(failures, name, label, actual)?; + } + Ok(()) + } + + /// The checkpoint a validator would name as its attestation target. + fn check_target( + &self, + failures: &mut Vec, + store: &Store, + labels: &Labels, + ) -> Result<(), String> { + if self.attestation_target_slot.is_none() && self.attestation_target_root_label.is_none() { + return Ok(()); + } + let target = attestation_target(store); + compare( + failures, + "attestationTargetSlot", + self.attestation_target_slot, + Some(target.slot.0), + ); + if let Some(label) = &self.attestation_target_root_label { + labels.check(failures, "attestationTargetRootLabel", label, target.root)?; + } + Ok(()) + } + + /// Which target slots each pool holds, and the pending pool's participant union. + fn check_pools(&self, failures: &mut Vec, store: &Store) { + let target_slots = |pool: &HashMap>| { + let mut slots: Vec = pool.keys().map(|data| data.target.slot.0).collect(); + slots.sort_unstable(); + slots.dedup(); + slots + }; + + let mut signature_slots: Vec = store + .attestation_signatures + .keys() + .map(|data| data.target.slot.0) + .collect(); + signature_slots.sort_unstable(); + signature_slots.dedup(); + compare( + failures, + "attestationSignatureTargetSlots", + self.attestation_signature_target_slots.clone(), + Some(signature_slots), + ); + compare( + failures, + "latestNewAggregatedTargetSlots", + self.latest_new_aggregated_target_slots.clone(), + Some(target_slots(&store.latest_new_aggregated_payloads)), + ); + compare( + failures, + "latestKnownAggregatedTargetSlots", + self.latest_known_aggregated_target_slots.clone(), + Some(target_slots(&store.latest_known_aggregated_payloads)), + ); + + if let Some(expected) = &self.new_pool_proof_participants { + for (slot, wanted) in expected { + let mut union: Vec = store + .latest_new_aggregated_payloads + .iter() + .filter(|(data, _)| data.target.slot.0.to_string() == *slot) + .flat_map(|(_, proofs)| proofs) + .flat_map(|proof| verity_chain::fork_choice::participants(&proof.participants)) + .map(|index| index.0) + .collect(); + union.sort_unstable(); + union.dedup(); + let mut wanted = wanted.clone(); + wanted.sort_unstable(); + compare( + failures, + &format!("newPoolProofParticipants[{slot}]"), + Some(wanted), + Some(union), + ); + } + } + } + + /// What the step's own block carries, and the label it was registered under. + fn check_block_body( + &self, + failures: &mut Vec, + step: &Step, + labels: &Labels, + ) -> Result<(), String> { + let needs_block = self.block_attestation_count.is_some() + || self.block_attestations.is_some() + || self.filled_block_root_label.is_some(); + if !needs_block { + return Ok(()); + } + let block = step + .block + .as_ref() + .ok_or("a block check on a step with no block")? + .build()?; + + compare( + failures, + "blockAttestationCount", + self.block_attestation_count, + Some(block.body.attestations.len()), + ); + if let Some(expected) = &self.block_attestations { + if expected.len() != block.body.attestations.len() { + failures.push(format!( + "blockAttestations: got {} entries, expected {}", + block.body.attestations.len(), + expected.len() + )); + } + for (check, attestation) in expected.iter().zip(block.body.attestations.iter()) { + let actual: Vec = + verity_chain::fork_choice::participants(&attestation.aggregation_bits) + .map(|index| index.0) + .collect(); + compare( + failures, + "blockAttestations.participants", + Some(check.participants.clone()), + Some(actual), + ); + compare( + failures, + "blockAttestations.attestationSlot", + check.attestation_slot, + Some(attestation.data.slot.0), + ); + compare( + failures, + "blockAttestations.targetSlot", + check.target_slot, + Some(attestation.data.target.slot.0), + ); + } + } + if let Some(label) = &self.filled_block_root_label { + compare( + failures, + "filledBlockRootLabel", + Some(labels.root(label)?), + Some(hash_tree_root(&block)), + ); + } + Ok(()) + } + + /// The two tiebreak assertions, plus the per-validator pool content checks. + fn check_ties( + &self, + failures: &mut Vec, + store: &Store, + labels: &Labels, + ) -> Result<(), String> { + if let Some(names) = &self.lexicographic_head_among { + let weights = block_weights(store); + let mut roots = Vec::new(); + for name in names { + let root = labels.root(name)?; + roots.push((weights.get(&root).copied().unwrap_or_default(), root)); + } + let tied = roots.iter().all(|(weight, _)| *weight == roots[0].0); + if !tied { + failures.push(format!("lexicographicHeadAmong: weights differ: {roots:?}")); + } + let winner = roots.iter().map(|(_, root)| *root).max(); + compare( + failures, + "lexicographicHeadAmong", + winner.map(|root| labels.name(root)), + Some(labels.name(store.head)), + ); + } + + if let Some(names) = &self.canonical_equivocation_head_among { + let mut best: Option<(Bytes32, Bytes32)> = None; + for name in names { + let root = labels.root(name)?; + let Some(vote) = store + .latest_known_aggregated_payloads + .keys() + .filter(|data| data.head.root == root) + .map(hash_tree_root) + .max() + else { + failures.push(format!("canonicalEquivocationHeadAmong: {name} unattested")); + continue; + }; + if best.is_none_or(|(seen, _)| vote > seen) { + best = Some((vote, root)); + } + } + compare( + failures, + "canonicalEquivocationHeadAmong", + best.map(|(_, root)| labels.name(root)), + Some(labels.name(store.head)), + ); + } + + if let Some(checks) = &self.attestation_checks { + for check in checks { + check.check(failures, store, labels)?; + } + } + Ok(()) + } +} + +impl AttestationCheck { + /// The vote a named pool records for one validator, by canonical precedence. + pub fn check( + &self, + failures: &mut Vec, + store: &Store, + labels: &Labels, + ) -> Result<(), String> { + let voter = ValidatorIndex(self.validator); + let winner = match self.location.as_str() { + "signatures" => best_vote(store.attestation_signatures.iter().filter_map( + |(data, entries)| { + entries + .iter() + .any(|entry| entry.validator_index == voter) + .then_some(*data) + }, + )), + "new" => best_vote(votes_for(&store.latest_new_aggregated_payloads, voter)), + "known" => best_vote(votes_for(&store.latest_known_aggregated_payloads, voter)), + other => return Err(format!("unknown attestation check location {other}")), + }; + let Some(vote) = winner else { + failures.push(format!( + "attestationChecks: validator {} not in the {} pool", + self.validator, self.location + )); + return Ok(()); + }; + + compare( + failures, + "attestationChecks.attestationSlot", + self.attestation_slot, + Some(vote.slot.0), + ); + compare( + failures, + "attestationChecks.headSlot", + self.head_slot, + Some(vote.head.slot.0), + ); + compare( + failures, + "attestationChecks.sourceSlot", + self.source_slot, + Some(vote.source.slot.0), + ); + compare( + failures, + "attestationChecks.targetSlot", + self.target_slot, + Some(vote.target.slot.0), + ); + if let Some(label) = &self.source_root_label { + compare( + failures, + "attestationChecks.sourceRootLabel", + Some(labels.root(label)?), + Some(vote.source.root), + ); + } + Ok(()) + } +} + +/// Every vote in a pool that names `voter` as a participant. +fn votes_for( + pool: &HashMap>, + voter: ValidatorIndex, +) -> impl Iterator + '_ { + pool.iter().filter_map(move |(data, proofs)| { + proofs + .iter() + .any(|proof| { + verity_chain::fork_choice::participants(&proof.participants).any(|i| i == voter) + }) + .then_some(*data) + }) +} + +/// The winner by canonical precedence: highest slot, then largest attestation-data root. +fn best_vote(votes: impl Iterator) -> Option { + votes.max_by_key(|data| (data.slot, hash_tree_root(data))) +} + +/// How many blocks the old head sat above its common ancestor with the new one. +fn reorg_depth(store: &Store, previous_head: Bytes32) -> usize { + let mut on_new_chain = HashSet::new(); + let mut cursor = store.head; + while let Some(block) = store.blocks.get(&cursor) { + if !on_new_chain.insert(cursor) { + break; + } + cursor = block.parent_root; + } + + let mut depth = 0; + let mut cursor = previous_head; + while let Some(block) = store.blocks.get(&cursor) { + if on_new_chain.contains(&cursor) { + break; + } + depth += 1; + cursor = block.parent_root; + } + depth +} diff --git a/crates/verity-chain/tests/fork_choice/labels.rs b/crates/verity-chain/tests/fork_choice/labels.rs new file mode 100644 index 0000000..eda1f20 --- /dev/null +++ b/crates/verity-chain/tests/fork_choice/labels.rs @@ -0,0 +1,63 @@ +//! Symbolic block names, as the generator's checks refer to blocks. + +use std::collections::HashMap; + +use crate::common::hex; +use verity_types::Bytes32; + +// --------------------------------------------------------------------------------------- +// Symbolic block names +// --------------------------------------------------------------------------------------- + +/// The generator's block labels — `"block_2b"`, `"fork_a"` — resolved to roots. +/// +/// The checks name blocks this way rather than by root, so a failure reads as "expected +/// `block_3`, got `block_2b`" instead of two hashes. +pub struct Labels(HashMap); + +impl Labels { + pub fn new(anchor_root: Bytes32) -> Self { + Self(HashMap::from([("genesis".to_string(), anchor_root)])) + } + + pub fn insert(&mut self, label: String, root: Bytes32) { + self.0.insert(label, root); + } + + pub fn root(&self, label: &str) -> Result { + self.0 + .get(label) + .copied() + .ok_or_else(|| format!("no block labelled {label}")) + } + + /// The label a root carries, or its hex when the generator never named it. + /// + /// Two labels can name one root: a fork built to be identical to another up to its label + /// hashes to the same block. Only failure messages read this, so an arbitrary pick among + /// them is fine — every assertion below compares roots. + pub fn name(&self, root: Bytes32) -> String { + self.0 + .iter() + .find_map(|(label, known)| (*known == root).then(|| label.clone())) + .unwrap_or_else(|| hex(&root)) + } + + /// Compares a root against the block a label names, reporting the mismatch by name. + pub fn check( + &self, + failures: &mut Vec, + field: &str, + label: &str, + actual: Bytes32, + ) -> Result<(), String> { + let expected = self.root(label)?; + if expected != actual { + failures.push(format!( + "{field}: got {}, expected {label}", + self.name(actual) + )); + } + Ok(()) + } +} diff --git a/crates/verity-chain/tests/fork_choice/mod.rs b/crates/verity-chain/tests/fork_choice/mod.rs new file mode 100644 index 0000000..6b18468 --- /dev/null +++ b/crates/verity-chain/tests/fork_choice/mod.rs @@ -0,0 +1,6 @@ +//! The fork-choice vector format, split by what each part is responsible for. + +pub mod checks; +pub mod labels; +pub mod shapes; +pub mod snapshot; diff --git a/crates/verity-chain/tests/fork_choice/shapes.rs b/crates/verity-chain/tests/fork_choice/shapes.rs new file mode 100644 index 0000000..2e7881c --- /dev/null +++ b/crates/verity-chain/tests/fork_choice/shapes.rs @@ -0,0 +1,93 @@ +//! The `fork_choice_test` vector, mirrored field for field. + +use crate::common::{AttestationDataJson, BlockJson, DataList, StateJson}; +use crate::fork_choice::checks::ChecksJson; +use crate::fork_choice::snapshot::StoreSnapshotJson; +use serde::Deserialize; + +// --------------------------------------------------------------------------------------- +// Fixture shapes +// --------------------------------------------------------------------------------------- + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct Case { + #[allow(dead_code)] + pub network: String, + #[allow(dead_code)] + pub lean_env: String, + #[allow(dead_code)] + pub proof_setting: u8, + pub anchor_state: StateJson, + pub anchor_block: BlockJson, + pub steps: Vec, + /// Set when building the store from the anchor is itself the thing expected to fail. + #[serde(default)] + pub rejection_reason: Option, + #[allow(dead_code)] + pub max_slot: u64, + #[serde(rename = "_info")] + #[allow(dead_code)] + pub info: serde_json::Value, +} + +/// One event, in the flat shape the generator emits for every step type. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields, default)] +pub struct Step { + pub step_type: String, + pub valid: bool, + pub store_snapshot: StoreSnapshotJson, + pub checks: Option, + pub rejection_reason: Option, + pub block: Option, + pub tick_to_slot: bool, + pub time: Option, + pub interval: Option, + pub has_proposal: bool, + pub attestation: Option, + pub is_aggregator: bool, +} + +impl Default for Step { + fn default() -> Self { + Self { + step_type: String::new(), + // Every emitted step carries `valid`; this only satisfies `default` above. + valid: true, + store_snapshot: StoreSnapshotJson::default(), + checks: None, + rejection_reason: None, + block: None, + tick_to_slot: false, + time: None, + interval: None, + attestation: None, + has_proposal: false, + is_aggregator: false, + } + } +} + +/// Both gossip shapes in one: a per-validator vote, and an aggregate over many. +#[derive(Debug, Default, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields, default)] +pub struct AttestationJson { + pub data: AttestationDataJson, + pub validator_index: Option, + pub signature: Option, + pub proof: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ProofJson { + pub participants: DataList, + pub proof: ProofBytesJson, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ProofBytesJson { + pub data: String, +} diff --git a/crates/verity-chain/tests/fork_choice/snapshot.rs b/crates/verity-chain/tests/fork_choice/snapshot.rs new file mode 100644 index 0000000..993bfed --- /dev/null +++ b/crates/verity-chain/tests/fork_choice/snapshot.rs @@ -0,0 +1,194 @@ +//! The ten observables leanSpec records after every step, and how they are compared. + +use std::collections::{HashMap, HashSet}; + +use crate::common::{self, compare, hex}; +use serde::Deserialize; +use verity_chain::{Store, block_weights, hash_tree_root}; +use verity_types::{AttestationData, SingleMessageAggregate}; + +// --------------------------------------------------------------------------------------- +// The store snapshot +// --------------------------------------------------------------------------------------- + +/// Every observable leanSpec records after a step, accepted or rejected. +#[derive(Debug, Default, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct StoreSnapshotJson { + pub time: u64, + pub head_root: String, + pub safe_target_root: String, + pub latest_justified: common::CheckpointJson, + pub latest_finalized: common::CheckpointJson, + pub block_roots: Vec, + pub block_weights: Vec, + pub attestation_signatures: Vec, + pub new_aggregated_payloads: Vec, + pub known_aggregated_payloads: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct BlockWeightJson { + pub root: String, + pub weight: u64, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct SignaturePoolJson { + pub data_root: String, + pub validator_indices: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct AggregatedPoolJson { + pub data_root: String, + pub participant_sets: Vec>, +} + +impl StoreSnapshotJson { + /// Compares all ten observables against the store. + pub fn check(&self, failures: &mut Vec, store: &Store) { + compare(failures, "time", Some(self.time), Some(store.time.0)); + compare( + failures, + "headRoot", + Some(self.head_root.clone()), + Some(hex(&store.head)), + ); + compare( + failures, + "safeTargetRoot", + Some(self.safe_target_root.clone()), + Some(hex(&store.safe_target)), + ); + compare( + failures, + "latestJustified", + Some(( + self.latest_justified.root.clone(), + self.latest_justified.slot, + )), + Some(( + hex(&store.latest_justified.root), + store.latest_justified.slot.0, + )), + ); + compare( + failures, + "latestFinalized", + Some(( + self.latest_finalized.root.clone(), + self.latest_finalized.slot, + )), + Some(( + hex(&store.latest_finalized.root), + store.latest_finalized.slot.0, + )), + ); + + let mut roots: Vec = store.blocks.keys().map(|root| hex(root)).collect(); + roots.sort(); + compare( + failures, + "blockRoots", + Some(self.block_roots.clone()), + Some(roots), + ); + + self.check_weights(failures, store); + self.check_pools(failures, store); + } + + /// Weights are recorded for every block above the finalized slot, zero included. + fn check_weights(&self, failures: &mut Vec, store: &Store) { + let weights = block_weights(store); + let mut actual: Vec<(String, u64)> = store + .blocks + .iter() + .filter(|(_, block)| block.slot.0 > store.latest_finalized.slot.0) + .map(|(root, _)| (hex(root), weights.get(root).copied().unwrap_or_default())) + .collect(); + actual.sort(); + + let expected: Vec<(String, u64)> = self + .block_weights + .iter() + .map(|entry| (entry.root.clone(), entry.weight)) + .collect(); + compare(failures, "blockWeights", Some(expected), Some(actual)); + } + + /// The three vote pools, as the coverage the snapshot records rather than as bytes. + fn check_pools(&self, failures: &mut Vec, store: &Store) { + let mut signatures: Vec<(String, Vec)> = store + .attestation_signatures + .iter() + .map(|(data, entries)| { + let mut indices: Vec = entries + .iter() + .map(|entry| entry.validator_index.0) + .collect(); + indices.sort_unstable(); + (hex(&hash_tree_root(data)), indices) + }) + .collect(); + signatures.sort(); + let expected: Vec<(String, Vec)> = self + .attestation_signatures + .iter() + .map(|entry| (entry.data_root.clone(), entry.validator_indices.clone())) + .collect(); + compare( + failures, + "attestationSignatures", + Some(expected), + Some(signatures), + ); + + compare( + failures, + "newAggregatedPayloads", + Some(pool_entries(&self.new_aggregated_payloads)), + Some(coverage(&store.latest_new_aggregated_payloads)), + ); + compare( + failures, + "knownAggregatedPayloads", + Some(pool_entries(&self.known_aggregated_payloads)), + Some(coverage(&store.latest_known_aggregated_payloads)), + ); + } +} + +fn pool_entries(entries: &[AggregatedPoolJson]) -> Vec<(String, Vec>)> { + entries + .iter() + .map(|entry| (entry.data_root.clone(), entry.participant_sets.clone())) + .collect() +} + +/// A pool as the snapshot sees it: participants per proof, sorted, proof bytes dropped. +fn coverage( + pool: &HashMap>, +) -> Vec<(String, Vec>)> { + let mut out: Vec<(String, Vec>)> = pool + .iter() + .map(|(data, proofs)| { + let mut sets: Vec> = proofs + .iter() + .map(|proof| { + verity_chain::fork_choice::participants(&proof.participants) + .map(|index| index.0) + .collect() + }) + .collect(); + sets.sort(); + (hex(&hash_tree_root(data)), sets) + }) + .collect(); + out.sort(); + out +} diff --git a/crates/verity-chain/tests/fork_choice_fixtures.rs b/crates/verity-chain/tests/fork_choice_fixtures.rs new file mode 100644 index 0000000..8f2c2a3 --- /dev/null +++ b/crates/verity-chain/tests/fork_choice_fixtures.rs @@ -0,0 +1,443 @@ +//! Conformance against leanSpec's fork-choice vectors (`fork_choice_test` format). +//! +//! Unlike the other suites, a case here is a **state machine**, not a single call. It opens +//! with a trusted anchor and then applies a list of steps — a block arriving, the clock +//! ticking, a vote or an aggregate reaching the node over gossip — carrying one store from +//! each step to the next. +//! +//! Every step, accepted or rejected, carries a `storeSnapshot`, and this replays against all +//! ten of its fields rather than the shallower `checks` block. leanSpec says why in +//! `StoreSnapshot`'s own docs: full block membership is what makes over- and under-pruning +//! observable, and the weights must agree "even where two clients agree on the head" — a +//! client whose weights are wrong but whose head happens to land right is the failure this +//! catches and `headSlot` does not. `checks` is asserted too, for what it adds on top: +//! symbolic block labels, the validator's attestation target, and the block body's own +//! contents. +//! +//! Two things the generator does are the harness's job here rather than the crate's, because +//! leanSpec puts them outside `Store`: +//! +//! - **Signature verification.** `verity-chain` splits it out of all three verifying entry +//! points (see that module's docs), so this replays the pure halves. Two steps in the whole +//! suite turn on a signature actually being wrong; they are listed in [`NEEDS_VERIFICATION`]. +//! - **Seeding the counted pool from a block's own votes.** The generator hands its store the +//! proofs the proposer had aggregated before it applies the block, so the snapshot expects +//! them. A replaying client only ever sees the block, and recovers the same coverage from +//! the body's aggregation bits — the proof bytes are not in the snapshot. `ethlambda`'s +//! runner does the same thing at the same point. +//! +//! Source: leanSpec `tests/consensus/lstar/fork_choice/`, filled into the +//! `fixtures-prod-scheme.tar.gz` release asset that `crates/verity-types/fixtures.sha256` +//! pins. leanSpec `main` @ `0588c2d215a955a516378677a92db2a5666802f3`. + +mod common; +mod fork_choice; + +use std::collections::HashSet; + +use common::bitlist; +use fork_choice::labels::Labels; +use fork_choice::shapes::{Case, Step}; +use verity_chain::{ + AttestationSignature, RejectionReason, Store, hash_tree_root, on_block, on_tick, + record_aggregated_payload, record_attestation_signature, +}; +use verity_types::config::INTERVALS_PER_SLOT; +use verity_types::{ + AggregationBits, ByteList512KiB, Interval, SignedAggregatedAttestation, SingleMessageAggregate, + ValidatorIndex, +}; + +/// Every suite under leanSpec's `fork_choice` fixture directory. +const SUITES: &[&str] = &[ + "test_ancestry_branches", + "test_attestation_source_divergence", + "test_attestation_target_selection", + "test_block_attestation_limits", + "test_block_future_horizon", + "test_block_genesis_self_vote", + "test_block_production", + "test_block_unknown_parent", + "test_checkpoint_sync", + "test_checkpoint_sync_window", + "test_duplicate_attestation_data", + "test_early_block_arrival", + "test_equivocation", + "test_fallback_pool_set_cover", + "test_finalization_mid_processing", + "test_finalized_safety", + "test_fork_choice_head", + "test_fork_choice_reorgs", + "test_gossip_aggregated_attestation_validation", + "test_gossip_aggregated_empty_participants", + "test_gossip_aggregated_registry_and_signature", + "test_gossip_attestation_validation", + "test_head_movement", + "test_lexicographic_tiebreaker", + "test_lmd_latest_message", + "test_prune_finalized_orphaned_branch", + "test_safe_target", + "test_safe_target_supermajority", + "test_signature_aggregation", + "test_store_pruning", + "test_tick_acceptance_branches", + "test_tick_system", +]; + +/// Cases whose rejection is the signature itself being wrong. +/// +/// `verity-chain` cannot produce `INVALID_SIGNATURE`: verification is the caller's, above +/// this crate (see `fork_choice`'s module docs). Both cases replay their earlier steps +/// normally; only the failing step is not asserted. They land with `verity-crypto`, which is +/// what supplies the missing check. +const NEEDS_VERIFICATION: &[&str] = &[ + "test_gossip_attestation_with_invalid_signature", + "test_aggregated_attestation_proof_verification_failure_rejected", +]; + +/// Vectors whose recorded vote pools hold more than the wire ever carried. +/// +/// The generator builds a block by aggregating the votes its proposer held, and merges that +/// pool into its store *before* applying the block. A replaying client only ever sees the +/// block, and a block body carries at most `MAX_ATTESTATIONS_DATA` distinct votes — in one of +/// these it carries none at all while the proposer's pool held three. Where the two diverge +/// the snapshot records local state that never reached the wire, and no client can reproduce +/// it. Everything else in those vectors is still asserted; see [`POOL_DERIVED`]. +const GENERATOR_POOL_EXCEEDS_WIRE: &[&str] = &[ + "test_attestation_target_justifiable_constraint", + "test_block_builder_fixed_point_advances_justification", + "test_block_builder_recovers_finality_after_non_zero_boundary_stall", + "test_produce_block_enforces_max_attestations_data_limit", + "test_produce_block_includes_pending_attestations", + "test_post_anchor_votes_can_finalize_above_anchor", + "test_fork_above_finalized_wins_at_or_below_loses", + "test_heavier_fork_below_finalized_slot_never_wins", +]; + +/// Vectors that turn on the aggregator duty at interval 2. +/// +/// Folding a slot's pooled signatures into proofs is a cryptographic operation and belongs to +/// `verity-crypto` (see `fork_choice::timeline`). Its absence shows up in the pools alone — +/// the signatures it would have drained, and the proofs it would have produced. It lands with +/// that crate, and these become ordinary vectors then. +const NEEDS_AGGREGATION: &[&str] = &[ + "test_interval_2_aggregator_aggregates_raw_signatures", + "test_aggregate_covers_union_of_priority_and_fallback_pools", + "test_tick_interval_0_skips_acceptance_when_not_proposer", +]; + +/// The observables that follow from the vote pools, and so from what a client was told. +/// +/// A vector on either list above is not asserted on these, and is asserted on everything +/// else: its head, checkpoints, block membership, clock, and block bodies all still have to +/// match. The safe target is here because it is weighed from the pending pool. +const POOL_DERIVED: &[&str] = &[ + "attestationSignatures", + "attestationSignatureTargetSlots", + "attestationChecks", + "blockWeights", + "knownAggregatedPayloads", + "latestKnownAggregatedTargetSlots", + "latestNewAggregatedTargetSlots", + "newAggregatedPayloads", + "newPoolProofParticipants", + "safeTargetRoot", + "safeTargetRootLabel", + "safeTargetSlot", +]; + +#[test] +fn should_match_leanspec_fork_choice_vectors_when_fixtures_are_present() { + let Some(root) = common::fixtures_dir() else { + eprintln!("skipping: set VERITY_FIXTURES to run leanSpec fork-choice vectors"); + return; + }; + + let mut failures = Vec::new(); + let mut matched: HashSet<&str> = HashSet::new(); + let mut relaxed_and_diverged: HashSet<&str> = HashSet::new(); + let mut checked = 0usize; + let mut steps = 0usize; + for suite in SUITES { + let files = common::collect_suite_json(&root, suite); + assert!( + !files.is_empty(), + "no JSON under {} (expected **/{suite}/*.json)", + root.display() + ); + for (id, case) in common::read_cases::(&files) { + let skip_last = listed(&id, NEEDS_VERIFICATION); + if let Some(name) = skip_last { + matched.insert(name); + } + let relaxed = + listed(&id, GENERATOR_POOL_EXCEEDS_WIRE).or(listed(&id, NEEDS_AGGREGATION)); + if let Some(name) = relaxed { + matched.insert(name); + } + checked += 1; + steps += case.steps.len(); + match replay(&case, skip_last.is_some()) { + Ok(()) => {} + Err(problems) => { + let (kept, dropped) = split_relaxed(problems, relaxed.is_some()); + if let (Some(name), true) = (relaxed, dropped) { + relaxed_and_diverged.insert(name); + } + if !kept.is_empty() { + failures.push(format!("{id}: {}", kept.join("\n "))); + } + } + } + } + } + + for name in NEEDS_VERIFICATION + .iter() + .chain(GENERATOR_POOL_EXCEEDS_WIRE) + .chain(NEEDS_AGGREGATION) + { + assert!( + matched.contains(*name), + "{name} matched no vector; leanSpec renamed or dropped it" + ); + } + // A listed vector that stopped diverging is one the list no longer has a reason to hold. + for name in GENERATOR_POOL_EXCEEDS_WIRE.iter().chain(NEEDS_AGGREGATION) { + assert!( + relaxed_and_diverged.contains(*name), + "{name} no longer diverges on its pools; take it off the list" + ); + } + assert!( + failures.is_empty(), + "{} of {checked} fork-choice vectors failed:\n{}", + failures.len(), + failures.join("\n") + ); + let relaxed_count = GENERATOR_POOL_EXCEEDS_WIRE.len() + NEEDS_AGGREGATION.len(); + eprintln!( + "fork choice: {checked} vectors matched, {steps} steps replayed, \ + {relaxed_count} not asserted on their vote pools" + ); +} + +/// The listed test function the leanSpec test id names, if any. +fn listed(id: &str, names: &[&'static str]) -> Option<&'static str> { + names.iter().copied().find(|name| id.contains(name)) +} + +/// Splits a vector's failures into those that stand and those [`POOL_DERIVED`] excuses. +fn split_relaxed(problems: Vec, relaxed: bool) -> (Vec, bool) { + if !relaxed { + return (problems, false); + } + let mut kept = Vec::new(); + let mut dropped = false; + for problem in problems { + let field = problem + .rsplit("): ") + .next() + .unwrap_or(&problem) + .split(['[', ':', '.']) + .next() + .unwrap_or_default(); + if POOL_DERIVED.contains(&field) { + dropped = true; + } else { + kept.push(problem); + } + } + (kept, dropped) +} + +/// Runs one case: build the anchor store, then apply every step in order. +/// +/// `skip_signature_step` drops the one step whose rejection this crate cannot produce; the +/// steps before it still run, since they build the store that step is posed against. +fn replay(case: &Case, skip_signature_step: bool) -> Result<(), Vec> { + let anchor_state = case.anchor_state.build().map_err(one)?; + let anchor_block = case.anchor_block.build().map_err(one)?; + let anchor = Store::new(&anchor_state, &anchor_block, Some(ValidatorIndex(0))); + + // A case may exist only to reject its own anchor, and then carries no steps at all. + if let Some(expected) = &case.rejection_reason { + return match anchor { + Err(reason) if reason.as_str() == expected => Ok(()), + Err(reason) => Err(one(format!( + "anchor rejected as {reason}, expected {expected}" + ))), + Ok(_) => Err(one(format!( + "anchor accepted, expected rejection {expected}" + ))), + }; + } + let mut store = anchor.map_err(|reason| one(format!("anchor rejected as {reason}")))?; + + let mut labels = Labels::new(hash_tree_root(&anchor_block)); + let mut failures = Vec::new(); + + for (index, step) in case.steps.iter().enumerate() { + if skip_signature_step && step.rejection_reason.as_deref() == Some("INVALID_SIGNATURE") { + continue; + } + let previous_head = store.head; + let outcome = apply(&mut store, step, &mut labels).map_err(one)?; + + let mut step_failures = Vec::new(); + check_outcome(&mut step_failures, step, outcome); + step.store_snapshot.check(&mut step_failures, &store); + if let Some(checks) = &step.checks { + checks + .check(&mut step_failures, &store, step, &labels, previous_head) + .map_err(one)?; + } + for failure in step_failures { + failures.push(format!("step {index} ({}): {failure}", step.step_type)); + } + } + + if failures.is_empty() { + Ok(()) + } else { + Err(failures) + } +} + +/// A single harness-level problem, in the shape the caller collects. +fn one(problem: String) -> Vec { + vec![problem] +} + +/// What applying a step produced: nothing, or the reason it was refused. +type Outcome = Option; + +/// Confirms a step was accepted or refused exactly as the vector says. +fn check_outcome(failures: &mut Vec, step: &Step, outcome: Outcome) { + match (step.valid, outcome) { + (true, None) => {} + (true, Some(reason)) => failures.push(format!("rejected as {reason}, expected accept")), + (false, None) => { + let expected = step.rejection_reason.as_deref().unwrap_or("a rejection"); + failures.push(format!("accepted, expected rejection {expected}")); + } + (false, Some(reason)) => { + if let Some(expected) = step.rejection_reason.as_deref() + && reason.as_str() != expected + { + failures.push(format!("rejected as {reason}, expected {expected}")); + } + } + } +} + +/// Applies one step to the store. +/// +/// A `Err` return is a defect in the vector or in this harness; a rejection by the spec comes +/// back as `Ok(Some(reason))` and is the vector's business, not this function's. +fn apply(store: &mut Store, step: &Step, labels: &mut Labels) -> Result { + match step.step_type.as_str() { + "block" => apply_block(store, step, labels), + "tick" => { + on_tick(store, tick_target(store, step)?, step.has_proposal); + Ok(None) + } + "attestation" => apply_attestation(store, step), + "gossipAggregatedAttestation" => apply_aggregated(store, step), + other => Err(format!("unknown step type {other}")), + } +} + +/// Ticks to the block's slot when the vector says to, imports it, then seeds its votes. +/// +/// The seeding is the second of the two harness-side jobs described in the module docs: the +/// block's aggregation bits name exactly the validators the proposer's proofs covered, which +/// is all the snapshot records of them. `on_block` files each vote with no proof behind it, +/// per leanSpec; the entry below adds the coverage on top. +fn apply_block(store: &mut Store, step: &Step, labels: &mut Labels) -> Result { + let json = step.block.as_ref().ok_or("block step carries no block")?; + let block = json.build()?; + + if step.tick_to_slot { + on_tick(store, Interval(block.slot.0 * INTERVALS_PER_SLOT), true); + } + if let Err(reason) = on_block(store, &block) { + return Ok(Some(reason)); + } + + for attestation in block.body.attestations.iter() { + store + .latest_known_aggregated_payloads + .entry(attestation.data) + .or_default() + .insert(SingleMessageAggregate { + participants: attestation.aggregation_bits.clone(), + proof: ByteList512KiB::default(), + }); + } + verity_chain::update_head(store); + + if let Some(label) = &json.block_root_label { + labels.insert(label.clone(), hash_tree_root(&block)); + } + Ok(None) +} + +fn apply_attestation(store: &mut Store, step: &Step) -> Result { + let json = step + .attestation + .as_ref() + .ok_or("attestation step is empty")?; + let data = json.data.build()?; + let validator_index = json + .validator_index + .ok_or("attestation step carries no validatorIndex")?; + let signature = json + .signature + .as_deref() + .ok_or("attestation step carries no signature")?; + + // leanSpec validates and verifies for every node, and records only for an aggregator. + // A non-aggregator's admission decision is the validation alone. + let outcome = if step.is_aggregator { + record_attestation_signature( + store, + ValidatorIndex(validator_index), + data, + AttestationSignature(common::unhex(signature)?), + ) + } else { + verity_chain::validate_attestation_signer(store, ValidatorIndex(validator_index), &data) + }; + Ok(outcome.err()) +} + +fn apply_aggregated(store: &mut Store, step: &Step) -> Result { + let json = step.attestation.as_ref().ok_or("aggregate step is empty")?; + let proof = json + .proof + .as_ref() + .ok_or("aggregate step carries no proof")?; + + let attestation = SignedAggregatedAttestation { + data: json.data.build()?, + proof: SingleMessageAggregate { + participants: bitlist::(&proof.participants.data)?, + proof: ByteList512KiB::try_from(common::unhex(&proof.proof.data)?) + .map_err(|error| format!("proof bytes: {error:?}"))?, + }, + }; + Ok(record_aggregated_payload(store, &attestation).err()) +} + +/// The interval a tick step advances to, from either the wire form it carries. +fn tick_target(store: &Store, step: &Step) -> Result { + if let Some(interval) = step.interval { + return Ok(Interval(interval)); + } + let time = step + .time + .ok_or("tick step carries neither time nor interval")?; + let clock = verity_chain::SlotClock::new(store.config.genesis_time); + Ok(clock.total_intervals(time * 1000)) +} diff --git a/crates/verity-chain/tests/state_transition_fixtures.rs b/crates/verity-chain/tests/state_transition_fixtures.rs index e529806..3560d78 100644 --- a/crates/verity-chain/tests/state_transition_fixtures.rs +++ b/crates/verity-chain/tests/state_transition_fixtures.rs @@ -6,9 +6,8 @@ //! mismatch says which field moved rather than only that a root differs. A rejecting case //! carries `rejectionReason` and no post-state. //! -//! The JSON containers are mirrored here rather than derived on the `verity-types` shapes. -//! Consensus values travel as SSZ, never as JSON; the `{"data": [...]}` wrappers and camelCase -//! names below are a test-generator convention, not part of any container's shape. +//! The JSON containers this reads are mirrored in `common` rather than derived on the +//! `verity-types` shapes — see that module for why. //! //! Source: leanSpec `tests/consensus/lstar/state_transition/`, filled into the //! `fixtures-prod-scheme.tar.gz` release asset that `crates/verity-types/fixtures.sha256` @@ -16,13 +15,10 @@ mod common; +use common::{BlockJson, DataList, StateJson, ValidatorJson, compare, flags, hex}; use serde::Deserialize; use verity_chain::{generate_genesis, hash_tree_root, process_block, state_transition}; -use verity_types::{ - AggregatedAttestation, AggregatedAttestations, AggregationBits, AttestationData, Block, - BlockBody, BlockHeader, Bytes32, Bytes52, Checkpoint, GenesisConfig, HistoricalBlockHashes, - JustificationValidators, JustifiedSlots, Slot, State, Validator, ValidatorIndex, Validators, -}; +use verity_types::State; /// Every suite under leanSpec's `state_transition` fixture directory. const SUITES: &[&str] = &[ @@ -234,165 +230,6 @@ struct Case { info: serde_json::Value, } -/// leanSpec wraps every SSZ collection in a `data` key. -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct DataList { - data: Vec, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -struct StateJson { - config: GenesisConfigJson, - slot: u64, - latest_block_header: BlockHeaderJson, - latest_justified: CheckpointJson, - latest_finalized: CheckpointJson, - historical_block_hashes: DataList, - justified_slots: DataList, - validators: DataList, - justifications_roots: DataList, - justifications_validators: DataList, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -struct GenesisConfigJson { - genesis_time: u64, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -struct BlockHeaderJson { - slot: u64, - proposer_index: u64, - parent_root: String, - state_root: String, - body_root: String, -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct CheckpointJson { - root: String, - slot: u64, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -struct ValidatorJson { - attestation_public_key: String, - proposal_public_key: String, - index: u64, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -struct BlockJson { - slot: u64, - proposer_index: u64, - parent_root: String, - state_root: String, - body: BlockBodyJson, -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct BlockBodyJson { - attestations: DataList, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -struct AggregatedAttestationJson { - aggregation_bits: DataList, - data: AttestationDataJson, -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct AttestationDataJson { - slot: u64, - head: CheckpointJson, - target: CheckpointJson, - source: CheckpointJson, -} - -impl StateJson { - fn build(&self) -> Result { - Ok(State { - config: GenesisConfig { - genesis_time: self.config.genesis_time, - }, - slot: Slot(self.slot), - latest_block_header: self.latest_block_header.build()?, - latest_justified: self.latest_justified.build()?, - latest_finalized: self.latest_finalized.build()?, - historical_block_hashes: roots(&self.historical_block_hashes)?, - justified_slots: bitlist::(&self.justified_slots.data)?, - validators: validators(&self.validators)?, - justifications_roots: roots(&self.justifications_roots)?, - justifications_validators: bitlist::( - &self.justifications_validators.data, - )?, - }) - } -} - -impl BlockHeaderJson { - fn build(&self) -> Result { - Ok(BlockHeader { - slot: Slot(self.slot), - proposer_index: ValidatorIndex(self.proposer_index), - parent_root: bytes32(&self.parent_root)?, - state_root: bytes32(&self.state_root)?, - body_root: bytes32(&self.body_root)?, - }) - } -} - -impl CheckpointJson { - fn build(&self) -> Result { - Ok(Checkpoint { - root: bytes32(&self.root)?, - slot: Slot(self.slot), - }) - } -} - -impl BlockJson { - fn build(&self) -> Result { - let mut attestations = AggregatedAttestations::default(); - for attestation in &self.body.attestations.data { - attestations - .push(attestation.build()?) - .map_err(|error| format!("attestations: {error:?}"))?; - } - Ok(Block { - slot: Slot(self.slot), - proposer_index: ValidatorIndex(self.proposer_index), - parent_root: bytes32(&self.parent_root)?, - state_root: bytes32(&self.state_root)?, - body: BlockBody { attestations }, - }) - } -} - -impl AggregatedAttestationJson { - fn build(&self) -> Result { - Ok(AggregatedAttestation { - aggregation_bits: bitlist::(&self.aggregation_bits.data)?, - data: AttestationData { - slot: Slot(self.data.slot), - head: self.data.head.build()?, - target: self.data.target.build()?, - source: self.data.source.build()?, - }, - }) - } -} - // --------------------------------------------------------------------------------------- // Partial post-state assertions // --------------------------------------------------------------------------------------- @@ -604,89 +441,3 @@ impl PostAssertions { } } } - -/// Records a mismatch when the case asserts a value and it disagrees. -fn compare( - failures: &mut Vec, - name: &str, - expected: Option, - actual: Option, -) { - let (Some(expected), Some(actual)) = (expected, actual) else { - return; - }; - if expected != actual { - failures.push(format!("{name}: got {actual:?}, expected {expected:?}")); - } -} - -// --------------------------------------------------------------------------------------- -// Primitive conversions -// --------------------------------------------------------------------------------------- - -fn flags(length: usize, read: impl Fn(usize) -> Option) -> Vec { - (0..length) - .map(|index| read(index).unwrap_or(false)) - .collect() -} - -fn hex(bytes: &[u8]) -> String { - let mut out = String::with_capacity(2 + bytes.len() * 2); - out.push_str("0x"); - for byte in bytes { - out.push_str(&format!("{byte:02x}")); - } - out -} - -fn unhex(text: &str) -> Result, String> { - let body = text - .strip_prefix("0x") - .ok_or_else(|| format!("{text}: missing 0x prefix"))?; - (0..body.len()) - .step_by(2) - .map(|index| { - u8::from_str_radix(&body[index..index + 2], 16) - .map_err(|error| format!("{text}: {error}")) - }) - .collect() -} - -fn bytes32(text: &str) -> Result { - unhex(text)? - .try_into() - .map_err(|_| format!("{text}: not 32 bytes")) -} - -fn bytes52(text: &str) -> Result { - unhex(text)? - .try_into() - .map_err(|_| format!("{text}: not 52 bytes")) -} - -fn roots(list: &DataList) -> Result { - let mut out = HistoricalBlockHashes::default(); - for text in &list.data { - out.push(bytes32(text)?) - .map_err(|error| format!("roots: {error:?}"))?; - } - Ok(out) -} - -fn validators(list: &DataList) -> Result { - let mut out = Validators::default(); - for entry in &list.data { - out.push(Validator { - attestation_public_key: bytes52(&entry.attestation_public_key)?, - proposal_public_key: bytes52(&entry.proposal_public_key)?, - index: ValidatorIndex(entry.index), - }) - .map_err(|error| format!("validators: {error:?}"))?; - } - Ok(out) -} - -fn bitlist>>(bits: &[bool]) -> Result { - T::try_from(bits.to_vec()) - .map_err(|_| format!("bitlist of {} bits exceeds its limit", bits.len())) -}