Skip to content

simulators/lean: correctly decode devnet5 blocks #1542

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
KolbyML merged 2 commits into from
Jun 15, 2026
Merged

simulators/lean: correctly decode devnet5 blocks #1542

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1770660
simulators/lean: decode devnet5 blocks with the merged-proof shape
MegaRedHand Jun 12, 2026
4ab4cfd
simulators/lean: support leanSpec's restructured package layout in th…
MegaRedHand Jun 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
169 changes: 130 additions & 39 deletions clients/lean-spec-client/lean_spec_runtime.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import shutil
import sys
import time
import typing
import types
from pathlib import Path
from typing import Any, Final
Expand Down Expand Up @@ -51,40 +52,88 @@ def decorate(func: object) -> object:

install_numba_njit_stub()

# leanSpec restructured its packages over time (subspecs/types -> node/spec).
# The devnet5 helper builds from leanSpec main (new layout) while the devnet4
# helper stays pinned on the old layout, so every import tries the new
# location first and falls back to the old ones.
try:
from lean_spec.spec.forks import DEFAULT_REGISTRY
except (ImportError, ModuleNotFoundError):
from lean_spec.forks import DEFAULT_REGISTRY
except ModuleNotFoundError:
DEFAULT_REGISTRY = None

try:
from lean_spec.subspecs.containers import Checkpoint
from lean_spec.subspecs.containers.validator import SubnetId, ValidatorIndex
from lean_spec.spec.forks.lstar.containers import Checkpoint, SubnetId, ValidatorIndex
except (ImportError, ModuleNotFoundError):
try:
from lean_spec.types import Checkpoint, SubnetId, ValidatorIndex
from lean_spec.subspecs.containers import Checkpoint
from lean_spec.subspecs.containers.validator import SubnetId, ValidatorIndex
except (ImportError, ModuleNotFoundError):
from lean_spec.forks.lstar.containers import Checkpoint
from lean_spec.forks.lstar.containers.validator import SubnetId, ValidatorIndex

from lean_spec.subspecs.chain.config import ATTESTATION_COMMITTEE_COUNT
from lean_spec.subspecs.genesis.config import GenesisConfig
from lean_spec.subspecs.networking.enr import ENR, keys as enr_keys
from lean_spec.subspecs.networking.gossipsub import GossipTopic
from lean_spec.subspecs.networking.reqresp.message import Status
from lean_spec.subspecs.networking.transport.identity import IdentityKeypair
from lean_spec.subspecs.networking.transport.identity.keypair import Secp256k1PublicKey
from lean_spec.subspecs.ssz.hash import hash_tree_root
from lean_spec.subspecs.xmss import SecretKey
from lean_spec.subspecs.xmss import aggregation as xmss_aggregation_module
from lean_spec.types import Bytes32
from lean_spec.types.collections import SSZList, SSZVector, _validate_offsets
from lean_spec.types.constants import OFFSET_BYTE_LENGTH
from lean_spec.types.container import Container
from lean_spec.types.exceptions import SSZSerializationError, SSZValueError
from lean_spec.types.uint import Uint32
try:
from lean_spec.types import Checkpoint, SubnetId, ValidatorIndex
except (ImportError, ModuleNotFoundError):
from lean_spec.forks.lstar.containers import Checkpoint
from lean_spec.forks.lstar.containers.validator import SubnetId, ValidatorIndex

try:
from lean_spec.node.genesis import GenesisConfig
from lean_spec.node.networking.enr import ENR, keys as enr_keys
from lean_spec.node.networking.gossipsub import GossipTopic
from lean_spec.node.networking.reqresp.message import Status
from lean_spec.node.networking.transport.identity import (
IdentityKeypair,
Secp256k1PublicKey,
)
from lean_spec.spec.crypto.merkleization import hash_tree_root
from lean_spec.spec.crypto.xmss import SecretKey
from lean_spec.spec.forks.lstar import aggregation as xmss_aggregation_module
from lean_spec.spec.forks.lstar.config import ATTESTATION_COMMITTEE_COUNT
from lean_spec.spec.ssz import (
Bytes32,
Container,
SSZList,
SSZSerializationError,
SSZValueError,
SSZVector,
Uint32,
)
from lean_spec.spec.ssz.collections import _validate_offsets
from lean_spec.spec.ssz.ssz_base import BYTES_PER_LENGTH_OFFSET as OFFSET_BYTE_LENGTH
except (ImportError, ModuleNotFoundError):
from lean_spec.subspecs.chain.config import ATTESTATION_COMMITTEE_COUNT
from lean_spec.subspecs.genesis.config import GenesisConfig
from lean_spec.subspecs.networking.enr import ENR, keys as enr_keys
from lean_spec.subspecs.networking.gossipsub import GossipTopic
from lean_spec.subspecs.networking.reqresp.message import Status
from lean_spec.subspecs.networking.transport.identity import IdentityKeypair
from lean_spec.subspecs.networking.transport.identity.keypair import Secp256k1PublicKey
from lean_spec.subspecs.ssz.hash import hash_tree_root
from lean_spec.subspecs.xmss import SecretKey
from lean_spec.subspecs.xmss import aggregation as xmss_aggregation_module
from lean_spec.types import Bytes32
from lean_spec.types.collections import SSZList, SSZVector, _validate_offsets
from lean_spec.types.constants import OFFSET_BYTE_LENGTH
from lean_spec.types.container import Container
from lean_spec.types.exceptions import SSZSerializationError, SSZValueError
from lean_spec.types.uint import Uint32

logger = logging.getLogger("lean_spec_client_runner")


def import_first_module(*module_names: str) -> types.ModuleType:
"""Import the first available module from a list of historical locations.

Mirrors the try/except import chains above for dynamically imported
modules: new-layout names come first, old-layout names act as fallbacks
for the pinned devnet4 helper checkout.
"""
for module_name in module_names[:-1]:
try:
return importlib.import_module(module_name)
except (ImportError, ModuleNotFoundError):
continue
return importlib.import_module(module_names[-1])


BOOTNODE_DIAL_TIMEOUT_SECS: Final = 10.0
STATUS_REFRESH_INTERVAL_SECS: Final = 1.0
PREPARED_ASSETS_DIR: Final = Path(
Expand Down Expand Up @@ -271,8 +320,14 @@ def container_deserialize_without_validation(
var_fields = []
bytes_read = 0

# Older leanSpec layouts resolve SSZ field types through a private
# helper; newer ones store the SSZ type directly on the annotation.
annotation_resolver = getattr(cls, "_get_ssz_field_type", None)
for field_name, field_info in cls.model_fields.items():
field_type = cls._get_ssz_field_type(field_info.annotation)
if annotation_resolver is not None:
field_type = annotation_resolver(field_info.annotation)
else:
field_type = field_info.annotation

if field_type.is_fixed_size():
size = field_type.get_byte_length()
Expand Down Expand Up @@ -328,19 +383,40 @@ def install_snappy_compress_fallback() -> None:
if _SNAPPY_COMPRESS_FALLBACK_INSTALLED:
return

snappy_package_module = importlib.import_module("lean_spec.snappy")
snappy_compress_module = importlib.import_module("lean_spec.snappy.compress")
snappy_framing_module = importlib.import_module("lean_spec.snappy.framing")
reqresp_codec_module = importlib.import_module("lean_spec.subspecs.networking.reqresp.codec")
networking_service_module = importlib.import_module(
"lean_spec.subspecs.networking.service.service"
snappy_package_module = import_first_module("lean_spec.node.snappy", "lean_spec.snappy")
snappy_compress_module = import_first_module(
"lean_spec.node.snappy.compress", "lean_spec.snappy.compress"
)
snappy_framing_module = import_first_module(
"lean_spec.node.snappy.framing", "lean_spec.snappy.framing"
)
reqresp_codec_module = import_first_module(
"lean_spec.node.networking.reqresp.codec",
"lean_spec.subspecs.networking.reqresp.codec",
)
networking_service_module = import_first_module(
"lean_spec.node.networking.service.service",
"lean_spec.subspecs.networking.service.service",
)

def encode_snappy_uncompressed_length(value: int) -> bytes:
# Standard unsigned LEB128, inlined because the varint helper moved
# modules (and changed name) across leanSpec layouts.
output = bytearray()
while True:
byte = value & 0x7F
value >>= 7
if value:
output.append(byte | 0x80)
else:
output.append(byte)
return bytes(output)

def literal_only_compress(data: bytes) -> bytes:
if not data:
return snappy_compress_module.encode_varint32(0)
return encode_snappy_uncompressed_length(0)

output = bytearray(snappy_compress_module.encode_varint32(len(data)))
output = bytearray(encode_snappy_uncompressed_length(len(data)))
offset = 0
while offset < len(data):
block_end = min(offset + snappy_compress_module.BLOCK_SIZE, len(data))
Expand Down Expand Up @@ -564,6 +640,21 @@ def write_validator_keys(
yaml.safe_dump(manifest, manifest_file, sort_keys=False)


def genesis_validator_pubkey_keys() -> tuple[str, str]:
"""Return the per-validator pubkey YAML keys the active GenesisConfig expects.

Newer leanSpec checkouts spell the keys out in full
(attestation_public_key); older ones use the abbreviated form
(attestation_pubkey). Introspect the validator entry model so the
emitted YAML always matches the checkout that will parse it.
"""
entry_annotation = GenesisConfig.model_fields["genesis_validators"].annotation
(entry_class,) = typing.get_args(entry_annotation)
if "attestation_public_key" in entry_class.model_fields:
return ("attestation_public_key", "proposal_public_key")
return ("attestation_pubkey", "proposal_pubkey")


def write_genesis_config(
validators: list[dict[str, str]],
genesis_time: int,
Expand All @@ -575,10 +666,11 @@ def write_genesis_config(
genesis.setdefault("NUM_VALIDATORS", len(validators))

if uses_latest_leanspec_format():
attestation_key, proposal_key = genesis_validator_pubkey_keys()
genesis["GENESIS_VALIDATORS"] = [
{
"attestation_pubkey": f"0x{validator[ATTESTATION_PUBKEY_FIELD]}",
"proposal_pubkey": f"0x{validator[PROPOSAL_PUBKEY_FIELD]}",
attestation_key: f"0x{validator[ATTESTATION_PUBKEY_FIELD]}",
proposal_key: f"0x{validator[PROPOSAL_PUBKEY_FIELD]}",
}
for validator in validators
]
Expand Down Expand Up @@ -635,10 +727,11 @@ def helper_genesis_metadata() -> dict[str, object]:

validators = genesis_config.get("GENESIS_VALIDATORS", [])
if uses_latest_leanspec_format():
attestation_key, proposal_key = genesis_validator_pubkey_keys()
genesis_validator_entries = [
{
"attestation_public_key": validator["attestation_pubkey"],
"proposal_public_key": validator["proposal_pubkey"],
"attestation_public_key": validator[attestation_key],
"proposal_public_key": validator[proposal_key],
}
for validator in validators
]
Expand Down Expand Up @@ -917,8 +1010,6 @@ def build_node_config(
}

if node_config_supports(node_config_type, "fork"):
if DEFAULT_REGISTRY is None:
raise RuntimeError("LeanSpec NodeConfig requires a fork, but no fork registry exists")
config["fork"] = DEFAULT_REGISTRY.current
config["network_name"] = gossip_network_name(fork_digest)
else:
Expand Down
Loading