Skip to content

refactor(testing): attach filler plugin state via typed StashKeys #908

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tcoratger merged 1 commit into from
Jun 10, 2026
Merged

refactor(testing): attach filler plugin state via typed StashKeys #908

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 41 additions & 22 deletions packages/testing/src/consensus_testing/pytest_plugins/filler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
import pytest

from consensus_testing.crypto_mode import AggregationProver, CryptoMode
from consensus_testing.forks import FORKS_BY_NAME
from consensus_testing.forks import FORKS_BY_NAME, BaseFork
from consensus_testing.keys import DEFAULT_MAX_SLOT, XmssKeyManager
from consensus_testing.test_fixtures import FIXTURE_FORMATS, FixtureInfo
from lean_spec.spec.forks import Slot, ValidatorIndex
Expand Down Expand Up @@ -103,9 +103,11 @@ def add_fixture(

if config is not None:
fixture_path = self.fixture_output_file(test_nodeid, fixture_format)
config.fixture_path_absolute = str(fixture_path.absolute()) # type: ignore[attribute-defined]
config.fixture_path_relative = str(fixture_path.relative_to(self.output_directory)) # type: ignore[attribute-defined]
config.fixture_format = fixture_format # type: ignore[attribute-defined]
config.stash[FIXTURE_PATH_ABSOLUTE_KEY] = str(fixture_path.absolute())
config.stash[FIXTURE_PATH_RELATIVE_KEY] = str(
fixture_path.relative_to(self.output_directory)
)
config.stash[FIXTURE_FORMAT_KEY] = fixture_format

def write_fixtures(self) -> None:
"""Write all collected fixtures to disk, grouped by test function."""
Expand All @@ -127,6 +129,22 @@ def write_fixtures(self) -> None:
json.dump(all_tests, output_handle, indent=4)


FIXTURE_COLLECTOR_KEY: pytest.StashKey[FixtureCollector] = pytest.StashKey()
"""Stash key for the session's fixture collector."""

TEST_FORK_CLASS_KEY: pytest.StashKey[type[BaseFork]] = pytest.StashKey()
"""Stash key for the fork class selected by the fork option."""

FIXTURE_PATH_ABSOLUTE_KEY: pytest.StashKey[str] = pytest.StashKey()
"""Stash key for the absolute path of the current test's fixture file."""

FIXTURE_PATH_RELATIVE_KEY: pytest.StashKey[str] = pytest.StashKey()
"""Stash key for the current test's fixture path relative to the output directory."""

FIXTURE_FORMAT_KEY: pytest.StashKey[str] = pytest.StashKey()
"""Stash key for the current test's fixture format name."""


def pytest_addoption(parser: pytest.Parser) -> None:
"""Add command-line options for fixture generation."""
group = parser.getgroup("fill", "leanSpec fixture generation")
Expand Down Expand Up @@ -219,8 +237,8 @@ def pytest_configure(config: pytest.Config) -> None:
)
pytest.exit("Missing required --fork option.", returncode=pytest.ExitCode.USAGE_ERROR)

fork_class = FORKS_BY_NAME.get(fork_name.lower())
if fork_class is None:
fork_name_normalized = fork_name.lower()
if fork_name_normalized not in FORKS_BY_NAME:
print(
f"Error: Unsupported fork: {fork_name}\n",
file=sys.stderr,
Expand All @@ -231,6 +249,8 @@ def pytest_configure(config: pytest.Config) -> None:
)
pytest.exit("Invalid fork specified.", returncode=pytest.ExitCode.USAGE_ERROR)

fork_class = FORKS_BY_NAME[fork_name_normalized]

# Check output directory
if output_directory.exists() and any(output_directory.iterdir()):
if not clean:
Expand All @@ -250,8 +270,8 @@ def pytest_configure(config: pytest.Config) -> None:

output_directory.mkdir(parents=True, exist_ok=True)

config.fixture_collector = FixtureCollector(output_directory, fork_name) # type: ignore[attribute-defined]
config.test_fork_class = fork_class # type: ignore[attribute-defined]
config.stash[FIXTURE_COLLECTOR_KEY] = FixtureCollector(output_directory, fork_name)
config.stash[TEST_FORK_CLASS_KEY] = fork_class


def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None:
Expand All @@ -261,10 +281,10 @@ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item
Real-crypto vectors cannot be mocked, so the fast mocked lane keeps only the
smoke subset and leaves the rest to the real lane.
"""
if not hasattr(config, "test_fork_class"):
if TEST_FORK_CLASS_KEY not in config.stash:
return

fork_class = config.test_fork_class
fork_class = config.stash[TEST_FORK_CLASS_KEY]
verbose = config.getoption("verbose")
mocking = AggregationProver.get_mode() == CryptoMode.MOCKED
deselected_items = []
Expand Down Expand Up @@ -349,8 +369,8 @@ def pytest_sessionstart(session: pytest.Session) -> None:

def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None:
"""Write all collected fixtures at the end of the session."""
if hasattr(session.config, "fixture_collector"):
session.config.fixture_collector.write_fixtures()
if FIXTURE_COLLECTOR_KEY in session.config.stash:
session.config.stash[FIXTURE_COLLECTOR_KEY].write_fixtures()


@pytest.hookimpl(tryfirst=True, hookwrapper=True)
Expand All @@ -360,17 +380,16 @@ def pytest_runtest_makereport(item: pytest.Item, call: pytest.CallInfo[None]) ->
report = outcome.get_result()

if call.when == "call":
if hasattr(item.config, "fixture_path_absolute") and hasattr(
item.config, "fixture_path_relative"
):
stash = item.config.stash
if FIXTURE_PATH_ABSOLUTE_KEY in stash and FIXTURE_PATH_RELATIVE_KEY in stash:
report.user_properties.append(
("fixture_path_absolute", item.config.fixture_path_absolute)
("fixture_path_absolute", stash[FIXTURE_PATH_ABSOLUTE_KEY])
)
report.user_properties.append(
("fixture_path_relative", item.config.fixture_path_relative)
("fixture_path_relative", stash[FIXTURE_PATH_RELATIVE_KEY])
)
if hasattr(item.config, "fixture_format"):
report.user_properties.append(("fixture_format", item.config.fixture_format))
if FIXTURE_FORMAT_KEY in stash:
report.user_properties.append(("fixture_format", stash[FIXTURE_FORMAT_KEY]))


@pytest.fixture
Expand Down Expand Up @@ -470,8 +489,8 @@ def fill_and_collect(**spec_fields: Any) -> Any:
network=fork.name(),
).model_copy(update={"proof_setting": proof_setting})

if hasattr(request.config, "fixture_collector"):
request.config.fixture_collector.add_fixture(
if FIXTURE_COLLECTOR_KEY in request.config.stash:
request.config.stash[FIXTURE_COLLECTOR_KEY].add_fixture(
fixture_format=spec_class.format_name,
fixture=filled_fixture,
test_nodeid=request.node.nodeid,
Expand All @@ -489,7 +508,7 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
if "fork" not in metafunc.fixturenames:
return

fork_class = metafunc.config.test_fork_class # type: ignore[attribute-defined]
fork_class = metafunc.config.stash[TEST_FORK_CLASS_KEY]

if not _check_markers_valid_for_fork(list(metafunc.definition.iter_markers()), fork_class):
verbose = metafunc.config.getoption("verbose")
Expand Down
Loading