fix(testing): make consensus vector generation deterministic across hash seeds

Some emitted vectors depended on set or dict iteration order, which Python
seeds per process, so two fills of identical input could produce different
bytes and break cross-client reproducibility.

- aggregation: build the new aggregate pool in insertion order rather than a
  hash-seeded set union, mirroring the fork-choice merge fix, so fork-choice
  weights are reproducible
- testing: collect a block's valid attestations in a list and emit gossipsub
  forwards in recipient order, so block bodies and outbound RPCs are stable

Add an order_sensitive marker on the vectors that exercise these orderings and
a fill-determinism check that regenerates the marked subset under two hash
seeds and diffs them, wired into CI so a regression in this class fails fast.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: tcoratger

.github/workflows/ci.yml

@@ -133,6 +133,9 @@ jobs:
       - name: Fill test fixtures
         run: just fill-ci
 
+      - name: Check fixture determinism (order-sensitive vectors)
+        run: just fill-determinism
+
   interop-tests:
     name: Interop tests - Multi-node consensus
     runs-on: macos-latest

Justfile

@@ -85,6 +85,31 @@ test-consensus *args:
 fill-ci *args:
     uv run --group test fill --fork=Lstar --clean -n auto --dist=worksteal "$@"
 
+# Generate the order-sensitive vectors twice under different hash seeds and diff.
+# Only the vectors marked order_sensitive run, so this stays cheap.
+# A difference means an emitted vector depends on set or dict iteration order,
+# which is hash-seeded and would break cross-client reproducibility.
+# Pass a path to widen the scope (for example the whole tests/consensus tree).
+[group('tests')]
+fill-determinism *args:
+    #!/usr/bin/env bash
+    set -euo pipefail
+    target="{{args}}"
+    [ -z "$target" ] && target="tests/consensus -m order_sensitive"
+    first="$(mktemp -d)"
+    second="$(mktemp -d)"
+    trap 'rm -rf "$first" "$second"' EXIT
+    # Single process: the marked subset is small, so xdist worker startup would
+    # cost more than it saves, and one process pins the hash seed cleanly.
+    PYTHONHASHSEED=1 uv run --group test fill --fork=Lstar --clean -n 0 -o "$first" $target -q
+    PYTHONHASHSEED=2 uv run --group test fill --fork=Lstar --clean -n 0 -o "$second" $target -q
+    if diff -rq "$first" "$second"; then
+        echo "Determinism check passed: fixtures are byte-identical across hash seeds."
+    else
+        echo "Determinism check FAILED: emitted vectors differ across PYTHONHASHSEED." >&2
+        exit 1
+    fi
+
 # Run API conformance tests against an external client
 [group('tests')]
 apitest server_url *args:

packages/testing/src/consensus_testing/pytest_plugins/filler.py

@@ -194,6 +194,11 @@ def pytest_configure(config: pytest.Config) -> None:
         "real_crypto(smoke=False): build and verify with the real prover, never the mock; "
         "smoke=True also keeps it in the fast mocked lane",
     )
+    config.addinivalue_line(
+        "markers",
+        "order_sensitive: emission could depend on set or dict iteration order; "
+        "the determinism check generates this vector twice and diffs the output",
+    )
 
     # Crypto mode is chosen explicitly and applies to either scheme.
     AggregationProver.set_mode(CryptoMode(config.getoption("--crypto")))

packages/testing/src/consensus_testing/test_fixtures/gossipsub_handler.py

@@ -476,8 +476,14 @@ async def _execute(self) -> GossipsubExpectation:
         # Each entry represents one RPC sent to a peer.
         # The structure mirrors the gossipsub RPC wire format.
         # Fixture consumers use this to assert exact outbound behavior.
+        # Emit outbound RPCs in canonical recipient order.
+        # The send order to mesh peers is set-driven.
+        # It is not consensus-relevant.
+        # Sorting by recipient keeps the emitted vector reproducible across runs.
         sent_rpcs = []
-        for recipient_peer_id, rpc in capture.sent:
+        for recipient_peer_id, rpc in sorted(
+            capture.sent, key=lambda entry: peer_names.get(entry[0], str(entry[0]))
+        ):
             subscriptions = (
                 [
                     SentSubscription(

packages/testing/src/consensus_testing/test_types/block_spec.py

@@ -178,7 +178,7 @@ def build_attestations(
     ) -> tuple[
         list[Attestation],
         dict[AttestationData, dict[ValidatorIndex, Signature]],
-        set[Attestation],
+        list[Attestation],
     ]:
         """
         Build attestations and signatures from this block's attestation specs.
@@ -195,11 +195,14 @@ def build_attestations(
                 - Subset of attestations that have valid (non-dummy) signatures
         """
         if self.attestations is None:
-            return [], {}, set()
+            return [], {}, []
 
         attestations: list[Attestation] = []
         signature_lookup: dict[AttestationData, dict[ValidatorIndex, Signature]] = {}
-        valid_attestations: set[Attestation] = set()
+        # A list, not a set.
+        # Build order is the canonical order consumers gossip in.
+        # That keeps the emitted block body reproducible across runs.
+        valid_attestations: list[Attestation] = []
 
         for aggregated_spec in self.attestations:
             # Build attestation data once.
@@ -224,7 +227,7 @@ def build_attestations(
                         validator_index,
                         attestation_data,
                     )
-                    valid_attestations.add(attestation)
+                    valid_attestations.append(attestation)
                 else:
                     signature = create_dummy_signature()
 

src/lean_spec/spec/forks/lstar/aggregation.py

@@ -113,9 +113,11 @@ def aggregate(self, store: LstarStore) -> tuple[LstarStore, list[SignedAggregate
         #
         # Known payloads cannot start a round alone, since re-aggregating them adds nothing.
         # They serve only as fallback building blocks once fresh evidence exists.
-        for attestation_data in (
-            store.latest_new_aggregated_payloads.keys() | store.attestation_signatures.keys()
-        ):
+        # Iterate in insertion order, in order to have a deterministic pool order.
+        for attestation_data in {
+            **store.latest_new_aggregated_payloads,
+            **store.attestation_signatures,
+        }:
             # Phase 1: Select.
             #
             # Reuse existing proofs first to keep the proof tree shallow.

tests/consensus/lstar/fork_choice/test_block_attestation_limits.py

@@ -14,7 +14,7 @@
 from lean_spec.spec.forks import RejectionReason, Slot, ValidatorIndex
 from lean_spec.spec.forks.lstar.config import MAX_ATTESTATIONS_DATA
 
-pytestmark = pytest.mark.valid_until("Lstar")
+pytestmark = [pytest.mark.valid_until("Lstar"), pytest.mark.order_sensitive]
 
 
 def _justifiable_slots(n: int) -> list[Slot]:

tests/consensus/lstar/networking/test_gossipsub_handlers.py

@@ -16,7 +16,7 @@
     PeerConfiguration,
 )
 
-pytestmark = pytest.mark.valid_until("Lstar")
+pytestmark = [pytest.mark.valid_until("Lstar"), pytest.mark.order_sensitive]
 
 TOPIC = "test_topic"
 PARAMS = GossipsubMeshParameters(d=4, d_low=3, d_high=6, d_lazy=3)