Merge remote-tracking branch 'origin/main' into feat/lean-api-events

# Conflicts:
#	bin/ethlambda/src/main.rs
#	crates/blockchain/src/lib.rs
#	crates/blockchain/src/store.rs
#	crates/net/rpc/src/lib.rs

Author: MegaRedHand

.github/actions/run-fixture-tests/action.yml

@@ -10,12 +10,43 @@ runs:
     - name: Get leanSpec fixtures release info
       id: fixtures-release
       shell: bash
+      env:
+        # Authenticate the API call: unauthenticated requests share the runner
+        # IP's low rate-limit pool, which is the usual cause of transient failures.
+        GH_TOKEN: ${{ github.token }}
       run: |
+        set -euo pipefail
         api_url="https://api.github.com/repos/leanEthereum/leanSpec/releases/latest"
-        json=$(curl -sL "$api_url")
-        fixtures_url=$(echo "$json" | python3 -c "import sys,json; j=json.load(sys.stdin); print(next(a.get('browser_download_url') for a in j.get('assets',[]) if a.get('name')=='fixtures-prod-scheme.tar.gz'))")
-        sha_url=$(echo "$json" | python3 -c "import sys,json; j=json.load(sys.stdin); print(next(a.get('browser_download_url') for a in j.get('assets',[]) if a.get('name')=='fixtures-prod-scheme.tar.gz.sha256'))")
-        sha=$(curl -sL "$sha_url" | cut -d' ' -f1)
+
+        # -f fails on HTTP errors (so an error body is never captured as data) and
+        # --retry rides out transient network/5xx hiccups. Fail loudly, since a
+        # command substitution in an assignment does not trip `set -e` on its own.
+        json=$(curl -sSL -f --retry 5 --retry-all-errors \
+          -H "Accept: application/vnd.github+json" \
+          -H "Authorization: Bearer $GH_TOKEN" \
+          "$api_url") \
+          || { echo "::error::Failed to fetch leanSpec latest release metadata from $api_url"; exit 1; }
+
+        asset_url() {
+          echo "$json" | python3 -c \
+            "import sys,json; name=sys.argv[1]; j=json.load(sys.stdin); print(next(a['browser_download_url'] for a in j.get('assets',[]) if a.get('name')==name))" \
+            "$1"
+        }
+        fixtures_url=$(asset_url "fixtures-prod-scheme.tar.gz") \
+          || { echo "::error::Release is missing the fixtures-prod-scheme.tar.gz asset"; exit 1; }
+        sha_url=$(asset_url "fixtures-prod-scheme.tar.gz.sha256") \
+          || { echo "::error::Release is missing the fixtures-prod-scheme.tar.gz.sha256 asset"; exit 1; }
+
+        sha=$(curl -sSL -f --retry 5 --retry-all-errors "$sha_url" | cut -d' ' -f1) \
+          || { echo "::error::Failed to download the fixtures checksum from $sha_url"; exit 1; }
+
+        # Validate before writing: a transient HTML error page captured as $sha
+        # previously corrupted $GITHUB_OUTPUT ("Invalid format '<!--'").
+        if [[ ! "$sha" =~ ^[0-9a-fA-F]{64}$ ]]; then
+          echo "::error::Expected a 64-char hex SHA256 for the fixtures archive, got: '$sha'"
+          exit 1
+        fi
+
         {
           echo "url=$fixtures_url"
           echo "sha_url=$sha_url"
@@ -39,8 +70,8 @@ runs:
         fixtures_url="${{ steps.fixtures-release.outputs.url }}"
         sha_url="${{ steps.fixtures-release.outputs.sha_url }}"
         echo "Downloading fixtures from $fixtures_url"
-        curl -L -f -o "$tmpdir/fixtures-prod-scheme.tar.gz" "$fixtures_url"
-        curl -L -f -o "$tmpdir/fixtures-prod-scheme.tar.gz.sha256" "$sha_url"
+        curl -L -f --retry 5 --retry-all-errors -o "$tmpdir/fixtures-prod-scheme.tar.gz" "$fixtures_url"
+        curl -L -f --retry 5 --retry-all-errors -o "$tmpdir/fixtures-prod-scheme.tar.gz.sha256" "$sha_url"
         expected=$(cut -d' ' -f1 "$tmpdir/fixtures-prod-scheme.tar.gz.sha256")
         actual=$(sha256sum "$tmpdir/fixtures-prod-scheme.tar.gz" | awk '{print $1}')
         if [ "$expected" != "$actual" ]; then

.github/workflows/docker_publish.yaml

@@ -1,6 +1,8 @@
 name: Publish Docker Image
 
 on:
+  push:
+    branches: [main]
   workflow_dispatch:
     inputs:
       tags:
@@ -12,21 +14,46 @@ permissions:
   contents: read
   packages: write
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: false
+
 env:
   REGISTRY: ghcr.io
   IMAGE_NAME: ${{ github.repository }}
+  # On push to main, inputs.tags is empty, so fall back to "unstable" so it
+  # tracks main. Manual dispatch still honors custom tags.
+  TAGS_INPUT: ${{ inputs.tags || 'unstable' }}
 
 jobs:
   build-image:
-    name: Build Docker image (${{ matrix.arch }})
+    name: Build Docker image (${{ matrix.arch.name }}, ${{ matrix.variant.name }})
     strategy:
       matrix:
-        include:
+        arch:
           - runner: ubuntu-latest
-            arch: amd64
+            name: amd64
           - runner: ubuntu-22.04-arm
-            arch: arm64
-    runs-on: ${{ matrix.runner }}
+            name: arm64
+        # Each tag is published twice: once as the regular image and once as a
+        # Shadow-simulator-compatible image (tag suffixed "-shadow"). The shadow
+        # build args mirror the `shadow-docker-build` Makefile target. LOCKED=
+        # builds the shadow variant unlocked (the quinn-udp [patch] is absent
+        # from Cargo.lock; see Dockerfile), so -shadow images are not
+        # reproducible and should not be treated as equivalent-security
+        # artifacts to the regular tags.
+        variant:
+          - name: default
+            suffix: ""
+            build_args: ""
+          - name: shadow
+            suffix: "-shadow"
+            build_args: |
+              SHADOW=1
+              FEATURES=shadow-integration
+              NO_DEFAULT_FEATURES=--no-default-features
+              LOCKED=
+    runs-on: ${{ matrix.arch.runner }}
 
     steps:
       - name: Checkout repository
@@ -44,11 +71,14 @@ jobs:
 
       - name: Prepare tags
         id: prep
+        env:
+          SUFFIX: ${{ matrix.variant.suffix }}
+          ARCH: ${{ matrix.arch.name }}
         run: |
           TAGS=""
-          IFS=',' read -ra TAG_ARRAY <<< "${{ inputs.tags }}"
+          IFS=',' read -ra TAG_ARRAY <<< "${TAGS_INPUT}"
           for t in "${TAG_ARRAY[@]}"; do
-            TAGS="${TAGS}${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${t}-${{ matrix.arch }},"
+            TAGS="${TAGS}${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${t}${SUFFIX}-${ARCH},"
           done
           TAGS="${TAGS%,}"  # Remove trailing comma
           echo "tags=${TAGS}" >> $GITHUB_OUTPUT
@@ -61,11 +91,12 @@ jobs:
           build-args: |
             GIT_COMMIT=${{ github.sha }}
             GIT_BRANCH=${{ github.ref_name }}
+            ${{ matrix.variant.build_args }}
           push: true
           tags: ${{ steps.prep.outputs.tags }}
-          platforms: linux/${{ matrix.arch }}
-          cache-from: type=gha,scope=${{ matrix.arch }}
-          cache-to: type=gha,scope=${{ matrix.arch }},mode=max
+          platforms: linux/${{ matrix.arch.name }}
+          cache-from: type=gha,scope=${{ matrix.arch.name }}-${{ matrix.variant.name }}
+          cache-to: type=gha,scope=${{ matrix.arch.name }}-${{ matrix.variant.name }},mode=max
 
   publish-manifest:
     name: Create and push multi-arch manifest
@@ -84,20 +115,26 @@ jobs:
         env:
           SHORT_SHA: ${{ github.sha }}
         run: |
-          IFS=',' read -ra TAG_ARRAY <<< "${{ inputs.tags }}"
+          IFS=',' read -ra TAG_ARRAY <<< "${TAGS_INPUT}"
           FIRST_TAG="${TAG_ARRAY[0]}"
+          SHORT="${SHORT_SHA::7}"
 
-          # Create manifest for first tag with SHA tag
-          docker buildx imagetools create \
-            -t ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${FIRST_TAG} \
-            -t ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:sha-${SHORT_SHA::7} \
-            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${FIRST_TAG}-amd64 \
-            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${FIRST_TAG}-arm64
-
-          # Create manifests for remaining tags
-          for t in "${TAG_ARRAY[@]:1}"; do
+          # Publish each tag twice: the regular image and its "-shadow" twin.
+          # These suffixes must stay in sync with the `variant` matrix suffixes
+          # in the build-image job above: a new variant must be added in both.
+          for suffix in "" "-shadow"; do
+            # First tag also gets an immutable sha-<sha> tag.
             docker buildx imagetools create \
-              -t ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${t} \
-              ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${t}-amd64 \
-              ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${t}-arm64
+              -t ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${FIRST_TAG}${suffix} \
+              -t ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:sha-${SHORT}${suffix} \
+              ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${FIRST_TAG}${suffix}-amd64 \
+              ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${FIRST_TAG}${suffix}-arm64
+
+            # Remaining tags.
+            for t in "${TAG_ARRAY[@]:1}"; do
+              docker buildx imagetools create \
+                -t ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${t}${suffix} \
+                ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${t}${suffix}-amd64 \
+                ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${t}${suffix}-arm64
+            done
           done

Cargo.lock

@@ -114,8 +114,8 @@ Docker images are published to `ghcr.io/lambdaclass/ethlambda` with the followin
 | Tag | Description |
 |-----|-------------|
 | `devnetX` | Stable image for a specific devnet (e.g. `devnet4`) |
-| `latest` | Alias for the stable image of the currently running devnet |
-| `unstable` | Development builds; promoted to `devnetX`/`latest` once tested |
+| `latest` | Alias for the latest stable image of the currently running devnet |
+| `unstable` | Built from the latest main commit; promoted to `devnetX`/`latest` once tested |
 | `sha-XXXXXXX` | Specific commit |
 
 [`RELEASE.md`](./RELEASE.md) has more details on our release process and how to tag new images.

README.md

@@ -21,17 +21,17 @@ exact commit it was built from.
 On top of that, the workflow accepts a comma-separated list of custom tags as a
 parameter (e.g. `latest,devnet2`). We use the following tagging convention:
 
-- `unstable` - the latest image built from the `main` branch, without any devnet-specific features
-- `latest` - the latest image built for the current devnet (`devnet3` at the time of writing)
-- `devnetX` - the latest image built with `devnetX` support (e.g. `devnet3`, `devnet4`)
+- `unstable` - the latest image built from the `main` branch
+- `latest` - the latest stable image built for the current devnet (`devnet5` at the time of writing)
+- `devnetX` - the latest stable image built with `devnetX` support (e.g. `devnet4`, `devnet5`)
 
 Future devnets will introduce new tags, with previous ones left without updates.
 
 ### Pulling an image
 
 ```bash
 docker pull ghcr.io/lambdaclass/ethlambda:unstable       # latest from main
-docker pull ghcr.io/lambdaclass/ethlambda:devnet3        # devnet3-compatible
+docker pull ghcr.io/lambdaclass/ethlambda:devnet5        # devnet5-compatible
 docker pull ghcr.io/lambdaclass/ethlambda:sha-12f8377    # pinned to a specific commit
 ```
 

RELEASE.md

@@ -16,10 +16,11 @@ jemalloc = ["dep:tikv-jemallocator"]
 # Shadow simulator compatibility: single-threaded tokio runtime and no jemalloc.
 # The quinn-udp UDP fallback is a Cargo `[patch]` (which cannot be feature-gated),
 # injected at build time by `shadow/build.sh` / `make shadow-build`.
-shadow-integration = []
+shadow-integration = ["ethlambda-crypto/shadow-integration"]
 
 [dependencies]
 ethlambda-blockchain.workspace = true
+ethlambda-crypto.workspace = true
 ethlambda-network-api.workspace = true
 ethlambda-p2p.workspace = true
 ethlambda-types.workspace = true

bin/ethlambda/Cargo.toml

@@ -107,4 +107,46 @@ pub(crate) struct CliOptions {
     /// `on_block`.
     #[arg(long, default_value = "3")]
     pub(crate) max_attestations_per_block: usize,
+    /// Shadow-simulator sim-cost + fake-XMSS flags (only under the
+    /// `shadow-integration` feature).
+    #[cfg(feature = "shadow-integration")]
+    #[command(flatten)]
+    pub(crate) shadow: ShadowOptions,
+}
+
+/// Shadow-simulator sim-cost + fake-XMSS flags. Compiled only under the
+/// `shadow-integration` feature.
+#[cfg(feature = "shadow-integration")]
+#[derive(Debug, clap::Args)]
+pub(crate) struct ShadowOptions {
+    /// Shadow sim only: replace the XMSS aggregation prover/verifier with a
+    /// deterministic stub (no leanVM proving/verifying). Off by default.
+    #[arg(long, default_value = "false")]
+    pub(crate) shadow_xmss_fake: bool,
+
+    /// Shadow sim only: signatures aggregated per second. Injects a sleep of
+    /// n/rate seconds into aggregation so its CPU cost shows up on Shadow's
+    /// virtual clock. Unset or <= 0 disables.
+    #[arg(long)]
+    pub(crate) shadow_xmss_aggregate_signatures_rate: Option<f64>,
+
+    /// Shadow sim only: signatures verified per aggregate per second; injects
+    /// a sleep of n/rate seconds into verification. Unset or <= 0 disables.
+    #[arg(long)]
+    pub(crate) shadow_xmss_verify_aggregated_signatures_rate: Option<f64>,
+
+    /// Shadow sim only: Type-1 components merged into a Type-2 per second;
+    /// injects a sleep of n/rate seconds into the proposal Type-2 merge.
+    /// Unset or <= 0 disables.
+    #[arg(long)]
+    pub(crate) shadow_xmss_merge_rate: Option<f64>,
+
+    /// Shadow sim only: byte length of each fake stub proof. Defaults to 32
+    /// KiB; capped at the 512 KiB on-wire proof limit.
+    #[arg(
+        long,
+        default_value_t = ethlambda_crypto::shadow_cost::DEFAULT_FAKE_PROOF_SIZE as u64,
+        value_parser = clap::value_parser!(u64).range(1..=524_288)
+    )]
+    pub(crate) shadow_xmss_fake_proof_size: u64,
 }