feat(cli): make benchmark reports comparable across runs and machines

Per-iteration rows show what one build cost; comparing an optimization
against a baseline needs three more things, which this adds.

Aggregate statistics per phase — count, min, mean, p50, p90, max, and a
coefficient of variation flagged above 10% so a noisy run is not read as
a result. Percentiles are nearest-rank, without interpolation: sample
counts are small, so an exact observed value beats a blend of two.
Outliers are never discarded, and the raw per-iteration rows stay above
the summary.

Build provenance — build.rs resolves the leansig and leanVM revisions
from Cargo.lock into the report. leansig is pinned to a moving branch and
leanVM does the signature aggregation, so either one moves the measured
crypto; two reports that disagree on them are not comparable, and without
this the report cannot say so. The per-[[package]] parse collects `name`
and `source` before extracting the rev, so it does not depend on TOML
field order.

Machine-readable output — `--format json` with a schema_version, and
`--output <path>` to write it alongside a human-readable run. Logs
already go to stderr, so the JSON pipes straight into jq. CI gains a
seconds-fast mock smoke step that asserts the contract, so a change to
the report shape cannot land unnoticed.

Author: pablodeymo

.github/workflows/ci.yml

@@ -75,3 +75,11 @@ jobs:
 
       - name: Run fixture-based tests
         uses: ./.github/actions/run-fixture-tests
+
+      # Reuses the release build from the test step; validates the benchmark
+      # harness end-to-end and its JSON output contract in a few seconds.
+      - name: Benchmark smoke (mock crypto)
+        run: |
+          cargo run --release --bin ethlambda -- benchmark synthetic --mock-crypto \
+            --num-validators 4 --warmup-slots 4 --iterations 3 --format json \
+            | jq -e '.schema_version == 1 and (.samples | length == 3)'

Cargo.lock

@@ -38,6 +38,7 @@ tracing.workspace = true
 tracing-subscriber = "0.3"
 
 serde.workspace = true
+serde_json.workspace = true
 serde_yaml_ng.workspace = true
 hex.workspace = true
 

bin/ethlambda/Cargo.toml

@@ -1,5 +1,13 @@
+use std::path::PathBuf;
+
 use vergen_git2::{Emitter, Git2Builder, RustcBuilder};
 
+/// Crate names whose resolved git revision is embedded in the binary, one per
+/// upstream crypto repository: `leansig` for leanSig, `lean-multisig` for
+/// leanVM (the direct dependency `ethlambda-crypto` builds against).
+const LEANSIG_PACKAGE: &str = "leansig";
+const LEANVM_PACKAGE: &str = "lean-multisig";
+
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     let git2 = Git2Builder::default().branch(true).sha(true).build()?;
     let rustc = RustcBuilder::default()
@@ -12,5 +20,70 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         .add_instructions(&git2)?
         .emit()?;
 
+    emit_crypto_revs();
+
     Ok(())
 }
+
+/// Embed the resolved leanSig and leanVM git revisions from the workspace
+/// Cargo.lock.
+///
+/// The crypto dependencies are pinned upstream (leansig to a moving branch,
+/// leanVM to a rev), so a `cargo update` or a rev bump changes the measured
+/// crypto with little or no ethlambda diff; benchmark reports embed these
+/// revisions to keep results interpretable across lock bumps.
+fn emit_crypto_revs() {
+    let revs = lockfile_git_revs();
+    for (package, env_var) in [
+        (LEANSIG_PACKAGE, "ETHLAMBDA_LEANSIG_REV"),
+        (LEANVM_PACKAGE, "ETHLAMBDA_LEANVM_REV"),
+    ] {
+        let rev = revs
+            .as_ref()
+            .and_then(|revs| revs.get(package).cloned())
+            .unwrap_or_else(|| "unknown".to_string());
+        println!("cargo:rustc-env={env_var}={rev}");
+    }
+    if let Some(lockfile) = workspace_lockfile() {
+        println!("cargo:rerun-if-changed={}", lockfile.display());
+    }
+}
+
+fn workspace_lockfile() -> Option<PathBuf> {
+    let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").ok()?;
+    Some(PathBuf::from(manifest_dir).join("../../Cargo.lock"))
+}
+
+/// Map each git-sourced package in the lockfile to its resolved revision.
+///
+/// Both fields of a `[[package]]` block are collected before the revision is
+/// extracted, so the result does not depend on TOML field order within the
+/// table (a lock-file reformatter emitting `source` before `name` would
+/// otherwise silently yield "unknown").
+fn lockfile_git_revs() -> Option<std::collections::HashMap<String, String>> {
+    let lockfile = std::fs::read_to_string(workspace_lockfile()?).ok()?;
+    let mut revs = std::collections::HashMap::new();
+    // A lockfile is a flat sequence of `[[package]]` blocks; splitting on the
+    // header gives one chunk per package (the first chunk is the file preamble,
+    // which has no `name` and is skipped).
+    for block in lockfile.split("[[package]]") {
+        let mut name = None;
+        let mut source = None;
+        for line in block.lines() {
+            let line = line.trim();
+            if let Some(value) = line.strip_prefix("name = ") {
+                name = Some(value.trim_matches('"').to_string());
+            } else if let Some(value) = line.strip_prefix("source = ") {
+                source = Some(value.trim_matches('"').to_string());
+            }
+        }
+        // source = "git+https://github.com/leanEthereum/leanSig?branch=devnet4#<rev>"
+        let (Some(name), Some(source)) = (name, source) else {
+            continue;
+        };
+        if let Some(rev) = source.strip_prefix("git+").and_then(|s| s.rsplit_once('#')) {
+            revs.insert(name, rev.1.to_string());
+        }
+    }
+    Some(revs)
+}

bin/ethlambda/build.rs

@@ -13,6 +13,7 @@ mod corpus;
 mod report;
 
 use std::collections::{BTreeMap, HashMap};
+use std::path::PathBuf;
 use std::time::Instant;
 
 use ethlambda_blockchain::block_builder::ProposerConfig;
@@ -84,6 +85,19 @@ struct CommonOptions {
     /// Mirrors the node flag: distinct AttestationData cap per built block.
     #[arg(long, default_value = "3")]
     max_attestations_per_block: usize,
+    /// Report format printed to stdout. Logs go to stderr, so JSON output can
+    /// be piped directly (e.g. into jq).
+    #[arg(long, value_enum, default_value_t = OutputFormat::Human)]
+    format: OutputFormat,
+    /// Also write the JSON report to this file.
+    #[arg(long)]
+    output: Option<PathBuf>,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)]
+enum OutputFormat {
+    Human,
+    Json,
 }
 
 pub(crate) fn run(options: BenchmarkOptions) -> eyre::Result<()> {
@@ -207,7 +221,16 @@ fn run_synthetic(options: SyntheticOptions) -> eyre::Result<()> {
         max_attestations_per_block: common.max_attestations_per_block,
     };
     let report = Report::new(Environment::collect(), params, samples);
-    println!("{}", report.human_table());
+
+    match common.format {
+        OutputFormat::Human => println!("{}", report.human_table()),
+        OutputFormat::Json => println!("{}", report.to_json()?),
+    }
+    if let Some(path) = &common.output {
+        std::fs::write(path, report.to_json()?)
+            .wrap_err_with(|| format!("failed to write report to {}", path.display()))?;
+        eprintln!("report written to {}", path.display());
+    }
 
     Ok(())
 }