From 9df5f5f69b358d354178b8d4be76060029b31234 Mon Sep 17 00:00:00 2001 From: Pablo Deymonnaz Date: Wed, 26 Aug 2026 16:48:11 -0300 Subject: [PATCH] feat(cli): make benchmark reports comparable across runs and machines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per-iteration rows show what one build cost; comparing an optimization against a baseline needs three more things, which this adds. Aggregate statistics per phase — count, min, mean, p50, p90, max, and a coefficient of variation flagged above 10% so a noisy run is not read as a result. Percentiles are nearest-rank, without interpolation: sample counts are small, so an exact observed value beats a blend of two. Outliers are never discarded, and the raw per-iteration rows stay above the summary. Build provenance — build.rs resolves the leansig and leanVM revisions from Cargo.lock into the report. leansig is pinned to a moving branch and leanVM does the signature aggregation, so either one moves the measured crypto; two reports that disagree on them are not comparable, and without this the report cannot say so. The per-[[package]] parse collects `name` and `source` before extracting the rev, so it does not depend on TOML field order. Machine-readable output — `--format json` with a schema_version, and `--output ` to write it alongside a human-readable run. Logs already go to stderr, so the JSON pipes straight into jq. CI gains a seconds-fast mock smoke step that asserts the contract, so a change to the report shape cannot land unnoticed. --- .github/workflows/ci.yml | 8 + Cargo.lock | 1 + bin/ethlambda/Cargo.toml | 1 + bin/ethlambda/build.rs | 73 +++++++++ bin/ethlambda/src/benchmark/mod.rs | 25 +++- bin/ethlambda/src/benchmark/report.rs | 205 +++++++++++++++++++++++++- 6 files changed, 305 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 50dc0765..05bedfc9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,3 +75,11 @@ jobs: - name: Run fixture-based tests uses: ./.github/actions/run-fixture-tests + + # Reuses the release build from the test step; validates the benchmark + # harness end-to-end and its JSON output contract in a few seconds. + - name: Benchmark smoke (mock crypto) + run: | + cargo run --release --bin ethlambda -- benchmark synthetic --mock-crypto \ + --num-validators 4 --warmup-slots 4 --iterations 3 --format json \ + | jq -e '.schema_version == 1 and (.samples | length == 3)' diff --git a/Cargo.lock b/Cargo.lock index b0c8efea..dacda8d5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1932,6 +1932,7 @@ dependencies = [ "libssz-types", "reqwest", "serde", + "serde_json", "serde_yaml_ng", "thiserror 2.0.18", "tikv-jemallocator", diff --git a/bin/ethlambda/Cargo.toml b/bin/ethlambda/Cargo.toml index e5853842..591490ca 100644 --- a/bin/ethlambda/Cargo.toml +++ b/bin/ethlambda/Cargo.toml @@ -38,6 +38,7 @@ tracing.workspace = true tracing-subscriber = "0.3" serde.workspace = true +serde_json.workspace = true serde_yaml_ng.workspace = true hex.workspace = true diff --git a/bin/ethlambda/build.rs b/bin/ethlambda/build.rs index ad4184ed..aea5f542 100644 --- a/bin/ethlambda/build.rs +++ b/bin/ethlambda/build.rs @@ -1,5 +1,13 @@ +use std::path::PathBuf; + use vergen_git2::{Emitter, Git2Builder, RustcBuilder}; +/// Crate names whose resolved git revision is embedded in the binary, one per +/// upstream crypto repository: `leansig` for leanSig, `lean-multisig` for +/// leanVM (the direct dependency `ethlambda-crypto` builds against). +const LEANSIG_PACKAGE: &str = "leansig"; +const LEANVM_PACKAGE: &str = "lean-multisig"; + fn main() -> Result<(), Box> { let git2 = Git2Builder::default().branch(true).sha(true).build()?; let rustc = RustcBuilder::default() @@ -12,5 +20,70 @@ fn main() -> Result<(), Box> { .add_instructions(&git2)? .emit()?; + emit_crypto_revs(); + Ok(()) } + +/// Embed the resolved leanSig and leanVM git revisions from the workspace +/// Cargo.lock. +/// +/// The crypto dependencies are pinned upstream (leansig to a moving branch, +/// leanVM to a rev), so a `cargo update` or a rev bump changes the measured +/// crypto with little or no ethlambda diff; benchmark reports embed these +/// revisions to keep results interpretable across lock bumps. +fn emit_crypto_revs() { + let revs = lockfile_git_revs(); + for (package, env_var) in [ + (LEANSIG_PACKAGE, "ETHLAMBDA_LEANSIG_REV"), + (LEANVM_PACKAGE, "ETHLAMBDA_LEANVM_REV"), + ] { + let rev = revs + .as_ref() + .and_then(|revs| revs.get(package).cloned()) + .unwrap_or_else(|| "unknown".to_string()); + println!("cargo:rustc-env={env_var}={rev}"); + } + if let Some(lockfile) = workspace_lockfile() { + println!("cargo:rerun-if-changed={}", lockfile.display()); + } +} + +fn workspace_lockfile() -> Option { + let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").ok()?; + Some(PathBuf::from(manifest_dir).join("../../Cargo.lock")) +} + +/// Map each git-sourced package in the lockfile to its resolved revision. +/// +/// Both fields of a `[[package]]` block are collected before the revision is +/// extracted, so the result does not depend on TOML field order within the +/// table (a lock-file reformatter emitting `source` before `name` would +/// otherwise silently yield "unknown"). +fn lockfile_git_revs() -> Option> { + let lockfile = std::fs::read_to_string(workspace_lockfile()?).ok()?; + let mut revs = std::collections::HashMap::new(); + // A lockfile is a flat sequence of `[[package]]` blocks; splitting on the + // header gives one chunk per package (the first chunk is the file preamble, + // which has no `name` and is skipped). + for block in lockfile.split("[[package]]") { + let mut name = None; + let mut source = None; + for line in block.lines() { + let line = line.trim(); + if let Some(value) = line.strip_prefix("name = ") { + name = Some(value.trim_matches('"').to_string()); + } else if let Some(value) = line.strip_prefix("source = ") { + source = Some(value.trim_matches('"').to_string()); + } + } + // source = "git+https://github.com/leanEthereum/leanSig?branch=devnet4#" + let (Some(name), Some(source)) = (name, source) else { + continue; + }; + if let Some(rev) = source.strip_prefix("git+").and_then(|s| s.rsplit_once('#')) { + revs.insert(name, rev.1.to_string()); + } + } + Some(revs) +} diff --git a/bin/ethlambda/src/benchmark/mod.rs b/bin/ethlambda/src/benchmark/mod.rs index 8d6da8f4..70e5d2fc 100644 --- a/bin/ethlambda/src/benchmark/mod.rs +++ b/bin/ethlambda/src/benchmark/mod.rs @@ -13,6 +13,7 @@ mod corpus; mod report; use std::collections::{BTreeMap, HashMap}; +use std::path::PathBuf; use std::time::Instant; use ethlambda_blockchain::block_builder::ProposerConfig; @@ -119,6 +120,19 @@ struct CommonOptions { /// Mirrors the node flag: distinct AttestationData cap per built block. #[arg(long, default_value = "3")] max_attestations_per_block: usize, + /// Report format printed to stdout. Logs go to stderr, so JSON output can + /// be piped directly (e.g. into jq). + #[arg(long, value_enum, default_value_t = OutputFormat::Human)] + format: OutputFormat, + /// Also write the JSON report to this file. + #[arg(long)] + output: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)] +enum OutputFormat { + Human, + Json, } pub(crate) fn run(options: BenchmarkOptions) -> eyre::Result<()> { @@ -168,7 +182,16 @@ fn run_synthetic(options: SyntheticOptions) -> eyre::Result<()> { ); let report = Report::new(Environment::collect(), Params::from(&options), samples); - println!("{}", report.human_table()); + + match common.format { + OutputFormat::Human => println!("{}", report.human_table()), + OutputFormat::Json => println!("{}", report.to_json()?), + } + if let Some(path) = &common.output { + std::fs::write(path, report.to_json()?) + .wrap_err_with(|| format!("failed to write report to {}", path.display()))?; + eprintln!("report written to {}", path.display()); + } Ok(()) } diff --git a/bin/ethlambda/src/benchmark/report.rs b/bin/ethlambda/src/benchmark/report.rs index 2fce8cc4..34e8014c 100644 --- a/bin/ethlambda/src/benchmark/report.rs +++ b/bin/ethlambda/src/benchmark/report.rs @@ -1,9 +1,9 @@ -//! Report emission for the block-building benchmark. +//! Statistics and report emission for the block-building benchmark. //! -//! Every measured iteration is reported on its own row: outliers are never -//! discarded (XMSS signing and OTS window advancement produce legitimate heavy -//! tails worth inspecting), and the per-iteration block roots let a -//! baseline-vs-optimized diff prove an optimization changed only speed, not +//! Raw per-iteration samples are always included in the JSON report: outliers +//! are never discarded (XMSS signing and OTS window advancement produce +//! legitimate heavy tails worth inspecting), and per-iteration block roots let +//! a baseline-vs-optimized diff prove an optimization changed only speed, not //! which attestations get selected. use std::collections::BTreeMap; @@ -13,6 +13,10 @@ use serde::Serialize; use crate::version; +/// Coefficient-of-variation threshold above which wall-time results are +/// flagged as too noisy to compare, per the benchmarking workflow standard. +const CV_WARN_THRESHOLD: f64 = 0.10; + #[derive(Debug, Serialize)] pub(crate) struct Sample { pub iteration: u64, @@ -37,6 +41,12 @@ pub(crate) struct Sample { #[derive(Debug, Serialize)] pub(crate) struct Environment { pub client_version: &'static str, + /// Resolved leansig git revision from Cargo.lock. leansig is pinned to a + /// moving branch, so results are not comparable across revisions. + pub leansig_rev: &'static str, + /// Resolved leanVM git revision from Cargo.lock. leanVM does the signature + /// aggregation, so a rev bump moves the measured crypto too. + pub leanvm_rev: &'static str, pub os: &'static str, pub arch: &'static str, pub available_parallelism: usize, @@ -46,6 +56,8 @@ impl Environment { pub(crate) fn collect() -> Self { Self { client_version: version::CLIENT_VERSION, + leansig_rev: env!("ETHLAMBDA_LEANSIG_REV"), + leanvm_rev: env!("ETHLAMBDA_LEANVM_REV"), os: std::env::consts::OS, arch: std::env::consts::ARCH, available_parallelism: std::thread::available_parallelism() @@ -68,22 +80,85 @@ pub(crate) struct Params { pub max_attestations_per_block: usize, } +#[derive(Debug, Serialize)] +pub(crate) struct Stats { + pub count: usize, + pub min_seconds: f64, + pub mean_seconds: f64, + pub p50_seconds: f64, + pub p90_seconds: f64, + pub max_seconds: f64, + /// Coefficient of variation (stddev / mean); NaN-free (0 when mean is 0). + pub cv: f64, +} + +#[derive(Debug, Serialize)] +pub(crate) struct Summary { + pub phases: BTreeMap, + pub overhead: Stats, + pub wall: Stats, +} + #[derive(Debug, Serialize)] pub(crate) struct Report { + pub schema_version: u32, pub environment: Environment, pub params: Params, pub samples: Vec, + pub summary: Summary, } impl Report { pub(crate) fn new(environment: Environment, params: Params, samples: Vec) -> Self { + let mut phases: BTreeMap = BTreeMap::new(); + if let Some(first) = samples.first() { + for phase in first.phases.keys() { + let values: Vec = samples + .iter() + .filter_map(|sample| sample.phases.get(phase).copied()) + .collect(); + phases.insert(phase.clone(), stats(&values)); + } + } + let overhead = stats( + &samples + .iter() + .map(|sample| sample.overhead_seconds) + .collect::>(), + ); + let wall = stats( + &samples + .iter() + .map(|sample| sample.wall_seconds) + .collect::>(), + ); + + if wall.cv > CV_WARN_THRESHOLD { + eprintln!( + "warning: wall-time coefficient of variation is {:.1}% (>{:.0}%); \ + results are noisy — check for background load or increase --iterations", + wall.cv * 100.0, + CV_WARN_THRESHOLD * 100.0 + ); + } + Self { + schema_version: 1, environment, params, samples, + summary: Summary { + phases, + overhead, + wall, + }, } } + pub(crate) fn to_json(&self) -> eyre::Result { + serde_json::to_string_pretty(self).map_err(Into::into) + } + pub(crate) fn human_table(&self) -> String { let mut out = String::new(); let params = &self.params; @@ -110,8 +185,13 @@ impl Report { ); let _ = writeln!( out, - " {} os={} arch={} threads={}", - env.client_version, env.os, env.arch, env.available_parallelism + " {} leansig={} leanvm={} os={} arch={} threads={}", + env.client_version, + env.leansig_rev, + env.leanvm_rev, + env.os, + env.arch, + env.available_parallelism ); let _ = writeln!(out); @@ -141,10 +221,121 @@ impl Report { &sample.block_root[..10], ); } + + let _ = writeln!(out); + let _ = writeln!( + out, + " {:<18} {:>5} {:>10} {:>10} {:>10} {:>10} {:>10}", + "phase", "count", "min", "mean", "p50", "p90", "max" + ); + for (phase, stats) in &self.summary.phases { + let _ = writeln!(out, "{}", stats_row(phase, stats)); + } + let _ = writeln!(out, "{}", stats_row("overhead", &self.summary.overhead)); + let _ = writeln!(out, "{}", stats_row("wall", &self.summary.wall)); out } } +fn stats_row(name: &str, stats: &Stats) -> String { + format!( + " {:<18} {:>5} {:>10} {:>10} {:>10} {:>10} {:>10}", + name, + stats.count, + format_ms(stats.min_seconds), + format_ms(stats.mean_seconds), + format_ms(stats.p50_seconds), + format_ms(stats.p90_seconds), + format_ms(stats.max_seconds), + ) +} + fn format_ms(seconds: f64) -> String { format!("{:.3}ms", seconds * 1e3) } + +fn stats(values: &[f64]) -> Stats { + if values.is_empty() { + return Stats { + count: 0, + min_seconds: 0.0, + mean_seconds: 0.0, + p50_seconds: 0.0, + p90_seconds: 0.0, + max_seconds: 0.0, + cv: 0.0, + }; + } + let mut sorted = values.to_vec(); + sorted.sort_by(|a, b| a.total_cmp(b)); + let count = sorted.len(); + let mean = sorted.iter().sum::() / count as f64; + let variance = sorted + .iter() + .map(|value| (value - mean).powi(2)) + .sum::() + / count as f64; + let cv = if mean > 0.0 { + variance.sqrt() / mean + } else { + 0.0 + }; + Stats { + count, + min_seconds: sorted[0], + mean_seconds: mean, + p50_seconds: percentile(&sorted, 0.50), + p90_seconds: percentile(&sorted, 0.90), + max_seconds: sorted[count - 1], + cv, + } +} + +/// Nearest-rank percentile over a sorted slice (no interpolation; sample +/// counts are small so exact sample values are preferable to blends). +fn percentile(sorted: &[f64], q: f64) -> f64 { + let index = ((sorted.len() - 1) as f64 * q).round() as usize; + sorted[index] +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn percentile_handles_single_sample() { + let sorted = [7.0]; + assert_eq!(percentile(&sorted, 0.0), 7.0); + assert_eq!(percentile(&sorted, 0.5), 7.0); + assert_eq!(percentile(&sorted, 1.0), 7.0); + } + + #[test] + fn percentile_odd_and_even_lengths() { + let odd = [1.0, 2.0, 3.0, 4.0, 5.0]; + assert_eq!(percentile(&odd, 0.5), 3.0); + assert_eq!(percentile(&odd, 1.0), 5.0); + let even = [1.0, 2.0, 3.0, 4.0]; + assert_eq!(percentile(&even, 0.5), 3.0); + assert_eq!(percentile(&even, 0.0), 1.0); + } + + #[test] + fn stats_on_known_values() { + let stats = stats(&[2.0, 4.0, 4.0, 4.0, 5.0, 5.0, 7.0, 9.0]); + assert_eq!(stats.count, 8); + assert_eq!(stats.min_seconds, 2.0); + assert_eq!(stats.max_seconds, 9.0); + assert_eq!(stats.mean_seconds, 5.0); + // population stddev of this classic set is 2.0 => cv = 0.4 + assert!((stats.cv - 0.4).abs() < 1e-12); + } + + #[test] + fn stats_on_empty_input_is_zeroed() { + let stats = stats(&[]); + assert_eq!(stats.count, 0); + assert_eq!(stats.mean_seconds, 0.0); + assert_eq!(stats.cv, 0.0); + } +}