Skip to content
.github/workflows/bench.yml

chore(deps): bump grevm to Galxe/grevm@eecc6fc (revm v40 landed upstr… #786

Sign in to view logs

chore(deps): bump grevm to Galxe/grevm@eecc6fc (revm v40 landed upstr…

chore(deps): bump grevm to Galxe/grevm@eecc6fc (revm v40 landed upstr… #786

Workflow file for this run

.github/workflows/bench.yml at 25fd1ec
# Runs benchmarks.
#
# The bench job replays real blocks via the Engine API against a reth node
# backed by a local snapshot managed with schelk.
#
# It runs the baseline binary and the feature (candidate) binary on the
# same block range (snapshot recovered between runs) to compare performance.
on:
<<<<<<< HEAD

Check failure on line 10 in .github/workflows/bench.yml

View workflow run for this annotation

GitHub Actions / .github/workflows/bench.yml

Invalid workflow file 

You have an error in your yaml syntax on line 10
workflow_dispatch:
# pull_request:
# # TODO: Disabled temporarily for https://github.com/CodSpeedHQ/runner/issues/55
# # merge_group:
# push:
# branches: [main]
env:
CARGO_TERM_COLOR: always
BASELINE: base
SEED: reth
=======
issue_comment:
types: [created]
workflow_dispatch:
inputs:
blocks:
description: "Number of blocks to benchmark (default: 500, big blocks: 30)"
required: false
default: ""
type: string
big_blocks:
description: "Use big blocks mode: false, true, or target gas like 100M/2G"
required: false
default: "false"
type: string
bal:
description: "Replay block access lists during big-block benchmarks"
required: false
default: "false"
type: choice
options:
- "false"
- "true"
- "feature"
- "baseline"
warmup:
description: "Number of warmup blocks (default: one-quarter of blocks)"
required: false
default: ""
type: string
baseline:
description: "Baseline git ref (default: merge-base)"
required: false
default: ""
type: string
feature:
description: "Feature git ref (default: branch head)"
required: false
default: ""
type: string
wait_time:
description: "Minimum interval between block submissions (e.g. 500ms, 1s)"
required: false
default: ""
type: string
baseline_args:
description: "Extra CLI args for the baseline reth node"
required: false
default: ""
type: string
feature_args:
description: "Extra CLI args for the feature reth node"
required: false
default: ""
type: string
samply:
description: "Enable samply profiling"
required: false
default: "false"
type: boolean
tracing_chrome:
description: "Enable Chrome trace recording"
required: false
default: "false"
type: boolean
cores:
description: "Limit reth to N CPU cores (0 = all available)"
required: false
default: "0"
type: string
slack:
description: "Slack notification policy"
required: false
default: "never"
type: choice
options:
- always
- on-win
- on-error
- never
run_pairs:
description: "Number of benchmark run pairs (default: 6, big blocks: 10)"
required: false
default: ""
type: string
otlp:
description: "Export OTLP traces and logs"
required: false
default: "true"
type: boolean
metrics:
description: "Upload txgen metrics to VictoriaMetrics"
required: false
default: "false"
type: boolean
env:
CARGO_TERM_COLOR: always
RUSTC_WRAPPER: "sccache"
BENCH_RUNNERS: 4
>>>>>>> v2.3.0
name: bench
permissions: {}
jobs:
<<<<<<< HEAD
codspeed:
runs-on:
group: Reth
steps:
- uses: actions/checkout@v5
with:
submodules: true
- uses: rui314/setup-mold@v1
=======
bench-ack:
if: |
(github.event_name == 'issue_comment' && github.event.issue.pull_request && (startsWith(github.event.comment.body, '@decofe bench') || startsWith(github.event.comment.body, 'derek bench'))) ||
github.event_name == 'workflow_dispatch'
name: bench-ack
runs-on: ubuntu-latest
permissions:
contents: read
pull-requests: write
outputs:
pr: ${{ steps.args.outputs.pr }}
actor: ${{ steps.args.outputs.actor }}
blocks: ${{ steps.args.outputs.blocks }}
warmup: ${{ steps.args.outputs.warmup }}
baseline: ${{ steps.args.outputs.baseline }}
feature: ${{ steps.args.outputs.feature }}
baseline-name: ${{ steps.args.outputs.baseline-name }}
feature-name: ${{ steps.args.outputs.feature-name }}
samply: ${{ steps.args.outputs.samply }}
tracing-chrome: ${{ steps.args.outputs.tracing-chrome }}
slack: ${{ steps.args.outputs.slack }}
metrics: ${{ steps.args.outputs.metrics }}
cores: ${{ steps.args.outputs.cores }}
big-blocks: ${{ steps.args.outputs.big-blocks }}
big-blocks-target-gas: ${{ steps.args.outputs.big-blocks-target-gas }}
bal: ${{ steps.args.outputs.bal }}
wait-time: ${{ steps.args.outputs.wait-time }}
baseline-args: ${{ steps.args.outputs.baseline-args }}
feature-args: ${{ steps.args.outputs.feature-args }}
run-pairs: ${{ steps.args.outputs.run-pairs }}
run-order: ${{ steps.args.outputs.run-order }}
otlp: ${{ steps.args.outputs.otlp }}
node-bin: ${{ steps.args.outputs.node-bin }}
comment-id: ${{ steps.ack.outputs.comment-id }}
pr-head-sha: ${{ steps.args.outputs.pr-head-sha }}
pr-head-ref: ${{ steps.args.outputs.pr-head-ref }}
pr-head-repo: ${{ steps.args.outputs.pr-head-repo }}
steps:
- name: Check org membership
if: github.event_name == 'issue_comment'
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
github-token: ${{ secrets.DEREK_BENCH_ACK_TOKEN }}
script: |
const org = 'paradigmxyz';
const allowedUsers = new Set(['rkrasiuk']);
const checkMembership = async (username) => {
if (allowedUsers.has(username)) return true;
try {
const { status } = await github.rest.orgs.checkMembershipForUser({ org, username });
return status === 204 || status === 302;
} catch {
return false;
}
};
const commenter = context.payload.comment.user.login;
if (!await checkMembership(commenter)) {
core.setFailed(`@${commenter} is not a member of ${org}`);
return;
}
const { data: pr } = await github.rest.pulls.get({
owner: context.repo.owner,
repo: context.repo.repo,
pull_number: context.issue.number,
});
const prAuthor = pr.user.login;
if (!await checkMembership(prAuthor)) {
core.setFailed(`PR author @${prAuthor} is not a member of ${org}`);
}
- name: Parse arguments
id: args
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
github-token: ${{ secrets.DEREK_PAT }}
script: |
const validBalModes = new Set(['false', 'true', 'feature', 'baseline']);
const validSlackModes = new Set(['always', 'on-win', 'on-error', 'never']);
const usage = '`@decofe bench [blocks=N] [big-blocks[=true|false|100M|2G]] [bal=true|false|feature|baseline] [warmup=N] [baseline=REF] [feature=REF] [samply] [tracing-chrome] [slack=always|on-win|on-error|never] [cores=N] [run-pairs=N] [otlp=true|false] [metrics[=true|false]] [wait-time=DURATION] [baseline-args="..."] [feature-args="..."]`';
let pr, actor, blocks, warmup, baseline, feature, samply, tracingChrome, cores, bigBlocks, bal;
let bigBlocksTargetGas = '';
let explicitBlocks = false;
let explicitWarmup = false;
let explicitRunPairs = false;
const parseBigBlocks = (value) => {
value = String(value || 'false');
if (value === 'true') return { enabled: 'true', targetGas: '' };
if (value === 'false') return { enabled: 'false', targetGas: '' };
if (/^\d+([kKmMgG])?$/.test(value)) return { enabled: 'true', targetGas: value };
return null;
};
const defaultBlocks = (isBigBlocks) => isBigBlocks ? '30' : '500';
const defaultWarmup = (blockCount) => String(Math.floor(Number(blockCount) / 4));
const defaultRunPairs = (isBigBlocks) => isBigBlocks ? '10' : '6';
const benchRunOrder = (runPairs) => {
const n = Number(runPairs);
if (!Number.isInteger(n) || n < 1) {
throw new Error('run pairs must be a positive integer');
}
return n % 2 === 0 ? 'ABBA'.repeat(n / 2) : 'AB'.repeat(n);
};
if (context.eventName === 'workflow_dispatch') {
actor = '${{ github.actor }}';
blocks = '${{ github.event.inputs.blocks }}' || '';
explicitBlocks = blocks !== '';
warmup = '${{ github.event.inputs.warmup }}' || '';
explicitWarmup = warmup !== '';
baseline = '${{ github.event.inputs.baseline }}';
feature = '${{ github.event.inputs.feature }}';
samply = '${{ github.event.inputs.samply }}' === 'true' ? 'true' : 'false';
tracingChrome = '${{ github.event.inputs.tracing_chrome }}' === 'true' ? 'true' : 'false';
var slack = '${{ github.event.inputs.slack }}' || 'never';
cores = '${{ github.event.inputs.cores }}' || '0';
const parsedBigBlocks = parseBigBlocks('${{ github.event.inputs.big_blocks }}');
if (!parsedBigBlocks) {
core.setFailed(`Invalid big_blocks value: ${{ github.event.inputs.big_blocks }} (must be false, true, or a gas value like 100M or 2G)`);
return;
}
bigBlocks = parsedBigBlocks.enabled;
bigBlocksTargetGas = parsedBigBlocks.targetGas;
bal = '${{ github.event.inputs.bal }}' || 'false';
var runPairs = '${{ github.event.inputs.run_pairs }}' || '';
explicitRunPairs = runPairs !== '';
var otlp = '${{ github.event.inputs.otlp }}' === 'false' ? 'false' : 'true';
var metrics = '${{ github.event.inputs.metrics }}' === 'true' ? 'true' : 'false';
var waitTime = '${{ github.event.inputs.wait_time }}' || '';
var baselineNodeArgs = '${{ github.event.inputs.baseline_args }}' || '';
var featureNodeArgs = '${{ github.event.inputs.feature_args }}' || '';
// Find PR for the selected branch
const branch = '${{ github.ref_name }}';
const { data: prs } = await github.rest.pulls.list({
owner: context.repo.owner,
repo: context.repo.repo,
head: `${context.repo.owner}:${branch}`,
state: 'open',
per_page: 1,
});
pr = prs.length ? String(prs[0].number) : '';
if (!pr) {
core.info(`No open PR found for branch '${branch}', results will be in job summary`);
}
} else {
pr = String(context.issue.number);
actor = context.payload.comment.user.login;
const body = context.payload.comment.body.trim();
const intArgs = new Set(['warmup', 'cores', 'blocks', 'run-pairs']);
const refArgs = new Set(['baseline', 'feature']);
const boolArgs = new Set(['samply', 'tracing-chrome', 'otlp', 'metrics']);
const enumArgs = new Map([['bal', validBalModes], ['slack', validSlackModes]]);
const durationArgs = new Set(['wait-time']);
const stringArgs = new Set(['baseline-args', 'feature-args']);
const defaults = { blocks: '', warmup: '', baseline: '', feature: '', samply: 'false', 'tracing-chrome': 'false', slack: 'always', 'big-blocks': 'false', bal: 'false', cores: '0', 'run-pairs': '', otlp: 'true', metrics: 'false', 'wait-time': '', 'baseline-args': '', 'feature-args': '' };
const unknown = [];
const invalid = [];
const args = body.replace(/^(?:@decofe|derek) bench\s*/, '');
// Parse args, handling quoted values like key="value with spaces"
const parts = [];
const argRegex = /(\S+?="[^"]*"|\S+?='[^']*'|\S+)/g;
let m;
while ((m = argRegex.exec(args)) !== null) parts.push(m[1]);
for (const part of parts) {
const eq = part.indexOf('=');
if (eq === -1) {
if (part === 'big-blocks') {
defaults[part] = 'true';
} else if (boolArgs.has(part)) {
defaults[part] = 'true';
} else {
unknown.push(part);
}
continue;
}
const key = part.slice(0, eq);
let value = part.slice(eq + 1);
// Strip surrounding quotes
if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) {
value = value.slice(1, -1);
}
if (key === 'big-blocks') {
const parsed = parseBigBlocks(value);
if (parsed) {
defaults[key] = value;
} else {
invalid.push(`\`${key}=${value}\` (must be false, true, or a gas value like 100M or 2G)`);
}
} else if (boolArgs.has(key)) {
if (value === 'true' || value === 'false') {
defaults[key] = value;
} else {
invalid.push(`\`${key}=${value}\` (must be true or false)`);
}
} else if (durationArgs.has(key)) {
if (/^\d+(ms|s|m)$/.test(value)) {
defaults[key] = value;
} else {
invalid.push(`\`${key}=${value}\` (must be a duration like 500ms, 1s, 2m)`);
}
} else if (enumArgs.has(key)) {
if (enumArgs.get(key).has(value)) {
defaults[key] = value;
} else {
invalid.push(`\`${key}=${value}\` (must be true, false, feature, or baseline)`);
}
} else if (intArgs.has(key)) {
if (!/^\d+$/.test(value)) {
invalid.push(`\`${key}=${value}\` (must be a positive integer)`);
} else {
defaults[key] = value;
if (key === 'blocks') explicitBlocks = true;
if (key === 'warmup') explicitWarmup = true;
if (key === 'run-pairs') explicitRunPairs = true;
}
} else if (refArgs.has(key)) {
if (!value) {
invalid.push(`\`${key}=\` (must be a git ref)`);
} else {
defaults[key] = value;
}
} else if (stringArgs.has(key)) {
defaults[key] = value;
} else {
unknown.push(key);
}
}
const errors = [];
if (unknown.length) errors.push(`Unknown argument(s): \`${unknown.join('`, `')}\``);
if (invalid.length) errors.push(`Invalid value(s): ${invalid.join(', ')}`);
if (errors.length) {
const msg = `❌ **Invalid bench command**\n\n${errors.join('\n')}\n\n**Usage:** ${usage}`;
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
body: msg,
});
core.setFailed(msg);
return;
}
blocks = defaults.blocks;
warmup = defaults.warmup;
baseline = defaults.baseline;
feature = defaults.feature;
samply = defaults.samply;
tracingChrome = defaults['tracing-chrome'];
var slack = defaults.slack;
cores = defaults.cores;
const parsedBigBlocks = parseBigBlocks(defaults['big-blocks']);
bigBlocks = parsedBigBlocks.enabled;
bigBlocksTargetGas = parsedBigBlocks.targetGas;
bal = defaults.bal;
var runPairs = defaults['run-pairs'];
var otlp = defaults.otlp;
var metrics = defaults.metrics;
var waitTime = defaults['wait-time'];
var baselineNodeArgs = defaults['baseline-args'];
var featureNodeArgs = defaults['feature-args'];
}
const isBigBlocks = bigBlocks === 'true';
if (!explicitBlocks) {
blocks = defaultBlocks(isBigBlocks);
}
if (!explicitRunPairs) {
runPairs = defaultRunPairs(isBigBlocks);
}
if (!/^\d+$/.test(blocks)) {
core.setFailed(`Invalid blocks value: ${blocks} (must be a positive integer)`);
return;
}
if (explicitWarmup && !/^\d+$/.test(warmup)) {
core.setFailed(`Invalid warmup value: ${warmup} (must be a positive integer)`);
return;
}
if (!explicitWarmup) {
warmup = defaultWarmup(blocks);
}
if (!/^[1-9]\d*$/.test(runPairs)) {
core.setFailed(`Invalid run-pairs value: ${runPairs} (must be a positive integer greater than zero)`);
return;
}
const runOrder = benchRunOrder(runPairs);
if (!validBalModes.has(bal)) {
core.setFailed(`Invalid bal mode: ${bal}`);
return;
}
const releaseRef = (ref) => /^v\d+\.\d+\.\d+/.test(ref || '');
if (releaseRef(baseline) || releaseRef(feature) || '${{ github.ref_type }}' === 'tag') {
otlp = 'false';
}
// Resolve display names for baseline/feature
let baselineName = baseline || 'main';
let featureName = feature;
if (pr) {
const { data: prData } = await github.rest.pulls.get({
owner: context.repo.owner,
repo: context.repo.repo,
pull_number: parseInt(pr),
});
core.setOutput('pr-head-sha', prData.head.sha);
core.setOutput('pr-head-ref', prData.head.ref);
core.setOutput('pr-head-repo', prData.head.repo.full_name);
if (!featureName) featureName = prData.head.ref;
} else {
if (!featureName) featureName = '${{ github.ref_name }}';
}
const nodeBin = bigBlocks === 'true' ? 'reth-bb' : 'reth';
core.setOutput('pr', pr || '');
core.setOutput('actor', actor);
core.setOutput('blocks', blocks);
core.setOutput('warmup', warmup);
core.setOutput('baseline', baseline);
core.setOutput('feature', feature);
core.setOutput('baseline-name', baselineName);
core.setOutput('feature-name', featureName);
core.setOutput('samply', samply);
core.setOutput('tracing-chrome', tracingChrome);
core.setOutput('slack', slack);
core.setOutput('cores', cores);
core.setOutput('big-blocks', bigBlocks);
core.setOutput('big-blocks-target-gas', bigBlocksTargetGas);
core.setOutput('bal', bal);
core.setOutput('wait-time', waitTime);
core.setOutput('baseline-args', baselineNodeArgs);
core.setOutput('feature-args', featureNodeArgs);
core.setOutput('run-pairs', runPairs);
core.setOutput('run-order', runOrder);
core.setOutput('otlp', otlp);
core.setOutput('metrics', metrics);
core.setOutput('node-bin', nodeBin);
- name: Acknowledge request
id: ack
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
github-token: ${{ secrets.DEREK_PAT }}
script: |
if (context.eventName === 'issue_comment') {
await github.rest.reactions.createForIssueComment({
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: context.payload.comment.id,
content: 'eyes',
});
}
const pr = '${{ steps.args.outputs.pr }}';
if (!pr) return;
const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
// Count queued/waiting bench runs ahead of this one.
// BENCH_RUNNERS is the number of self-hosted runners available.
let queueMsg = '';
let ahead = 0;
const numRunners = parseInt(process.env.BENCH_RUNNERS) || 1;
try {
const statuses = ['queued', 'in_progress', 'waiting', 'requested', 'pending'];
const allRuns = [];
for (const status of statuses) {
const { data: { workflow_runs: r } } = await github.rest.actions.listWorkflowRuns({
owner: context.repo.owner,
repo: context.repo.repo,
workflow_id: 'bench.yml',
status,
per_page: 100,
});
allRuns.push(...r);
}
const benchRuns = allRuns.filter(r => r.event === 'issue_comment' || r.event === 'workflow_dispatch');
const thisRun = benchRuns.find(r => r.id === context.runId);
const thisCreatedAt = thisRun ? new Date(thisRun.created_at) : new Date();
const totalAhead = benchRuns.filter(r => r.id !== context.runId && new Date(r.created_at) <= thisCreatedAt).length;
ahead = Math.max(0, totalAhead - numRunners + 1);
if (ahead > 0) {
queueMsg = `\n🔢 **Queue position:** ${ahead} job(s) ahead (${numRunners} runner(s))`;
}
} catch (e) {
core.info(`Skipping queue tracking: ${e.message}`);
}
const actor = '${{ steps.args.outputs.actor }}';
const blocks = '${{ steps.args.outputs.blocks }}';
const warmup = '${{ steps.args.outputs.warmup }}';
const baseline = '${{ steps.args.outputs.baseline-name }}';
const feature = '${{ steps.args.outputs.feature-name }}';
const samply = '${{ steps.args.outputs.samply }}' === 'true';
const tracingChrome = '${{ steps.args.outputs.tracing-chrome }}' === 'true';
const slack = '${{ steps.args.outputs.slack }}' || 'always';
const bigBlocks = '${{ steps.args.outputs.big-blocks }}' === 'true';
const bal = '${{ steps.args.outputs.bal }}' || 'false';
const samplyNote = samply ? ', samply: `enabled`' : '';
const tracingChromeNote = tracingChrome ? ', tracing-chrome: `enabled`' : '';
const slackNote = slack !== 'always' ? `, slack: \`${slack}\`` : '';
const balNote = bal !== 'false' ? `, BAL: \`${bal}\`` : '';
const cores = '${{ steps.args.outputs.cores }}';
const coresNote = cores && cores !== '0' ? `, cores: \`${cores}\`` : '';
const runPairs = '${{ steps.args.outputs.run-pairs }}' || '6';
const runOrder = '${{ steps.args.outputs.run-order }}';
const runPairsNote = `, run-pairs: \`${runPairs}\`${runOrder ? ` (${runOrder})` : ''}`;
const otlpEnabled = '${{ steps.args.outputs.otlp }}' !== 'false';
const otlpNote = !otlpEnabled ? ', otlp: `disabled`' : '';
const waitTimeVal = '${{ steps.args.outputs.wait-time }}';
const waitTimeNote = waitTimeVal ? `, wait-time: \`${waitTimeVal}\`` : '';
const baselineArgsVal = '${{ steps.args.outputs.baseline-args }}';
const baselineArgsNote = baselineArgsVal ? `, baseline-args: \`${baselineArgsVal}\`` : '';
const featureArgsVal = '${{ steps.args.outputs.feature-args }}';
const featureArgsNote = featureArgsVal ? `, feature-args: \`${featureArgsVal}\`` : '';
const blocksDesc = bigBlocks ? `${blocks} big blocks, ${warmup} warmup blocks` : `${blocks} blocks, ${warmup} warmup blocks`;
const config = `**Config:** ${blocksDesc}, baseline: \`${baseline}\`, feature: \`${feature}\`${samplyNote}${tracingChromeNote}${slackNote}${balNote}${coresNote}${runPairsNote}${otlpNote}${waitTimeNote}${baselineArgsNote}${featureArgsNote}`;
const { data: comment } = await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: parseInt(pr),
body: `cc @${actor}\n\n🚀 Benchmark queued! [View run](${runUrl})\n\n⏳ **Status:** Waiting for runner...${queueMsg}\n\n${config}`,
});
core.setOutput('comment-id', String(comment.id));
core.setOutput('queue-position', String(ahead || 0));
- name: Poll queue position
if: steps.ack.outputs.comment-id && steps.ack.outputs.queue-position != '0'
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
github-token: ${{ secrets.DEREK_PAT }}
script: |
const pr = '${{ steps.args.outputs.pr }}';
const commentId = parseInt('${{ steps.ack.outputs.comment-id }}');
const actor = '${{ steps.args.outputs.actor }}';
const blocks = '${{ steps.args.outputs.blocks }}';
const warmup = '${{ steps.args.outputs.warmup }}';
const baseline = '${{ steps.args.outputs.baseline-name }}';
const feature = '${{ steps.args.outputs.feature-name }}';
const samply = '${{ steps.args.outputs.samply }}' === 'true';
const tracingChrome = '${{ steps.args.outputs.tracing-chrome }}' === 'true';
const slack = '${{ steps.args.outputs.slack }}' || 'always';
const bigBlocks = '${{ steps.args.outputs.big-blocks }}' === 'true';
const bal = '${{ steps.args.outputs.bal }}' || 'false';
const samplyNote = samply ? ', samply: `enabled`' : '';
const tracingChromeNote = tracingChrome ? ', tracing-chrome: `enabled`' : '';
const slackNote = slack !== 'always' ? `, slack: \`${slack}\`` : '';
const balNote = bal !== 'false' ? `, BAL: \`${bal}\`` : '';
const cores = '${{ steps.args.outputs.cores }}';
const coresNote = cores && cores !== '0' ? `, cores: \`${cores}\`` : '';
const runPairs = '${{ steps.args.outputs.run-pairs }}' || '6';
const runOrder = '${{ steps.args.outputs.run-order }}';
const runPairsNote = `, run-pairs: \`${runPairs}\`${runOrder ? ` (${runOrder})` : ''}`;
const otlpEnabled = '${{ steps.args.outputs.otlp }}' !== 'false';
const otlpNote = !otlpEnabled ? ', otlp: `disabled`' : '';
const waitTimeVal = '${{ steps.args.outputs.wait-time }}';
const waitTimeNote = waitTimeVal ? `, wait-time: \`${waitTimeVal}\`` : '';
const baselineArgsVal = '${{ steps.args.outputs.baseline-args }}';
const baselineArgsNote = baselineArgsVal ? `, baseline-args: \`${baselineArgsVal}\`` : '';
const featureArgsVal = '${{ steps.args.outputs.feature-args }}';
const featureArgsNote = featureArgsVal ? `, feature-args: \`${featureArgsVal}\`` : '';
const blocksDesc = bigBlocks ? `${blocks} big blocks, ${warmup} warmup blocks` : `${blocks} blocks, ${warmup} warmup blocks`;
const config = `**Config:** ${blocksDesc}, baseline: \`${baseline}\`, feature: \`${feature}\`${samplyNote}${tracingChromeNote}${slackNote}${balNote}${coresNote}${runPairsNote}${otlpNote}${waitTimeNote}${baselineArgsNote}${featureArgsNote}`;
const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
const numRunners = parseInt(process.env.BENCH_RUNNERS) || 1;
async function getQueuePosition() {
const statuses = ['queued', 'in_progress', 'waiting', 'requested', 'pending'];
const allRuns = [];
for (const status of statuses) {
const { data: { workflow_runs: r } } = await github.rest.actions.listWorkflowRuns({
owner: context.repo.owner,
repo: context.repo.repo,
workflow_id: 'bench.yml',
status,
per_page: 100,
});
allRuns.push(...r);
}
const benchRuns = allRuns.filter(r => r.event === 'issue_comment' || r.event === 'workflow_dispatch');
const thisRun = benchRuns.find(r => r.id === context.runId);
const thisCreatedAt = thisRun ? new Date(thisRun.created_at) : new Date();
const totalAhead = benchRuns.filter(r => r.id !== context.runId && new Date(r.created_at) <= thisCreatedAt).length;
return { ahead: Math.max(0, totalAhead - numRunners + 1), numRunners };
}
let lastPosition = parseInt('${{ steps.ack.outputs.queue-position }}');
const sleep = ms => new Promise(r => setTimeout(r, ms));
while (true) {
await sleep(10_000);
try {
const { ahead, numRunners } = await getQueuePosition();
if (ahead !== lastPosition) {
lastPosition = ahead;
const queueMsg = ahead > 0
? `\n🔢 **Queue position:** ${ahead} job(s) ahead (${numRunners} runner(s))`
: '';
await github.rest.issues.updateComment({
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: commentId,
body: `cc @${actor}\n\n🚀 Benchmark queued! [View run](${runUrl})\n\n⏳ **Status:** Waiting for runner...${queueMsg}\n\n${config}`,
});
}
if (ahead === 0) break;
} catch (e) {
core.info(`Queue poll error: ${e.message}`);
}
}
bench-txgen:
needs: bench-ack
name: bench-txgen
runs-on: [self-hosted, Linux, X64, available]
permissions:
contents: read
pull-requests: write
timeout-minutes: 120
env:
BENCH_RPC_URL: http://ethereum-mainnet-stable-reth-greedy-goose:8545
SCHELK_MOUNT: /reth-bench
RETH_SCOPE: reth-bench.scope
BENCH_WORK_DIR: ${{ github.workspace }}/bench-work
BENCH_PR: ${{ needs.bench-ack.outputs.pr }}
BENCH_PR_HEAD_SHA: ${{ needs.bench-ack.outputs.pr-head-sha }}
BENCH_PR_HEAD_REF: ${{ needs.bench-ack.outputs.pr-head-ref }}
BENCH_PR_HEAD_REPO: ${{ needs.bench-ack.outputs.pr-head-repo }}
BENCH_ACTOR: ${{ needs.bench-ack.outputs.actor }}
BENCH_BLOCKS: ${{ needs.bench-ack.outputs.blocks }}
BENCH_WARMUP_BLOCKS: ${{ needs.bench-ack.outputs.warmup }}
BENCH_SAMPLY: ${{ needs.bench-ack.outputs.samply }}
BENCH_TRACING_CHROME: ${{ needs.bench-ack.outputs.tracing-chrome }}
BENCH_CORES: ${{ needs.bench-ack.outputs.cores }}
BENCH_BIG_BLOCKS: ${{ needs.bench-ack.outputs.big-blocks }}
BENCH_BIG_BLOCKS_TARGET_GAS: ${{ needs.bench-ack.outputs.big-blocks-target-gas }}
BENCH_BAL: ${{ needs.bench-ack.outputs.bal }}
BENCH_WAIT_TIME: ${{ needs.bench-ack.outputs.wait-time }}
BENCH_BASELINE_ARGS: ${{ needs.bench-ack.outputs.baseline-args }}
BENCH_FEATURE_ARGS: ${{ needs.bench-ack.outputs.feature-args }}
BENCH_RUN_PAIRS: ${{ needs.bench-ack.outputs.run-pairs }}
BENCH_RUN_ORDER: ${{ needs.bench-ack.outputs.run-order }}
BENCH_OTLP: ${{ needs.bench-ack.outputs.otlp }}
BENCH_METRICS: ${{ needs.bench-ack.outputs.metrics }}
BENCH_COMMENT_ID: ${{ needs.bench-ack.outputs.comment-id }}
BENCH_SLACK: ${{ needs.bench-ack.outputs.slack }}
BENCH_NODE_BIN: ${{ needs.bench-ack.outputs.node-bin }}
BENCH_SNAPSHOT_MANIFEST_URL: ${{ secrets.BENCH_SNAPSHOT_MANIFEST_URL }}
BENCH_METRICS_ADDR: "127.0.0.1:9001"
BENCH_TARGET_METRICS_CONFIG: .github/config/bench-metrics-targets.json
BENCH_TARGET_METRICS_SCRAPE_INTERVAL_MS: "200"
BENCH_OTLP_TRACES_ENDPOINT: ${{ needs.bench-ack.outputs.otlp != 'false' && secrets.BENCH_OTLP_TRACES_ENDPOINT || '' }}
BENCH_OTLP_LOGS_ENDPOINT: ${{ needs.bench-ack.outputs.otlp != 'false' && secrets.BENCH_OTLP_LOGS_ENDPOINT || '' }}
BENCH_VICTORIAMETRICS_URL: ${{ needs.bench-ack.outputs.metrics == 'true' && secrets.BENCH_VICTORIAMETRICS_URL || '' }}
steps:
- name: Clean up previous bench-work
run: sudo rm -rf "$BENCH_WORK_DIR" 2>/dev/null || true
- name: Resolve checkout ref
id: checkout-ref
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
script: |
if (!process.env.BENCH_PR) {
core.setOutput('repository', '${{ github.repository }}');
core.setOutput('ref', '${{ github.ref }}');
return;
}
const sha = process.env.BENCH_PR_HEAD_SHA;
if (!sha) {
core.setFailed('BENCH_PR_HEAD_SHA is not set — ack job must pin the PR head SHA');
return;
}
const repository = process.env.BENCH_PR_HEAD_REPO || '${{ github.repository }}';
core.info(`PR #${process.env.BENCH_PR}, using ${repository}@${sha}`);
core.setOutput('repository', repository);
core.setOutput('ref', sha);
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
repository: ${{ steps.checkout-ref.outputs.repository }}
persist-credentials: false
submodules: true
fetch-depth: 0
ref: ${{ steps.checkout-ref.outputs.ref }}
- name: Resolve job URL and update status
if: env.BENCH_COMMENT_ID
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
github-token: ${{ secrets.DEREK_PAT }}
script: |
const { data: jobs } = await github.rest.actions.listJobsForWorkflowRun({
owner: context.repo.owner,
repo: context.repo.repo,
run_id: context.runId,
});
const expectedJobName = 'bench-txgen';
const job = jobs.jobs.find(j => j.name === expectedJobName);
const jobUrl = job ? job.html_url : `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
core.exportVariable('BENCH_JOB_URL', jobUrl);
const blocks = process.env.BENCH_BLOCKS;
const warmup = process.env.BENCH_WARMUP_BLOCKS;
const baseline = '${{ needs.bench-ack.outputs.baseline-name }}';
const feature = '${{ needs.bench-ack.outputs.feature-name }}';
const samply = process.env.BENCH_SAMPLY === 'true';
const tracingChrome = process.env.BENCH_TRACING_CHROME === 'true';
const slack = process.env.BENCH_SLACK || 'always';
const bigBlocks = process.env.BENCH_BIG_BLOCKS === 'true';
const bal = process.env.BENCH_BAL || 'false';
const samplyNote = samply ? ', samply: `enabled`' : '';
const tracingChromeNote = tracingChrome ? ', tracing-chrome: `enabled`' : '';
const slackNote = slack !== 'always' ? `, slack: \`${slack}\`` : '';
const balNote = bal !== 'false' ? `, BAL: \`${bal}\`` : '';
const cores = process.env.BENCH_CORES || '0';
const coresNote = cores && cores !== '0' ? `, cores: \`${cores}\`` : '';
const runPairs = process.env.BENCH_RUN_PAIRS || '6';
const runOrder = process.env.BENCH_RUN_ORDER || '';
const runPairsNote = `, run-pairs: \`${runPairs}\`${runOrder ? ` (${runOrder})` : ''}`;
const otlpEnabled = (process.env.BENCH_OTLP || 'false') !== 'false';
const otlpNote = !otlpEnabled ? ', otlp: `disabled`' : '';
const waitTimeVal = process.env.BENCH_WAIT_TIME || '';
const waitTimeNote = waitTimeVal ? `, wait-time: \`${waitTimeVal}\`` : '';
const baselineArgsVal = process.env.BENCH_BASELINE_ARGS || '';
const baselineArgsNote = baselineArgsVal ? `, baseline-args: \`${baselineArgsVal}\`` : '';
const featureArgsVal = process.env.BENCH_FEATURE_ARGS || '';
const featureArgsNote = featureArgsVal ? `, feature-args: \`${featureArgsVal}\`` : '';
const blocksDesc = bigBlocks ? `${blocks} big blocks, ${warmup} warmup blocks` : `${blocks} blocks, ${warmup} warmup blocks`;
core.exportVariable('BENCH_CONFIG', `**Config:** ${blocksDesc}, baseline: \`${baseline}\`, feature: \`${feature}\`${samplyNote}${tracingChromeNote}${slackNote}${balNote}${coresNote}${runPairsNote}${otlpNote}${waitTimeNote}${baselineArgsNote}${featureArgsNote}`);
const { buildBody } = require('./.github/scripts/bench-update-status.js');
await github.rest.issues.updateComment({
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: parseInt(process.env.BENCH_COMMENT_ID),
body: buildBody('Building binaries...'),
});
>>>>>>> v2.3.0
- uses: dtolnay/rust-toolchain@stable
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
continue-on-error: true
- uses: astral-sh/setup-uv@bd01e18f51369d5a26f1651c3cb451d3417e3bba # v6.3.1
- name: Mask OTLP endpoints
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
<<<<<<< HEAD
cache-on-failure: true
- name: Install cargo-codspeed
uses: taiki-e/install-action@v2
with:
tool: cargo-codspeed
- name: Build the benchmark target(s)
run: ./.github/scripts/codspeed-build.sh
- name: Run the benchmarks
uses: CodSpeedHQ/action@v4
with:
run: cargo codspeed run --workspace
mode: instrumentation
token: ${{ secrets.CODSPEED_TOKEN }}
=======
script: |
for (const envName of ['BENCH_OTLP_TRACES_ENDPOINT', 'BENCH_OTLP_LOGS_ENDPOINT']) {
const endpoint = process.env[envName] || '';
if (!endpoint) {
continue;
}
core.setSecret(endpoint);
try {
const url = new URL(endpoint);
url.username = '';
url.password = '';
core.setSecret(url.toString());
} catch {
// The CLI parser will report malformed endpoints later; this step only masks.
}
}
- name: Install dependencies
env:
DEREK_TOKEN: ${{ secrets.DEREK_TOKEN }}
run: |
mkdir -p "$HOME/.local/bin"
# apt packages
sudo apt-get update -qq
sudo apt-get install -y --no-install-recommends \
python3 make jq zstd curl dmsetup \
linux-tools-"$(uname -r)" || \
sudo apt-get install -y --no-install-recommends linux-tools-generic
# mc (MinIO client)
if ! command -v mc &>/dev/null; then
curl -sSfL https://dl.min.io/client/mc/release/linux-amd64/mc -o "$HOME/.local/bin/mc"
chmod +x "$HOME/.local/bin/mc"
fi
# uv (Python package manager)
if ! command -v uv &>/dev/null; then
curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR="$HOME/.local/bin" sh
fi
# Configure git auth for private repos
git config --global url."https://x-access-token:${DEREK_TOKEN}@github.com/".insteadOf "https://github.com/"
# thin-provisioning-tools (era_invalidate, required by schelk)
if ! command -v era_invalidate &>/dev/null; then
git clone --depth 1 https://github.com/jthornber/thin-provisioning-tools /tmp/tpt
sudo make -C /tmp/tpt install
rm -rf /tmp/tpt
fi
# schelk (snapshot rollback tool, invoked via sudo)
if ! sudo sh -c 'command -v schelk' &>/dev/null; then
cargo install --git https://github.com/tempoxyz/schelk --locked
sudo install "$HOME/.cargo/bin/schelk" /usr/local/bin/
fi
# samply (optional CPU profiler, invoked via sudo)
if [ "${BENCH_SAMPLY:-false}" = "true" ] && ! sudo sh -c 'command -v samply' &>/dev/null; then
cargo install samply --git https://github.com/DaniPopes/samply --branch edge --locked
sudo install "$HOME/.cargo/bin/samply" /usr/local/bin/
fi
# Verify all required tools are available
- name: Check dependencies
run: |
export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$PATH"
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
missing=()
for cmd in mc schelk cpupower taskset stdbuf python3 curl make uv jq; do
command -v "$cmd" &>/dev/null || missing+=("$cmd")
done
if [ ${#missing[@]} -gt 0 ]; then
echo "::error::Missing required tools: ${missing[*]}"
exit 1
fi
echo "All dependencies found"
# Build binaries
- name: Resolve PR head branch
id: pr-info
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
script: |
if (process.env.BENCH_PR) {
const ref = process.env.BENCH_PR_HEAD_REF || '${{ github.ref_name }}';
const sha = process.env.BENCH_PR_HEAD_SHA || '${{ github.sha }}';
core.setOutput('head-ref', ref);
core.setOutput('head-sha', sha);
} else {
core.setOutput('head-ref', '${{ github.ref_name }}');
core.setOutput('head-sha', '${{ github.sha }}');
}
- name: Resolve refs
id: refs
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
script: |
const { execSync } = require('child_process');
const run = (cmd) => execSync(cmd, { encoding: 'utf8' }).trim();
const baselineArg = '${{ needs.bench-ack.outputs.baseline }}';
const featureArg = '${{ needs.bench-ack.outputs.feature }}';
let baselineRef, baselineName, featureRef, featureName;
if (baselineArg) {
try { run(`git fetch origin "${baselineArg}" --quiet`); } catch {}
try {
baselineRef = run(`git rev-parse "${baselineArg}"`);
} catch {
baselineRef = run(`git rev-parse "origin/${baselineArg}"`);
}
baselineName = baselineArg;
} else {
try {
baselineRef = run('git merge-base HEAD origin/main');
} catch {
baselineRef = '${{ github.sha }}';
}
baselineName = 'main';
}
if (featureArg) {
try { run(`git fetch origin "${featureArg}" --quiet`); } catch {}
try {
featureRef = run(`git rev-parse "${featureArg}"`);
} catch {
featureRef = run(`git rev-parse "origin/${featureArg}"`);
}
featureName = featureArg;
} else {
featureRef = '${{ steps.pr-info.outputs.head-sha }}';
featureName = '${{ steps.pr-info.outputs.head-ref }}';
}
core.setOutput('baseline-ref', baselineRef);
core.setOutput('baseline-name', baselineName);
core.setOutput('feature-ref', featureRef);
core.setOutput('feature-name', featureName);
- name: Prepare source dirs
run: |
prepare_source_dir() {
local dir="$1"
local ref="$2"
if [ -d "$dir" ]; then
git -C "$dir" reset --hard HEAD
git -C "$dir" clean -fdx
git -C "$dir" fetch origin "$ref"
else
git clone . "$dir"
fi
git -C "$dir" checkout --force "$ref"
}
BASELINE_REF="${{ steps.refs.outputs.baseline-ref }}"
prepare_source_dir ../reth-baseline "$BASELINE_REF"
FEATURE_REF="${{ steps.refs.outputs.feature-ref }}"
prepare_source_dir ../reth-feature "$FEATURE_REF"
- name: Build binaries
id: build
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
TXGEN_DEPLOY_KEY: ${{ secrets.TXGEN_DEPLOY_KEY }}
TXGEN_TOKEN: ${{ secrets.TXGEN_TOKEN }}
GH_PROJECT_TOKEN: ${{ secrets.GH_PROJECT_TOKEN }}
DEREK_PAT: ${{ secrets.DEREK_PAT }}
DEREK_TOKEN: ${{ secrets.DEREK_TOKEN }}
BENCH_REPO: ${{ github.repository }}
run: |
BASELINE_DIR="$(cd ../reth-baseline && pwd)"
FEATURE_DIR="$(cd ../reth-feature && pwd)"
.github/scripts/bench-txgen-install.sh
.github/scripts/bench-txgen-build.sh baseline "${BASELINE_DIR}" "${{ steps.refs.outputs.baseline-ref }}" &
PID_BASELINE=$!
.github/scripts/bench-txgen-build.sh feature "${FEATURE_DIR}" "${{ steps.refs.outputs.feature-ref }}" &
PID_FEATURE=$!
FAIL=0
wait $PID_BASELINE || FAIL=1
wait $PID_FEATURE || FAIL=1
if [ $FAIL -ne 0 ]; then
echo "::error::One or more build tasks failed"
exit 1
fi
- name: Sync snapshot
id: snapshot-check
run: |
BENCH_RETH_BINARY="$(pwd)/../reth-feature/target/profiling/${BENCH_NODE_BIN}" \
.github/scripts/bench-reth-snapshot.sh
# System tuning for reproducible benchmarks
- name: System setup
run: |
# Switch amd-pstate to passive mode so the kernel governor
# controls frequency directly (EPP is ignored in passive).
echo passive | sudo tee /sys/devices/system/cpu/amd_pstate/status 2>/dev/null || true
sudo cpupower frequency-set -g performance || true
# Pin all cores to the nominal (base) frequency from CPPC.
NOMINAL_KHZ=""
if [ -f /sys/devices/system/cpu/cpu0/acpi_cppc/nominal_freq ]; then
NOMINAL_MHZ=$(cat /sys/devices/system/cpu/cpu0/acpi_cppc/nominal_freq)
NOMINAL_KHZ=$((NOMINAL_MHZ * 1000))
elif [ -f /sys/devices/system/cpu/cpu0/cpufreq/base_frequency ]; then
NOMINAL_KHZ=$(cat /sys/devices/system/cpu/cpu0/cpufreq/base_frequency)
fi
if [ -n "$NOMINAL_KHZ" ] && [ "$NOMINAL_KHZ" -gt 0 ] 2>/dev/null; then
echo "Pinning all cores to nominal frequency: $((NOMINAL_KHZ / 1000)) MHz"
for f in /sys/devices/system/cpu/cpu*/cpufreq/scaling_max_freq; do
echo "$NOMINAL_KHZ" | sudo tee "$f" > /dev/null
done
for f in /sys/devices/system/cpu/cpu*/cpufreq/scaling_min_freq; do
echo "$NOMINAL_KHZ" | sudo tee "$f" > /dev/null
done
fi
sudo swapoff -a || true
echo 0 | sudo tee /proc/sys/kernel/randomize_va_space || true
# Disable SMT (hyperthreading)
for cpu in /sys/devices/system/cpu/cpu*/topology/thread_siblings_list; do
first=$(cut -d, -f1 < "$cpu" | cut -d- -f1)
current=$(echo "$cpu" | grep -o 'cpu[0-9]*' | grep -o '[0-9]*')
if [ "$current" != "$first" ]; then
echo 0 | sudo tee "/sys/devices/system/cpu/cpu${current}/online" || true
fi
done
echo "Online CPUs: $(nproc)"
# Disable transparent huge pages (compaction causes latency spikes)
for p in /sys/kernel/mm/transparent_hugepage /sys/kernel/mm/transparent_hugepages; do
[ -d "$p" ] && echo never | sudo tee "$p/enabled" && echo never | sudo tee "$p/defrag" && break
done || true
# Replace any stale PM QoS holders left behind by earlier benchmark jobs.
sudo pkill -f '^bench-cpu-dma-latency' 2>/dev/null || true
# Prevent deep C-states (avoids wake-up latency jitter)
sudo bash -c 'exec 3<>/dev/cpu_dma_latency; printf "\0\0\0\0" >&3; exec -a bench-cpu-dma-latency sleep infinity' &
echo "BENCH_CPU_DMA_LATENCY_PID=$!" >> "$GITHUB_ENV"
# Move all IRQs to core 0 (housekeeping core)
for irq in /proc/irq/*/smp_affinity_list; do
echo 0 | sudo tee "$irq" 2>/dev/null || true
done
# Stop noisy background services
sudo systemctl stop \
irqbalance cron atd unattended-upgrades snapd \
prometheus-node-exporter-apt.timer prometheus-node-exporter-apt.service \
prometheus-node-exporter-nvme.timer prometheus-node-exporter-nvme.service \
prometheus-node-exporter-ipmitool-sensor.timer prometheus-node-exporter-ipmitool-sensor.service \
sysstat-collect.timer sysstat-collect.service \
sysstat-summary.timer sysstat-summary.service \
2>/dev/null || true
# Log environment for reproducibility
echo "=== Benchmark environment ==="
uname -r
lscpu | grep -E 'Model name|CPU\(s\)|MHz|NUMA'
cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor
cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq
echo "scaling_min_freq: $(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_min_freq)"
echo "scaling_max_freq: $(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_max_freq)"
cat /sys/kernel/mm/transparent_hugepage/enabled 2>/dev/null || cat /sys/kernel/mm/transparent_hugepages/enabled 2>/dev/null || echo "THP: unknown"
free -h
# Clean up any leftover state
- name: Pre-flight cleanup
run: |
sudo systemctl stop "$RETH_SCOPE" 2>/dev/null || true
sudo systemctl reset-failed "$RETH_SCOPE" 2>/dev/null || true
sudo schelk recover -y --kill || sudo schelk full-recover -y || true
rm -rf "$BENCH_WORK_DIR"
mkdir -p "$BENCH_WORK_DIR"
- name: Initialize benchmark metrics labels
run: |
if [ -n "${BENCH_VICTORIAMETRICS_URL:-}" ]; then
echo "::add-mask::$BENCH_VICTORIAMETRICS_URL"
fi
quote_arg() {
local value="$1"
if [[ "$value" =~ ^[A-Za-z0-9._/:=@+-]+$ ]]; then
printf '%s' "$value"
else
printf "'%s'" "${value//\'/\'\\\'\'}"
fi
}
derek_cmd=(derek bench)
[ "$BENCH_BIG_BLOCKS" = "true" ] && derek_cmd+=(big-blocks="${BENCH_BIG_BLOCKS_TARGET_GAS:-true}")
derek_cmd+=(
blocks="$BENCH_BLOCKS"
warmup="$BENCH_WARMUP_BLOCKS"
baseline="${{ needs.bench-ack.outputs.baseline-name }}"
feature="${{ needs.bench-ack.outputs.feature-name }}"
bal="$BENCH_BAL"
cores="$BENCH_CORES"
run-pairs="$BENCH_RUN_PAIRS"
otlp="$BENCH_OTLP"
metrics="$BENCH_METRICS"
slack="$BENCH_SLACK"
)
[ "$BENCH_SAMPLY" = "true" ] && derek_cmd+=(samply)
[ "$BENCH_TRACING_CHROME" = "true" ] && derek_cmd+=(tracing-chrome)
[ -n "$BENCH_WAIT_TIME" ] && derek_cmd+=(wait-time="$BENCH_WAIT_TIME")
[ -n "$BENCH_BASELINE_ARGS" ] && derek_cmd+=(baseline-args="$BENCH_BASELINE_ARGS")
[ -n "$BENCH_FEATURE_ARGS" ] && derek_cmd+=(feature-args="$BENCH_FEATURE_ARGS")
DEREK_BENCH_COMMAND=""
for arg in "${derek_cmd[@]}"; do
if [ -n "$DEREK_BENCH_COMMAND" ]; then
DEREK_BENCH_COMMAND+=" "
fi
DEREK_BENCH_COMMAND+="$(quote_arg "$arg")"
done
echo "DEREK_BENCH_COMMAND=${DEREK_BENCH_COMMAND}" >> "$GITHUB_ENV"
BENCH_ID="ci-${{ github.run_id }}"
BENCH_REFERENCE_EPOCH=$(date +%s)
echo "BENCH_ID=${BENCH_ID}" >> "$GITHUB_ENV"
echo "BENCH_REFERENCE_EPOCH=${BENCH_REFERENCE_EPOCH}" >> "$GITHUB_ENV"
LABELS_FILE="$(mktemp "${RUNNER_TEMP:-/tmp}/bench-metrics-labels.XXXXXX")"
echo '{}' > "$LABELS_FILE"
echo "BENCH_LABELS_FILE=${LABELS_FILE}" >> "$GITHUB_ENV"
# Extract txgen payloads once so they are reused across all benchmark
# runs instead of re-fetching from the remote RPC for every run.
- name: Extract txgen payloads
run: |
PAYLOADS_DIR="$BENCH_WORK_DIR/txgen-payloads"
.github/scripts/bench-txgen-extract.sh \
"../reth-feature/target/profiling/${BENCH_NODE_BIN}" \
"$PAYLOADS_DIR"
echo "TXGEN_PAYLOADS_DIR=${PAYLOADS_DIR}" >> "$GITHUB_ENV"
# Interleaved run order (A=feature, B=baseline) to reduce systematic bias
# from thermal drift and cache warming.
- name: Run benchmarks
id: run-benchmarks
env:
BASELINE_REF: ${{ steps.refs.outputs.baseline-ref }}
FEATURE_REF: ${{ steps.refs.outputs.feature-ref }}
BENCH_GH_TOKEN: ${{ secrets.DEREK_PAT }}
run: |
set -euo pipefail
build_run_order() {
local run_pairs="$1"
if ! [[ "$run_pairs" =~ ^[1-9][0-9]*$ ]]; then
echo "run pairs must be a positive integer" >&2
return 1
fi
if [ $((run_pairs % 2)) -eq 0 ]; then
printf 'ABBA%.0s' $(seq 1 $((run_pairs / 2)))
else
printf 'AB%.0s' $(seq 1 "$run_pairs")
fi
echo
}
RUN_ORDER="$(build_run_order "${BENCH_RUN_PAIRS:-6}")"
echo "BENCH_RUN_ORDER=${RUN_ORDER}" >> "$GITHUB_ENV"
echo "${RUN_ORDER}" > "$BENCH_WORK_DIR/run_order.txt"
echo "Run order: ${RUN_ORDER} (A=feature, B=baseline)"
run_total="${#RUN_ORDER}"
run_progress=0
feature_index=0
baseline_index=0
for ((i = 0; i < ${#RUN_ORDER}; i++)); do
side="${RUN_ORDER:i:1}"
if [ "$side" = "A" ]; then
run_type="feature"
feature_index=$((feature_index + 1))
run_index="$feature_index"
ref="$FEATURE_REF"
binary="../reth-feature/target/profiling/${BENCH_NODE_BIN}"
elif [ "$side" = "B" ]; then
run_type="baseline"
baseline_index=$((baseline_index + 1))
run_index="$baseline_index"
ref="$BASELINE_REF"
binary="../reth-baseline/target/profiling/${BENCH_NODE_BIN}"
else
echo "::error::Invalid run-order entry '${side}'"
exit 1
fi
run_progress=$((run_progress + 1))
run_label="${run_type}-${run_index}"
.github/scripts/bench-update-status.sh "Running benchmark: ${run_label} (${run_progress}/${run_total})..."
LAST_RUN_START=$(date +%s)
echo "BENCH_LAST_RUN_START=${LAST_RUN_START}" >> "$GITHUB_ENV"
export OTEL_RESOURCE_ATTRIBUTES="benchmark_id=${BENCH_ID},benchmark_run=${run_label},run_type=${run_type},git_ref=${ref}"
cat > "$BENCH_LABELS_FILE" <<LABELS
{"benchmark_run":"${run_label}","run_type":"${run_type}","git_ref":"${ref}","bench_sha":"${ref}","benchmark_id":"${BENCH_ID}","run_start_epoch":"${LAST_RUN_START}","reference_epoch":"${BENCH_REFERENCE_EPOCH}"}
LABELS
taskset -c 0 .github/scripts/bench-txgen-run.sh "$run_type" "$binary" "$BENCH_WORK_DIR/${run_label}"
done
.github/scripts/bench-update-status.sh "Finalizing results..."
- name: Generate observability URLs
id: metrics
if: "!cancelled()"
run: |
FROM_MS=$(( BENCH_REFERENCE_EPOCH * 1000 ))
TO_MS=$(( $(date +%s) * 1000 ))
GRAFANA_URL="https://tempoxyz.grafana.net/d/reth-bench-ghr/reth-bench-ghr?orgId=1&from=${FROM_MS}&to=${TO_MS}&timezone=browser&var-datasource=efk1hcn87dnnkd&var-job=github-reth-bench&var-benchmark_id=${BENCH_ID}&var-benchmark_run=\$__all"
OBSERVABILITY_URLS="$(python3 - "$BENCH_ID" "$FROM_MS" "$TO_MS" <<'PY'
from datetime import datetime, timezone
import json
import sys
from urllib.parse import quote
bench_id, from_ms, to_ms = sys.argv[1:4]
logs_uid = "cfk1hzy08xekgd"
traces_uid = "dfk1hrnxca9s0a"
def iso_from_epoch_ms(ms):
dt = datetime.fromtimestamp(int(ms) / 1000, tz=timezone.utc)
return dt.isoformat(timespec="milliseconds").replace("+00:00", "Z")
from_time = iso_from_epoch_ms(from_ms)
to_time = iso_from_epoch_ms(to_ms)
def explore_url(panes):
encoded = quote(json.dumps(panes, separators=(",", ":")), safe="")
return f"https://tempoxyz.grafana.net/explore?schemaVersion=1&panes={encoded}&orgId=1"
logs_pane = {
"pw4": {
"datasource": logs_uid,
"queries": [{
"refId": "A",
"datasource": {"type": "victoriametrics-logs-datasource", "uid": logs_uid},
"editorMode": "code",
"expr": f"benchmark_id:{bench_id}",
"queryType": "instant",
}],
"range": {"from": from_time, "to": to_time},
"panelsState": {
"logs": {
"sortOrder": "Descending",
"columns": ["Time", "Line"],
"visualisationType": "table",
"labelFieldName": "labels",
},
},
"compact": False,
},
}
traceql_query = "{resource.benchmark_id=" + json.dumps(bench_id) + "}"
traces_pane = {
"g39": {
"datasource": traces_uid,
"queries": [{
"refId": "A",
"datasource": {"type": "tempo", "uid": traces_uid},
"queryType": "traceqlSearch",
"serviceMapUseNativeHistograms": False,
"filters": [
{
"id": "benchmark-id",
"operator": "=",
"scope": "resource",
"tag": "benchmark_id",
"value": [bench_id],
"valueType": "string",
"isCustomValue": True,
},
],
"query": traceql_query,
}],
"range": {"from": from_time, "to": to_time},
"panelsState": {"logs": {"visualisationType": "table"}},
"compact": False,
},
}
print(explore_url(logs_pane))
print(explore_url(traces_pane))
PY
)"
LOGS_URL="$(printf '%s\n' "$OBSERVABILITY_URLS" | sed -n '1p')"
TRACES_URL="$(printf '%s\n' "$OBSERVABILITY_URLS" | sed -n '2p')"
echo "grafana-url=${GRAFANA_URL}" >> "$GITHUB_OUTPUT"
echo "logs-url=${LOGS_URL}" >> "$GITHUB_OUTPUT"
echo "traces-url=${TRACES_URL}" >> "$GITHUB_OUTPUT"
echo "observability-from-ms=${FROM_MS}" >> "$GITHUB_OUTPUT"
echo "observability-to-ms=${TO_MS}" >> "$GITHUB_OUTPUT"
echo "Grafana URL: ${GRAFANA_URL}"
echo "Logs URL: ${LOGS_URL}"
echo "Traces URL: ${TRACES_URL}"
- name: Scan logs for errors
if: "!cancelled()"
run: |
ERRORS_FILE="$BENCH_WORK_DIR/errors.md"
found=false
mapfile -t RUN_DIRS < <(find "$BENCH_WORK_DIR" -mindepth 1 -maxdepth 1 -type d \( -name 'baseline-*' -o -name 'feature-*' \) -printf '%f\n' | sort -V)
for run_dir in "${RUN_DIRS[@]}"; do
LOG="$BENCH_WORK_DIR/$run_dir/node.log"
if [ ! -f "$LOG" ]; then continue; fi
panics=$(grep -c -E 'panicked at' "$LOG" || true)
errors=$(grep -c ' ERROR ' "$LOG" || true)
if [ "$panics" -gt 0 ] || [ "$errors" -gt 0 ]; then
if [ "$found" = false ]; then
printf '### ⚠️ Node Errors\n\n' >> "$ERRORS_FILE"
found=true
fi
printf '<details><summary><b>%s</b>: %d panic(s), %d error(s)</summary>\n\n' "$run_dir" "$panics" "$errors" >> "$ERRORS_FILE"
if [ "$panics" -gt 0 ]; then
printf '**Panics:**\n```\n' >> "$ERRORS_FILE"
grep -E 'panicked at' "$LOG" | head -10 >> "$ERRORS_FILE"
printf '```\n' >> "$ERRORS_FILE"
fi
if [ "$errors" -gt 0 ]; then
printf '**Errors (first 20):**\n```\n' >> "$ERRORS_FILE"
grep ' ERROR ' "$LOG" | head -20 >> "$ERRORS_FILE"
printf '```\n' >> "$ERRORS_FILE"
fi
printf '\n</details>\n\n' >> "$ERRORS_FILE"
fi
done
- name: Upload samply profiles
if: success() && env.BENCH_SAMPLY == 'true'
run: |
PROFILER_API="https://api.profiler.firefox.com"
PROFILER_ACCEPT="Accept: application/vnd.firefox-profiler+json;version=1.0"
mapfile -t RUN_DIRS < <(find "$BENCH_WORK_DIR" -mindepth 1 -maxdepth 1 -type d \( -name 'baseline-*' -o -name 'feature-*' \) -printf '%f\n' | sort -V)
for run_dir in "${RUN_DIRS[@]}"; do
PROFILE="$BENCH_WORK_DIR/$run_dir/samply-profile.json.gz"
if [ ! -f "$PROFILE" ]; then continue; fi
PROFILE_SIZE=$(du -h "$PROFILE" | cut -f1)
echo "Uploading $run_dir samply profile (${PROFILE_SIZE}) to Firefox Profiler..."
# Upload compressed profile and get JWT back
JWT=$(curl -sf -X POST \
-H "Content-Type: application/octet-stream" \
-H "$PROFILER_ACCEPT" \
--data-binary "@$PROFILE" \
"$PROFILER_API/compressed-store") || {
echo "::warning::Failed to upload $run_dir profile to Firefox Profiler"
continue
}
# Extract profileToken from JWT payload (header.payload.signature)
PAYLOAD=$(echo "$JWT" | cut -d. -f2)
# Fix base64 padding
case $(( ${#PAYLOAD} % 4 )) in
2) PAYLOAD="${PAYLOAD}==" ;;
3) PAYLOAD="${PAYLOAD}=" ;;
esac
PROFILE_TOKEN=$(echo "$PAYLOAD" | base64 -d 2>/dev/null | python3 -c "import sys,json; print(json.load(sys.stdin)['profileToken'])")
PROFILE_URL="https://profiler.firefox.com/public/${PROFILE_TOKEN}"
echo "Profile uploaded: $PROFILE_URL"
# Shorten the URL
SHORT_URL=$(curl -sf -X POST \
-H "Content-Type: application/json" \
-H "$PROFILER_ACCEPT" \
-d "{\"longUrl\":\"$PROFILE_URL\"}" \
"$PROFILER_API/shorten" | python3 -c "import sys,json; print(json.load(sys.stdin)['shortUrl'])" 2>/dev/null) || SHORT_URL="$PROFILE_URL"
echo "$SHORT_URL" > "$BENCH_WORK_DIR/$run_dir/samply-profile-url.txt"
echo "Short profile URL for $run_dir: $SHORT_URL"
done
# Results & charts
- name: Upload raw results
if: success()
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
with:
name: bench-raw-results
path: ${{ env.BENCH_WORK_DIR }}
- name: Parse results
id: results
if: success()
env:
BASELINE_REF: ${{ steps.refs.outputs.baseline-ref }}
BASELINE_NAME: ${{ steps.refs.outputs.baseline-name }}
FEATURE_NAME: ${{ steps.refs.outputs.feature-name }}
FEATURE_REF: ${{ steps.refs.outputs.feature-ref }}
run: |
git fetch origin "${BASELINE_NAME}" --quiet 2>/dev/null || true
BASELINE_HEAD=$(git rev-parse "origin/${BASELINE_NAME}" 2>/dev/null || echo "")
BEHIND_BASELINE=0
if [ -n "$BASELINE_HEAD" ] && [ "$BASELINE_REF" != "$BASELINE_HEAD" ]; then
BEHIND_BASELINE=$(git rev-list --count "${BASELINE_REF}..${BASELINE_HEAD}" 2>/dev/null || echo "0")
fi
SUMMARY_ARGS="--output-summary $BENCH_WORK_DIR/summary.json"
SUMMARY_ARGS="$SUMMARY_ARGS --output-markdown $BENCH_WORK_DIR/comment.md"
SUMMARY_ARGS="$SUMMARY_ARGS --repo ${{ github.repository }}"
SUMMARY_ARGS="$SUMMARY_ARGS --baseline-ref ${BASELINE_REF}"
SUMMARY_ARGS="$SUMMARY_ARGS --baseline-name ${BASELINE_NAME}"
SUMMARY_ARGS="$SUMMARY_ARGS --feature-name ${FEATURE_NAME}"
SUMMARY_ARGS="$SUMMARY_ARGS --feature-ref ${FEATURE_REF}"
mapfile -t BASELINE_CSVS < <(find "$BENCH_WORK_DIR" -path "$BENCH_WORK_DIR/baseline-*/combined_latency.csv" -type f | sort -V)
mapfile -t FEATURE_CSVS < <(find "$BENCH_WORK_DIR" -path "$BENCH_WORK_DIR/feature-*/combined_latency.csv" -type f | sort -V)
if [ "${#BASELINE_CSVS[@]}" -eq 0 ] || [ "${#FEATURE_CSVS[@]}" -eq 0 ]; then
echo "::error::Missing baseline or feature benchmark CSVs"
exit 1
fi
SUMMARY_ARGS="$SUMMARY_ARGS --baseline-csv ${BASELINE_CSVS[*]}"
SUMMARY_ARGS="$SUMMARY_ARGS --feature-csv ${FEATURE_CSVS[*]}"
SUMMARY_ARGS="$SUMMARY_ARGS --run-pairs ${BENCH_RUN_PAIRS:-6}"
if [ "$BEHIND_BASELINE" -gt 0 ]; then
SUMMARY_ARGS="$SUMMARY_ARGS --behind-baseline $BEHIND_BASELINE"
fi
if [ "${BENCH_BIG_BLOCKS:-false}" = "true" ]; then
SUMMARY_ARGS="$SUMMARY_ARGS --big-blocks"
fi
if [ -n "${BENCH_WARMUP_BLOCKS:-}" ]; then
SUMMARY_ARGS="$SUMMARY_ARGS --warmup-blocks $BENCH_WARMUP_BLOCKS"
fi
if [ -n "${BENCH_WAIT_TIME:-}" ]; then
SUMMARY_ARGS="$SUMMARY_ARGS --wait-time $BENCH_WAIT_TIME"
fi
if [ -n "${BENCH_BAL:-}" ] && [ "${BENCH_BAL}" != "false" ]; then
SUMMARY_ARGS="$SUMMARY_ARGS --bal-mode $BENCH_BAL"
fi
if [ -n "${BENCH_ID:-}" ]; then
SUMMARY_ARGS="$SUMMARY_ARGS --benchmark-id $BENCH_ID"
fi
GRAFANA_URL='${{ steps.metrics.outputs.grafana-url }}'
if [ "${BENCH_METRICS:-false}" = "true" ] && [ -n "$GRAFANA_URL" ]; then
SUMMARY_ARGS="$SUMMARY_ARGS --grafana-url $GRAFANA_URL"
fi
LOGS_URL='${{ steps.metrics.outputs.logs-url }}'
if [ -n "$LOGS_URL" ]; then
SUMMARY_ARGS="$SUMMARY_ARGS --logs-url $LOGS_URL"
fi
TRACES_URL='${{ steps.metrics.outputs.traces-url }}'
if [ -n "$TRACES_URL" ]; then
SUMMARY_ARGS="$SUMMARY_ARGS --traces-url $TRACES_URL"
fi
OBSERVABILITY_FROM_MS='${{ steps.metrics.outputs.observability-from-ms }}'
OBSERVABILITY_TO_MS='${{ steps.metrics.outputs.observability-to-ms }}'
if [ -n "$OBSERVABILITY_FROM_MS" ] && [ -n "$OBSERVABILITY_TO_MS" ]; then
SUMMARY_ARGS="$SUMMARY_ARGS --observability-from-ms $OBSERVABILITY_FROM_MS"
SUMMARY_ARGS="$SUMMARY_ARGS --observability-to-ms $OBSERVABILITY_TO_MS"
fi
if [ -n "${BENCH_TARGET_METRICS_CONFIG:-}" ]; then
SUMMARY_ARGS="$SUMMARY_ARGS --target-metrics-config $BENCH_TARGET_METRICS_CONFIG"
fi
# shellcheck disable=SC2086
python3 .github/scripts/bench-reth-summary.py $SUMMARY_ARGS
- name: Generate charts
if: success()
env:
BASELINE_NAME: ${{ steps.refs.outputs.baseline-name }}
FEATURE_NAME: ${{ steps.refs.outputs.feature-name }}
run: |
CHART_ARGS="--output-dir $BENCH_WORK_DIR/charts"
mapfile -t FEATURE_CSVS < <(find "$BENCH_WORK_DIR" -path "$BENCH_WORK_DIR/feature-*/combined_latency.csv" -type f | sort -V)
mapfile -t BASELINE_CSVS < <(find "$BENCH_WORK_DIR" -path "$BENCH_WORK_DIR/baseline-*/combined_latency.csv" -type f | sort -V)
if [ "${#BASELINE_CSVS[@]}" -eq 0 ] || [ "${#FEATURE_CSVS[@]}" -eq 0 ]; then
echo "::error::Missing baseline or feature benchmark CSVs"
exit 1
fi
CHART_ARGS="$CHART_ARGS --feature ${FEATURE_CSVS[*]}"
CHART_ARGS="$CHART_ARGS --baseline ${BASELINE_CSVS[*]}"
CHART_ARGS="$CHART_ARGS --baseline-name ${BASELINE_NAME}"
CHART_ARGS="$CHART_ARGS --feature-name ${FEATURE_NAME}"
# shellcheck disable=SC2086
uv run --with matplotlib python3 .github/scripts/bench-reth-charts.py $CHART_ARGS
- name: Upload results
if: "!cancelled()"
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
with:
name: bench-results
path: ${{ env.BENCH_WORK_DIR }}
- name: Push benchmark artifacts
id: push-charts
if: success()
env:
DEREK_TOKEN: ${{ secrets.DEREK_TOKEN }}
RUN_ID: ${{ github.run_id }}
run: |
PR_NUMBER="${BENCH_PR:-0}"
CHART_DIR="pr/${PR_NUMBER}/${RUN_ID}"
CHARTS_REPO="https://x-access-token:${DEREK_TOKEN}@github.com/decofe/reth-bench-charts.git"
TMP_DIR=$(mktemp -d)
if git clone --depth 1 --filter=blob:none --sparse "${CHARTS_REPO}" "${TMP_DIR}" 2>/dev/null; then
git -C "${TMP_DIR}" sparse-checkout set --no-cone "${CHART_DIR}"
true
else
git init "${TMP_DIR}"
git -C "${TMP_DIR}" remote add origin "${CHARTS_REPO}"
fi
.github/scripts/configure-git-token-user.sh "${TMP_DIR}" "${DEREK_TOKEN}"
mkdir -p "${TMP_DIR}/${CHART_DIR}"
cp "$BENCH_WORK_DIR"/charts/*.png "${TMP_DIR}/${CHART_DIR}/"
if [ "${BENCH_TRACING_CHROME:-false}" = "true" ]; then
TRACE_DIR="${CHART_DIR}/tracing-chrome"
mkdir -p "${TMP_DIR}/${TRACE_DIR}"
mapfile -t RUN_DIRS < <(find "$BENCH_WORK_DIR" -mindepth 1 -maxdepth 1 -type d \( -name 'baseline-*' -o -name 'feature-*' \) -printf '%f\n' | sort -V)
for run_dir in "${RUN_DIRS[@]}"; do
TRACE="$BENCH_WORK_DIR/$run_dir/tracing-chrome-profile.json"
if [ ! -f "$TRACE" ]; then continue; fi
TRACE_SIZE=$(du -h "$TRACE" | cut -f1)
echo "Adding $run_dir Chrome trace (${TRACE_SIZE}) to benchmark artifacts..."
gzip -c "$TRACE" > "${TMP_DIR}/${TRACE_DIR}/${run_dir}.json.gz"
done
fi
git -C "${TMP_DIR}" add "${CHART_DIR}"
if git -C "${TMP_DIR}" diff --cached --quiet; then
echo "Artifacts for ${CHART_DIR} are already present, skipping push"
PUSH_SHA="$(git -C "${TMP_DIR}" rev-parse HEAD)"
else
git -C "${TMP_DIR}" commit -m "bench artifacts for PR #${PR_NUMBER} run ${RUN_ID}"
for attempt in 1 2 3 4 5; do
if git -C "${TMP_DIR}" push origin HEAD:main; then
break
fi
if [ "$attempt" -eq 5 ]; then
echo "::error::Failed to push benchmark artifacts after ${attempt} attempts"
rm -rf "${TMP_DIR}"
exit 1
fi
sleep "$attempt"
git -C "${TMP_DIR}" fetch origin main
git -C "${TMP_DIR}" rebase origin/main
done
PUSH_SHA="$(git -C "${TMP_DIR}" rev-parse HEAD)"
fi
if [ "${BENCH_TRACING_CHROME:-false}" = "true" ]; then
RAW_BASE="https://raw.githubusercontent.com/decofe/reth-bench-charts/${PUSH_SHA}/${CHART_DIR}/tracing-chrome"
mapfile -t RUN_DIRS < <(find "$BENCH_WORK_DIR" -mindepth 1 -maxdepth 1 -type d \( -name 'baseline-*' -o -name 'feature-*' \) -printf '%f\n' | sort -V)
for run_dir in "${RUN_DIRS[@]}"; do
if [ ! -f "${TMP_DIR}/${CHART_DIR}/tracing-chrome/${run_dir}.json.gz" ]; then continue; fi
RAW_URL="${RAW_BASE}/${run_dir}.json.gz"
PERFETTO_URL="$(python3 - "$RAW_URL" "${run_dir}.json.gz" <<'PY'
import sys
from urllib.parse import quote
raw_url, file_name = sys.argv[1:3]
print(
"https://ui.perfetto.dev/#!/?url="
+ quote(raw_url, safe="")
+ "&fileName="
+ quote(file_name, safe="")
)
PY
)"
echo "$PERFETTO_URL" > "$BENCH_WORK_DIR/$run_dir/tracing-chrome-profile-url.txt"
echo "Perfetto URL for $run_dir: $PERFETTO_URL"
done
fi
echo "sha=${PUSH_SHA}" >> "$GITHUB_OUTPUT"
rm -rf "${TMP_DIR}"
- name: Compare & comment
if: success() && env.BENCH_COMMENT_ID
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
github-token: ${{ secrets.DEREK_PAT }}
script: |
const fs = require('fs');
let comment = '';
try {
comment = fs.readFileSync(process.env.BENCH_WORK_DIR + '/comment.md', 'utf8');
} catch (e) {
comment = '⚠️ Engine benchmark completed but failed to generate comparison.';
}
const sha = '${{ steps.push-charts.outputs.sha }}';
const prNumber = process.env.BENCH_PR || '0';
const runId = '${{ github.run_id }}';
if (sha) {
const baseUrl = `https://raw.githubusercontent.com/decofe/reth-bench-charts/${sha}/pr/${prNumber}/${runId}`;
const charts = [
{ file: 'latency_throughput.png', label: 'Latency, Throughput & Diff' },
{ file: 'wait_breakdown.png', label: 'Wait Time Breakdown' },
{ file: 'gas_vs_latency.png', label: 'Gas vs Latency' },
];
let chartMarkdown = '\n\n### Charts\n\n';
for (const chart of charts) {
chartMarkdown += `<details><summary>${chart.label}</summary>\n\n`;
chartMarkdown += `![${chart.label}](${baseUrl}/${chart.file})\n\n`;
chartMarkdown += `</details>\n\n`;
}
comment += chartMarkdown;
}
// Samply profile links (URLs point directly to Firefox Profiler)
if (process.env.BENCH_SAMPLY === 'true') {
const { loadSamplyUrls } = require('./.github/scripts/bench-utils');
const samplyUrls = loadSamplyUrls(process.env.BENCH_WORK_DIR);
const links = [];
for (const [run, url] of Object.entries(samplyUrls)) {
links.push(`- **${run}**: [Firefox Profiler](${url})`);
}
if (links.length > 0) {
comment += `\n\n### Samply Profiles\n\n${links.join('\n')}\n`;
}
}
if (process.env.BENCH_TRACING_CHROME === 'true') {
const { loadTracingChromeUrls } = require('./.github/scripts/bench-utils');
const tracingChromeUrls = loadTracingChromeUrls(process.env.BENCH_WORK_DIR);
const links = [];
for (const [run, url] of Object.entries(tracingChromeUrls)) {
links.push(`- **${run}**: [Perfetto](${url})`);
}
if (links.length > 0) {
comment += `\n\n### Chrome Traces\n\n${links.join('\n')}\n`;
}
}
// Node errors (panics / ERROR logs)
try {
const errors = fs.readFileSync(process.env.BENCH_WORK_DIR + '/errors.md', 'utf8');
if (errors.trim()) {
comment += '\n\n' + errors;
}
} catch (e) {}
const jobUrl = process.env.BENCH_JOB_URL || `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
const body = `cc @${process.env.BENCH_ACTOR}\n\n✅ Benchmark complete! [View job](${jobUrl})\n\n${comment}`;
const ackCommentId = process.env.BENCH_COMMENT_ID;
if (ackCommentId) {
await github.rest.issues.updateComment({
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: parseInt(ackCommentId),
body,
});
}
- name: Write job summary
if: success()
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
script: |
const jobSummary = require('./.github/scripts/bench-job-summary.js');
await jobSummary({
core,
context,
chartSha: '${{ steps.push-charts.outputs.sha }}',
logsUrl: '${{ steps.metrics.outputs.logs-url }}',
tracesUrl: '${{ steps.metrics.outputs.traces-url }}',
runId: '${{ github.run_id }}',
});
- name: Send Slack notification (success)
if: success() && (env.BENCH_SLACK == 'always' || env.BENCH_SLACK == 'on-win')
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
env:
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
with:
script: |
const notify = require('./.github/scripts/bench-slack-notify.js');
await notify.success({ core, context });
- name: Update status (failed)
if: failure() && env.BENCH_COMMENT_ID
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
github-token: ${{ secrets.DEREK_PAT }}
script: |
const steps_status = [
['building binaries', '${{ steps.build.outcome }}'],
['syncing snapshot', '${{ steps.snapshot-check.outcome }}'],
['running benchmarks', '${{ steps.run-benchmarks.outcome }}'],
['parsing results', '${{ steps.results.outcome }}'],
];
const failed = steps_status.find(([, o]) => o === 'failure');
const failedStep = failed ? failed[0] : 'unknown step';
const fs = require('fs');
let errorDetails = '';
try {
const errors = fs.readFileSync(process.env.BENCH_WORK_DIR + '/errors.md', 'utf8');
if (errors.trim()) {
errorDetails = '\n\n' + errors;
}
} catch (e) {}
const jobUrl = process.env.BENCH_JOB_URL || `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
await github.rest.issues.updateComment({
owner: context.repo.owner, repo: context.repo.repo,
comment_id: parseInt(process.env.BENCH_COMMENT_ID),
body: `cc @${process.env.BENCH_ACTOR}\n\n❌ Benchmark failed while ${failedStep}. [View logs](${jobUrl})${errorDetails}`,
});
- name: Send Slack notification (failure)
if: failure() && env.BENCH_SLACK != 'never' && env.BENCH_SLACK != 'on-win'
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
env:
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
with:
script: |
const steps_status = [
['building binaries', '${{ steps.build.outcome }}'],
['syncing snapshot', '${{ steps.snapshot-check.outcome }}'],
['running benchmarks', '${{ steps.run-benchmarks.outcome }}'],
['parsing results', '${{ steps.results.outcome }}'],
];
const failed = steps_status.find(([, o]) => o === 'failure');
const failedStep = failed ? failed[0] : 'unknown step';
const notify = require('./.github/scripts/bench-slack-notify.js');
await notify.failure({ core, context, failedStep });
- name: Update status (cancelled)
if: cancelled() && env.BENCH_COMMENT_ID
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
github-token: ${{ secrets.DEREK_PAT }}
script: |
const jobUrl = process.env.BENCH_JOB_URL || `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
await github.rest.issues.updateComment({
owner: context.repo.owner, repo: context.repo.repo,
comment_id: parseInt(process.env.BENCH_COMMENT_ID),
body: `cc @${process.env.BENCH_ACTOR}\n\n⚠️ Benchmark cancelled. [View logs](${jobUrl})`,
});
- name: Clean build outputs
if: always()
run: |
sudo rm -rf ../reth-baseline/target ../reth-feature/target "$BENCH_WORK_DIR" 2>/dev/null || true
- name: Restore system settings
if: always()
run: |
# Restore frequency scaling to full range
for f in /sys/devices/system/cpu/cpu*/cpufreq/scaling_min_freq; do
cat "$(dirname "$f")/cpuinfo_min_freq" | sudo tee "$f" > /dev/null 2>&1 || true
done
for f in /sys/devices/system/cpu/cpu*/cpufreq/scaling_max_freq; do
cat "$(dirname "$f")/cpuinfo_max_freq" | sudo tee "$f" > /dev/null 2>&1 || true
done
# Restore amd-pstate to active (EPP) mode with powersave governor
echo active | sudo tee /sys/devices/system/cpu/amd_pstate/status 2>/dev/null || true
if [ -n "${BENCH_CPU_DMA_LATENCY_PID:-}" ]; then
sudo kill "$BENCH_CPU_DMA_LATENCY_PID" 2>/dev/null || true
fi
sudo pkill -f '^bench-cpu-dma-latency' 2>/dev/null || true
sudo cpupower frequency-set -g powersave 2>/dev/null || true
sudo systemctl start irqbalance cron atd 2>/dev/null || true
>>>>>>> v2.3.0