From 7d296aac3579453ee1e4d387cf5ed8125df2b52b Mon Sep 17 00:00:00 2001 From: qqrm <4681325+qqrm@users.noreply.github.com> Date: Thu, 15 Oct 2026 06:59:16 +0300 Subject: [PATCH 01/12] Require measured completed-frame WebGPU performance --- .github/workflows/PIPELINES.md | 54 +++++--- .github/workflows/benchmark.yml | 127 +++++++++++------- .github/workflows/build.yml | 2 +- .github/workflows/ci.yml | 10 +- .github/workflows/perf.yml | 56 -------- Cargo.toml | 1 + DOCS/PIPELINES.md | 8 +- DOCS/TESTS.md | 4 +- benches/webgpu_bench.rs | 2 +- scripts/compare_fps.py | 79 ++++++----- scripts/parse_perf_log.py | 53 +++++--- scripts/perf_schema.py | 22 +++ .../rendering/renderer/performance.rs | 23 +++- tests/compare_fps_script.py | 62 +++++++-- tests/parse_perf_log_script.py | 59 +++++--- tests/performance_limit.rs | 54 ++++---- 16 files changed, 364 insertions(+), 252 deletions(-) delete mode 100644 .github/workflows/perf.yml create mode 100644 scripts/perf_schema.py diff --git a/.github/workflows/PIPELINES.md b/.github/workflows/PIPELINES.md index 26cd911..2812102 100644 --- a/.github/workflows/PIPELINES.md +++ b/.github/workflows/PIPELINES.md @@ -1,26 +1,46 @@ # Performance Tracking -## Improvement Checklist -- [ ] Profile GPU memory usage -- [x] Reduce vertex buffer updates -- [x] Cache computations on the GPU side -- [ ] Minimize CPU/GPU synchronization +The required CI performance job runs a release build of `tests/performance_limit.rs` +on Ubuntu 24.04 with Rust 1.99.0, Chrome 155.0.8059.79 and SwiftShader. +It uses deterministic 1,000 / 50,000 / 1,000,000-candle base datasets, an +800x600 canvas and the production renderer's LOD and 4x MSAA. -### Additional Changes -- [x] `CandleSeries` switched to `VecDeque` -- [x] Store `Chart` in state - - [x] Log vector on `VecDeque` +Each dataset receives 10 completed warmup frames followed by five samples of +30 frames. Every frame calls the real renderer and awaits the WebGPU queue +completion callback. A missing adapter, failed render, empty geometry, missing +sample, duplicate sample, zero, negative or non-finite value fails the job. -## WebGPU Performance Benchmark -- [x] Measure FPS for different data volumes -- [x] Log render pass times through `wgpu::CommandEncoder::finish` -- [x] Use `Performance.now()` in the browser for frame timing -- [x] Automate running a fixed-candle test scene +This measures serialized, cached-chart throughput including CPU submission, +browser scheduling and GPU completion latency. It does not measure display FPS, +initial bulk-load cost, live updates, physical GPU speed or memory consumption. +The million source candles are reduced by the production LOD; they are not a +million individual GPU draw calls. -The benchmark is run with: +`scripts/parse_perf_log.py` keeps the three workloads separate. The comparator +requires five positive measurements for every workload in both the result and +the reviewed baseline. Any workload median more than 30% below its baseline +fails. This deliberately broad guard detects large regressions on variable +shared CI hosts; it is not a 5% performance guarantee. The baseline is never +created or updated automatically, and a fast workload cannot hide a slower one. + +Run the same workload with a matching wasm-bindgen runner and ChromeDriver: ```bash -wasm-pack test --chrome --headless +WASM_BINDGEN_TEST_ONLY_WEB=1 WASM_BINDGEN_TEST_TIMEOUT=180 \ +WASM_BINDGEN_TEST_WEBDRIVER_JSON=webdriver.json \ +CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUNNER=wasm-bindgen-test-runner \ +cargo test --release --locked --features performance --test performance_limit -- --nocapture 2>&1 | tee perf.log +python3 scripts/parse_perf_log.py perf.log benchmark_result.json +python3 scripts/compare_fps.py benchmark_result.json DOCS/perf_baseline.json perf_result.json 30 ``` -Workflow `perf.yml` stores the `perf-log` file. +`benchmark.yml` is called by CI for every PR and main push; it can also be +dispatched manually. It always retains raw samples, comparison and environment +metadata as `completed-frame-performance`. The duplicate unmeasured `perf.yml` +workflow has been removed. These measurement jobs have read-only permissions +and no notification credentials. + +A baseline update must retain raw samples and link the successful measurement +run, tested SHA, browser/compiler versions and software backend. Review changes +to workload or environment before recalibrating; do not copy a regressed result +into the baseline just to make CI green. diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index d46df84..422b421 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -1,73 +1,96 @@ name: benchmark on: + workflow_call: workflow_dispatch: - push: - branches: [main] permissions: contents: read -concurrency: - group: benchmark-${{ github.ref }} - cancel-in-progress: true jobs: - perf: - runs-on: ubuntu-latest + measure: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + env: + CARGO_INCREMENTAL: '0' steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master with: - toolchain: stable + toolchain: '1.99.0' targets: wasm32-unknown-unknown - - name: Cache cargo - uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5 + - name: Read the locked WASM runner version + id: bindgen + run: | + python3 - <<'PY' + import os + import tomllib + from pathlib import Path + packages = tomllib.loads(Path('Cargo.lock').read_text())['package'] + versions = {p['version'] for p in packages if p['name'] == 'wasm-bindgen'} + assert len(versions) == 1, versions + with open(os.environ['GITHUB_OUTPUT'], 'a') as output: + output.write(f'version={versions.pop()}\n') + PY + - uses: taiki-e/install-action@f7e5d7c961414b23f5b25b2da9294395d08513ad # v2.87.26 with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo- - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + tool: wasm-bindgen@${{ steps.bindgen.outputs.version }} + fallback: none + - uses: browser-actions/setup-chrome@48ad923757ca74d66703209fe939badbdf80f2f4 # v2 + id: chrome with: - python-version: '3.x' - - name: Install wasm-pack - run: cargo install wasm-pack --version 0.13.1 --locked - - name: Run benchmarks + chrome-version: '155.0.8059.79' + install-chromedriver: true + - name: Select fixed software WebGPU browser and record environment env: - WASM_BINDGEN_TEST_BROWSER_ARGS: "--no-sandbox --enable-unsafe-webgpu" - run: | - set -o pipefail - wasm-pack test --chrome --headless -- --locked | tee perf.log - python scripts/parse_perf_log.py perf.log benchmark_result.json - - name: Analyze FPS - run: | - python scripts/compare_fps.py benchmark_result.json DOCS/perf_baseline.json DOCS/perf_result.json 5 - - name: Update perf markdown + CHROME_BINARY: ${{ steps.chrome.outputs.chrome-path }} + TESTED_SHA: ${{ github.sha }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} run: | - python - <<'EOF' - import json, pathlib - data = json.load(open('DOCS/perf_result.json')) - lines = ['# Latest metrics', '', '| Metric | Value |', '| --- | --- |'] - for k, v in data.items(): - lines.append(f'| {k} | {v} |') - pathlib.Path('DOCS/perf.md').write_text('\n'.join(lines) + '\n') - EOF - - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 - with: - name: perf_results - path: DOCS/perf_result.json - - name: Send benchmark results to bot - continue-on-error: true + python3 - <<'PY' + import json + import os + import platform + import subprocess + from pathlib import Path + path = Path('webdriver.json') + config = json.loads(path.read_text()) + config['goog:chromeOptions']['binary'] = os.environ['CHROME_BINARY'] + path.write_text(json.dumps(config)) + metadata = { + 'tested_sha': os.environ['TESTED_SHA'], + 'run_url': os.environ['RUN_URL'], + 'browser': subprocess.check_output([os.environ['CHROME_BINARY'], '--version'], text=True).strip(), + 'rust': subprocess.check_output(['rustc', '--version'], text=True).strip(), + 'os': platform.platform(), + 'cpu': platform.processor(), + 'backend': 'SwiftShader software WebGPU', + 'browser_flags': config['goog:chromeOptions']['args'], + } + Path('perf-environment.json').write_text(json.dumps(metadata, indent=2) + '\n') + PY + - name: Measure completed frames for each deterministic dataset env: - TELEGRAM_BOT_TOKEN: ${{ secrets.TELEGRAM_BOT_TOKEN }} - TELEGRAM_CHAT_ID: ${{ secrets.TELEGRAM_CHAT_ID }} + CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUNNER: wasm-bindgen-test-runner + CHROMEDRIVER: ${{ steps.chrome.outputs.chromedriver-path }} + WASM_BINDGEN_TEST_ONLY_WEB: '1' + WASM_BINDGEN_TEST_TIMEOUT: '180' + WASM_BINDGEN_TEST_WEBDRIVER_JSON: webdriver.json run: | - if [[ -z "$TELEGRAM_BOT_TOKEN" || -z "$TELEGRAM_CHAT_ID" ]]; then - echo "Telegram secrets are not configured; skipping notification." - exit 0 - fi - scripts/send_to_bot.sh DOCS/perf_result.json "Benchmark results" + set -o pipefail + cargo test --release --locked --features performance --test performance_limit -- --nocapture 2>&1 | tee perf.log + python3 scripts/parse_perf_log.py perf.log benchmark_result.json + - name: Reject per-workload throughput regressions + run: python3 scripts/compare_fps.py benchmark_result.json DOCS/perf_baseline.json perf_result.json 30 + - name: Preserve measurements and environment, including failures + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: completed-frame-performance + path: | + perf.log + perf-environment.json + benchmark_result.json + perf_result.json + if-no-files-found: error diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4397ce3..42e24bf 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,7 +22,7 @@ jobs: toolchain: stable targets: wasm32-unknown-unknown - name: Cache cargo - uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: | ~/.cargo/registry diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dae6a28..de297f1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -103,7 +103,7 @@ jobs: with: toolchain: stable targets: wasm32-unknown-unknown - - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: '24' - name: Read the locked WASM runner version @@ -142,7 +142,7 @@ jobs: with: toolchain: stable targets: wasm32-unknown-unknown - - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: '24' cache: npm @@ -176,9 +176,11 @@ jobs: fallback: none - name: Audit the committed lockfile against current RustSec advisories run: cargo audit --deny unsound + performance: + uses: ./.github/workflows/benchmark.yml ci-gate: if: always() - needs: [checks, render-tests, node-tests, browser, dependency-audit] + needs: [checks, render-tests, node-tests, browser, dependency-audit, performance] runs-on: ubuntu-latest timeout-minutes: 2 steps: @@ -189,9 +191,11 @@ jobs: NODE: ${{ needs.node-tests.result }} BROWSER: ${{ needs.browser.result }} AUDIT: ${{ needs.dependency-audit.result }} + PERFORMANCE: ${{ needs.performance.result }} run: | test "$CHECKS" = success test "$RENDER" = success test "$NODE" = success test "$BROWSER" = success test "$AUDIT" = success + test "$PERFORMANCE" = success diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml deleted file mode 100644 index c9f784b..0000000 --- a/.github/workflows/perf.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: perf -on: - push: - branches: [main] - workflow_dispatch: - -permissions: - contents: read - -concurrency: - group: perf-${{ github.ref }} - cancel-in-progress: true - -jobs: - bench: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master - with: - toolchain: stable - targets: wasm32-unknown-unknown - - name: Cache cargo - uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo- - - name: Install wasm-pack - run: cargo install wasm-pack --version 0.13.1 --locked - - name: Run benchmarks - run: | - set -o pipefail - wasm-pack test --chrome --headless -- --locked | tee perf.log - - name: Upload results - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 - with: - name: perf-log - path: perf.log - - name: Send results to bot - continue-on-error: true - env: - TELEGRAM_BOT_TOKEN: ${{ secrets.TELEGRAM_BOT_TOKEN }} - TELEGRAM_CHAT_ID: ${{ secrets.TELEGRAM_CHAT_ID }} - run: | - if [[ -z "$TELEGRAM_BOT_TOKEN" || -z "$TELEGRAM_CHAT_ID" ]]; then - echo "Telegram secrets are not configured; skipping notification." - exit 0 - fi - scripts/send_to_bot.sh perf.log "Performance results" diff --git a/Cargo.toml b/Cargo.toml index b1f0eb4..f43cc1c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,6 +43,7 @@ quickcheck_macros = "1" parallel = ["rayon"] logic-only = [] render = [] +performance = ["render"] [profile.dev] panic = "unwind" # Preserves stack traces diff --git a/DOCS/PIPELINES.md b/DOCS/PIPELINES.md index c102978..46be053 100644 --- a/DOCS/PIPELINES.md +++ b/DOCS/PIPELINES.md @@ -1,13 +1,13 @@ # GitHub Actions pipelines -- CI runs locked stable Rust checks and strict clippy, Rust browser tests, Node logic tests, deterministic WebGPU end-to-end tests and a current RustSec audit of the committed lockfile for every pull request and main push. The ci-gate requires all five jobs to succeed; skipped or failed jobs block merge. The audit rejects vulnerabilities and unsoundness advisories; informational maintenance/yanked warnings remain visible. +- CI runs locked stable Rust checks and strict clippy, Rust browser tests, Node logic tests, deterministic WebGPU end-to-end tests, measured completed-frame performance and a current RustSec audit of the committed lockfile for every pull request and main push. The ci-gate requires all six jobs to succeed; skipped or failed jobs block merge. The audit rejects vulnerabilities and unsoundness advisories; informational maintenance/yanked warnings remain visible. - Workflow Security runs actionlint, zizmor and signed Dependabot commit provenance checks. Its gate requires all three results. - Weekly Dependabot maintenance batches cover Cargo, npm and Actions. Native auto-merge targets the exact tested head after both gates, with branch protection supplying the final requirements. No review or cron merge sweep is used. - build.yml publishes the development bundle after a main push; release.yml keeps its published-release/manual triggers. Build jobs have read-only repository access; only the separate Pages deployment jobs can write contents. -- benchmark.yml and perf.yml retain performance artifacts. Benchmark results no longer write directly to the protected main branch. Existing optional Telegram notifications remain restricted to those main/release workflows. +- benchmark.yml is the reusable read-only performance job required by CI. It retains measurements, comparison and environment artifacts, including on failures. The duplicate unmeasured perf.yml has been removed. Browser execution runs on hosted CI. The software Vulkan/WebGPU backend validates behavior, not physical-device GPU performance. Geometry fixtures contain no GPU resources and are safe to drop. Actual WebGPU behavior is validated separately by the hosted UI suite. Rust browser tests block Binance DNS; stream abort tests use local handles. -## Known performance reporting issue +## Measured performance guard -The legacy `perf` and `benchmark` workflows currently prove smoke-test/pipeline execution only. Missing FPS samples are converted to zero by `parse_perf_log.py`, and the zero baseline does not enforce performance. Their green status must not be interpreted as a performance guarantee. A follow-up needs a representative measured workload and failure on missing samples; alternatively, these workflows should explicitly become smoke tests without an FPS gate. +The fixed release-mode workload measures five samples for each of three chart sizes, waiting for GPU completion after every frame. Missing/invalid samples fail closed; medians are compared separately against a reviewed measured baseline with a 30% shared-runner tolerance. This is cached-chart serialized throughput on SwiftShader, not physical-device or display FPS. See [the workload and calibration contract](../.github/workflows/PIPELINES.md). diff --git a/DOCS/TESTS.md b/DOCS/TESTS.md index 0887335..83cb50c 100644 --- a/DOCS/TESTS.md +++ b/DOCS/TESTS.md @@ -36,7 +36,9 @@ Failure screenshots, traces, and the HTML report are written to `test-results/` ## Required CI and coverage boundaries -`ci-gate` requires locked stable compilation, strict Clippy, Python regression and dependency-policy tests, browser Rust tests, Node logic tests, and the Playwright WebGPU suite. Browser Rust tests explicitly select the Chrome binary paired with ChromeDriver. Tests run with `--no-fail-fast` so a failing test executable does not hide later failures. +`ci-gate` requires locked stable compilation, strict Clippy, Python regression and dependency-policy tests, browser Rust tests, Node logic tests, the Playwright WebGPU suite, dependency auditing, and measured completed-frame performance. Browser Rust tests explicitly select the Chrome binary paired with ChromeDriver. Tests run with `--no-fail-fast` so a failing test executable does not hide later failures. + +`tests/performance_limit.rs` runs separately in release mode with the `performance` feature. It requires a real WebGPU adapter (software SwiftShader on CI), rejects invalid inputs and reports all 15 completed-frame samples. Ten Python regression tests reject absent/invalid/duplicate samples, zero or mismatched baselines, missing baseline files, and per-workload regressions; medians tolerate individual outliers. See [measurement details](../.github/workflows/PIPELINES.md). `tests/app_context.rs` checks independent UI/ECS owners, disposal/remount, chart lifetime, and callback ownership. The renderer and render queue remain shared services; these tests do not establish independent GPU renderers for multiple application roots. Browser tests use software WebGPU and do not establish physical-device performance. diff --git a/benches/webgpu_bench.rs b/benches/webgpu_bench.rs index 464a881..691713c 100644 --- a/benches/webgpu_bench.rs +++ b/benches/webgpu_bench.rs @@ -44,7 +44,7 @@ async fn benchmark_fps() { for &count in &[100usize, 1000usize] { let chart = sample_chart(count); - let fps = renderer.measure_fps(&chart, 30); + let fps = renderer.measure_fps(&chart, 30).await.expect("completed GPU frames"); web_sys::console::log_1(&format!("{count} candles: {fps:.2} FPS").into()); } } diff --git a/scripts/compare_fps.py b/scripts/compare_fps.py index b820908..f30e048 100644 --- a/scripts/compare_fps.py +++ b/scripts/compare_fps.py @@ -1,39 +1,44 @@ +"""Compare each workload median against a separately reviewed measured baseline.""" import json +import math import sys -import os - -if len(sys.argv) != 5: - print("Usage: compare_fps.py NEW BASELINE OUTPUT THRESHOLD") - sys.exit(1) - -new_file, baseline_file, output_file, threshold = sys.argv[1:] -threshold = float(threshold) - -if not os.path.exists(new_file): - print("benchmark_result.json not found, run parse_perf_log.py first") - sys.exit(1) - -with open(new_file) as f: - new_data = json.load(f) -new_fps = float(new_data.get("fps", 0)) - -if new_fps == 0: - print("Warning: FPS is zero, logs might be incorrect") - -if os.path.exists(baseline_file): - with open(baseline_file) as f: - base_data = json.load(f) - base_fps = float(base_data.get("fps", 0)) -else: - base_fps = new_fps - with open(baseline_file, "w") as f: - json.dump(new_data, f) - -with open(output_file, "w") as f: - json.dump(new_data, f) - -if base_fps and new_fps < base_fps * (1 - threshold / 100.0): - print(f"FPS decreased from {base_fps} to {new_fps}") - sys.exit(1) -else: - print(f"FPS {new_fps}, baseline {base_fps}") +from pathlib import Path +from statistics import median + +from perf_schema import validate + + +def compare(current, baseline, threshold): + validate(current) + validate(baseline) + if not math.isfinite(threshold) or not 0 <= threshold < 100: + raise ValueError("threshold must be finite and in [0, 100)") + results = {} + for count, samples in current["cases"].items(): + actual, expected = median(samples), median(baseline["cases"][count]) + results[count] = { + "median_fps": actual, + "baseline_median_fps": expected, + "change_percent": (actual / expected - 1) * 100, + "passed": actual >= expected * (1 - threshold / 100), + } + return {"threshold_percent": threshold, "cases": results, + "passed": all(case["passed"] for case in results.values())} + + +def main(): + if len(sys.argv) != 5: + raise ValueError("Usage: compare_fps.py NEW BASELINE OUTPUT THRESHOLD") + current, baseline, output, threshold = sys.argv[1:] + result = compare(json.loads(Path(current).read_text(encoding="utf-8")), + json.loads(Path(baseline).read_text(encoding="utf-8")), float(threshold)) + Path(output).write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + print(json.dumps(result, indent=2)) + return 0 if result["passed"] else 1 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except (ValueError, TypeError, KeyError, OSError) as error: + sys.exit(f"Invalid performance comparison: {error}") diff --git a/scripts/parse_perf_log.py b/scripts/parse_perf_log.py index bd6cf20..4655cd2 100644 --- a/scripts/parse_perf_log.py +++ b/scripts/parse_perf_log.py @@ -1,28 +1,41 @@ +"""Parse every expected completed-frame sample; missing measurements are errors.""" import json -import re import sys +from pathlib import Path -if len(sys.argv) != 3: - print("Usage: parse_perf_log.py LOG OUTPUT") - sys.exit(1) +from perf_schema import COUNTS, SAMPLES, FRAMES, WORKLOAD, validate -log_path, out_path = sys.argv[1:3] -fps_values = [] -with open(log_path) as f: - for line in f: - m = re.search(r"\d+ candles: ([0-9.]+) FPS", line) - if m: - fps_values.append(float(m.group(1))) +def parse(log): + cases = {str(count): [None] * SAMPLES for count in COUNTS} + for line in log.splitlines(): + if "PERF_SAMPLE " not in line: + continue + data = json.loads(line.split("PERF_SAMPLE ", 1)[1]) + count, sample = data["candles"], data["sample"] + if type(count) is not int or count not in COUNTS: + raise ValueError("unexpected candle count") + if type(sample) is not int or sample not in range(SAMPLES): + raise ValueError("unexpected sample index") + if data["frames"] != FRAMES or type(data["frames"]) is not int: + raise ValueError("unexpected frame count") + if cases[str(count)][sample] is not None: + raise ValueError("duplicate performance sample") + cases[str(count)][sample] = data["fps"] + result = {"schema": 1, "workload": WORKLOAD, "cases": cases} + validate(result) + return result -if not fps_values: - print("No FPS data found in log, assuming zero") - # Zero FPS is allowed to keep the pipeline green without measurements - with open(out_path, "w") as f: - json.dump({"fps": 0.0}, f) - sys.exit(0) -fps = sum(fps_values) / len(fps_values) if fps_values else 0.0 +def main(): + if len(sys.argv) != 3: + raise ValueError("Usage: parse_perf_log.py LOG OUTPUT") + result = parse(Path(sys.argv[1]).read_text(encoding="utf-8")) + Path(sys.argv[2]).write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") -with open(out_path, "w") as f: - json.dump({"fps": fps}, f) + +if __name__ == "__main__": + try: + main() + except (ValueError, KeyError, TypeError, OSError) as error: + sys.exit(f"Invalid performance measurements: {error}") diff --git a/scripts/perf_schema.py b/scripts/perf_schema.py new file mode 100644 index 0000000..bf1ca5e --- /dev/null +++ b/scripts/perf_schema.py @@ -0,0 +1,22 @@ +"""Versioned workload contract shared by measurement parsing and comparison.""" +import math + +COUNTS = (1000, 50000, 1000000) +SAMPLES = 5 +FRAMES = 30 +WORKLOAD = "cached-chart-completed-frames-800x600-msaa4-v1" + + +def validate(data): + if data.get("schema") != 1 or data.get("workload") != WORKLOAD: + raise ValueError("incompatible performance schema or workload") + cases = data.get("cases") + if not isinstance(cases, dict) or set(cases) != {str(count) for count in COUNTS}: + raise ValueError("missing or unexpected workloads") + for count, samples in cases.items(): + if not isinstance(samples, list) or len(samples) != SAMPLES: + raise ValueError(f"expected {SAMPLES} samples for {count} candles") + for fps in samples: + if type(fps) not in (int, float) or not math.isfinite(fps) or fps <= 0: + raise ValueError(f"missing, non-finite or non-positive FPS for {count} candles") + return data diff --git a/src/infrastructure/rendering/renderer/performance.rs b/src/infrastructure/rendering/renderer/performance.rs index 7543ff2..ace21ed 100644 --- a/src/infrastructure/rendering/renderer/performance.rs +++ b/src/infrastructure/rendering/renderer/performance.rs @@ -1,16 +1,31 @@ use super::*; impl WebGpuRenderer { - /// Measure average FPS for the given number of frames - pub fn measure_fps(&mut self, chart: &Chart, num_frames: u32) -> f64 { + /// Measure completed frames, including CPU submission and GPU queue latency. + /// This is serialized throughput, not display refresh rate or a GPU timestamp. + pub async fn measure_fps(&mut self, chart: &Chart, num_frames: u32) -> Result { + if num_frames == 0 || self.gpu.is_none() || chart.get_candle_count() == 0 { + return Err(JsValue::from_str("benchmark needs frames, candles and a real GPU")); + } let window = web_sys::window().expect("no window"); let performance = window.performance().expect("performance not available"); let start = performance.now(); for _ in 0..num_frames { - let _ = self.render(chart); + self.render(chart)?; + if self.cached_vertices.is_empty() { + return Err(JsValue::from_str("benchmark submitted no chart geometry")); + } + let (done, completed) = futures::channel::oneshot::channel(); + self.gpu().queue.on_submitted_work_done(move || { + let _ = done.send(()); + }); + completed.await.map_err(|_| JsValue::from_str("GPU completion callback lost"))?; } let end = performance.now(); let elapsed = (end - start) / 1000.0; - if elapsed > 0.0 { num_frames as f64 / elapsed } else { 0.0 } + if !elapsed.is_finite() || elapsed <= 0.0 { + return Err(JsValue::from_str("invalid benchmark elapsed time")); + } + Ok(num_frames as f64 / elapsed) } } diff --git a/tests/compare_fps_script.py b/tests/compare_fps_script.py index 896ed18..12e853c 100644 --- a/tests/compare_fps_script.py +++ b/tests/compare_fps_script.py @@ -1,21 +1,57 @@ +import copy +import json import subprocess import sys +import unittest from pathlib import Path from tempfile import TemporaryDirectory -import unittest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) +from compare_fps import compare +from perf_schema import COUNTS, WORKLOAD + + +def data(fps=100): + return {"schema": 1, "workload": WORKLOAD, + "cases": {str(count): [fps] * 5 for count in COUNTS}} class ScriptTests(unittest.TestCase): - def test_missing_file(self): + def test_each_workload_is_required(self): + current = data(200) + current["cases"]["50000"] = [60] * 5 + self.assertFalse(compare(current, data(), 30)["passed"]) + + def test_median_rejects_outlier_influence(self): + current = data() + current["cases"]["1000"] = [1, 98, 99, 100, 10000] + self.assertTrue(compare(current, data(), 5)["passed"]) + + def test_threshold_boundary(self): + self.assertTrue(compare(data(70), data(), 30)["passed"]) + self.assertFalse(compare(data(69.9), data(), 30)["passed"]) + + def test_invalid_baselines_and_thresholds(self): + invalid = [data(0), data(float("nan")), {"fps": 0}, {}] + mismatch = copy.deepcopy(data()) + mismatch["workload"] = "different" + invalid.append(mismatch) + for baseline in invalid: + with self.subTest(baseline=baseline), self.assertRaises(ValueError): + compare(data(), baseline, 30) + for threshold in (-1, 100, float("nan"), float("inf")): + with self.subTest(threshold=threshold), self.assertRaises(ValueError): + compare(data(), data(), threshold) + + def test_missing_baseline_does_not_self_initialize(self): with TemporaryDirectory() as directory: - tmp_path = Path(directory) - missing = tmp_path / "benchmark_result.json" - baseline = tmp_path / "baseline.json" - output = tmp_path / "result.json" - result = subprocess.run( - [sys.executable, str(Path(__file__).resolve().parent.parent / "scripts" / "compare_fps.py"), - str(missing), str(baseline), str(output), "5"], - capture_output=True, text=True - ) - assert result.returncode == 1 - assert "benchmark_result.json not found" in result.stdout + current = Path(directory) / "current.json" + baseline = Path(directory) / "missing.json" + output = Path(directory) / "result.json" + current.write_text(json.dumps(data()), encoding="utf-8") + result = subprocess.run([sys.executable, str(Path(__file__).resolve().parents[1] + / "scripts" / "compare_fps.py"), str(current), + str(baseline), str(output), "30"], capture_output=True) + self.assertNotEqual(result.returncode, 0) + self.assertFalse(baseline.exists()) + self.assertFalse(output.exists()) diff --git a/tests/parse_perf_log_script.py b/tests/parse_perf_log_script.py index 92492be..d895ea7 100644 --- a/tests/parse_perf_log_script.py +++ b/tests/parse_perf_log_script.py @@ -1,23 +1,48 @@ import json -import subprocess import sys -from pathlib import Path -from tempfile import TemporaryDirectory import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) +from parse_perf_log import parse +from perf_schema import COUNTS, SAMPLES, FRAMES + + +def measurements(): + return [{"candles": count, "sample": sample, "frames": FRAMES, "fps": 60 + sample} + for count in COUNTS for sample in range(SAMPLES)] + + +def log(rows): + return "\n".join("PERF_SAMPLE " + json.dumps(row) for row in rows) class ScriptTests(unittest.TestCase): - def test_no_fps_in_log(self): - with TemporaryDirectory() as directory: - tmp_path = Path(directory) - log = tmp_path / "log.txt" - log.write_text("some output without fps\n") - output = tmp_path / "result.json" - result = subprocess.run( - [sys.executable, str(Path(__file__).resolve().parent.parent / "scripts" / "parse_perf_log.py"), - str(log), str(output)], - capture_output=True, text=True - ) - assert result.returncode == 0 - assert json.loads(output.read_text()) == {"fps": 0.0} - assert "No FPS data found in log" in result.stdout + def test_complete_measurements(self): + data = parse("unrelated log\n" + log(measurements())) + self.assertEqual(data["cases"]["1000"], [60, 61, 62, 63, 64]) + + def test_missing_samples_or_workloads(self): + for rows in ([], measurements()[:-1], measurements()[5:]): + with self.subTest(rows=len(rows)), self.assertRaises(ValueError): + parse(log(rows)) + + def test_invalid_samples(self): + for value in (0, -1, float("nan"), float("inf"), None, True, "60"): + rows = measurements() + rows[0]["fps"] = value + with self.subTest(value=value), self.assertRaises(ValueError): + parse(log(rows)) + + def test_wrong_count_index_frames_and_duplicates(self): + for field, value in (("candles", 100), ("sample", 5), ("sample", True), ("frames", 0)): + rows = measurements() + rows[0][field] = value + with self.subTest(field=field), self.assertRaises(ValueError): + parse(log(rows)) + with self.assertRaises(ValueError): + parse(log(measurements() + measurements()[:1])) + + def test_malformed_sample(self): + with self.assertRaises(ValueError): + parse(log(measurements()) + '\nPERF_SAMPLE {broken}') diff --git a/tests/performance_limit.rs b/tests/performance_limit.rs index 2abc320..d3e7498 100644 --- a/tests/performance_limit.rs +++ b/tests/performance_limit.rs @@ -1,5 +1,7 @@ -#![cfg(feature = "render")] +#![cfg(feature = "performance")] wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser); +use leptos::SignalSet; +use price_chart_wasm::domain::market_data::TimeInterval; use price_chart_wasm::domain::{ chart::{Chart, ChartType}, market_data::{Candle, OHLCV, Price, Timestamp, Volume}, @@ -45,35 +47,35 @@ fn sample_chart(count: usize) -> Chart { } #[wasm_bindgen_test(async)] -async fn fps_degradation_logging() { - if !WebGpuRenderer::is_webgpu_supported().await { - web_sys::console::log_1(&"Skipping test: WebGPU not supported".into()); - return; - } +async fn completed_frame_throughput() { + price_chart_wasm::app::current_interval().set(TimeInterval::TwoSeconds); + assert!(WebGpuRenderer::is_webgpu_supported().await, "WebGPU is required"); setup_canvas("perf-canvas", 800, 600); - let mut renderer = match WebGpuRenderer::new("perf-canvas", 800, 600).await { - Ok(r) => r, - Err(e) => { - web_sys::console::log_1(&format!("Skipping test: {e:?}").into()); - return; - } - }; - - let performance = web_sys::window().unwrap().performance().unwrap(); + let mut renderer = + WebGpuRenderer::new("perf-canvas", 800, 600).await.expect("benchmark GPU initialization"); let counts = [1_000usize, 50_000, 1_000_000]; for &count in &counts { let chart = sample_chart(count); - let start = performance.now(); - let _ = renderer.render(&chart); - let first_render_ms = performance.now() - start; - let fps = renderer.measure_fps(&chart, 30); - web_sys::console::log_1( - &format!("{count} candles: {first_render_ms:.2} ms first render").into(), - ); - if fps < 30.0 { - web_sys::console::log_1(&format!("⚠ {count} candles: {fps:.2} FPS").into()); - } else { - web_sys::console::log_1(&format!("{count} candles: {fps:.2} FPS").into()); + renderer.measure_fps(&chart, 10).await.expect("warmup frames"); + for sample in 0..5 { + let fps = renderer.measure_fps(&chart, 30).await.expect("measured frames"); + assert!(fps.is_finite() && fps > 0.0); + // Structured output is consumed by parse_perf_log.py. Each dataset + // retains its own samples; averaging different workloads hides regressions. + console_log!( + "PERF_SAMPLE {}", + serde_json::json!({"candles": count, "sample": sample, "frames": 30, "fps": fps}) + ); } } } + +#[wasm_bindgen_test(async)] +async fn invalid_benchmark_inputs_fail() { + let mut renderer = price_chart_wasm::infrastructure::rendering::renderer::dummy_renderer(); + assert!(renderer.measure_fps(&sample_chart(10), 30).await.is_err()); + setup_canvas("invalid-perf-canvas", 800, 600); + let mut renderer = WebGpuRenderer::new("invalid-perf-canvas", 800, 600).await.unwrap(); + assert!(renderer.measure_fps(&sample_chart(10), 0).await.is_err()); + assert!(renderer.measure_fps(&sample_chart(0), 30).await.is_err()); +} From e7d2ef6d1e798eacb59b4ec449348846cde5ed43 Mon Sep 17 00:00:00 2001 From: qqrm <4681325+qqrm@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:03:25 +0300 Subject: [PATCH 02/12] Wait for supported WebGPU readback and reject no-op frames --- .github/workflows/PIPELINES.md | 2 +- .../rendering/renderer/initialization.rs | 1 + src/infrastructure/rendering/renderer/mod.rs | 2 ++ .../rendering/renderer/performance.rs | 25 ++++++++++++++++--- .../rendering/renderer/render_loop.rs | 1 + tests/performance_limit.rs | 7 ++++++ 6 files changed, 33 insertions(+), 5 deletions(-) diff --git a/.github/workflows/PIPELINES.md b/.github/workflows/PIPELINES.md index 2812102..2904c35 100644 --- a/.github/workflows/PIPELINES.md +++ b/.github/workflows/PIPELINES.md @@ -7,7 +7,7 @@ It uses deterministic 1,000 / 50,000 / 1,000,000-candle base datasets, an Each dataset receives 10 completed warmup frames followed by five samples of 30 frames. Every frame calls the real renderer and awaits the WebGPU queue -completion callback. A missing adapter, failed render, empty geometry, missing +completion via a four-byte readback fence. This includes the fence and mapping overhead; wgpu 25 does not implement its queue-completion callback on WebGPU. A missing adapter, failed render, empty geometry, missing sample, duplicate sample, zero, negative or non-finite value fails the job. This measures serialized, cached-chart throughput including CPU submission, diff --git a/src/infrastructure/rendering/renderer/initialization.rs b/src/infrastructure/rendering/renderer/initialization.rs index dd73939..6e0e718 100644 --- a/src/infrastructure/rendering/renderer/initialization.rs +++ b/src/infrastructure/rendering/renderer/initialization.rs @@ -261,6 +261,7 @@ impl WebGpuRenderer { zoom_level: 1.0, pan_offset: 0.0, render_time_log: VecDeque::new(), + submitted_frames: 0, line_visibility: LineVisibility::default(), }; diff --git a/src/infrastructure/rendering/renderer/mod.rs b/src/infrastructure/rendering/renderer/mod.rs index 2a04890..9e7ee42 100644 --- a/src/infrastructure/rendering/renderer/mod.rs +++ b/src/infrastructure/rendering/renderer/mod.rs @@ -96,6 +96,7 @@ pub struct WebGpuRenderer { // ⏱️ Performance metrics render_time_log: VecDeque, + submitted_frames: u64, // 📊 Indicator line visibility line_visibility: LineVisibility, @@ -134,6 +135,7 @@ pub fn dummy_renderer() -> WebGpuRenderer { zoom_level: 1.0, pan_offset: 0.0, render_time_log: VecDeque::new(), + submitted_frames: 0, line_visibility: LineVisibility::default(), } } diff --git a/src/infrastructure/rendering/renderer/performance.rs b/src/infrastructure/rendering/renderer/performance.rs index ace21ed..37a7e2b 100644 --- a/src/infrastructure/rendering/renderer/performance.rs +++ b/src/infrastructure/rendering/renderer/performance.rs @@ -9,17 +9,34 @@ impl WebGpuRenderer { } let window = web_sys::window().expect("no window"); let performance = window.performance().expect("performance not available"); + // wgpu 25's WebGPU backend does not implement on_submitted_work_done. + // A mapped readback after a same-queue buffer clear is a completion fence: + // the mapping resolves only after that submission and prior render work. + let fence = self.gpu().device.create_buffer(&wgpu::BufferDescriptor { + label: Some("Benchmark completion fence"), + size: 4, + usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, + mapped_at_creation: false, + }); let start = performance.now(); for _ in 0..num_frames { + let before = self.submitted_frames; self.render(chart)?; - if self.cached_vertices.is_empty() { + if self.submitted_frames == before { return Err(JsValue::from_str("benchmark submitted no chart geometry")); } + let mut encoder = self.gpu().device.create_command_encoder(&Default::default()); + encoder.clear_buffer(&fence, 0, None); + self.gpu().queue.submit([encoder.finish()]); let (done, completed) = futures::channel::oneshot::channel(); - self.gpu().queue.on_submitted_work_done(move || { - let _ = done.send(()); + fence.slice(..).map_async(wgpu::MapMode::Read, move |result| { + let _ = done.send(result); }); - completed.await.map_err(|_| JsValue::from_str("GPU completion callback lost"))?; + completed + .await + .map_err(|_| JsValue::from_str("GPU completion callback lost"))? + .map_err(|_| JsValue::from_str("GPU completion readback failed"))?; + fence.unmap(); } let end = performance.now(); let elapsed = (end - start) / 1000.0; diff --git a/src/infrastructure/rendering/renderer/render_loop.rs b/src/infrastructure/rendering/renderer/render_loop.rs index 7d804b8..1601256 100644 --- a/src/infrastructure/rendering/renderer/render_loop.rs +++ b/src/infrastructure/rendering/renderer/render_loop.rs @@ -162,6 +162,7 @@ impl WebGpuRenderer { } self.gpu().queue.submit(std::iter::once(encoder.finish())); + self.submitted_frames = self.submitted_frames.wrapping_add(1); if let Some(start) = start_pass && let Some(window) = web_sys::window() diff --git a/tests/performance_limit.rs b/tests/performance_limit.rs index d3e7498..42907ba 100644 --- a/tests/performance_limit.rs +++ b/tests/performance_limit.rs @@ -78,4 +78,11 @@ async fn invalid_benchmark_inputs_fail() { let mut renderer = WebGpuRenderer::new("invalid-perf-canvas", 800, 600).await.unwrap(); assert!(renderer.measure_fps(&sample_chart(10), 0).await.is_err()); assert!(renderer.measure_fps(&sample_chart(0), 30).await.is_err()); + price_chart_wasm::app::current_interval().set(TimeInterval::TwoSeconds); + let chart = sample_chart(10); + renderer.measure_fps(&chart, 1).await.unwrap(); + // The base series is populated, but the selected derived series is empty. + // A previous cached frame must not make a no-op render count as a new frame. + price_chart_wasm::app::current_interval().set(TimeInterval::OneMinute); + assert!(renderer.measure_fps(&chart, 1).await.is_err()); } From 89266f5b1036bdb77bddbd1f601f6175a8b0736a Mon Sep 17 00:00:00 2001 From: qqrm <4681325+qqrm@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:06:56 +0300 Subject: [PATCH 03/12] Surface GPU validation failures from measured frames --- .../rendering/renderer/performance.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/infrastructure/rendering/renderer/performance.rs b/src/infrastructure/rendering/renderer/performance.rs index 37a7e2b..aa47d01 100644 --- a/src/infrastructure/rendering/renderer/performance.rs +++ b/src/infrastructure/rendering/renderer/performance.rs @@ -7,6 +7,19 @@ impl WebGpuRenderer { if num_frames == 0 || self.gpu.is_none() || chart.get_candle_count() == 0 { return Err(JsValue::from_str("benchmark needs frames, candles and a real GPU")); } + self.gpu().device.push_error_scope(wgpu::ErrorFilter::Validation); + let result = self.measure_completed_frames(chart, num_frames).await; + if let Some(error) = self.gpu().device.pop_error_scope().await { + return Err(JsValue::from_str(&format!("benchmark GPU validation: {error}"))); + } + result + } + + async fn measure_completed_frames( + &mut self, + chart: &Chart, + num_frames: u32, + ) -> Result { let window = web_sys::window().expect("no window"); let performance = window.performance().expect("performance not available"); // wgpu 25's WebGPU backend does not implement on_submitted_work_done. From 2cabb3343b6b9c74247d995208fa76036c3baa4d Mon Sep 17 00:00:00 2001 From: qqrm <4681325+qqrm@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:11:01 +0300 Subject: [PATCH 04/12] Retain browser-side mapping and device-loss diagnostics --- tests/performance_limit.rs | 41 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/tests/performance_limit.rs b/tests/performance_limit.rs index 42907ba..81590ff 100644 --- a/tests/performance_limit.rs +++ b/tests/performance_limit.rs @@ -10,6 +10,35 @@ use price_chart_wasm::infrastructure::rendering::renderer::WebGpuRenderer; use wasm_bindgen::JsCast; use wasm_bindgen_test::*; +#[wasm_bindgen::prelude::wasm_bindgen(inline_js = " +export function traceMapErrors() { + if (globalThis.__perfMapErrors) return; + globalThis.__perfMapErrors = []; + const requestDevice = GPUAdapter.prototype.requestDevice; + GPUAdapter.prototype.requestDevice = function(...args) { + return requestDevice.apply(this, args).then(device => { + device.lost.then(info => globalThis.__perfMapErrors.push( + `Device lost: ${info.reason}: ${info.message}`)); + return device; + }); + }; + const original = GPUBuffer.prototype.mapAsync; + GPUBuffer.prototype.mapAsync = function(...args) { + return original.apply(this, args).catch(error => { + globalThis.__perfMapErrors.push(String(error)); + throw error; + }); + }; +} +export function mapErrors() { return JSON.stringify(globalThis.__perfMapErrors); } +")] +extern "C" { + #[wasm_bindgen::prelude::wasm_bindgen(js_name = traceMapErrors)] + fn trace_map_errors(); + #[wasm_bindgen::prelude::wasm_bindgen(js_name = mapErrors)] + fn map_errors() -> String; +} + fn setup_canvas(id: &str, width: u32, height: u32) { let window = web_sys::window().unwrap(); let document = window.document().unwrap(); @@ -48,6 +77,7 @@ fn sample_chart(count: usize) -> Chart { #[wasm_bindgen_test(async)] async fn completed_frame_throughput() { + trace_map_errors(); price_chart_wasm::app::current_interval().set(TimeInterval::TwoSeconds); assert!(WebGpuRenderer::is_webgpu_supported().await, "WebGPU is required"); setup_canvas("perf-canvas", 800, 600); @@ -56,7 +86,10 @@ async fn completed_frame_throughput() { let counts = [1_000usize, 50_000, 1_000_000]; for &count in &counts { let chart = sample_chart(count); - renderer.measure_fps(&chart, 10).await.expect("warmup frames"); + renderer + .measure_fps(&chart, 10) + .await + .unwrap_or_else(|error| panic!("warmup frames: {error:?}; browser: {}", map_errors())); for sample in 0..5 { let fps = renderer.measure_fps(&chart, 30).await.expect("measured frames"); assert!(fps.is_finite() && fps > 0.0); @@ -72,6 +105,7 @@ async fn completed_frame_throughput() { #[wasm_bindgen_test(async)] async fn invalid_benchmark_inputs_fail() { + trace_map_errors(); let mut renderer = price_chart_wasm::infrastructure::rendering::renderer::dummy_renderer(); assert!(renderer.measure_fps(&sample_chart(10), 30).await.is_err()); setup_canvas("invalid-perf-canvas", 800, 600); @@ -80,7 +114,10 @@ async fn invalid_benchmark_inputs_fail() { assert!(renderer.measure_fps(&sample_chart(0), 30).await.is_err()); price_chart_wasm::app::current_interval().set(TimeInterval::TwoSeconds); let chart = sample_chart(10); - renderer.measure_fps(&chart, 1).await.unwrap(); + renderer + .measure_fps(&chart, 1) + .await + .unwrap_or_else(|error| panic!("positive control: {error:?}; browser: {}", map_errors())); // The base series is populated, but the selected derived series is empty. // A previous cached frame must not make a no-op render count as a new frame. price_chart_wasm::app::current_interval().set(TimeInterval::OneMinute); From 566d3fefe37a6eb90f50f4f9a4d60c4f7f13fde6 Mon Sep 17 00:00:00 2001 From: qqrm <4681325+qqrm@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:14:47 +0300 Subject: [PATCH 05/12] Retain the WebGPU instance for the renderer lifetime --- src/infrastructure/rendering/renderer/initialization.rs | 1 + src/infrastructure/rendering/renderer/mod.rs | 2 ++ 2 files changed, 3 insertions(+) diff --git a/src/infrastructure/rendering/renderer/initialization.rs b/src/infrastructure/rendering/renderer/initialization.rs index 6e0e718..1eebce9 100644 --- a/src/infrastructure/rendering/renderer/initialization.rs +++ b/src/infrastructure/rendering/renderer/initialization.rs @@ -249,6 +249,7 @@ impl WebGpuRenderer { uniform_bind_group, msaa_texture, msaa_view, + _instance: instance, }), template_vertices: 0, cached_vertices: Vec::new(), diff --git a/src/infrastructure/rendering/renderer/mod.rs b/src/infrastructure/rendering/renderer/mod.rs index 9e7ee42..77d3a78 100644 --- a/src/infrastructure/rendering/renderer/mod.rs +++ b/src/infrastructure/rendering/renderer/mod.rs @@ -69,6 +69,8 @@ struct GpuResources { uniform_bind_group: wgpu::BindGroup, msaa_texture: wgpu::Texture, msaa_view: wgpu::TextureView, + // Drop last: the browser instance must outlive every device/queue resource. + _instance: wgpu::Instance, } /// Actual WebGPU renderer for candles From 1b79e9df3d64542d367a1291273a48dff5ce57a6 Mon Sep 17 00:00:00 2001 From: qqrm <4681325+qqrm@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:18:49 +0300 Subject: [PATCH 06/12] Select and verify the software WebGPU adapter explicitly --- .github/workflows/benchmark.yml | 5 ++++ .../rendering/renderer/initialization.rs | 1 - src/infrastructure/rendering/renderer/mod.rs | 2 -- tests/performance_limit.rs | 25 +++++++++++++++++++ 4 files changed, 30 insertions(+), 3 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 422b421..cb3b46d 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -56,6 +56,11 @@ jobs: from pathlib import Path path = Path('webdriver.json') config = json.loads(path.read_text()) + config['goog:chromeOptions']['args'].extend([ + '--use-webgpu-adapter=swiftshader', + '--use-vulkan=swiftshader', + '--enable-unsafe-swiftshader', + ]) config['goog:chromeOptions']['binary'] = os.environ['CHROME_BINARY'] path.write_text(json.dumps(config)) metadata = { diff --git a/src/infrastructure/rendering/renderer/initialization.rs b/src/infrastructure/rendering/renderer/initialization.rs index 1eebce9..6e0e718 100644 --- a/src/infrastructure/rendering/renderer/initialization.rs +++ b/src/infrastructure/rendering/renderer/initialization.rs @@ -249,7 +249,6 @@ impl WebGpuRenderer { uniform_bind_group, msaa_texture, msaa_view, - _instance: instance, }), template_vertices: 0, cached_vertices: Vec::new(), diff --git a/src/infrastructure/rendering/renderer/mod.rs b/src/infrastructure/rendering/renderer/mod.rs index 77d3a78..9e7ee42 100644 --- a/src/infrastructure/rendering/renderer/mod.rs +++ b/src/infrastructure/rendering/renderer/mod.rs @@ -69,8 +69,6 @@ struct GpuResources { uniform_bind_group: wgpu::BindGroup, msaa_texture: wgpu::Texture, msaa_view: wgpu::TextureView, - // Drop last: the browser instance must outlive every device/queue resource. - _instance: wgpu::Instance, } /// Actual WebGPU renderer for candles diff --git a/tests/performance_limit.rs b/tests/performance_limit.rs index 81590ff..6bdc211 100644 --- a/tests/performance_limit.rs +++ b/tests/performance_limit.rs @@ -31,12 +31,35 @@ export function traceMapErrors() { }; } export function mapErrors() { return JSON.stringify(globalThis.__perfMapErrors); } +export async function verifyBrowserGpu() { + const adapter = await navigator.gpu.requestAdapter(); + if (!adapter) throw new Error('No browser WebGPU adapter'); + const device = await adapter.requestDevice(); + const buffer = device.createBuffer({ size: 256, usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ }); + const encoder = device.createCommandEncoder(); + encoder.clearBuffer(buffer); + device.queue.submit([encoder.finish()]); + await buffer.mapAsync(GPUMapMode.READ); + buffer.unmap(); + const info = { + vendor: adapter.info.vendor, + architecture: adapter.info.architecture, + device: adapter.info.device, + description: adapter.info.description, + }; + if (!JSON.stringify(info).toLowerCase().includes('swiftshader')) { + throw new Error('Benchmark requires SwiftShader: ' + JSON.stringify(info)); + } + return JSON.stringify(info); +} ")] extern "C" { #[wasm_bindgen::prelude::wasm_bindgen(js_name = traceMapErrors)] fn trace_map_errors(); #[wasm_bindgen::prelude::wasm_bindgen(js_name = mapErrors)] fn map_errors() -> String; + #[wasm_bindgen::prelude::wasm_bindgen(catch, js_name = verifyBrowserGpu)] + async fn verify_browser_gpu() -> Result; } fn setup_canvas(id: &str, width: u32, height: u32) { @@ -78,6 +101,8 @@ fn sample_chart(count: usize) -> Chart { #[wasm_bindgen_test(async)] async fn completed_frame_throughput() { trace_map_errors(); + let adapter = verify_browser_gpu().await.expect("browser GPU readback preflight"); + console_log!("PERF_ADAPTER {:?}", adapter); price_chart_wasm::app::current_interval().set(TimeInterval::TwoSeconds); assert!(WebGpuRenderer::is_webgpu_supported().await, "WebGPU is required"); setup_canvas("perf-canvas", 800, 600); From 1fca258b090f61848cdc14b9b46593a6d71e3572 Mon Sep 17 00:00:00 2001 From: qqrm <4681325+qqrm@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:22:00 +0300 Subject: [PATCH 07/12] Record measured SwiftShader baseline and reproducible browser settings --- .github/workflows/PIPELINES.md | 11 +++++-- .github/workflows/benchmark.yml | 9 ++---- DOCS/perf.md | 24 +++++++++++++-- DOCS/perf_baseline.json | 54 ++++++++++++++++++++++++++++++++- DOCS/perf_result.json | 1 - webdriver-perf.json | 15 +++++++++ 6 files changed, 100 insertions(+), 14 deletions(-) delete mode 100644 DOCS/perf_result.json create mode 100644 webdriver-perf.json diff --git a/.github/workflows/PIPELINES.md b/.github/workflows/PIPELINES.md index 2904c35..e7f51dc 100644 --- a/.github/workflows/PIPELINES.md +++ b/.github/workflows/PIPELINES.md @@ -7,7 +7,9 @@ It uses deterministic 1,000 / 50,000 / 1,000,000-candle base datasets, an Each dataset receives 10 completed warmup frames followed by five samples of 30 frames. Every frame calls the real renderer and awaits the WebGPU queue -completion via a four-byte readback fence. This includes the fence and mapping overhead; wgpu 25 does not implement its queue-completion callback on WebGPU. A missing adapter, failed render, empty geometry, missing +completion via a four-byte readback fence. This includes fence and mapping +overhead; wgpu 25 does not implement its queue-completion callback on WebGPU. +A missing adapter, failed render, empty geometry, missing sample, duplicate sample, zero, negative or non-finite value fails the job. This measures serialized, cached-chart throughput including CPU submission, @@ -27,7 +29,7 @@ Run the same workload with a matching wasm-bindgen runner and ChromeDriver: ```bash WASM_BINDGEN_TEST_ONLY_WEB=1 WASM_BINDGEN_TEST_TIMEOUT=180 \ -WASM_BINDGEN_TEST_WEBDRIVER_JSON=webdriver.json \ +WASM_BINDGEN_TEST_WEBDRIVER_JSON=webdriver-perf.json \ CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUNNER=wasm-bindgen-test-runner \ cargo test --release --locked --features performance --test performance_limit -- --nocapture 2>&1 | tee perf.log python3 scripts/parse_perf_log.py perf.log benchmark_result.json @@ -44,3 +46,8 @@ A baseline update must retain raw samples and link the successful measurement run, tested SHA, browser/compiler versions and software backend. Review changes to workload or environment before recalibrating; do not copy a regressed result into the baseline just to make CI green. + +The dedicated `webdriver-perf.json` selects SwiftShader for both ANGLE and Dawn +WebGPU/Vulkan. Selecting only ANGLE allowed a browser adapter to be created but +its device was destroyed before readback. A raw browser preflight now requires +successful mapping and verifies the SwiftShader adapter before measuring Rust. diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index cb3b46d..bb4c8f0 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -54,13 +54,8 @@ jobs: import platform import subprocess from pathlib import Path - path = Path('webdriver.json') + path = Path('webdriver-perf.json') config = json.loads(path.read_text()) - config['goog:chromeOptions']['args'].extend([ - '--use-webgpu-adapter=swiftshader', - '--use-vulkan=swiftshader', - '--enable-unsafe-swiftshader', - ]) config['goog:chromeOptions']['binary'] = os.environ['CHROME_BINARY'] path.write_text(json.dumps(config)) metadata = { @@ -81,7 +76,7 @@ jobs: CHROMEDRIVER: ${{ steps.chrome.outputs.chromedriver-path }} WASM_BINDGEN_TEST_ONLY_WEB: '1' WASM_BINDGEN_TEST_TIMEOUT: '180' - WASM_BINDGEN_TEST_WEBDRIVER_JSON: webdriver.json + WASM_BINDGEN_TEST_WEBDRIVER_JSON: webdriver-perf.json run: | set -o pipefail cargo test --release --locked --features performance --test performance_limit -- --nocapture 2>&1 | tee perf.log diff --git a/DOCS/perf.md b/DOCS/perf.md index 530ac51..8d634f7 100644 --- a/DOCS/perf.md +++ b/DOCS/perf.md @@ -1,5 +1,23 @@ -# Latest metrics +# Recorded performance baseline -| Metric | Value | +This is a fixed calibration, not a live production FPS report. + +| Source candles | Median completed frames/s | | --- | --- | -| fps | 0.0 | +| 1,000 | 160.54 | +| 50,000 | 159.21 | +| 1,000,000 | 161.75 | + +Measured on Ubuntu 24.04, Chrome 155.0.8059.79, Rust 1.99.0 and the verified +Google SwiftShader adapter. Each workload has five samples of 30 completed frames +after ten warmup frames, with production LOD, an 800x600 canvas and 4x MSAA. +Readback synchronization overhead is included; display FPS and physical GPU +performance are not measured. + +[Calibration run](https://github.com/qqrm/webgpu-candles/actions/runs/38680812974) on head +`1b79e9df3d64542d367a1291273a48dff5ce57a6`. The measurement step passed; the initial +comparison intentionally failed against the invalid legacy zero baseline. + +Raw samples and environment provenance are retained in [perf_baseline.json](perf_baseline.json). +Subsequent CI results are in the `completed-frame-performance` artifact of each CI run. +See [the measurement contract](../.github/workflows/PIPELINES.md) before changing the baseline. diff --git a/DOCS/perf_baseline.json b/DOCS/perf_baseline.json index 80edfbd..3e0edae 100644 --- a/DOCS/perf_baseline.json +++ b/DOCS/perf_baseline.json @@ -1 +1,53 @@ -{"fps": 0.0} \ No newline at end of file +{ + "schema": 1, + "workload": "cached-chart-completed-frames-800x600-msaa4-v1", + "cases": { + "1000": [ + 160.54370802450515, + 159.9360255897666, + 159.0330788804076, + 164.40157825515493, + 165.23008289042278 + ], + "50000": [ + 159.5872011064708, + 157.0187375693399, + 163.89860139860713, + 147.41290354282725, + 159.2103168285241 + ], + "1000000": [ + 161.7992071838904, + 161.75122661346748, + 162.42115806285545, + 159.1174286623582, + 159.11320904822702 + ] + }, + "provenance": { + "tested_sha": "d95f58f76e286296b87fc2b05a6751e4eef347b8", + "run_url": "https://github.com/qqrm/webgpu-candles/actions/runs/38680812974", + "browser": "Google Chrome for Testing 155.0.8059.79", + "rust": "rustc 1.99.0 (b940084d7 2026-09-28)", + "os": "Linux-6.17.0-1022-azure-x86_64-with-glibc2.39", + "cpu": "x86_64", + "backend": "SwiftShader software WebGPU", + "browser_flags": [ + "--no-sandbox", + "--enable-unsafe-webgpu", + "--enable-features=Vulkan", + "--use-angle=swiftshader", + "--disable-vulkan-surface", + "--host-resolver-rules=MAP api.binance.com ~NOTFOUND, MAP stream.binance.com ~NOTFOUND", + "--use-webgpu-adapter=swiftshader", + "--use-vulkan=swiftshader", + "--enable-unsafe-swiftshader" + ], + "measurement_head": "1b79e9df3d64542d367a1291273a48dff5ce57a6", + "adapter": { + "vendor": "google", + "architecture": "swiftshader" + }, + "measurement_status": "2 browser tests and all 15 samples passed; comparator rejected legacy zero baseline" + } +} diff --git a/DOCS/perf_result.json b/DOCS/perf_result.json deleted file mode 100644 index 80edfbd..0000000 --- a/DOCS/perf_result.json +++ /dev/null @@ -1 +0,0 @@ -{"fps": 0.0} \ No newline at end of file diff --git a/webdriver-perf.json b/webdriver-perf.json new file mode 100644 index 0000000..c438b41 --- /dev/null +++ b/webdriver-perf.json @@ -0,0 +1,15 @@ +{ + "goog:chromeOptions": { + "args": [ + "--no-sandbox", + "--enable-unsafe-webgpu", + "--enable-features=Vulkan", + "--use-angle=swiftshader", + "--disable-vulkan-surface", + "--host-resolver-rules=MAP api.binance.com ~NOTFOUND, MAP stream.binance.com ~NOTFOUND", + "--use-webgpu-adapter=swiftshader", + "--use-vulkan=swiftshader", + "--enable-unsafe-swiftshader" + ] + } +} From 9c89f7271e7c7f17c5c1a9448a1e25004885ccdc Mon Sep 17 00:00:00 2001 From: qqrm <4681325+qqrm@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:25:05 +0300 Subject: [PATCH 08/12] Distinguish measured PR head from workflow merge provenance --- .github/workflows/PIPELINES.md | 4 ++++ .github/workflows/benchmark.yml | 4 ++++ DOCS/perf_baseline.json | 5 +++-- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/PIPELINES.md b/.github/workflows/PIPELINES.md index e7f51dc..dc4eec2 100644 --- a/.github/workflows/PIPELINES.md +++ b/.github/workflows/PIPELINES.md @@ -51,3 +51,7 @@ The dedicated `webdriver-perf.json` selects SwiftShader for both ANGLE and Dawn WebGPU/Vulkan. Selecting only ANGLE allowed a browser adapter to be created but its device was destroyed before readback. A raw browser preflight now requires successful mapping and verifies the SwiftShader adapter before measuring Rust. + +Provenance records both `tested_sha` (the checked-out workflow SHA, a synthetic +merge commit for pull requests) and `tested_head_sha` (the PR head; identical to +`tested_sha` for pushes/manual runs). Baseline calibration preserves both. diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index bb4c8f0..781adab 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -46,6 +46,8 @@ jobs: env: CHROME_BINARY: ${{ steps.chrome.outputs.chrome-path }} TESTED_SHA: ${{ github.sha }} + TESTED_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + EVENT_NAME: ${{ github.event_name }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} run: | python3 - <<'PY' @@ -60,6 +62,8 @@ jobs: path.write_text(json.dumps(config)) metadata = { 'tested_sha': os.environ['TESTED_SHA'], + 'tested_head_sha': os.environ['TESTED_HEAD_SHA'], + 'event': os.environ['EVENT_NAME'], 'run_url': os.environ['RUN_URL'], 'browser': subprocess.check_output([os.environ['CHROME_BINARY'], '--version'], text=True).strip(), 'rust': subprocess.check_output(['rustc', '--version'], text=True).strip(), diff --git a/DOCS/perf_baseline.json b/DOCS/perf_baseline.json index 3e0edae..289995c 100644 --- a/DOCS/perf_baseline.json +++ b/DOCS/perf_baseline.json @@ -43,11 +43,12 @@ "--use-vulkan=swiftshader", "--enable-unsafe-swiftshader" ], - "measurement_head": "1b79e9df3d64542d367a1291273a48dff5ce57a6", "adapter": { "vendor": "google", "architecture": "swiftshader" }, - "measurement_status": "2 browser tests and all 15 samples passed; comparator rejected legacy zero baseline" + "measurement_status": "2 browser tests and all 15 samples passed; comparator rejected legacy zero baseline", + "tested_head_sha": "1b79e9df3d64542d367a1291273a48dff5ce57a6", + "event": "pull_request" } } From 489162ec49794664fcfd50b2c8dc9f19af238093 Mon Sep 17 00:00:00 2001 From: qqrm <4681325+qqrm@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:27:23 +0300 Subject: [PATCH 09/12] Materialize benchmark profile for the locked WASM runner --- .github/workflows/PIPELINES.md | 2 ++ .github/workflows/benchmark.yml | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/PIPELINES.md b/.github/workflows/PIPELINES.md index dc4eec2..25d96f6 100644 --- a/.github/workflows/PIPELINES.md +++ b/.github/workflows/PIPELINES.md @@ -51,6 +51,8 @@ The dedicated `webdriver-perf.json` selects SwiftShader for both ANGLE and Dawn WebGPU/Vulkan. Selecting only ANGLE allowed a browser adapter to be created but its device was destroyed before readback. A raw browser preflight now requires successful mapping and verifies the SwiftShader adapter before measuring Rust. +The job writes that profile to `webdriver.json`, the only filename supported by +the locked wasm-bindgen 0.2.100 runner. Other jobs have independent checkouts. Provenance records both `tested_sha` (the checked-out workflow SHA, a synthetic merge commit for pull requests) and `tested_head_sha` (the PR head; identical to diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 781adab..345fedd 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -56,10 +56,11 @@ jobs: import platform import subprocess from pathlib import Path - path = Path('webdriver-perf.json') - config = json.loads(path.read_text()) + config = json.loads(Path('webdriver-perf.json').read_text()) config['goog:chromeOptions']['binary'] = os.environ['CHROME_BINARY'] - path.write_text(json.dumps(config)) + # The locked wasm-bindgen 0.2.100 runner reads only webdriver.json. + # Materialize the dedicated profile in this isolated job checkout. + Path('webdriver.json').write_text(json.dumps(config)) metadata = { 'tested_sha': os.environ['TESTED_SHA'], 'tested_head_sha': os.environ['TESTED_HEAD_SHA'], @@ -80,7 +81,6 @@ jobs: CHROMEDRIVER: ${{ steps.chrome.outputs.chromedriver-path }} WASM_BINDGEN_TEST_ONLY_WEB: '1' WASM_BINDGEN_TEST_TIMEOUT: '180' - WASM_BINDGEN_TEST_WEBDRIVER_JSON: webdriver-perf.json run: | set -o pipefail cargo test --release --locked --features performance --test performance_limit -- --nocapture 2>&1 | tee perf.log From c311f8b15bbf6e4e08a9a5647ce55f5fc19b6ad8 Mon Sep 17 00:00:00 2001 From: qqrm <4681325+qqrm@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:27:54 +0300 Subject: [PATCH 10/12] Document reproducible setup for the locked browser runner --- .github/workflows/PIPELINES.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/PIPELINES.md b/.github/workflows/PIPELINES.md index 25d96f6..314423f 100644 --- a/.github/workflows/PIPELINES.md +++ b/.github/workflows/PIPELINES.md @@ -25,11 +25,21 @@ fails. This deliberately broad guard detects large regressions on variable shared CI hosts; it is not a 5% performance guarantee. The baseline is never created or updated automatically, and a fast workload cannot hide a slower one. -Run the same workload with a matching wasm-bindgen runner and ChromeDriver: +Run the same workload in a disposable checkout with the pinned compiler/browser, +a matching wasm-bindgen runner and ChromeDriver on PATH. Set `CHROME_BINARY` to +the browser executable; the preparation replaces only this checkout's profile: ```bash +set -o pipefail +python3 - <<'PY' +import json +import os +from pathlib import Path +config = json.loads(Path('webdriver-perf.json').read_text()) +config['goog:chromeOptions']['binary'] = os.environ['CHROME_BINARY'] +Path('webdriver.json').write_text(json.dumps(config)) +PY WASM_BINDGEN_TEST_ONLY_WEB=1 WASM_BINDGEN_TEST_TIMEOUT=180 \ -WASM_BINDGEN_TEST_WEBDRIVER_JSON=webdriver-perf.json \ CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUNNER=wasm-bindgen-test-runner \ cargo test --release --locked --features performance --test performance_limit -- --nocapture 2>&1 | tee perf.log python3 scripts/parse_perf_log.py perf.log benchmark_result.json From c7026b9067c55fff7bb2d738fa276ce0266e046a Mon Sep 17 00:00:00 2001 From: qqrm <4681325+qqrm@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:32:34 +0300 Subject: [PATCH 11/12] Compare reference and candidate on the same software GPU runner --- .github/workflows/PIPELINES.md | 35 ++++++++++++++++++++++----------- .github/workflows/benchmark.yml | 28 +++++++++++++++++++++++++- DOCS/PIPELINES.md | 2 +- DOCS/perf.md | 4 +++- 4 files changed, 55 insertions(+), 14 deletions(-) diff --git a/.github/workflows/PIPELINES.md b/.github/workflows/PIPELINES.md index 314423f..1657749 100644 --- a/.github/workflows/PIPELINES.md +++ b/.github/workflows/PIPELINES.md @@ -20,12 +20,20 @@ million individual GPU draw calls. `scripts/parse_perf_log.py` keeps the three workloads separate. The comparator requires five positive measurements for every workload in both the result and -the reviewed baseline. Any workload median more than 30% below its baseline -fails. This deliberately broad guard detects large regressions on variable -shared CI hosts; it is not a 5% performance guarantee. The baseline is never -created or updated automatically, and a fast workload cannot hide a slower one. +the reference. The reference is pinned to reviewed commit +`1b79e9df3d64542d367a1291273a48dff5ce57a6` and measured first in a separate +checkout on the same runner, with the same compiler, browser and profile as the +candidate. Any workload median more than 30% below that reference fails. Missing +reference measurements fail too; a fast workload cannot hide a slower one. -Run the same workload in a disposable checkout with the pinned compiler/browser, +An absolute cross-host baseline was unsuitable: identical renderer code measured +about 160 and 91-109 completed frames/s on different hosted machines. Paired +measurements reduce that host variance without increasing the threshold. The +broad guard still cannot establish a 5% guarantee or eliminate within-run load +changes. `DOCS/perf_baseline.json` retains the original calibration as historical +evidence; its absolute numbers are not the CI acceptance threshold. + +Run a candidate measurement in a disposable checkout with the pinned compiler/browser, a matching wasm-bindgen runner and ChromeDriver on PATH. Set `CHROME_BINARY` to the browser executable; the preparation replaces only this checkout's profile: @@ -43,19 +51,22 @@ WASM_BINDGEN_TEST_ONLY_WEB=1 WASM_BINDGEN_TEST_TIMEOUT=180 \ CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUNNER=wasm-bindgen-test-runner \ cargo test --release --locked --features performance --test performance_limit -- --nocapture 2>&1 | tee perf.log python3 scripts/parse_perf_log.py perf.log benchmark_result.json -python3 scripts/compare_fps.py benchmark_result.json DOCS/perf_baseline.json perf_result.json 30 ``` +The reusable workflow performs the complete paired comparison, using +`reference_result.json` from its same-runner reference measurement. Dispatch that +workflow to reproduce the CI gate instead of comparing unrelated host results. + `benchmark.yml` is called by CI for every PR and main push; it can also be dispatched manually. It always retains raw samples, comparison and environment metadata as `completed-frame-performance`. The duplicate unmeasured `perf.yml` workflow has been removed. These measurement jobs have read-only permissions and no notification credentials. -A baseline update must retain raw samples and link the successful measurement -run, tested SHA, browser/compiler versions and software backend. Review changes -to workload or environment before recalibrating; do not copy a regressed result -into the baseline just to make CI green. +Changing the pinned reference requires review with raw paired samples, both +tested SHAs, browser/compiler versions and software backend. Review changes to +workload or environment before recalibrating; do not advance the reference to +a regressed implementation just to make CI green. The dedicated `webdriver-perf.json` selects SwiftShader for both ANGLE and Dawn WebGPU/Vulkan. Selecting only ANGLE allowed a browser adapter to be created but @@ -66,4 +77,6 @@ the locked wasm-bindgen 0.2.100 runner. Other jobs have independent checkouts. Provenance records both `tested_sha` (the checked-out workflow SHA, a synthetic merge commit for pull requests) and `tested_head_sha` (the PR head; identical to -`tested_sha` for pushes/manual runs). Baseline calibration preserves both. +`tested_sha` for pushes/manual runs), the pinned reference SHA and CPU details. +Reference/candidate logs and complete sample sets are retained separately. +Baseline calibration preserves its original tested workflow SHA and PR head. diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 345fedd..0012329 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -16,6 +16,13 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: Check out the reviewed measurement reference + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: qqrm/webgpu-candles + ref: 1b79e9df3d64542d367a1291273a48dff5ce57a6 + path: _perf_reference + persist-credentials: false - uses: dtolnay/rust-toolchain@e2a55d2ffb04f378e9626c28d38b36d230d1e12f # master with: toolchain: '1.99.0' @@ -61,6 +68,10 @@ jobs: # The locked wasm-bindgen 0.2.100 runner reads only webdriver.json. # Materialize the dedicated profile in this isolated job checkout. Path('webdriver.json').write_text(json.dumps(config)) + Path('_perf_reference/webdriver.json').write_text(json.dumps(config)) + reference_sha = subprocess.check_output( + ['git', '-C', '_perf_reference', 'rev-parse', 'HEAD'], text=True).strip() + assert reference_sha == '1b79e9df3d64542d367a1291273a48dff5ce57a6' metadata = { 'tested_sha': os.environ['TESTED_SHA'], 'tested_head_sha': os.environ['TESTED_HEAD_SHA'], @@ -72,9 +83,22 @@ jobs: 'cpu': platform.processor(), 'backend': 'SwiftShader software WebGPU', 'browser_flags': config['goog:chromeOptions']['args'], + 'cpu_details': json.loads(subprocess.check_output(['lscpu', '--json'], text=True)), + 'reference_sha': reference_sha, + 'comparison': 'candidate versus pinned reference measured on this same runner', } Path('perf-environment.json').write_text(json.dumps(metadata, indent=2) + '\n') PY + - name: Measure the pinned reference on this runner + env: + CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUNNER: wasm-bindgen-test-runner + CHROMEDRIVER: ${{ steps.chrome.outputs.chromedriver-path }} + WASM_BINDGEN_TEST_ONLY_WEB: '1' + WASM_BINDGEN_TEST_TIMEOUT: '180' + run: | + set -o pipefail + (cd _perf_reference && cargo test --release --locked --features performance --test performance_limit -- --nocapture) 2>&1 | tee reference-perf.log + python3 scripts/parse_perf_log.py reference-perf.log reference_result.json - name: Measure completed frames for each deterministic dataset env: CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUNNER: wasm-bindgen-test-runner @@ -86,7 +110,7 @@ jobs: cargo test --release --locked --features performance --test performance_limit -- --nocapture 2>&1 | tee perf.log python3 scripts/parse_perf_log.py perf.log benchmark_result.json - name: Reject per-workload throughput regressions - run: python3 scripts/compare_fps.py benchmark_result.json DOCS/perf_baseline.json perf_result.json 30 + run: python3 scripts/compare_fps.py benchmark_result.json reference_result.json perf_result.json 30 - name: Preserve measurements and environment, including failures if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 @@ -94,6 +118,8 @@ jobs: name: completed-frame-performance path: | perf.log + reference-perf.log + reference_result.json perf-environment.json benchmark_result.json perf_result.json diff --git a/DOCS/PIPELINES.md b/DOCS/PIPELINES.md index 46be053..c9b18f9 100644 --- a/DOCS/PIPELINES.md +++ b/DOCS/PIPELINES.md @@ -10,4 +10,4 @@ Browser execution runs on hosted CI. The software Vulkan/WebGPU backend validate ## Measured performance guard -The fixed release-mode workload measures five samples for each of three chart sizes, waiting for GPU completion after every frame. Missing/invalid samples fail closed; medians are compared separately against a reviewed measured baseline with a 30% shared-runner tolerance. This is cached-chart serialized throughput on SwiftShader, not physical-device or display FPS. See [the workload and calibration contract](../.github/workflows/PIPELINES.md). +The fixed release-mode workload measures five samples for each of three chart sizes, waiting for GPU completion after every frame. Missing/invalid samples fail closed; medians are compared separately against a pinned reference measured on the same runner, with a 30% tolerance. Cross-host calibration values remain historical evidence only. This is cached-chart serialized throughput on SwiftShader, not physical-device or display FPS. See [the workload and calibration contract](../.github/workflows/PIPELINES.md). diff --git a/DOCS/perf.md b/DOCS/perf.md index 8d634f7..1109838 100644 --- a/DOCS/perf.md +++ b/DOCS/perf.md @@ -1,6 +1,8 @@ # Recorded performance baseline -This is a fixed calibration, not a live production FPS report. +This is a historical calibration, not a live production FPS report or CI threshold. +CI measures the pinned reference and candidate on the same runner to avoid +comparing absolute SwiftShader throughput across different hosted CPUs. | Source candles | Median completed frames/s | | --- | --- | From ed16cfcfe344181a21aec68202bf14a3064e5695 Mon Sep 17 00:00:00 2001 From: qqrm <4681325+qqrm@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:33:49 +0300 Subject: [PATCH 12/12] Reject mismatched reference and candidate measurement workloads --- .github/workflows/PIPELINES.md | 3 +++ .github/workflows/benchmark.yml | 11 +++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/PIPELINES.md b/.github/workflows/PIPELINES.md index 1657749..84e7d1b 100644 --- a/.github/workflows/PIPELINES.md +++ b/.github/workflows/PIPELINES.md @@ -25,6 +25,9 @@ the reference. The reference is pinned to reviewed commit checkout on the same runner, with the same compiler, browser and profile as the candidate. Any workload median more than 30% below that reference fails. Missing reference measurements fail too; a fast workload cannot hide a slower one. +The candidate and reference measurement harness must match byte for byte before +either runs. Its hash and the shared browser-profile hash are recorded; changing +dataset/canvas/warmup/sample parameters requires a reviewed reference update. An absolute cross-host baseline was unsuitable: identical renderer code measured about 160 and 91-109 completed frames/s on different hosted machines. Paired diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 0012329..7e5bf9d 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -58,17 +58,22 @@ jobs: RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} run: | python3 - <<'PY' + import hashlib import json import os import platform import subprocess from pathlib import Path + harness = Path('tests/performance_limit.rs').read_bytes() + reference_harness = Path('_perf_reference/tests/performance_limit.rs').read_bytes() + assert harness == reference_harness, 'Workload changed: review and update the pinned measurement reference' config = json.loads(Path('webdriver-perf.json').read_text()) config['goog:chromeOptions']['binary'] = os.environ['CHROME_BINARY'] + profile = json.dumps(config) # The locked wasm-bindgen 0.2.100 runner reads only webdriver.json. # Materialize the dedicated profile in this isolated job checkout. - Path('webdriver.json').write_text(json.dumps(config)) - Path('_perf_reference/webdriver.json').write_text(json.dumps(config)) + Path('webdriver.json').write_text(profile) + Path('_perf_reference/webdriver.json').write_text(profile) reference_sha = subprocess.check_output( ['git', '-C', '_perf_reference', 'rev-parse', 'HEAD'], text=True).strip() assert reference_sha == '1b79e9df3d64542d367a1291273a48dff5ce57a6' @@ -85,6 +90,8 @@ jobs: 'browser_flags': config['goog:chromeOptions']['args'], 'cpu_details': json.loads(subprocess.check_output(['lscpu', '--json'], text=True)), 'reference_sha': reference_sha, + 'harness_sha256': hashlib.sha256(harness).hexdigest(), + 'browser_profile_sha256': hashlib.sha256(profile.encode()).hexdigest(), 'comparison': 'candidate versus pinned reference measured on this same runner', } Path('perf-environment.json').write_text(json.dumps(metadata, indent=2) + '\n')