diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d74e191..bb44ed8 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -69,7 +69,8 @@ jobs: "submission_safe"] REQUIRED_PHASES = ["0_preflight", "1_inventory", "2_battery", "3_sensors", "4_cpu_variance", "5_thermal_load", "6_display", - "7_physical", "8_apple_diagnostics", "9_idle_drain"] + "7_physical", "8_apple_diagnostics", "9_idle_drain", + "10_race_bench", "11_ssd_test"] FILENAME_RE = re.compile(r"^\d{4}-\d{2}-\d{2}-[a-z0-9-]+-[0-9a-f]{4}\.json$") def find_keys(obj, predicate, path=""): diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f9bd3f..8d3d1d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ All notable changes to this project are documented here. Format follows [Keep a ## [Unreleased] +### Added: performance benchmarks (wave 6) + +- **Phase 10 race benchmark (`race-bench.sh`).** Compresses a 200 MB incompressible random blob with `xz -9 -T` and reports wall-clock seconds plus MB/s throughput. Unlike Phase 4 (SHA-256, hardware-accelerated on Apple Silicon), LZMA is a general-purpose CPU workload that does not benefit from SHA-NI or Apple's crypto coprocessor, so the number is comparable across chassis families. Runs cold (before Phase 4) so boost headroom isn't already burned. Skipped gracefully if `xz` isn't on PATH. +- **Phase 11 SSD sequential read/write (`ssd-test.sh`).** Writes 2 GB of incompressible random data, drops the page cache via `sudo purge`, reads it back. Reports write_mb_per_s and read_mb_per_s. Uses `os.urandom`-generated random chunks since APFS transparently compresses zeros (a `dd if=/dev/zero` write would report fictional throughput). Free-space safety: skipped if less than 2× test size available. In `--no-sudo` mode the phase still runs with `ALLOW_NO_PURGE=1` but `page_cache_dropped: false` flags the read number as RAM-speed not SSD-speed. +- **Schema bump to 1.1.** New phases are additive. Backward compatible: pre-1.1 reports validate against the 1.0 shape. CI submission audit now requires the two new phase keys; legacy submissions need to be re-run on wave-6 to pass. +- **Race + SSD verdicts are `"info"` in v0.2.** No pass/fail thresholds yet (calibration corpus is empty). Numbers contribute to the corpus; v0.3 sets chassis-family thresholds once enough submissions accumulate. +- **Run time impact.** Adds ~1.5 min total. Phase 10 is 30-60 s depending on chassis; Phase 11 is ~10-15 s on Apple Silicon SSDs. + ### Removed - **`AGENTS.md` and `CLAUDE.md`**. The orchestrator (`./run`) handles all the automated phases end-to-end without an LLM, sudo and the y/N confirmation prompt mean an agent can't actually drive `./run` anyway, and the manual phases (display test, physical inspection, Apple Diagnostics, idle drain) are checklist work that a runbook covers directly. The cross-tool agent convention added churn without enabling anything. All agent framing dropped from README, Runbook, CONTRIBUTING, SECURITY, Reports/SCHEMA, targets/README, examples/m5-2026/README, Shakedown Brain, bug-report template, and cpu-variance.sh comments. @@ -78,7 +86,8 @@ All notable changes to this project are documented here. Format follows [Keep a - The methodology has not yet been validated against a confirmed-defective unit. Thresholds are derived from public reports and a small number of presumed-good runs. - Phase 4 uses SHA-256 which is hardware-accelerated on Apple Silicon and Coffee Lake+ Intel. It stresses the SHA engines, scheduling, and thermal mass, but not the integer pipelines or memory bandwidth that Cinebench / Geekbench probe more thoroughly. The variance methodology transfers cleanly to any sustained workload; the SHA choice is for zero-install portability. A non-accelerated workload pass is on the roadmap. -- GPU not yet covered. Memory bandwidth not yet covered. NVMe SSD performance not yet covered (only SMART status). +- GPU not yet covered. Memory bandwidth not yet covered. +- NVMe SSD sequential read/write covered as of wave 6 (Phase 11), but the numbers are informational only in v0.2; chassis-family thresholds land in v0.3. - 14" and 16" MBP currently share the `active-cooled-pro` threshold table. The calibration's [Thermal Throttling note](examples/m5-2026/Issues/Thermal%20Throttling.md) describes 14" M5 Max throttling as design behavior, so a working 14" may land in the warn band of the current thresholds. A separate 14" sub-class is on the roadmap. ## [0.1.0] 2026-04-30 diff --git a/README.md b/README.md index fef0fe4..8f01ac0 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ Or without a preset, which auto-detects chassis class from `system_profiler` (fi ./run ``` -The orchestrator runs the four automated phases (preflight → inventory → battery → CPU variance → thermal load), asks for sudo once upfront (Phase 5 needs `powermetrics`), and writes a SCHEMA-compliant report to `Reports/local/` plus a sanitized PR-able copy to `Reports/submissions/`. Runtime ~18 min on Intel, ~25 min on Air, ~45 min on MacBook Pro. +The orchestrator runs the six automated phases (preflight → inventory → battery → race benchmark → SSD test → CPU variance → thermal load), asks for sudo once upfront (Phase 5 and the SSD page-cache drop need it), and writes a SCHEMA-compliant report to `Reports/local/` plus a sanitized PR-able copy to `Reports/submissions/`. Runtime ~20 min on Intel, ~27 min on Air, ~47 min on MacBook Pro. `./run --no-sudo` skips the 10-min thermal phase (the only phase that needs sudo) for a half-runtime no-password variance-only pass. @@ -59,6 +59,8 @@ See [`examples/sample-report-illustrative/`](examples/sample-report-illustrative | 1 | Hardware identity vs. target | `system_profiler` + `sysctl` → `inventory.sh` | | 2 | Battery health (cycles, capacity, condition) | `ioreg AppleSmartBattery` → `battery.sh` | | 3 | Sensor & port inventory | (uses Phase 1 output) | +| 10 | Race benchmark (cold) | `xz -9 -T

` of a 200 MB random blob → `race-bench.sh` | +| 11 | SSD sequential read/write (cold) | 2 GB incompressible random data, page cache dropped between → `ssd-test.sh` | | 4 | **CPU performance variance** | 5 s burst + chassis-class-aware warmup (300 s on Pro, 60 s on Air) + 5 × 60 s timed iterations, parallel SHA-256 → `cpu-variance.sh` | | 5 | Sustained thermal load (chassis-class-aware thresholds) | 10 min continuous + `powermetrics` sampling → `thermal-load.sh` | | 6 | Display visual inspection | fullscreen color cycle in Safari → `display-test.sh` | @@ -66,6 +68,8 @@ See [`examples/sample-report-illustrative/`](examples/sample-report-illustrative | 8 | Apple Diagnostics | reboot + Cmd-D | | 9 | Optional idle drain | 30 min sleep, measure %/30 min | +Phases 10 (race) and 11 (SSD) run before the heavy phases so the chassis is cold. They produce calibration numbers (verdict `"info"`); pass/fail thresholds land in schema v0.3 once the submission corpus is non-empty. Unlike Phase 4 (SHA-256, hardware-accelerated), Phase 10 (LZMA) is fair across chassis families since it doesn't hit any crypto coprocessor. + ## Specifying your target Two ways: @@ -111,6 +115,8 @@ mac-shakedown/ │ ├── run-shakedown.sh # the orchestrator (`./run` execs this) │ ├── inventory.sh # system_profiler + sysctl → JSON │ ├── battery.sh # ioreg battery health → JSON +│ ├── race-bench.sh # cold xz race → JSON +│ ├── ssd-test.sh # sequential SSD read/write → JSON (sudo for purge) │ ├── cpu-variance.sh # burst + warmup + 5×60s timed iters → JSON │ ├── thermal-load.sh # 10-min sustained + powermetrics → JSON (sudo) │ └── display-test.sh # fullscreen color cycle (HTML) @@ -170,7 +176,7 @@ The manual phases (display, physical inspection, Apple Diagnostics, idle drain) - **Hosted aggregator.** Eventually, submission via API to a public site so reports aren't reviewed by hand. Until then, the PR-submission flow above *is* the aggregator. Slower, but no infra, and PR review catches PII before merge. - **Non-accelerated workload pass.** Optional Phase 4b that runs a workload without hardware acceleration (e.g. unaccelerated AES, BLAKE2b in pure Python, or a pinned matrix-multiply kernel) so the test stresses integer pipelines and memory bandwidth too. Catches batch defects that don't show up under SHA-NI / Apple Silicon's crypto engines. - **GPU variance test.** Currently CPU-only. M5 Max GPU is the bigger thermal contributor and a Metal compute load would be much more aggressive than CPU SHA-256. -- **NVMe SSD performance.** Currently we only check SMART status. Apple has shipped 256 GB single-die SSD perf regressions on past gens, worth catching. +- **NVMe SSD performance thresholds.** Wave 6 added sequential read/write measurement (Phase 11). Numbers are informational in v0.2; chassis-family pass/fail thresholds land in v0.3 once the submission corpus has baselines. Apple has shipped 256 GB single-die SSD perf regressions on past gens, worth catching. - **Memory bandwidth.** STREAM-style benchmark. - **Per-core pinning.** macOS lacks public CPU affinity APIs, so we can't pin workers to specific cores. A defective single core gets averaged out across N P-cores. Reporting `worker_imbalance_pct_per_iter` is a partial mitigation; investigating workarounds (ASIA-style fence, pthread_qos hints) is on the list. - **More generation calibrations.** Apple Silicon M1–M4 (the scripts work today; only the calibration notes and target presets need filling), Intel-era issues (T2 chip, butterfly keyboards 2018–19, GPU stutter 2019). diff --git a/Reports/SCHEMA.md b/Reports/SCHEMA.md index 313533e..f1ab1cb 100644 --- a/Reports/SCHEMA.md +++ b/Reports/SCHEMA.md @@ -9,7 +9,7 @@ Every QA run produces two artifacts in `Reports/`: | Field | Type | Required | Notes | |---|---|---|---| -| `schema_version` | string | yes | Semver of the schema. Currently `"1.0"`. | +| `schema_version` | string | yes | Semver of the schema. Currently `"1.1"`. | | `shakedown_version` | string | yes | Tool version that produced the report (e.g. `"0.1.0"`). | | `timestamp` | string | yes | ISO 8601 UTC. | | `illustrative` | bool | no | `true` for hand-crafted samples (under `examples/sample-report-*`). Aggregator must filter these out. | @@ -136,6 +136,75 @@ The `intel_*` fields are populated only on Intel Macs (where the chassis class i `raw_log_path` points at a per-user tempfile under `/var/folders/...`. `./run` strips it from the canonical submission JSON. +## Phase 10 (`10_race_bench`): full detail + +Produced by `race-bench.sh`. Fixed-work CPU race: compresses a 200 MB incompressible random blob with `xz -9 -T`. Unlike Phase 4 (SHA-256, hardware-accelerated on Apple Silicon), this workload is a fair cross-chassis-family comparison since LZMA does not benefit from SHA-NI or the Apple crypto coprocessor. + +```json +{ + "verdict": "info", + "duration_s": 42, + "details": { + "workload": "xz -9 -T12 compression of 200MB random data", + "blob_size_mb": 200, + "threads": 12, + "preset": 9, + "wall_seconds": 42.317, + "input_bytes": 209715200, + "output_bytes": 209665024, + "compression_ratio": 0.9998, + "throughput_mb_per_s": 4.73, + "data_quality": "ok", + "data_quality_notes": [] + }, + "verdict_reasons": [ + "compressed 200MB in 42.32s (4.7 MB/s, output 100.0% of input)", + "informational in v0.2; calibrated pass/fail thresholds land in v0.3" + ] +} +``` + +Race numbers are informational in v0.2: there are no chassis-family thresholds yet. Once submissions populate a baseline corpus, v0.3 sets pass/fail bands per chassis family. + +`verdict: "skipped"` if `xz` is not on PATH (pre-Catalina) or `xz` returned a non-zero exit code. + +## Phase 11 (`11_ssd_test`): full detail + +Produced by `ssd-test.sh`. Sequential write+read benchmark on incompressible random data, with page cache dropped (`sudo purge`) between write and read. + +```json +{ + "verdict": "info", + "duration_s": 8, + "details": { + "workload": "sequential write+read of 2GB incompressible random data", + "size_gb": 2, + "chunk_mb": 8, + "write_seconds": 4.213, + "write_mb_per_s": 485.7, + "page_cache_dropped": true, + "read_seconds": 3.892, + "read_mb_per_s": 525.8, + "data_quality": "ok", + "data_quality_notes": [] + }, + "verdict_reasons": [ + "write 486 MB/s, read 526 MB/s", + "informational in v0.2; calibrated pass/fail thresholds land in v0.3" + ] +} +``` + +Why incompressible random data: APFS transparently compresses zero-blocks, so `dd if=/dev/zero` writes report fictional throughput (the OS never touches the SSD for compressible sections). The script generates one 8 MB random chunk via `os.urandom` and writes it repeatedly. + +Why `sudo purge` between write and read: without dropping the page cache, the read measures RAM bandwidth (~10–50 GB/s on modern Macs), not SSD. In `--no-sudo` mode, the phase still runs but `page_cache_dropped: false` and `data_quality: "few_samples"` flag the inflated read number. + +`verdict: "skipped"` if free disk space is less than 2× the test size. + +## Note on phase key ordering + +The phase keys (`0_preflight`, `1_inventory`, ... `10_race_bench`, `11_ssd_test`) sort alphabetically rather than numerically (`10` comes before `2` lexicographically). JSON output preserves the orchestrator's insertion order so human-readable reports render in run order: preflight, inventory, battery, sensors, race, ssd, cpu_variance, thermal_load, then the skipped manual phases. Parsers iterating phase names should not assume numeric sort order. + ## Privacy / submission-safety Reports default to submission-safe: @@ -159,3 +228,8 @@ A future hosted aggregator should rotate to HMAC-SHA-256 with a per-deployment s - **Major bumps (1.0 → 2.0)** are breaking. Aggregator must explicitly support each major version. If a test methodology changes (e.g. variance test gains a new metric, or thresholds shift), bump at least the patch version and document the change in [CHANGELOG.md](../CHANGELOG.md). Methodology changes that affect comparability across submissions should bump the minor or major. + +### History + +- **1.0** → initial release schema (phases 0-9). +- **1.1** → added phases 10_race_bench and 11_ssd_test (informational; pass/fail thresholds pending v0.3 calibration corpus). Backward compatible: reports without these phases still validate as 1.0 shape. diff --git a/Verification/scripts/race-bench.sh b/Verification/scripts/race-bench.sh new file mode 100755 index 0000000..cc9cdc8 --- /dev/null +++ b/Verification/scripts/race-bench.sh @@ -0,0 +1,129 @@ +#!/bin/bash +# race-bench.sh: fixed-work CPU race benchmark. +# +# Compresses a 200 MB incompressible-random blob with xz -9 -T, measures +# wall-clock seconds and throughput. Unlike cpu-variance.sh (SHA-256, hardware- +# accelerated on Apple Silicon), xz/LZMA is a general-purpose CPU workload that +# does not benefit from SHA-NI or the AMX/crypto coprocessor. The number is +# therefore comparable across chassis families (Apple Silicon vs Intel). +# +# Output: JSON to stdout. Progress to stderr. +# +# Env knobs: +# BLOB_MB default 200 (size of random blob to compress) +# PRESET default 9 (xz compression level, 0-9; 9 = slow/strong) +# THREADS default = P-cores + +set -euo pipefail + +BLOB_MB=${BLOB_MB:-200} +PRESET=${PRESET:-9} + +# THREADS: probe P-cores then fall back; treat empty string as unset. +if [[ -z "${THREADS:-}" ]]; then + THREADS=$(sysctl -n hw.perflevel0.physicalcpu 2>/dev/null \ + || sysctl -n hw.physicalcpu 2>/dev/null \ + || sysctl -n hw.ncpu 2>/dev/null \ + || echo "") +fi +if ! [[ "$THREADS" =~ ^[0-9]+$ ]] || (( THREADS < 1 )); then + echo "race-bench.sh: could not determine thread count (sysctl failed)" >&2 + exit 2 +fi + +# xz ships in macOS since Catalina (10.15). Guard anyway. +if ! command -v xz >/dev/null 2>&1; then + python3 - <<'PYEOF' +import json +print(json.dumps({ + "workload": "xz -T compression race", + "verdict": "skipped", + "data_quality": "skipped", + "data_quality_notes": ["xz not found on PATH (race benchmark needs xz; ships with Catalina and later)"], + "verdict_reasons": ["xz unavailable; phase skipped"], +}, indent=2)) +PYEOF + exit 0 +fi + +echo "race-bench: ${BLOB_MB}MB random blob, xz -${PRESET} -T${THREADS}" >&2 + +BLOB=$(mktemp -t shakedown-race.XXXXXX) +OUTPUT=$(mktemp -t shakedown-race-out.XXXXXX) +trap 'rm -f "$BLOB" "$OUTPUT"' EXIT INT TERM + +python3 - "$BLOB" "$OUTPUT" "$BLOB_MB" "$PRESET" "$THREADS" <<'PYEOF' +import json +import os +import subprocess +import sys +import time + +blob_path = sys.argv[1] +output_path = sys.argv[2] +blob_mb = int(sys.argv[3]) +preset = int(sys.argv[4]) +threads = int(sys.argv[5]) + +# Generate incompressible random blob. Must be fresh urandom per chunk: LZMA +# detects identical repeated blocks and compresses them ~10× tighter, which +# would defeat the "non-accelerated CPU-bound workload" property of this test. +# Blob generation is outside the timed region below, so the extra urandom cost +# does not affect the benchmark. +with open(blob_path, "wb") as f: + for _ in range(blob_mb): + f.write(os.urandom(1024 * 1024)) +input_bytes = os.path.getsize(blob_path) + +print(f" generated {blob_mb}MB random data; starting xz...", file=sys.stderr) + +# Time the compression. perf_counter rather than /usr/bin/time -p avoids +# format-parsing edge cases (BSD vs GNU output ordering). +with open(output_path, "wb") as out: + t = time.perf_counter() + proc = subprocess.run( + ["xz", "-c", f"-{preset}", f"-T{threads}", blob_path], + stdout=out, stderr=subprocess.PIPE, + ) + wall = time.perf_counter() - t + +if proc.returncode != 0: + err = proc.stderr.decode("utf-8", errors="replace")[:500] + print(json.dumps({ + "workload": f"xz -{preset} -T{threads} compression of {blob_mb}MB random data", + "blob_size_mb": blob_mb, + "threads": threads, + "preset": preset, + "verdict": "skipped", + "data_quality": "skipped", + "data_quality_notes": [f"xz exited rc={proc.returncode}", err], + "verdict_reasons": ["xz failed; phase skipped"], + }, indent=2)) + sys.exit(0) + +output_bytes = os.path.getsize(output_path) +mb_per_s = (input_bytes / (1024 * 1024)) / wall if wall > 0 else 0 +ratio = output_bytes / input_bytes if input_bytes else None + +result = { + "workload": f"xz -{preset} -T{threads} compression of {blob_mb}MB random data", + "blob_size_mb": blob_mb, + "threads": threads, + "preset": preset, + "wall_seconds": round(wall, 3), + "input_bytes": input_bytes, + "output_bytes": output_bytes, + "compression_ratio": round(ratio, 4) if ratio is not None else None, + "throughput_mb_per_s": round(mb_per_s, 2), + "data_quality": "ok", + "data_quality_notes": [], + "verdict": "info", + "verdict_reasons": [ + f"compressed {blob_mb}MB in {wall:.2f}s ({mb_per_s:.1f} MB/s, " + f"output {ratio*100:.1f}% of input)" if ratio is not None + else f"compressed {blob_mb}MB in {wall:.2f}s ({mb_per_s:.1f} MB/s)", + "informational in v0.2; calibrated pass/fail thresholds land in v0.3" + ], +} +print(json.dumps(result, indent=2)) +PYEOF diff --git a/Verification/scripts/run-shakedown.sh b/Verification/scripts/run-shakedown.sh index d8d5ad2..108e312 100755 --- a/Verification/scripts/run-shakedown.sh +++ b/Verification/scripts/run-shakedown.sh @@ -211,9 +211,9 @@ fi if [[ -z "${SHAKEDOWN_YES:-}" ]]; then if [[ "$NO_SUDO" -eq 1 ]]; then - duration_hint="~8 min of sustained 100% CPU load (Phase 4 variance)" + duration_hint="~10 min total: race + SSD benchmarks (~1.5 min) plus Phase 4 variance (~8 min)" else - duration_hint="~18 min of sustained 100% CPU load (Phase 4 variance + Phase 5 thermal)" + duration_hint="~20 min total: race + SSD benchmarks (~1.5 min) plus Phase 4 variance (~8 min) plus Phase 5 thermal (~10 min)" fi cat <&2 @@ -253,6 +253,8 @@ trap 'if [[ -n "$HEARTBEAT_PID" ]]; then kill "$HEARTBEAT_PID" 2>/dev/null || tr PREFLIGHT_TXT="$WORK/preflight.txt" INVENTORY_JSON="$WORK/inventory.json" BATTERY_JSON="$WORK/battery.json" +RACE_JSON="$WORK/race.json" +SSD_JSON="$WORK/ssd.json" VARIANCE_JSON="$WORK/variance.json" THERMAL_JSON="$WORK/thermal.json" @@ -274,6 +276,19 @@ ignite "Phase 1 — inventory" ignite "Phase 2 — battery" "$SCRIPT_DIR/battery.sh" > "$BATTERY_JSON" +# Run race + SSD benchmarks while the chassis is still cold. Cold race captures +# peak boost throughput unobscured by thermal saturation. SSD numbers are +# similarly cleaner before NVMe controllers warm up under chassis heat soak. +ignite "Phase 10 — race benchmark (xz compression, ~30-60s)" +"$SCRIPT_DIR/race-bench.sh" > "$RACE_JSON" + +ignite "Phase 11 — SSD sequential read/write (~30s)" +if [[ "$NO_SUDO" -eq 1 ]]; then + ALLOW_NO_PURGE=1 "$SCRIPT_DIR/ssd-test.sh" > "$SSD_JSON" +else + "$SCRIPT_DIR/ssd-test.sh" > "$SSD_JSON" +fi + ignite "Phase 4 — CPU variance (~6-10 min depending on chassis)" start_heartbeat "$SCRIPT_DIR/cpu-variance.sh" > "$VARIANCE_JSON" @@ -301,6 +316,8 @@ python3 - \ "$PREFLIGHT_TXT" \ "$INVENTORY_JSON" \ "$BATTERY_JSON" \ + "$RACE_JSON" \ + "$SSD_JSON" \ "$VARIANCE_JSON" \ "$THERMAL_JSON" \ "$REPO_ROOT/Reports/local" \ @@ -315,11 +332,12 @@ import os import re import sys -(target_file, preflight_txt, inv_path, bat_path, var_path, thr_path, - local_dir, submissions_dir, target_name, notes) = sys.argv[1:11] +(target_file, preflight_txt, inv_path, bat_path, race_path, ssd_path, + var_path, thr_path, + local_dir, submissions_dir, target_name, notes) = sys.argv[1:13] SHAKEDOWN_VERSION = "0.1.0" -SCHEMA_VERSION = "1.0" +SCHEMA_VERSION = "1.1" def load(path): with open(path) as f: @@ -328,6 +346,8 @@ def load(path): target = load(target_file) if target_file else None inventory = load(inv_path) battery = load(bat_path) +race = load(race_path) +ssd = load(ssd_path) variance = load(var_path) thermal = load(thr_path) @@ -481,6 +501,17 @@ thermal_verdict = thermal.get("verdict", "fail") thermal_reasons = thermal.get("verdict_reasons") or [] thermal_details = {k: v for k, v in thermal.items() if k not in ("verdict", "verdict_reasons", "raw_log_path")} +# Race + SSD benchmarks default to "info" verdict in v0.2 (no pass/fail +# thresholds yet — they're calibration inputs). "info" is treated as not-failing +# and not-warning in the overall result computation below. +race_verdict = race.get("verdict", "info") +race_reasons = race.get("verdict_reasons") or [] +race_details = {k: v for k, v in race.items() if k not in ("verdict", "verdict_reasons")} + +ssd_verdict = ssd.get("verdict", "info") +ssd_reasons = ssd.get("verdict_reasons") or [] +ssd_details = {k: v for k, v in ssd.items() if k not in ("verdict", "verdict_reasons")} + skipped_phases = { "6_display": "run ./Verification/scripts/display-test.sh and fill in manual_responses", "7_physical": "follow Runbook Phase 7 manual checklist", @@ -535,6 +566,8 @@ phases = { "1_inventory": phase_block(inv_verdict, 1, {"asserts": inv_asserts}, inv_reasons or ["target asserts matched"]), "2_battery": phase_block(bat_verdict, 1, battery_details, bat_reasons), "3_sensors": phase_block(sensors_verdict, 1, {"expected_present": present, "missing": missing}, sensors_reasons), + "10_race_bench": phase_block(race_verdict, int(race_details.get("wall_seconds") or 0), race_details, race_reasons), + "11_ssd_test": phase_block(ssd_verdict, int((ssd_details.get("write_seconds") or 0) + (ssd_details.get("read_seconds") or 0)), ssd_details, ssd_reasons), "4_cpu_variance": phase_block(variance_verdict, variance_details.get("warmup_sec", 0) + variance_details.get("iterations", 0) * variance_details.get("seconds_per_iter", 0) + variance_details.get("burst_sec", 0), variance_details, variance_reasons), "5_thermal_load": phase_block(thermal_verdict, thermal_details.get("duration_s", 600), thermal_details, thermal_reasons), } diff --git a/Verification/scripts/ssd-test.sh b/Verification/scripts/ssd-test.sh new file mode 100755 index 0000000..e45614b --- /dev/null +++ b/Verification/scripts/ssd-test.sh @@ -0,0 +1,163 @@ +#!/bin/bash +# ssd-test.sh: sequential SSD write+read benchmark. +# +# Writes SIZE_GB of incompressible random data to a temp file, runs `sudo purge` +# to drop the page cache, reads it back. Reports write and read throughput. +# +# Incompressible random data matters because APFS transparently compresses +# zeros (a `dd if=/dev/zero` write would report fictional GB/s). Page-cache +# drop matters because without it the read is RAM speed, not SSD. +# +# Output: JSON to stdout. Progress to stderr. +# +# Env knobs: +# SIZE_GB default 2 (test file size; needs 2× free space headroom) +# ALLOW_NO_PURGE default 0 (if 1, skip sudo purge; read numbers will +# include page cache and be unrealistically high) + +set -euo pipefail + +SIZE_GB=${SIZE_GB:-2} +ALLOW_NO_PURGE=${ALLOW_NO_PURGE:-0} + +if ! [[ "$SIZE_GB" =~ ^[0-9]+$ ]] || (( SIZE_GB < 1 )); then + echo "ssd-test.sh: SIZE_GB must be a positive integer (got '$SIZE_GB')" >&2 + exit 2 +fi + +# Free-space safety: require 2× the test size on the volume holding the temp dir. +# df -g reports in GB (1024^3) on macOS; column 4 is "available". +FREE_GB=$(df -g . 2>/dev/null | awk 'NR==2 {print $4}') +NEEDED_GB=$((SIZE_GB * 2)) +if ! [[ "$FREE_GB" =~ ^[0-9]+$ ]] || (( FREE_GB < NEEDED_GB )); then + python3 - "$FREE_GB" "$NEEDED_GB" "$SIZE_GB" <<'PYEOF' +import json, sys +free, needed, size_gb = sys.argv[1], sys.argv[2], int(sys.argv[3]) +print(json.dumps({ + "workload": f"sequential write+read of {size_gb}GB", + "size_gb": size_gb, + "verdict": "skipped", + "data_quality": "skipped", + "data_quality_notes": [ + f"insufficient free space: {free or 'unknown'} GB available, " + f"need {needed} GB (2× test size headroom)" + ], + "verdict_reasons": ["insufficient free space; phase skipped"], +}, indent=2)) +PYEOF + exit 0 +fi + +TEMP_DIR=$(mktemp -d -t shakedown-ssd) +trap 'rm -rf "$TEMP_DIR"' EXIT INT TERM + +TEST_FILE="$TEMP_DIR/test.bin" + +echo "ssd-test: writing ${SIZE_GB}GB incompressible random data" >&2 + +python3 - "$TEST_FILE" "$SIZE_GB" "$ALLOW_NO_PURGE" <<'PYEOF' +import json +import os +import subprocess +import sys +import time + +test_file = sys.argv[1] +size_gb = int(sys.argv[2]) +allow_no_purge = sys.argv[3] == "1" + +# Reuse a single random 8 MB chunk across writes. Generating fresh urandom for +# every chunk costs measurable time and isn't needed: APFS deduplication does +# not look across writes to a single growing file, and compression sees blocks +# individually. One incompressible 8 MB chunk repeated is plenty. +CHUNK_MB = 8 +chunk = os.urandom(CHUNK_MB * 1024 * 1024) +chunks = (size_gb * 1024) // CHUNK_MB +total_bytes = chunks * CHUNK_MB * 1024 * 1024 + +data_quality_notes = [] + +# --- Write benchmark --- +t = time.perf_counter() +with open(test_file, "wb") as f: + for _ in range(chunks): + f.write(chunk) + f.flush() + os.fsync(f.fileno()) +write_seconds = time.perf_counter() - t +write_mb_per_s = (total_bytes / (1024 * 1024)) / write_seconds if write_seconds > 0 else 0 + +print(f" wrote {total_bytes // (1024*1024)}MB in {write_seconds:.2f}s " + f"= {write_mb_per_s:.0f} MB/s", file=sys.stderr) + +# --- Drop page cache so the read measures SSD, not RAM --- +purge_ok = False +if allow_no_purge: + data_quality_notes.append( + "ALLOW_NO_PURGE=1 set; page cache not dropped, so read numbers will " + "reflect RAM speed instead of SSD speed" + ) +else: + print(" dropping page cache (sudo purge)...", file=sys.stderr) + try: + subprocess.run(["sudo", "-n", "/usr/sbin/purge"], + check=True, timeout=60) + purge_ok = True + except FileNotFoundError: + data_quality_notes.append( + "/usr/sbin/purge not found; page cache not dropped, so read " + "numbers will include cache and be artificially high" + ) + except subprocess.TimeoutExpired: + data_quality_notes.append( + "sudo purge timed out after 60s; page cache likely not fully " + "dropped, so read numbers may be inflated" + ) + except subprocess.CalledProcessError as e: + data_quality_notes.append( + f"sudo purge failed (rc={e.returncode}; sudo session may have " + f"expired), so read numbers will include page cache" + ) + +# --- Read benchmark --- +t = time.perf_counter() +bytes_read = 0 +with open(test_file, "rb") as f: + while True: + buf = f.read(CHUNK_MB * 1024 * 1024) + if not buf: + break + bytes_read += len(buf) +read_seconds = time.perf_counter() - t +read_mb_per_s = (bytes_read / (1024 * 1024)) / read_seconds if read_seconds > 0 else 0 + +print(f" read {bytes_read // (1024*1024)}MB in {read_seconds:.2f}s " + f"= {read_mb_per_s:.0f} MB/s " + f"({'cache dropped' if purge_ok else 'CACHE NOT DROPPED'})", + file=sys.stderr) + +# --- Verdict & data quality --- +# v0.2: SSD numbers are informational. They contribute to calibration but don't +# pass/fail on their own until v0.3 sets chassis-family thresholds. +data_quality = "ok" if purge_ok else "few_samples" + +result = { + "workload": f"sequential write+read of {size_gb}GB incompressible random data", + "size_gb": size_gb, + "chunk_mb": CHUNK_MB, + "write_seconds": round(write_seconds, 3), + "write_mb_per_s": round(write_mb_per_s, 2), + "page_cache_dropped": purge_ok, + "read_seconds": round(read_seconds, 3), + "read_mb_per_s": round(read_mb_per_s, 2), + "data_quality": data_quality, + "data_quality_notes": data_quality_notes, + "verdict": "info", + "verdict_reasons": [ + f"write {write_mb_per_s:.0f} MB/s, read {read_mb_per_s:.0f} MB/s" + + ("" if purge_ok else " (cache not dropped, so read is RAM speed)"), + "informational in v0.2; calibrated pass/fail thresholds land in v0.3" + ], +} +print(json.dumps(result, indent=2)) +PYEOF