|
| 1 | +#!/usr/bin/env bash |
| 2 | +# run-tests-parallel.sh — run every registered test suite as parallel |
| 3 | +# processes of the already-built test-runner. |
| 4 | +# |
| 5 | +# ZERO-LOSS CONTRACT (gate quality must be identical to the sequential run): |
| 6 | +# 1. The suite list comes from `test-runner --list-suites`, which is printed |
| 7 | +# by the SAME macro table that executes suites — the list cannot drift |
| 8 | +# from reality by construction. |
| 9 | +# 2. UNION GUARD: after the run, the set of suites that actually produced a |
| 10 | +# result is compared against that list; any difference (a suite that |
| 11 | +# never ran, or ran twice) fails the gate loudly. A newly added suite is |
| 12 | +# picked up automatically on the next invocation. |
| 13 | +# 3. Per-suite pass/fail/skip counts are summed and reported in the same |
| 14 | +# "N passed[, M failed][, K skipped]" shape as the sequential runner, so |
| 15 | +# before/after totals are directly comparable. |
| 16 | +# 4. ANY suite failing, crashing (nonzero exit), or missing ⇒ exit 1. |
| 17 | +# |
| 18 | +# Usage: run-tests-parallel.sh <path-to-test-runner> [jobs] |
| 19 | +# jobs defaults to CBM_TEST_PAR_JOBS, then the CPU count. |
| 20 | + |
| 21 | +set -uo pipefail |
| 22 | + |
| 23 | +RUNNER="${1:?usage: run-tests-parallel.sh <path-to-test-runner> [jobs]}" |
| 24 | +JOBS="${2:-${CBM_TEST_PAR_JOBS:-}}" |
| 25 | + |
| 26 | +if [ -z "$JOBS" ]; then |
| 27 | + if command -v nproc >/dev/null 2>&1; then |
| 28 | + JOBS=$(nproc) |
| 29 | + elif command -v sysctl >/dev/null 2>&1; then |
| 30 | + JOBS=$(sysctl -n hw.ncpu 2>/dev/null || echo 4) |
| 31 | + else |
| 32 | + JOBS=4 |
| 33 | + fi |
| 34 | +fi |
| 35 | + |
| 36 | +LOGDIR="$(dirname "$RUNNER")/test-logs" |
| 37 | +rm -rf "$LOGDIR" |
| 38 | +mkdir -p "$LOGDIR" |
| 39 | + |
| 40 | +SUITES_FILE="$LOGDIR/suites.txt" |
| 41 | +RESULTS_FILE="$LOGDIR/results.txt" |
| 42 | + |
| 43 | +# tr strips the CR that the Windows CRT appends to every stdout line — a |
| 44 | +# suites file with CRLF endings made the runner reject every name |
| 45 | +# ("arena\r" is an unknown suite) and fail all 104 suites on CI. |
| 46 | +if ! "$RUNNER" --list-suites | tr -d '\r' > "$SUITES_FILE"; then |
| 47 | + echo "FAIL: test-runner --list-suites exited nonzero" >&2 |
| 48 | + exit 1 |
| 49 | +fi |
| 50 | +NSUITES=$(wc -l < "$SUITES_FILE" | tr -d ' ') |
| 51 | +if [ "$NSUITES" -lt 1 ] || grep -qvE '^[a-z0-9_]+$' "$SUITES_FILE"; then |
| 52 | + echo "FAIL: suite list empty or malformed (runner too old for --list-suites?)" >&2 |
| 53 | + exit 1 |
| 54 | +fi |
| 55 | +# Timing-sensitive suites run SEQUENTIALLY after the parallel wave: they |
| 56 | +# spawn subprocesses / watch the filesystem / bind ports with fixed |
| 57 | +# deadlines, and a saturated 4-core CI runner starves those deadlines into |
| 58 | +# flakes (3 cli-suite failures on the ubuntu legs of the first CI run). |
| 59 | +# Same suites, same tests, same gates — only the schedule differs; the |
| 60 | +# union guard below still checks the COMBINED result set. |
| 61 | +# stack_overflow_a/b/c: their giant-recursion ASan allocations stall ~100x |
| 62 | +# when co-STARTED with a large wave on Apple Silicon (2s staggered vs ~230s |
| 63 | +# simultaneous — a local scheduler/zone quirk, not contention: job count |
| 64 | +# does not change it). Staggered in the tail they cost seconds. |
| 65 | +SERIAL_SUITES="cli subprocess watcher incremental httpd ui index_resilience mcp \ |
| 66 | + stack_overflow_a stack_overflow_b stack_overflow_c" |
| 67 | +is_serial() { |
| 68 | + case " $SERIAL_SUITES " in *" $1 "*) return 0 ;; *) return 1 ;; esac |
| 69 | +} |
| 70 | +PAR_FILE="$LOGDIR/suites-parallel.txt" |
| 71 | +SER_FILE="$LOGDIR/suites-serial.txt" |
| 72 | +: > "$PAR_FILE" |
| 73 | +: > "$SER_FILE" |
| 74 | +while IFS= read -r sname; do |
| 75 | + if is_serial "$sname"; then |
| 76 | + echo "$sname" >> "$SER_FILE" |
| 77 | + else |
| 78 | + echo "$sname" >> "$PAR_FILE" |
| 79 | + fi |
| 80 | +done < "$SUITES_FILE" |
| 81 | +echo "=== parallel test run: $NSUITES suites ($(wc -l < "$SER_FILE" | tr -d ' ') serial-tail), $JOBS jobs ===" |
| 82 | + |
| 83 | +export RUNNER LOGDIR RESULTS_FILE |
| 84 | +run_one() { |
| 85 | + s="$1" |
| 86 | + t0=$SECONDS |
| 87 | + "$RUNNER" "$s" > "$LOGDIR/$s.log" 2>&1 |
| 88 | + rc=$? |
| 89 | + secs=$((SECONDS - t0)) |
| 90 | + summary=$(grep -E '^ [0-9]+ passed' "$LOGDIR/$s.log" | tail -1) |
| 91 | + pass=$(printf '%s' "$summary" | sed -n 's/^ \([0-9]*\) passed.*/\1/p') |
| 92 | + failn=$(printf '%s' "$summary" | sed -n 's/.* \([0-9]*\) failed.*/\1/p') |
| 93 | + skip=$(printf '%s' "$summary" | sed -n 's/.* \([0-9]*\) skipped.*/\1/p') |
| 94 | + # A single short echo line is an atomic append (< PIPE_BUF). |
| 95 | + echo "$s rc=$rc pass=${pass:-0} fail=${failn:-0} skip=${skip:-0} secs=$secs" >> "$RESULTS_FILE" |
| 96 | +} |
| 97 | +export -f run_one |
| 98 | + |
| 99 | +xargs -P "$JOBS" -I{} bash -c 'run_one "$@"' _ {} < "$PAR_FILE" |
| 100 | +# Serial tail: quiet machine for the deadline-sensitive suites. |
| 101 | +while IFS= read -r sname; do |
| 102 | + run_one "$sname" |
| 103 | +done < "$SER_FILE" |
| 104 | + |
| 105 | +# ── Union guard: every listed suite produced exactly one result ── |
| 106 | +MISSING=$(comm -23 <(sort "$SUITES_FILE") <(awk '{print $1}' "$RESULTS_FILE" | sort -u)) |
| 107 | +DUPES=$(awk '{print $1}' "$RESULTS_FILE" | sort | uniq -d) |
| 108 | +if [ -n "$MISSING" ] || [ -n "$DUPES" ]; then |
| 109 | + echo "FAIL: shard union does not match --list-suites (GATE-QUALITY LOSS)" >&2 |
| 110 | + [ -n "$MISSING" ] && echo " never ran: $MISSING" >&2 |
| 111 | + [ -n "$DUPES" ] && echo " ran twice: $DUPES" >&2 |
| 112 | + exit 1 |
| 113 | +fi |
| 114 | + |
| 115 | +TOTAL_PASS=$(awk -F'pass=' '{split($2,a," "); s+=a[1]} END{print s+0}' "$RESULTS_FILE") |
| 116 | +TOTAL_FAIL=$(awk -F'fail=' '{split($2,a," "); s+=a[1]} END{print s+0}' "$RESULTS_FILE") |
| 117 | +TOTAL_SKIP=$(awk -F'skip=' '{split($2,a," "); s+=a[1]} END{print s+0}' "$RESULTS_FILE") |
| 118 | +BAD_RC=$(grep -cv ' rc=0 ' "$RESULTS_FILE" || true) |
| 119 | + |
| 120 | +echo "── 8 slowest suites ──" |
| 121 | +sort -t= -k6 -rn "$RESULTS_FILE" | head -8 |
| 122 | +grep -v ' rc=0 ' "$RESULTS_FILE" || true |
| 123 | +for f in $(grep -v ' rc=0 ' "$RESULTS_FILE" | awk '{print $1}'); do |
| 124 | + echo "──── $f: every failure site ────" |
| 125 | + grep -B2 -A8 "FAIL" "$LOGDIR/$f.log" | head -120 |
| 126 | + echo "──── $f: last 15 lines ────" |
| 127 | + tail -15 "$LOGDIR/$f.log" |
| 128 | +done |
| 129 | + |
| 130 | +echo "────────────────────────────────────────────" |
| 131 | +echo " $TOTAL_PASS passed, $TOTAL_FAIL failed, $TOTAL_SKIP skipped ($NSUITES suites, $JOBS jobs)" |
| 132 | +echo "────────────────────────────────────────────" |
| 133 | + |
| 134 | +if [ "$TOTAL_FAIL" -gt 0 ] || [ "$BAD_RC" -gt 0 ]; then |
| 135 | + exit 1 |
| 136 | +fi |
| 137 | +exit 0 |
0 commit comments