|
| 1 | +#!/usr/bin/env python3 |
| 2 | +""" |
| 3 | +audit_translation.py — deterministic faithfulness audit of a UTS spec against its |
| 4 | +generated Kotlin test, so the per-spec review (SKILL.md Step 7) reconciles a concrete |
| 5 | +extracted ledger instead of eyeballing two files. |
| 6 | +
|
| 7 | +Usage: |
| 8 | + python3 audit_translation.py <spec-file.md> <GeneratedTest.kt> |
| 9 | +
|
| 10 | +It does ZERO semantic judgement — only mechanical extraction with regex, so the same |
| 11 | +inputs always give the same report: |
| 12 | +
|
| 13 | + 1. Test-ID coverage. Every `**Test ID**: \`<id>\`` in the spec vs every `@UTS <id>` |
| 14 | + in the Kotlin file. |
| 15 | + - missingInKotlin: a spec Test ID with no matching @UTS tag → a whole test |
| 16 | + case is absent; implement it (or consciously exclude it and explain why). |
| 17 | + - orphanInKotlin: an @UTS tag with no matching spec Test ID → a stale or |
| 18 | + hand-edited tag. Investigate. |
| 19 | +
|
| 20 | + 2. Per-test line ledger. Within each spec test block (from one Test ID to the next), |
| 21 | + every non-blank, non-comment code line inside the ```pseudo fences is extracted |
| 22 | + verbatim, grouped by its section (Setup / Test Steps / Assertions / …) — so setup, |
| 23 | + operations AND assertions are all enumerated, nothing escapes the ledger. Each line |
| 24 | + is tagged: "assert" (ASSERT*), "await" (AWAIT* / EXPECT), or "step" (everything else |
| 25 | + — setup, mock construction, operations). For convenience the ASSERT* and AWAIT* |
| 26 | + lines are also surfaced flat as specAsserts / specAwaits. |
| 27 | + Alongside them, the count of assertion / await / poll calls in the matching Kotlin |
| 28 | + method is reported as a tripwire: Kotlin assertions < spec ASSERTs for a test is a |
| 29 | + strong signal an assertion was silently dropped. |
| 30 | +
|
| 31 | +Robustness contract: this tool must never crash mid-run on any spec/test pair. It is a |
| 32 | +review aid — if it can't extract something it degrades to "couldn't verify" (fewer lines |
| 33 | +in the ledger) rather than throwing. Whatever happens it emits ONE parseable JSON object, |
| 34 | +never a traceback, so a caller can always rely on the output shape. |
| 35 | +
|
| 36 | +Exit status: |
| 37 | + 0 — clean audit (no missing/orphan Test IDs) |
| 38 | + 2 — audit ran, but there are missing/orphan Test IDs (gateable) |
| 39 | + 64 — could not run: bad usage, unreadable file, or an internal error (JSON carries `error`) |
| 40 | +""" |
| 41 | + |
| 42 | +import json |
| 43 | +import re |
| 44 | +import sys |
| 45 | +from pathlib import Path |
| 46 | + |
| 47 | +# Spec markers ------------------------------------------------------------- |
| 48 | +TEST_ID_RE = re.compile(r"\*\*Test ID\*\*:\s*`([^`]+)`") |
| 49 | +HEADING_RE = re.compile(r"^#{1,4}\s+(.*\S)\s*$") |
| 50 | +FENCE_RE = re.compile(r"^\s*```") |
| 51 | +# Imperative spec keywords. Order matters: longest / most specific first so AWAIT_STATE |
| 52 | +# is classified before the bare AWAIT. |
| 53 | +DIRECTIVE_RE = re.compile(r"\b(ASSERT_[A-Z_]+|ASSERT|AWAIT_STATE|AWAIT_ERROR|AWAIT_ALL|AWAIT|EXPECT)\b") |
| 54 | + |
| 55 | +# Kotlin markers ----------------------------------------------------------- |
| 56 | +UTS_TAG_RE = re.compile(r"@UTS\s+(\S+)") |
| 57 | +KOTLIN_ASSERT_RE = re.compile( |
| 58 | + r"\b(assertEquals|assertNotEquals|assertNull|assertNotNull|assertTrue|assertFalse|" |
| 59 | + r"assertIs|assertIsNot|assertContains|assertFailsWith|assertFails|assertSame|" |
| 60 | + r"assertNotSame|awaitState|awaitChannelState|pollUntil)\b" |
| 61 | +) |
| 62 | + |
| 63 | + |
| 64 | +def fail(code, message, status=64, **extra): |
| 65 | + """Emit a structured error (same JSON shape as resolve_uts.py) and exit. `status` |
| 66 | + defaults to 64 — "couldn't run" — kept distinct from the audit's own 0 (clean) and |
| 67 | + 2 (missing/orphan IDs) outcomes so callers can tell the two apart.""" |
| 68 | + print(json.dumps({"ok": False, "error": code, "message": message, **extra}, indent=2)) |
| 69 | + sys.exit(status) |
| 70 | + |
| 71 | + |
| 72 | +def read_lines(path): |
| 73 | + """Read a file into lines, tolerant of encoding issues — a stray non-UTF-8 byte in a |
| 74 | + spec must never crash the audit, so undecodable bytes are replaced, not raised on.""" |
| 75 | + return Path(path).read_text(encoding="utf-8", errors="replace").splitlines() |
| 76 | + |
| 77 | + |
| 78 | +def classify(keyword): |
| 79 | + if keyword.startswith("ASSERT"): |
| 80 | + return "assert" |
| 81 | + if keyword.startswith("AWAIT") or keyword == "EXPECT": |
| 82 | + return "await" |
| 83 | + return "other" |
| 84 | + |
| 85 | + |
| 86 | +def line_kind(stripped): |
| 87 | + """Tag a pseudocode line: assert / await / step.""" |
| 88 | + m = DIRECTIVE_RE.search(stripped) |
| 89 | + if m: |
| 90 | + k = classify(m.group(1)) |
| 91 | + if k in ("assert", "await"): |
| 92 | + return k |
| 93 | + return "step" |
| 94 | + |
| 95 | + |
| 96 | +def parse_spec(path): |
| 97 | + """Return an ordered list of test-id dicts, each: |
| 98 | + {testId, title, sections:[{heading, lines:[{text, kind}]}], asserts[], awaits[], codeLineTotal}.""" |
| 99 | + lines = read_lines(path) |
| 100 | + |
| 101 | + # locate each Test ID line and the nearest preceding heading (its human title) |
| 102 | + tests = [] |
| 103 | + last_heading = "" |
| 104 | + for i, line in enumerate(lines): |
| 105 | + h = HEADING_RE.match(line) |
| 106 | + if h: |
| 107 | + last_heading = h.group(1) |
| 108 | + m = TEST_ID_RE.search(line) |
| 109 | + if m: |
| 110 | + tests.append({"testId": m.group(1), "title": last_heading, "_start": i}) |
| 111 | + |
| 112 | + # block boundaries: each test runs to the next test's start (or EOF) |
| 113 | + for idx, t in enumerate(tests): |
| 114 | + start = t["_start"] |
| 115 | + end = tests[idx + 1]["_start"] if idx + 1 < len(tests) else len(lines) |
| 116 | + |
| 117 | + sections = [] # [{heading, lines:[{text, kind}]}] |
| 118 | + cur = None # current section being filled |
| 119 | + in_fence = False |
| 120 | + for line in lines[start:end]: |
| 121 | + h = HEADING_RE.match(line) |
| 122 | + if h and not in_fence: |
| 123 | + cur = {"heading": h.group(1), "lines": []} |
| 124 | + sections.append(cur) |
| 125 | + continue |
| 126 | + if FENCE_RE.match(line): |
| 127 | + in_fence = not in_fence |
| 128 | + continue |
| 129 | + if not in_fence: |
| 130 | + continue |
| 131 | + stripped = line.strip() |
| 132 | + if not stripped or stripped.startswith("#"): # blank / pseudocode comment |
| 133 | + continue |
| 134 | + if cur is None: # code before any heading in the block (rare) |
| 135 | + cur = {"heading": "(preamble)", "lines": []} |
| 136 | + sections.append(cur) |
| 137 | + cur["lines"].append({"text": stripped, "kind": line_kind(stripped)}) |
| 138 | + |
| 139 | + # drop sections with no code (prose-only headings, requirement tables, the title line) |
| 140 | + sections = [s for s in sections if s["lines"]] |
| 141 | + asserts = [ln["text"] for s in sections for ln in s["lines"] if ln["kind"] == "assert"] |
| 142 | + awaits = [ln["text"] for s in sections for ln in s["lines"] if ln["kind"] == "await"] |
| 143 | + t["sections"] = sections |
| 144 | + t["asserts"] = asserts |
| 145 | + t["awaits"] = awaits |
| 146 | + t["codeLineTotal"] = sum(len(s["lines"]) for s in sections) |
| 147 | + del t["_start"] |
| 148 | + return tests |
| 149 | + |
| 150 | + |
| 151 | +def parse_kotlin(path): |
| 152 | + """Return dict: uts-id -> {assertionCalls[], assertionCount}. Method block for a tag |
| 153 | + runs from its @UTS line to the next @UTS line (or EOF).""" |
| 154 | + lines = read_lines(path) |
| 155 | + tags = [(i, m.group(1)) for i, line in enumerate(lines) for m in [UTS_TAG_RE.search(line)] if m] |
| 156 | + out = {} |
| 157 | + for idx, (start, tag) in enumerate(tags): |
| 158 | + end = tags[idx + 1][0] if idx + 1 < len(tags) else len(lines) |
| 159 | + calls = [] |
| 160 | + for line in lines[start:end]: |
| 161 | + for m in KOTLIN_ASSERT_RE.finditer(line): |
| 162 | + calls.append(m.group(1)) |
| 163 | + out[tag] = {"assertionCalls": calls, "assertionCount": len(calls)} |
| 164 | + return out |
| 165 | + |
| 166 | + |
| 167 | +def main(): |
| 168 | + if len(sys.argv) != 3: |
| 169 | + fail("USAGE", "Usage: audit_translation.py <spec-file.md> <GeneratedTest.kt>") |
| 170 | + |
| 171 | + spec_path, kt_path = sys.argv[1], sys.argv[2] |
| 172 | + for p in (spec_path, kt_path): |
| 173 | + if not Path(p).is_file(): |
| 174 | + fail("FILE_NOT_FOUND", f"{p!r} not found.") |
| 175 | + |
| 176 | + # Robustness backstop: never let an unexpected parsing/IO error surface as a traceback. |
| 177 | + # fail() emits structured JSON and exits 64 (distinct from the 0/2 audit outcomes) so |
| 178 | + # callers and the model can tell "couldn't run" apart from "ran, found gaps". |
| 179 | + try: |
| 180 | + spec_tests = parse_spec(spec_path) |
| 181 | + kotlin = parse_kotlin(kt_path) |
| 182 | + report = build_report(spec_path, kt_path, spec_tests, kotlin) |
| 183 | + except Exception as exc: # noqa: BLE001 — deliberate catch-all; this tool must not crash |
| 184 | + fail("INTERNAL_ERROR", f"{type(exc).__name__}: {exc}", spec=spec_path, kotlin=kt_path) |
| 185 | + |
| 186 | + print(json.dumps(report, indent=2)) |
| 187 | + cov = report["idCoverage"] |
| 188 | + sys.exit(2 if (cov["missingInKotlin"] or cov["orphanInKotlin"]) else 0) |
| 189 | + |
| 190 | + |
| 191 | +def build_report(spec_path, kt_path, spec_tests, kotlin): |
| 192 | + spec_ids = [t["testId"] for t in spec_tests] |
| 193 | + kt_ids = list(kotlin.keys()) |
| 194 | + spec_id_set, kt_id_set = set(spec_ids), set(kt_ids) |
| 195 | + missing = [i for i in spec_ids if i not in kt_id_set] |
| 196 | + orphan = [i for i in kt_ids if i not in spec_id_set] |
| 197 | + |
| 198 | + per_test = [] |
| 199 | + for t in spec_tests: |
| 200 | + kt = kotlin.get(t["testId"]) |
| 201 | + per_test.append({ |
| 202 | + "testId": t["testId"], |
| 203 | + "title": t["title"], |
| 204 | + # every code line of the spec test, grouped by section (setup / steps / assertions), |
| 205 | + # each tagged assert | await | step — the full "did I translate every line?" ledger |
| 206 | + "sections": t["sections"], |
| 207 | + "specCodeLineTotal": t["codeLineTotal"], |
| 208 | + # flat convenience views of the observable lines |
| 209 | + "specAsserts": t["asserts"], |
| 210 | + "specAwaits": t["awaits"], |
| 211 | + "specAssertCount": len(t["asserts"]), |
| 212 | + "specAwaitCount": len(t["awaits"]), |
| 213 | + "kotlinPresent": kt is not None, |
| 214 | + "kotlinAssertionCalls": kt["assertionCalls"] if kt else [], |
| 215 | + "kotlinAssertionCount": kt["assertionCount"] if kt else 0, |
| 216 | + # tripwire: fewer Kotlin assertions than spec ASSERTs => likely a dropped assertion |
| 217 | + "assertionShortfall": (len(t["asserts"]) - kt["assertionCount"]) if kt else len(t["asserts"]), |
| 218 | + }) |
| 219 | + |
| 220 | + report = { |
| 221 | + "ok": True, |
| 222 | + "spec": spec_path, |
| 223 | + "kotlin": kt_path, |
| 224 | + "idCoverage": { |
| 225 | + "specCount": len(spec_ids), |
| 226 | + "kotlinCount": len(kt_ids), |
| 227 | + "missingInKotlin": missing, |
| 228 | + "orphanInKotlin": orphan, |
| 229 | + }, |
| 230 | + "perTest": per_test, |
| 231 | + "summary": { |
| 232 | + "specAssertTotal": sum(len(t["asserts"]) for t in spec_tests), |
| 233 | + "specAwaitTotal": sum(len(t["awaits"]) for t in spec_tests), |
| 234 | + "kotlinAssertionTotal": sum(k["assertionCount"] for k in kotlin.values()), |
| 235 | + "testsWithShortfall": [p["testId"] for p in per_test if p["assertionShortfall"] > 0], |
| 236 | + }, |
| 237 | + } |
| 238 | + return report |
| 239 | + |
| 240 | + |
| 241 | +if __name__ == "__main__": |
| 242 | + main() |
0 commit comments