Skip to content

Commit b03edbf

Browse files
authored
Merge pull request #1214 from ably/feature/path-based-liveobjects-implementation
[Feature] Path based LiveObjects Implementation
2 parents abcd022 + a44020d commit b03edbf

233 files changed

Lines changed: 16315 additions & 9927 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.claude/skills/uts-to-kotlin/SKILL.md‎

Lines changed: 322 additions & 93 deletions
Large diffs are not rendered by default.

‎.claude/skills/uts-to-kotlin/references/objects-mapping.md‎

Lines changed: 677 additions & 0 deletions
Large diffs are not rendered by default.
Lines changed: 242 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,242 @@
1+
#!/usr/bin/env python3
2+
"""
3+
audit_translation.py — deterministic faithfulness audit of a UTS spec against its
4+
generated Kotlin test, so the per-spec review (SKILL.md Step 7) reconciles a concrete
5+
extracted ledger instead of eyeballing two files.
6+
7+
Usage:
8+
python3 audit_translation.py <spec-file.md> <GeneratedTest.kt>
9+
10+
It does ZERO semantic judgement — only mechanical extraction with regex, so the same
11+
inputs always give the same report:
12+
13+
1. Test-ID coverage. Every `**Test ID**: \`<id>\`` in the spec vs every `@UTS <id>`
14+
in the Kotlin file.
15+
- missingInKotlin: a spec Test ID with no matching @UTS tag → a whole test
16+
case is absent; implement it (or consciously exclude it and explain why).
17+
- orphanInKotlin: an @UTS tag with no matching spec Test ID → a stale or
18+
hand-edited tag. Investigate.
19+
20+
2. Per-test line ledger. Within each spec test block (from one Test ID to the next),
21+
every non-blank, non-comment code line inside the ```pseudo fences is extracted
22+
verbatim, grouped by its section (Setup / Test Steps / Assertions / …) — so setup,
23+
operations AND assertions are all enumerated, nothing escapes the ledger. Each line
24+
is tagged: "assert" (ASSERT*), "await" (AWAIT* / EXPECT), or "step" (everything else
25+
— setup, mock construction, operations). For convenience the ASSERT* and AWAIT*
26+
lines are also surfaced flat as specAsserts / specAwaits.
27+
Alongside them, the count of assertion / await / poll calls in the matching Kotlin
28+
method is reported as a tripwire: Kotlin assertions < spec ASSERTs for a test is a
29+
strong signal an assertion was silently dropped.
30+
31+
Robustness contract: this tool must never crash mid-run on any spec/test pair. It is a
32+
review aid — if it can't extract something it degrades to "couldn't verify" (fewer lines
33+
in the ledger) rather than throwing. Whatever happens it emits ONE parseable JSON object,
34+
never a traceback, so a caller can always rely on the output shape.
35+
36+
Exit status:
37+
0 — clean audit (no missing/orphan Test IDs)
38+
2 — audit ran, but there are missing/orphan Test IDs (gateable)
39+
64 — could not run: bad usage, unreadable file, or an internal error (JSON carries `error`)
40+
"""
41+
42+
import json
43+
import re
44+
import sys
45+
from pathlib import Path
46+
47+
# Spec markers -------------------------------------------------------------
48+
TEST_ID_RE = re.compile(r"\*\*Test ID\*\*:\s*`([^`]+)`")
49+
HEADING_RE = re.compile(r"^#{1,4}\s+(.*\S)\s*$")
50+
FENCE_RE = re.compile(r"^\s*```")
51+
# Imperative spec keywords. Order matters: longest / most specific first so AWAIT_STATE
52+
# is classified before the bare AWAIT.
53+
DIRECTIVE_RE = re.compile(r"\b(ASSERT_[A-Z_]+|ASSERT|AWAIT_STATE|AWAIT_ERROR|AWAIT_ALL|AWAIT|EXPECT)\b")
54+
55+
# Kotlin markers -----------------------------------------------------------
56+
UTS_TAG_RE = re.compile(r"@UTS\s+(\S+)")
57+
KOTLIN_ASSERT_RE = re.compile(
58+
r"\b(assertEquals|assertNotEquals|assertNull|assertNotNull|assertTrue|assertFalse|"
59+
r"assertIs|assertIsNot|assertContains|assertFailsWith|assertFails|assertSame|"
60+
r"assertNotSame|awaitState|awaitChannelState|pollUntil)\b"
61+
)
62+
63+
64+
def fail(code, message, status=64, **extra):
65+
"""Emit a structured error (same JSON shape as resolve_uts.py) and exit. `status`
66+
defaults to 64 — "couldn't run" — kept distinct from the audit's own 0 (clean) and
67+
2 (missing/orphan IDs) outcomes so callers can tell the two apart."""
68+
print(json.dumps({"ok": False, "error": code, "message": message, **extra}, indent=2))
69+
sys.exit(status)
70+
71+
72+
def read_lines(path):
73+
"""Read a file into lines, tolerant of encoding issues — a stray non-UTF-8 byte in a
74+
spec must never crash the audit, so undecodable bytes are replaced, not raised on."""
75+
return Path(path).read_text(encoding="utf-8", errors="replace").splitlines()
76+
77+
78+
def classify(keyword):
79+
if keyword.startswith("ASSERT"):
80+
return "assert"
81+
if keyword.startswith("AWAIT") or keyword == "EXPECT":
82+
return "await"
83+
return "other"
84+
85+
86+
def line_kind(stripped):
87+
"""Tag a pseudocode line: assert / await / step."""
88+
m = DIRECTIVE_RE.search(stripped)
89+
if m:
90+
k = classify(m.group(1))
91+
if k in ("assert", "await"):
92+
return k
93+
return "step"
94+
95+
96+
def parse_spec(path):
97+
"""Return an ordered list of test-id dicts, each:
98+
{testId, title, sections:[{heading, lines:[{text, kind}]}], asserts[], awaits[], codeLineTotal}."""
99+
lines = read_lines(path)
100+
101+
# locate each Test ID line and the nearest preceding heading (its human title)
102+
tests = []
103+
last_heading = ""
104+
for i, line in enumerate(lines):
105+
h = HEADING_RE.match(line)
106+
if h:
107+
last_heading = h.group(1)
108+
m = TEST_ID_RE.search(line)
109+
if m:
110+
tests.append({"testId": m.group(1), "title": last_heading, "_start": i})
111+
112+
# block boundaries: each test runs to the next test's start (or EOF)
113+
for idx, t in enumerate(tests):
114+
start = t["_start"]
115+
end = tests[idx + 1]["_start"] if idx + 1 < len(tests) else len(lines)
116+
117+
sections = [] # [{heading, lines:[{text, kind}]}]
118+
cur = None # current section being filled
119+
in_fence = False
120+
for line in lines[start:end]:
121+
h = HEADING_RE.match(line)
122+
if h and not in_fence:
123+
cur = {"heading": h.group(1), "lines": []}
124+
sections.append(cur)
125+
continue
126+
if FENCE_RE.match(line):
127+
in_fence = not in_fence
128+
continue
129+
if not in_fence:
130+
continue
131+
stripped = line.strip()
132+
if not stripped or stripped.startswith("#"): # blank / pseudocode comment
133+
continue
134+
if cur is None: # code before any heading in the block (rare)
135+
cur = {"heading": "(preamble)", "lines": []}
136+
sections.append(cur)
137+
cur["lines"].append({"text": stripped, "kind": line_kind(stripped)})
138+
139+
# drop sections with no code (prose-only headings, requirement tables, the title line)
140+
sections = [s for s in sections if s["lines"]]
141+
asserts = [ln["text"] for s in sections for ln in s["lines"] if ln["kind"] == "assert"]
142+
awaits = [ln["text"] for s in sections for ln in s["lines"] if ln["kind"] == "await"]
143+
t["sections"] = sections
144+
t["asserts"] = asserts
145+
t["awaits"] = awaits
146+
t["codeLineTotal"] = sum(len(s["lines"]) for s in sections)
147+
del t["_start"]
148+
return tests
149+
150+
151+
def parse_kotlin(path):
152+
"""Return dict: uts-id -> {assertionCalls[], assertionCount}. Method block for a tag
153+
runs from its @UTS line to the next @UTS line (or EOF)."""
154+
lines = read_lines(path)
155+
tags = [(i, m.group(1)) for i, line in enumerate(lines) for m in [UTS_TAG_RE.search(line)] if m]
156+
out = {}
157+
for idx, (start, tag) in enumerate(tags):
158+
end = tags[idx + 1][0] if idx + 1 < len(tags) else len(lines)
159+
calls = []
160+
for line in lines[start:end]:
161+
for m in KOTLIN_ASSERT_RE.finditer(line):
162+
calls.append(m.group(1))
163+
out[tag] = {"assertionCalls": calls, "assertionCount": len(calls)}
164+
return out
165+
166+
167+
def main():
168+
if len(sys.argv) != 3:
169+
fail("USAGE", "Usage: audit_translation.py <spec-file.md> <GeneratedTest.kt>")
170+
171+
spec_path, kt_path = sys.argv[1], sys.argv[2]
172+
for p in (spec_path, kt_path):
173+
if not Path(p).is_file():
174+
fail("FILE_NOT_FOUND", f"{p!r} not found.")
175+
176+
# Robustness backstop: never let an unexpected parsing/IO error surface as a traceback.
177+
# fail() emits structured JSON and exits 64 (distinct from the 0/2 audit outcomes) so
178+
# callers and the model can tell "couldn't run" apart from "ran, found gaps".
179+
try:
180+
spec_tests = parse_spec(spec_path)
181+
kotlin = parse_kotlin(kt_path)
182+
report = build_report(spec_path, kt_path, spec_tests, kotlin)
183+
except Exception as exc: # noqa: BLE001 — deliberate catch-all; this tool must not crash
184+
fail("INTERNAL_ERROR", f"{type(exc).__name__}: {exc}", spec=spec_path, kotlin=kt_path)
185+
186+
print(json.dumps(report, indent=2))
187+
cov = report["idCoverage"]
188+
sys.exit(2 if (cov["missingInKotlin"] or cov["orphanInKotlin"]) else 0)
189+
190+
191+
def build_report(spec_path, kt_path, spec_tests, kotlin):
192+
spec_ids = [t["testId"] for t in spec_tests]
193+
kt_ids = list(kotlin.keys())
194+
spec_id_set, kt_id_set = set(spec_ids), set(kt_ids)
195+
missing = [i for i in spec_ids if i not in kt_id_set]
196+
orphan = [i for i in kt_ids if i not in spec_id_set]
197+
198+
per_test = []
199+
for t in spec_tests:
200+
kt = kotlin.get(t["testId"])
201+
per_test.append({
202+
"testId": t["testId"],
203+
"title": t["title"],
204+
# every code line of the spec test, grouped by section (setup / steps / assertions),
205+
# each tagged assert | await | step — the full "did I translate every line?" ledger
206+
"sections": t["sections"],
207+
"specCodeLineTotal": t["codeLineTotal"],
208+
# flat convenience views of the observable lines
209+
"specAsserts": t["asserts"],
210+
"specAwaits": t["awaits"],
211+
"specAssertCount": len(t["asserts"]),
212+
"specAwaitCount": len(t["awaits"]),
213+
"kotlinPresent": kt is not None,
214+
"kotlinAssertionCalls": kt["assertionCalls"] if kt else [],
215+
"kotlinAssertionCount": kt["assertionCount"] if kt else 0,
216+
# tripwire: fewer Kotlin assertions than spec ASSERTs => likely a dropped assertion
217+
"assertionShortfall": (len(t["asserts"]) - kt["assertionCount"]) if kt else len(t["asserts"]),
218+
})
219+
220+
report = {
221+
"ok": True,
222+
"spec": spec_path,
223+
"kotlin": kt_path,
224+
"idCoverage": {
225+
"specCount": len(spec_ids),
226+
"kotlinCount": len(kt_ids),
227+
"missingInKotlin": missing,
228+
"orphanInKotlin": orphan,
229+
},
230+
"perTest": per_test,
231+
"summary": {
232+
"specAssertTotal": sum(len(t["asserts"]) for t in spec_tests),
233+
"specAwaitTotal": sum(len(t["awaits"]) for t in spec_tests),
234+
"kotlinAssertionTotal": sum(k["assertionCount"] for k in kotlin.values()),
235+
"testsWithShortfall": [p["testId"] for p in per_test if p["assertionShortfall"] > 0],
236+
},
237+
}
238+
return report
239+
240+
241+
if __name__ == "__main__":
242+
main()

0 commit comments

Comments
 (0)