diff --git a/benchmarks/voice/.gitignore b/benchmarks/voice/.gitignore new file mode 100644 index 00000000..79ff55b8 --- /dev/null +++ b/benchmarks/voice/.gitignore @@ -0,0 +1,2 @@ +cache/ +results/ diff --git a/benchmarks/voice/README.md b/benchmarks/voice/README.md new file mode 100644 index 00000000..67982811 --- /dev/null +++ b/benchmarks/voice/README.md @@ -0,0 +1,1556 @@ +# Voice replay harness + +Scripted conversations synthesized to audio with ElevenLabs and replayed into a +real `VoiceSession` at real-time pace, as if spoken into a mic. + +> **This directory breaks the rules of `benchmarks/`.** Everything else here +> measures pure framework overhead and needs no API keys. This is a *behavioral +> eval* of the voice stack: it needs `ELEVENLABS_API_KEY` and `DEEPGRAM_API_KEY`, +> it makes real network calls, and it runs in real time. Never wire it into the +> default test path. + +Full design: `planning/VOICE_REPLAY_HARNESS_PLAN.md`. + +## Why + +Every voice test in `python/tests/voice/` injects `TranscriptEvent`s through a +mock STT. That covers the session state machine and nothing below it — no audio, +no real STT connection, no VAD, no endpointing. The bugs that actually reach +production live in that gap. + +## Status: 38 scenarios across 5 domains, matrix, scoring and gating in place + +| File | Role | +|---|---| +| `synth.py` | ElevenLabs → cached PCM16 16k, fluent-utterance slicing, PCM/frame/WAV helpers | +| `scenario.py` | script primitives, declarative expectations, scenario library | +| `harness.py` | fake browser: paced feeder, reactive driver, playback ack pump | +| `score.py` | JSONL records, scorecard aggregation, baseline diffing | +| `cli.py` | runner, reporting, regression gate | + +### The echo suppressor fails on echo it cannot read + +`--aec-leak GAIN` mixes the assistant's own output back into the mic, standing in +for an echo canceller that does not fully cancel. Until it existed, all ~3,000 +runs fed clean user-only audio, so `_likely_stt_echo` — the guard that stops the +assistant interrupting itself on speaker bleed — had never once been exercised. +`coding_barge_in_echo` only ever proved it does not fire on genuine speech. + +Measured on `deepgram-flux/local` with `support_echo_silence`, where the user +says nothing at all for the whole reply: + +| leak gain | outcome | +|---|---| +| 0.0 (every prior run) | clean | +| 0.15 | clean | +| 0.20 | 1 failure in 3 | +| 0.30 | **4 failures in 6** | + +Failure means the assistant cut itself off mid-reply and committed its own words +as a user turn. Real barge-in still passes at 0.30, so this is under-suppression, +not over-suppression. + +**The mechanism is the interesting part.** The committed ghosts were `'Our +retailers.'`, `'Arise retailers.'` and `'has aroused retailers.'` — all manglings +of the same phrase from the reply's tail, "...authorized retailers." The +suppressor is a text-similarity check against what the assistant just said, so +echo the STT transcribes *badly enough* stops resembling its source and passes +the filter built to catch it. Clean echo gets suppressed; garbled echo does not, +which inverts the intuition that louder bleed is the dangerous case. + +That also explains why the boundary is probabilistic rather than a threshold: it +depends on how the STT happens to mis-hear a given phrase, not on the gain alone. + +**The "clean at 0.15" row above is an artifact of measuring one scenario.** The +leak axis had only ever been run against `support_echo_silence`; running the whole +suite at 0.15 on `deepgram-flux/local` gives 57/78 with **12 ghost turns and 9 +scenarios failing**, every one of them `interrupted=True` with a fragment of the +assistant's own reply committed as a user turn — `'Add memory access.'` from "a bad +memory access", `'exactly once.'`, `'retailers. Stop.'`, `'Hopefully, you should see +a doctor.'`. `support_simple` — a single question, one reply, no pauses — has the +assistant interrupting itself on 3 of 4 repeats. So a merely mediocre echo canceller +does not degrade turn-taking at the margins; it breaks the ordinary case, and the +suite's clean bill of health came entirely from never having fed it echo. + +**Fixed by comparing against a length-matched window.** The cap above is entirely an +artifact of scoring a short commit against a long tail, so the repair is to score it +against a slice of the tail its own length, aligned on the longest run the two share: + +```python +block = SequenceMatcher(None, c, tail).find_longest_match(0, len(c), 0, len(tail)) +start = max(0, min(len(tail) - len(c), block.b - block.a)) +return SequenceMatcher(None, c, tail[start : start + len(c)]).ratio() >= 0.65 +``` + +Anchoring beats sliding every window, which is the obvious alternative and measurably +worse: a weak anchor yields a badly-matched window, so coincidental resemblance +elsewhere in the reply cannot inflate the score. On the adversarial case it is the +difference between 0.75 and 0.58. The threshold is calibrated on transcripts from leak +runs rather than picked — garbled echo of a known reply scores 0.68–1.00 and genuine +barge-ins against the same replies score 0.24–0.58, two classes that do not overlap — +and 0.65 sits in that gap nearer the echo side deliberately, since suppressing real +speech makes the assistant uninterruptible, the worse of the two failures. + +| | before | after | +|---|---|---| +| leak 0.15, gated | 44/48 and 55/60 (~92%) | **40/40 (100%)** | +| leak 0.15, ghost turns | 12 per 78 runs | **6** | +| clean, 4 baselined cells | 100% | **100%, no regressions** (126 runs) | + +Those numbers stood for a few hours. The clean-run check below missed the case that +undoes them — see "the suppressor ate a real turn on a clean run". + +The clean-run check is the one that matters for shipping it: a suppressor that catches +more echo can start eating genuine interruptions. All seven baselined cells were run, +and the decisive one is `deepgram-flux/provider`, where `_likely_stt_echo` is the *only* +semantic filter between a partial and cancelling the reply — no VAD corroboration, no +EOU, nothing to catch an over-suppression. It scores 53/54 with all ten barge-in +scenarios passing every repeat, `coding_barge_in_echo` included, while +`support_echo_silence` stays uninterrupted. Its one flake is `food_long_pause` with +`interrupted=False`, so the assistant never spoke over it and the echo check was never +consulted. + +The ElevenLabs cells look worse (17/24 and 21/24 on the barge-in subset) and are not: +the genuine barge-in is heard, and the extra turn is `'Yes.'` or `'Yeah.'` — the +provider's documented hallucination on trailing silence, the same family as the `'Bye.'` +above. 17/24 is also up from the 15/24 measured there before this change. + +Reading that comparison surfaced a gate defect worth knowing about. Per-scenario rates +are matched by name and safe on a filtered run, but ghost turns, latency and dead air +are aggregates over whatever subset ran, so comparing them against a full-suite +baseline compares populations. The barge-in subset alone carries most of the suite's +ghost turns and duly reported "ghost turns 1 → 3" while being clean; a subset that +*excludes* them would have reported an improvement just as falsely. `compare()` now +takes `partial` and skips aggregates for filtered runs, which is the same reasoning +that already makes `--update-baseline` refuse them. + +**The residual had two causes, and the second was self-inflicted.** Both call sites +gated the check behind `state.assistant_active`, so echo committing just *after* +playback ended was never tested. `coding_followup_after_reply` was the clean +demonstration: it commits `'memory access.'`, an exact substring of its own reply, and +nothing looks. `TurnState` now carries `seconds_since_assistant_active` and the check +runs for 2s past playback — bounded, because `assistant_text` outlives its turn, so an +unbounded window would let a user utterance resembling the last reply be dropped, and +dropping it means no new turn starts to clear that text. + +That alone fixed nothing, which is the interesting part. Tracing the scenario showed the +suppressor working perfectly and the rescue path undoing it: + +``` +stt_committed_received 'memory access.' audio_playing=True +stt_commit_ignored reason=echo <- suppressed correctly +stt_stale_partial_commit mic_quiet=True stale_secs=1.9 +stt_stale_partial_synthesized 'memory access.' <- resurrected 3s later +stt_committed_accepted action=new_turn <- ghost turn +``` + +An IGNOREd commit does not bump `_last_commit_at`, so suppressed echo still looks like a +stranded partial to the stale-partial watchdog, which synthesizes it back once the grace +window has closed. The watchdog exists to rescue user speech that an over-eager AEC +ducked below the provider's commit threshold — which is the one thing echo is not — so +it now runs the same echo check before rescuing. Together: `coding_followup_after_reply` +0/3 → 3/3, and ghost turns across the whole 78-run leak suite 12 → 2. + +Worth noting the shape of this bug, since the codebase invites more of it: two fixes +that are individually correct, landed a day apart, in different files, that cancel each +other. Neither test suite could catch it because each mechanism is right in isolation. + +| leak 0.15, deepgram-flux/local | ghosts (78 runs) | `coding_followup_after_reply` | +| --- | --- | --- | +| before the suppressor fix | 12 | 0/2 | +| window-anchored suppressor | 6 | 0/3 | +| \+ grace window | 6 | 0/3 | +| \+ rescue guard | **2** | **3/3** | + +Eleven of the thirteen leak markers were retired by this. One of the two that remained is +different in kind and survives everything below: `support_barge_in_one_word` gets the echo +tail and the user's word in a *single* commit (`'retailers. Stop.'`), so suppressing it +loses the barge-in the scenario tests — that needs the echo span located inside the commit, +not a verdict on the whole string. + +**Then the suppressor ate a real turn on a clean run, which reframes everything above.** +`food_long_pause` on `elevenlabs/lexical` went 4/4 → 0/4. The user says "Pepperoni pizza, +please." after a reply of "Got it — a large pepperoni pizza.", and the check drops it — +on a run with no leak injected at all, suppressing echo that could not exist. It scores +0.67, inside the 0.68–1.00 band the threshold was calibrated on. + +The calibration was not wrong about its samples; it was drawn only from barge-ins, where +the user is *changing* the subject and shares little vocabulary with the reply. But +confirmations, disambiguation and digit readback all put the user's next words into the +reply by design, and there the populations interleave — genuine speech at 0.67 and 0.78 +against real echo at 0.76 and 0.79. No threshold splits that, so this is not a tuning +problem, and a per-utterance "is this echo?" has no reliable text-only answer. + +**So resemblance was latched behind proof.** The check is now two branches with different +standing: a verbatim substring of what is playing is evidence and always counts, while +garbled resemblance is a guess, armed only once the same session has produced verbatim +echo. The argument was that base rate decides it — when AEC works there is no echo in the +signal at all, so every fuzzy suppression is a false positive by construction, and when it +leaks the verbatim branch will see it. + +**It never arms, and the reason invalidates the design rather than its tuning.** Every +remaining ghost under leak is garbled and not one is verbatim: `'surprised retailers.'`, +`'Veroni, anything else?'`, `'Nash exactly once.'`, `'Not anything else?'`. Leak severity +determines transcription quality. A leak quiet enough to be garbled is garbled +*consistently* — the STT is transcribing faint, distorted audio, so of course every copy +comes back wrong — while verbatim echo belongs to a louder, cleaner leak. The two are +near-disjoint regimes rather than sequential stages, so "verbatim echo seen" does not +predict "garbled echo present", and a 20-second scenario never earns its proof. + +| leak 0.15, deepgram-flux/local | gated pass | ghost turns | +| --- | --- | --- | +| unconditional resemblance | ~100% (13 markers) | 2 | +| latched behind proof | 85% (2 markers) | 10 | + +**Kept regardless, because it changes which error you get.** Unconditional, a *working* +AEC drops genuine user speech and the assistant answers something the user never said. +Latched, a *leaking* AEC produces ghost turns. A ghost turn on a broken microphone is +plainly the lesser harm, so the latch dominates on worst case even though it gives the +leak numbers back. The real fix is timing, which is the only signal that survives both +regimes: echo of a word arrives while that word is playing, and a user repeating it does +so afterwards. Text similarity cannot see that and never will. + +One narrowing worth recording, since the earlier entry over-claimed: the latch restored +`food_long_pause` on `elevenlabs/lexical` only. It still fails on `deepgram-flux/provider` +and `deepgram-nova/lexical`, so those are a different cause and folding them into the echo +story was wrong. + +**The axis baselines now, and the old refusal rule was what blocked it.** At `--repeat 4` +it scores 85% with per-scenario rates recorded, and only `banking_digits` and +`banking_digits_pause` fail every repeat. The previous rule — refuse if anything fails — +demanded that a probabilistic axis be perfect before it could gate at all, so the axis +never gated and its markers were fitted to whichever run was in front of me. Refusal now +triggers only on a scenario that fails *every* repeat; flaky ones are recorded at their +measured rate. The repeat count matters more than expected: at `--repeat 2` five scenarios +looked hard and three of those five turned out to be intermittent at 4. + +Digit readback is the honest residual, and it is the worst case from both directions at +once. Digits never produce verbatim echo — a run of digits garbles into other digits — so +the filter never arms. And the obvious repair, trusting resemblance here because a readback +plainly echoes, is exactly backwards: the reply *is* the user's digits, so a genuine +correction ("Four four eight." against "…account four four seven.") scores 0.75. It is +where resemblance is least informative and most tempting. + +Still open on this axis: 4 session errors per 120 runs under leak, untriaged. + +**The latch then armed itself on a clean run, and the mechanism refutes the claim above.** +`food_long_pause` went 100% → 0% on `deepgram-flux/provider` and `deepgram-nova/lexical`, +and the trace shows `'pepperoni pizza, please.'` suppressed as echo while the reply is +still playing — which only the *resemblance* branch can do, so the latch was armed on a +run with no leak. What armed it was the user's own partials. The STT emits `Pepperoni`, +then `Pepperoni pizza`, and `'pepperoni pizza'` is a verbatim substring of "Got it — a +large pepperoni pizza." + +So "a verbatim substring cannot also be produced by a user saying the same thing" is +false, and confirmation flows are exactly where it is false: the reply restates the +request, so the user's words are a substring of it by construction. Length does not +separate the cases either — the echo this exists to catch is `'memory access'` at 13 +characters and the false proof is `'pepperoni pizza'` at 15. + +What does separate them is *what kind of event* the evidence is. A partial is text the STT +is still revising, and reading it as evidence about the microphone is a category error; +a commit is a claim the provider has settled on. Only commits arm the latch now. +Suppressing an echo-shaped partial still happens and predates all of this — that was never +the problem, arming on it was. `food_long_pause` returns to 100% on both cells, and +`banking_correction` and `support_barge_in`, which regressed the same way, clear with it. + +**Full matrix, 12 cells × 39 scenarios × 3 repeats: zero ghost turns.** Not one in 1386 +runs, against a suite where they were routine this morning. That is the clearest signal +that the day's filters — resemblance latched to commits, the unvoiced-commit guard, the +watchdog's echo check — are subtracting spurious turns without eating real ones. + +It also prices the hold path for the first time, because five cells had never been +baselined and four of them are holdless. Raw rates, before the pause family was marked +on those cells — they read 100% now, which is the point of marking, so these numbers +only exist here: + +| detector | ElevenLabs | Flux | Nova | +|---|---|---|---| +| `heuristic` (no hold) | 69% | **90%** | 67% | +| `provider` (no hold) | 68% | 96% | 65% | +| `lexical` | 98% | 100% | 96% | +| `local` | 100% | 100% | 99% | + +The holdless cells fail one family — pause merging — and fail it identically: `nova/heuristic` +and `nova/provider` miss the same 12 scenarios, both ElevenLabs cells the same 11. With no +hold, every fragment the STT commits is a turn boundary and nothing downstream can merge +it, so roughly 30 points of the suite is what holding buys. + +`deepgram-flux/heuristic` at 90% is the exception that makes it an STT × detector fact +rather than a detector one: Flux holds the fragment provider-side before any detector sees +it, so the same holdless detector merges nine of the scenarios Nova's cannot. That is why +the markers are written per cell — a blanket "holdless cannot merge" rule would mark those +nine as expected failures on Flux and hide any future regression in them. + +**All twelve cells now gate.** Previously seven did, and two of those seven compared against +`pass=1.0` entries recorded before the day's changes, so they would have passed a gate they +should have failed. What is left is flaky rather than broken and recorded at measured rates: +`coding_double_pause` on three cells, `medical_barge_in` and `banking_digits_pause` on +`elevenlabs/lexical`, `food_long_pause` and `coding_pause` on `deepgram-nova/local`. One +ghost turn survives, on `deepgram-flux/provider` in `medical_simple` — the only one in the +matrix, and untriaged. + +Two things this run does *not* establish, worth stating so the baseline is not over-read. +Latency and dead air were measured at `--jobs 6`, so they gate only against future runs at +the same concurrency. And the leak axis has its own baseline at 85%, which is the number to +watch for echo work — the clean matrix says nothing about it. + +Both mechanisms that consume mic energy were ruled out as causes by disabling them: +the hold's VAD extension (`HOLD_VAD_MAX_EXTENSION_SECS = 0`) and the stale-partial +watchdog's mic-quiet anchor, separately and together, fail at the same rate. That +matters because both look like plausible suspects — echo carries energy, so in +principle it can defer a hold and can go quiet in a way that triggers a rescue — and +neither is responsible. The under-suppression is the whole story. + +**The fuzzy branch of that check is unreachable, provably.** `_likely_stt_echo` +falls back to `SequenceMatcher(c, tail).ratio() >= 0.68`, where `tail` is +`max(3*len(c), 100)` characters. Since `ratio()` is `2M/(len(c)+len(tail))` with +`M <= len(c)`, sizing the tail at three times the commit caps the score at +exactly **0.50** — a *perfect* match scores well under the threshold: + +| commit chars | tail chars | best possible ratio | +|---:|---:|---:| +| 10 | 100 | 0.182 | +| 33 | 100 | 0.496 | +| 60 | 180 | 0.500 | +| 200 | 400 | 0.667 | + +It can only fire while the assistant's whole reply is still shorter than the +~100-char window. Echo is guarded during the opening of a reply and unguarded for +the rest of it, with nothing but exact-substring matching left — which is why +every ghost came from the tail of a long reply and every one was a near-miss. + +**A window-matching fix was tried and reverted.** Replacing the whole-string +ratio with a best-window comparison separates the recorded ghosts (0.615–0.963) +from clean user phrases (0.340–0.500) with a clear gap, and at a 0.58 threshold +it takes `support_echo_silence` at 0.30 leak from 4-in-6 failing to 6/6 passing. +It also breaks barge-in outright: `support_barge_in`, which passes at 0.30 leak +today, fails every run, and the assistant becomes uninterruptible — the worse +failure of the two. + +The flaw was in how the threshold was set. Scoring *clean* user phrases against +the reply is the wrong sample: under leak the STT transcribes a **blend** of user +speech and echo, and the blend lands between the two distributions rather than +inside the genuine one. No threshold on that comparison separates them, because +the thing being measured is not two populations but a continuum. + +Which leaves the audio reference. Real AEC correlates the mic against recently +played output, and the harness already has both signals. Note that Silero is +*not* the answer despite being available: `_vad_vetoes_barge_in` says plainly +that speaker echo carries energy, so VAD cannot tell it from speech. + +## What it found + +**The biggest thing it found was a bug in the harness, not in Timbal.** +Synthesizing each `Say()` on its own gives every fragment sentence-final +intonation, because ElevenLabs renders a fragment as a complete sentence. A +speaker who pauses mid-sentence does no such thing. Measured with Smart Turn v3 on +identical words ending at the same boundary, "I want to return an item I bought" +scores **0.986 (finished)** rendered standalone and **0.019 (mid-thought)** cut out +of the fluent whole-sentence render — a swing of 0.967 from intonation alone. + +Three of the known failures were that artifact. `fluent()` now renders such +sentences in one TTS call and slices them at the character timestamps ElevenLabs +returns alongside the audio; `food_list_pause`, `support_pause` and +`banking_digits_pause` went green across every detector, and median latency +improved ~10% because turns stopped being split and re-run. + +The lesson generalizes beyond this harness: a synthetic voice fixture tests TTS +phrasing unless you deliberately build it to test turn-taking. Anything previously +concluded about prosody from per-fragment audio is worth re-checking. + +**It also invalidated a fix that had looked good.** Before fluent synthesis, +`local` failed `medical_hesitant_pause` because Smart Turn scored the fragment +0.395 — correctly mid-thought — armed a HOLD, and a text-confidence tier then cut +that hold from 3.0s to 0.35s against a 1.5s pause, because Flux's auto-appended +period made the transcript read finished. Gating the shrink on a *confidently* +wrong audio score fixed it and held 3/3: under-scored closers sat near the floor +("That's all." → 0.115) and real fragments just under the threshold. On faithful +audio that same fragment scores **0.054** — below the closer — so the bands invert +and the threshold turns out to have been fitting the artifact. Reverted rather +than shipped. + +**Trailing-modifier continuations are still unsolved, but only one case is left, +and it is a genuine tradeoff rather than a bug.** `medical_hesitant_pause` splits +on every detector. Namo scores "The pain started" 1.00 complete with or without the +trailing period, so this was never about provider-added punctuation; the audio EOU +hears the continuation clearly (0.054) but the text-complete tier cuts its hold to +0.35s, under the 1.5s pause. `local` merges it about one run in five. + +Disabling that tier makes it merge 5/5 with no other scenario regressing — but the +suite had nothing covering what the tier protects, so that result was measured +blind. `support_closer` now covers it: Smart Turn under-scores "That's all." at +0.115 while the text reads finished, and removing the tier costs **2.7s of dead +air** there (speech end to reply: 0.91s with the tier, 3.59s without). Thirteen of +fourteen closers measured score above 0.5 and never reach the tier at all, so the +population it protects is small — but 2.7s is far too expensive to trade for the +fragment case, so the shipped behavior stands. + +Gating the tier on the audio score does not resolve it either. At the tier, closers +score {0.115} and fragments {0.054, 0.376}: the fragments straddle the closer, so +no threshold separates them. + +**Two timings, and confusing them will mislead you.** `eou→audio` is measured +from the *accepted* commit, so it does not include hold time: a detector that +holds three seconds and then answers in 200ms reports 200ms. `dead air` is +speech end → turn accepted, and is the only metric that can see a hold. Both are +gated per cell against the baseline. A change can move one without the other — +removing the text-complete hold tier leaves `eou→audio` flat while adding 2.6s of +dead air to six barge-in cells. + +Use `MaxDeadAir` on any scenario where the speaker has genuinely finished, so +that buying a merge with silence has to be declared rather than slipping through +as a free win. + +**Flux's `end_of_turn_confidence` does not rescue it.** Across all 28 commits in a +suite run, mid-thought fragments scored 0.690–0.920 and genuine turn ends +0.701–0.904 — overlapping almost completely, and the most confident "end of turn" +in the suite was a fragment mid-account-number. The best available threshold +catches 4 of 5 fragments while wrongly holding 35% of real turn ends, roughly +1.5s of dead air on a third of all turns. It measures "has speech stopped", not +"has the thought finished". Measured and dropped rather than built. + +**The punctuation rescue in `effective_text_eou` is load-bearing in both +directions.** It promotes a Namo-incomplete verdict whenever the text ends in +terminal punctuation, which wrongly rescues `support_pause`'s fragment (Namo 0.00, +correct) and rightly rescues "Pepperoni pizza, please." (Namo 0.00, wrong). +Restricting it to question marks fixes the first and breaks the second, one for +one — both are Namo 0.00 plus a provider-added period, so nothing separates them. +Left alone. + +`banking_confirmation` intermittently splits "Yes, that's correct." in two with a +self-barge-in, and `banking_digits_pause` under `provider` emits the one genuine +ghost turn the suite reproduces. Both are recorded as known failures with written +reasons, scoped to the detectors where they actually occur, so they report on +every run without blocking the gate. + +The digit-string one is a good argument for `--repeat`. A single run showed it +passing and the marker looked stale; at `--repeat 6` it merged 5 times and split +once, emitting a ghost `'I'`. A rate like that is invisible to any single run in +either direction — it would read as fixed five times out of six and as reliably +broken the sixth. It is marked `intermittent` so passing no longer reports as an +unexpected pass, since passing genuinely proves nothing here. + +**Adding a second STT backend broke seven cells that were working correctly.** +`banking_digits` failed in seven of twelve cells while committing exactly one turn +in all twelve: Flux writes "four four seven two nine one" and Nova writes "447291" +for the same audio. Perfect turn-taking, reported as a turn-taking defect. The +suite had already been bitten by this once and fixed it by rewriting the script to +match Flux's spelling, which is precisely the fix that does not survive a second +provider. `normalize()` now spells digit runs out, so the comparison is +representation-independent. + +**A scenario was quietly grading providers against each other's habits.** +`food_long_pause` asserted a merge under `lexical` and `local` because that had +been *observed* on Flux, so Nova failed it for splitting a 3.0s pause — which is +perfectly defensible behavior, and which the trace showed Flux never even gave a +detector the chance to get wrong. At that gap neither answer is knowably right: +holding costs 3s of dead air, splitting sends a fragment to the LLM. It now asserts +`ContentPreserved` — everything spoken arrived, across however many turns — which +still fails ElevenLabs for dropping half the order while letting both turn-taking +policies pass. Any expectation that encodes an observation rather than a +requirement will do this as soon as the matrix widens. + +Three cells are baselined so far: + +| cell | gated | p50 | p95 | known failures | +|---|---|---:|---:|---:| +| `deepgram-flux/local` | 19/19 | 214ms | 283ms | 2 | +| `deepgram-flux/lexical` | 19/19 | 235ms | 328ms | 2 | +| `deepgram-flux/provider` | 16/16 | 234ms | 324ms | 3 | + +The three are indistinguishable at the median: the 21ms spread is inside the ±6% +drift two identical serial runs show (see below), so nobody should read a winner +out of this column. + +### Measuring only Flux was measuring Flux + +The Flux column cannot tell detectors apart, because Flux does turn detection +itself. Every detector scores 95–100% on it, and the reasonable-looking conclusion +from that column alone — "the audio EOU buys nothing Namo doesn't already provide" +— was an artifact of the choice of backend. Running the same 21 scenarios against +Nova and ElevenLabs, which return transcripts without deciding turns, separates +them immediately (2 repeats, 456 runs): + +| detector | Flux | Nova | ElevenLabs | +|---|---:|---:|---:| +| `heuristic` | 95% | 79% | 79% | +| `local` | **100%** | **89%** | **95%** | +| `lexical` | 100% | 76% | 87% | +| `provider` | 97% | 75% | 75% | + +`local` wins every column. Restricting to the mid-sentence pause merges, the only +family where detectors differ at all, makes the mechanism plain: + +| detector | Flux | Nova | ElevenLabs | +|---|---:|---:|---:| +| `heuristic` | 80% | 20% | 20% | +| `local` | 100% | **60%** | **80%** | +| `lexical` | 100% | 20% | 70% | +| `provider` | 100% | **0%** | **0%** | + +`provider` goes from perfect to zero. That is the expected result stated plainly: +it defers to the STT, and off Flux there is nothing to defer to. Every other +scenario family is flat — barge-in is 90–100% in all twelve cells — so pause +merging carries the entire discriminating signal in the suite. + +The real finding is that **Smart Turn earns its keep precisely where the STT does +not endpoint for you**, which is invisible on Flux. `local` beats `lexical` by 40 +points on Nova and 10 on ElevenLabs: hearing that the speaker's pitch is still +mid-phrase survives a transcript that reads like a finished sentence, and text +alone does not. If Timbal is ever pointed at a plain streaming ASR, `local` is the +default and `provider` is close to unusable for pauses. + +One caveat: ElevenLabs drops content on hard inputs independently of turn-taking +(it committed "I'd like to order a large" and silently lost the pizza), which +`ContentPreserved` now catches but which is an STT-quality difference rather than +a turn-taking one. + +### Seven cells gate, five do not, and that is deliberate + +A census of the nine ungated cells at `--repeat 3` (1,041 runs) turned up 77 +failing scenario-cells. Marking all 77 would have been the wrong answer, because +they are not 77 findings: + +| detector | unmarked failing cells | +|---|---:| +| `heuristic` | 29 | +| `provider` | 24 | +| `lexical` | 14 | +| `local` | 10 | + +`heuristic` has no EOU model at all and `provider` delegates to the STT, so +neither can hold a mid-sentence pause on a backend that does not endpoint for +them. Their 53 failures are one architectural fact recorded 53 times, and +marking them would turn `known_failure` from a record of defects into wallpaper. + +So the four cells where the detector does real work — `deepgram-nova` and +`elevenlabs` × `local` and `lexical` — are now baselined, taking the gate from 3 +cells to **7**. That cost 24 cell-scoped markers, each citing a measured rate. +The remaining five (`deepgram-flux/heuristic`, and `heuristic`/`provider` on both +other backends) are measured on every run and reported, but not gated: their +pause-family failures are a property of choosing a detector with nothing to hold +with, which is what the detector table above is for. + +The markers cluster into three named reasons rather than 24 bespoke ones — +`_TEXT_ONLY_PAUSE_SHORTFALL` (Namo alone cannot hold the pause off Flux), +`_AUDIO_PAUSE_SHORTFALL` (Smart Turn hears it but the hold does not survive), and +`_EL_OVERMERGE` — so a fix to any one of them shows up as a block of unexpected +passes rather than a scatter. + +The ranking survived the suite growing to 38 (numbers are not comparable to the +table above — the added scenarios are deliberately harder): `local` 92/81/92, +`lexical` 94/71/89, `heuristic` 89/65/76, `provider` 91/63/75. `local` still wins +both non-Flux columns by 10 points or more; `lexical` edges it by 2 on Flux, where +the backend is doing the work anyway. + +### Harder cases: 21 → 38 scenarios + +Barge-in scored 90–100% in all twelve cells, which meant five scenarios were +spending real time confirming something already known. Seventeen were added to +attack the edges instead: barge-in at 150ms instead of a comfortable 600, a +one-word "Stop.", two interruptions in one session, an interruption made of the +assistant's own words, a backchannel that must *not* interrupt, three-part +sentences, self-corrections, a 12-second run-on, two finished sentences 0.9s +apart that must not merge, and four seconds of pure silence. + +**Barge-in survived all of it.** Instant, one-word, double and echoing barge-ins +pass in all twelve cells, as do pure silence, a bare "No." and a follow-up taken +after the assistant stops. The one-word case is the surprising pass: +`MIN_BARGE_IN_PARTIAL_WORDS` drops single-word *partials* as mic blips, so "Stop." +should have been ignored — the commit still arrives and still cuts the assistant +off. The gate delays a one-word barge-in rather than losing it. + +Every new failure is a stop mid-thought, and the detector rankings barely move +(17 new scenarios, 12 cells): + +| detector | Flux | Nova | ElevenLabs | +|---|---:|---:|---:| +| `heuristic` | 82% | 47% | 70% | +| `local` | 82% | **70%** | **88%** | +| `lexical` | 88% | 64% | 88% | +| `provider` | 82% | 47% | 70% | + +**The metric was hiding failures.** `coding_double_pause` passed on ElevenLabs +having committed "The build fails when I import the module" — "inside a test" +simply gone. Whole-string similarity scored that 0.84 against the full sentence, +over the 0.8 bar, so a third of the utterance vanished and the suite called it a +clean merge. The same bug was passing `banking_digits_pause` on ElevenLabs at 0.85 +while "291" was dropped from an account number. `Merged` and `ContentPreserved` +now check each spoken fragment against its best-matching window of the transcript, +where a missing part matches nothing and scores 0.40. Both of those are now +correctly red. A similarity threshold over a long string cannot see a dropped +tail, and the longer the utterance the blinder it gets. + +Three findings are the STT's, identical under all four detectors: + +- **Nova commits backchannels.** "Mm-hmm" over the assistant is transcribed and + committed, and every detector then treats it as a barge-in and stops the reply. + Flux and ElevenLabs swallow the sound and pass. Timbal has no notion of a + backchannel, so on any STT that transcribes one, an acknowledgement silences the + assistant. +- **Nova endpoints at sentence boundaries.** The 12-second run-on comes back as + three turns, split at clause boundaries inside continuous speech. +- **ElevenLabs merges finished sentences.** "I'll have a coffee." and "Actually, + make it two." 0.9s apart come back as one turn in all four cells. Its endpointer + waits longer than Flux's, which helps on pauses and hurts here — the suite had + no scenario punishing an over-merge before this, so a detector could have scored + well by holding everything. + +Two findings sharpen older ones. The spurious single-token `'I'` turn under Flux + +`provider` had been attributed to digit strings for as long as +`banking_digits_pause` was the only scenario producing it; `medical_filler_midway` +now produces it verbatim on ordinary words, so it is the path, not the content. +And running the baselined cells at `--repeat 3` caught `coding_pause` splitting +once in three under `provider` — a scenario baselined as a clean pass, unchanged, +that had only ever been run once per cell. + +The trailing-modifier failure now has four instances rather than one, and they +agree: `medical_hesitant_pause`, `medical_self_correction`, `banking_correction` +and `coding_double_pause` all commit a fragment that is a complete sentence in +isolation, and no text signal argues for holding. `banking_correction` is the one +with teeth — split, the agent transfers to account 447 when the caller said 448 — +and it is also the clearest win for Namo anywhere in the suite: `lexical` merges it +3/3 on Flux where `local` and `provider` never do. + +Known failures went from 3 to 10 of 38, which is worth watching. Eight of the ten +are scoped to specific cells rather than blanket-marked, and eight are +`intermittent` — but a suite that marks everything interesting gates nothing, and +this is the point at which that stops being a theoretical concern. + +### The suite could not see the cost of a hold + +`eou→audio` starts counting at the *accepted* commit, so every second a hold +spends deciding costs nothing measurable. Pass/fail only ever saw whether a turn +eventually merged. Between them, the two metrics were blind to turn-taking +timing — and that blindness nearly shipped a bad change. + +The text-complete hold tier shortens a hold to 0.35s when the audio model says +"incomplete" but the transcript reads finished. Disabling it looks like an +unambiguous win: **11 scenario-cells fixed, zero regressions**, every `local` +cell converging on 95% (Flux 89→95, Nova 79→95, ElevenLabs 87→95), including +`medical_hesitant_pause` on all three backends. + +Timing speech end to commit tells the other half. The tier fires on **21% of +scenario-cells**, and the cost lands almost entirely on cases that were already +correct: + +| scenario-cell | cost of removing the tier | was it failing before? | +|---|---:|---| +| `medical_barge_in`, `medical_barge_in_twice` (6 cells) | +2.6s | no — pure loss | +| `support_closer` (3 cells) | +2.6s | no — pure loss | +| `support_pause` (3 cells) | +2.4s | no — pure loss | +| `medical_long_utterance` (Nova) | **+8.2s** | counted as a *fix* | + +Six barge-in cells taking 2.6s longer to answer an ordinary question is not +visible anywhere in pass/fail, and an 8.2s "fix" is not a fix. So the tier stays +— but **0.35 was the wrong value, and only a sweep could show that**, because +both endpoints look bad from where we were standing. + +`MaxDeadAir` and a per-cell dead-air gate now exist, measuring speech end → turn +accepted, and `sweep.py` can vary a detector parameter without editing product +constants. Sweeping the tier over 0.35 / 0.8 / 1.2 / 2.0 / 3.0 found the shape +nobody had looked for: + +| tier timeout | pause merges | dead air p50 | p95 | +|---|---:|---:|---:| +| 0.35 (was) | 58% | 852ms | 2270ms | +| **1.2** (now) | 81% | 889ms | 1944ms | +| 3.0 | 90% | 844ms | 3617ms | + +**Median dead air is flat across the whole range.** The tier only fires when +audio says incomplete and text says finished — a minority of turns — so the +median cannot move and the cost is entirely a tail effect. That is exactly why +0.35 read as free and went eight months unexamined: the metric that would have +priced it did not exist, and the one that did could not see it. + +Confirmed on the full suite at `--repeat 3`, 684 runs across all three backends +under `local`: correctness **84% → 92%**, with every scenario the tier exists to +protect — `support_closer`, all four barge-ins, `banking_short_reject`, +`coding_followup_after_reply` — unchanged at 100%. `coding_double_pause` goes +33→100%, `medical_long_utterance` 67→100%, `medical_hesitant_pause` 0→67%. The +cost is p95 dead air 1969 → 2683ms. 3.0 scores higher on merges still and is not +worth it: p95 3617ms and `support_pause_short` starts failing. + +The default is now 1.2. The gated Flux cells stayed at 100% and their dead-air p50 +improved, so they were re-baselined serially against the new default rather than +left holding pre-retune numbers the gate would have had slack against +(725→668, 819→656, 550→489ms). + +What the sweep also settled is where this knob *stops*. `medical_self_correction` +tops out at 33% and `banking_correction` at 22% — at every value tried. Those are +the two trailing-modifier cases with real consequences, and no hold duration +fixes them, because their problem is not how long the hold runs but that nothing +arms one when both signals read finished. A fixed timeout is the wrong shape for +that, and a threshold on the audio score cannot discriminate either: the fragment +scores 0.054 and the closer it must not catch scores 0.115. + +**Dead air immediately found something unrelated.** On +`deepgram-nova/lexical` a spoken account number sits at **7.5s** — reproducible, +and against 0.4–2.1s in all eleven other cells. Root cause: +`_maybe_start_endpointer` arms the VAD endpointing fast path only when the +detector exposes an audio EOU model, so `LexicalTurnDetector` (text-only) never +arms it, nothing ever sends Deepgram a `Finalize`, and the commit waits on Nova's +own endpointing. `local` commits the same audio in 750ms. Flux hides it entirely +by endpointing itself. The fast path's value — telling the STT to stop — is +independent of *how* EOU was decided, so the one configuration that needs it +most is the one architecturally forbidden from having it. + +Getting there killed three hypotheses, each of which looked right: + +- **Namo mis-scoring digit strings.** It scores them 0.987–1.000, complete. +- **The stale-partial watchdog rescuing a stranded turn.** The committed text is + numeral-formatted (`"447291"`) while every partial was word-formatted, so the + commit is Nova's own `smart_format` final, not a synthesized rescue. +- **`_endpoint_text_score` routing past Namo.** Real bug — the chain checks + `effective_text_eou` then `fallback_text_eou`, and `LexicalTurnDetector` names + its predictor `text_eou`, so the endpointer scored partials with the + punctuation baseline. Fixing it changed dead air by 0ms, because that method is + only ever called by an armed endpointer, which under `lexical` never exists. + Dead code for the detector it names. + +None of the three was visible in pass/fail or in `eou→audio`. + +**Parallelism turned out to be nearly free, which was not the expectation.** +`--jobs` was built assuming concurrent sessions would contend for Silero and Smart +Turn inference and inflate `eou→audio`, so the gate refuses to compare runs +measured at different concurrency. Then the full suite was run four times, twice +each way — p50 per cell, in run order: + +| cell | serial | serial | `--jobs 4` | `--jobs 4` | +|---|---:|---:|---:|---:| +| `deepgram-flux/local` | 228ms | 214ms | 214ms | 217ms | +| `deepgram-flux/lexical` | 230ms | 235ms | 213ms | 214ms | +| `deepgram-flux/provider` | 222ms | 234ms | 222ms | 215ms | + +Two serial runs of the same cell differ by up to 6%, which is as large as any +serial-to-parallel gap — and the parallel runs are, if anything, slightly faster +and tighter (213–222ms, against 214–235ms serial). There is no separation to find +here. + +Real-time pacing is why: a replayed session spends nearly all of its wall clock +asleep between 20ms frames, so four of them interleave in the gaps rather than +competing for CPU. The same 54 runs take 426s serially and 114s at `--jobs 4`, a +3.7× speedup that costs nothing measurable. + +The guard stays anyway. It costs one serial run before committing a baseline, and +the headroom above is specific to this machine, this suite size and `--jobs 4` — +none of which the gate can verify at compare time. It fails toward reporting a +number rather than gating on one. + +The same table is a warning about the latency figures generally: a 15% gate on a +statistic that drifts 6% between identical runs has less headroom than it looks +like, and any cross-detector comparison under ~20ms is noise. + +### ElevenLabs was doing the merging, and that hid a defect in ours + +ElevenLabs' dead air sits at p50 ~1.5s against Deepgram's 0.5–0.7s, which read as +a provider property until someone looked: `default_voice_config_from_env` ships +`vad_silence_threshold_secs=1.2` while Nova ships `endpointing=300`. A 4x +asymmetry, in *our* config, with a written justification on neither. The obvious +move was to cut it toward Nova's and collect the second. + +Sweeping it says the opposite, and says it flatly: + +| `vad_silence_threshold_secs` | pause merges | dead air p50 | p95 | +|---|---:|---:|---:| +| 0.3 | 47% | 728ms | 2388ms | +| 0.5 | 44% | 757ms | 2722ms | +| 0.8 | 59% | 1024ms | 3125ms | +| 1.2 (ships) | **86%** | 1409ms | 2721ms | + +Every millisecond of that 1.5s is buying a merge: 39 points of them for 681ms. +And the per-scenario column says who was earning it — `medical_self_correction` +and `banking_correction`, the two cases the hold-tier sweep above could never fix +at any value, score 100% here and 0% at 0.3. **ElevenLabs' VAD was holding them, +not our detector.** Take its 1.2s away and Timbal's own merge path is exposed at +47%, which is also roughly where `deepgram-nova/lexical` has been sitting all +along at `endpointing=300`. Two findings filed separately were one defect, with a +provider's patience masking it on one backend and not the other. + +That also killed a day of planned work before it started. Switching ElevenLabs to +`commit_strategy="manual"` — Timbal owning endpointing end to end, which sounds +strictly better — is the 0.3 column with extra steps, plus `rate_limited` being a +*fatal* STT message type and the endpointer silently dropping any commit it skips +for `session_gate` or `commit_interval`. + +The defect itself is one line of the `coding_double_pause` trace at 0.3: + +``` +17:19:00 commit "The build fails when I-" audio p=0.007 → HOLD 3.0s +17:19:02 commit "I import the module." audio p=0.035 → HOLD 1.2s +17:19:03 [user starts speaking "inside a test."] +17:19:03 stt_hold_expired ← mid-speech +17:19:04 commit "Inside a test." → 2 turns, FAIL +``` + +`_arm_hold` already refuses to fire mid-utterance — but it keys that on a new STT +*partial* since the commit, and lets Silero only corroborate one, never trigger. +When the provider commits on a short silence the next fragment's audio is in +flight for over a second before any partial exists, so the guard has nothing to +hold on and the timer wins. Silero saw that speech the whole time; the log line +`vad_speech_stop utterance_secs=0.96` sits four lines above the expiry. The +fastest signal in the pipeline was subordinated to the slowest. + +Holds now also extend on mic energy alone (`_vad_hears_speech_now`), capped at +3.0s, re-checked every 500ms: + +| at `vad_silence_threshold_secs=0.3` | pause merges | dead air p50 | p95 | +|---|---:|---:|---:| +| before | 47% | 728ms | 2388ms | +| after | **77%** | 742ms | 2230ms | + +Thirty points for fourteen milliseconds — the tell that those merges were never +being bought with time. They were being lost to a timer firing while the user was +still talking, and waiting for the *user* rather than for a *duration* costs +nothing when the user has in fact stopped. At 1.2 the change is inert (86% → 83%, +inside the noise at n=78), exactly as it should be: ElevenLabs isn't splitting +utterances there, so there is nothing to catch. + +The cap is not decoration. Echo surviving an imperfect canceller carries energy — +see the echo-suppressor section — so without it the assistant's own playback could +hold a turn open for as long as it speaks. + +**What this changes is the shape of the config question, not just a number.** The +gap between an aggressive threshold and the shipped one has gone from 39 points to +six: + +| | pause merges | dead air p50 | +|---|---:|---:| +| 0.3 | 77% | 742ms | +| 1.2 | 83% | 1417ms | + +Six points for 675ms on every turn is a trade worth arguing about; 39 points for +the same 675ms was not. Unresolved deliberately: n=78 on one backend, and the +pause family excludes every barge-in by construction, which is precisely where +holding longer would cost. + +**Confirming the fix on the full matrix caught the harness scoring it wrong.** +The gate reported `ghost turns 1 → 3` (or 4) against baseline in four cells — and +those are exactly the four where merging improved. `count_ghost_turns` asked +whether a committed turn resembled *one* script entry at ≥0.6 similarity, so a +correct three-fragment merge, being about 2.5x longer than any fragment it was +compared against, resembled none of them: + +``` +deepgram-nova/local coding_double_pause turns=1 ghosts=1 passed=True failures=[] + committed: ['The build fails when I import the module. Inside a test.'] + script: ['The build fails when I', 'import the module', 'inside a test.'] +``` + +One turn, zero failures, and the metric calls it invented. Three repeats × the one +scenario in the suite with three fragments, plus one pre-existing real ghost, +accounts for every unit of that "regression" — and it gates. It mis-scored the +inverse too: `medical_long_utterance` split into three commits counted two ghosts, +because a piece of a run-on does not resemble the whole. + +Turns are now matched against a *window* of the whole script (`best_window`, which +already existed for `HeardPrefix`) rather than any single entry, since turn +boundaries are the thing under test and are not expected to line up with script +boundaries. Turns under three words keep the per-entry rule: a one-word needle +finds a passable window in almost any script, and a spurious lone `"I"` turn — +which Flux really does emit — has to stay countable. Across all 38 speaking +scenarios, replayed both as one perfect merge and as perfect per-entry turns, +neither shape now registers a ghost. The rule lives in `scenario.attributable` +and is shared by the `NoGhostTurns` expectation and the gated count — the +expectation had silently kept the per-entry comparison, so the same run could +fail `NoGhostTurns` on a correct merge while gating zero ghost turns, or the +reverse on a split. + +**Chasing two apparent flakes found the harness had been mis-measuring +ElevenLabs on the whole pause family.** 69 failures in the matrix read +`fragments never heard` — the second half of an utterance absent entirely, the +worst failure class here. 68 were ElevenLabs, across 13 scenarios, concentrated +in `provider` (31) and `heuristic` (29): exactly the two detectors with no +Timbal-side endpointing, so exactly the two that wait on the provider's own +commit. + +`AwaitCommit` waited for `len(committed) > base`, with `base` sampled when the +last `Say` *started*. ElevenLabs commits ~1.6s after speech ends, so the +*previous* fragment's commit landed after that sample and satisfied the wait +meant for the fragment under test. What remained was the 0.8s tail plus a 0.5s +drain, against a commit needing ~1.0s. The turn was lost to teardown: + +``` +[+ 3.19s] committed "The pain is, um..." <- fragment 1, 1.7s after it was spoken +[+ 4.05s] await_commit <- satisfied instantly by the line above +[+ 4.67s] partial "Mostly at night." <- fragment 2 arrives + <- teardown ~4.85s. Never committed. +``` + +Nova commits in ~600ms, before the next fragment is even spoken, so its +watermark stays honest — which is why Nova showed real splits while ElevenLabs +showed phantom content loss. The bias tracked provider commit latency, so the +slower provider was penalised for being slow *twice*. + +A count cannot distinguish a commit for the speech just fed from a late one for +earlier speech, so waits are now timed. Anything arriving after the last `Say`'s +audio finished feeding is the answer, because mic audio is paced at realtime and +the provider cannot have been sent a clip it has not received yet. + +That alone is not enough, and the comment it replaced said why: "a fast commit +can land while the clip is still draining". Two different events look like that. +One is this bug — an earlier utterance's commit, arriving late — but the other is +real: the provider can have all the audio it needs before the harness finishes +handing the clip over, and `food_simple` commits `'…a large coffee and a +croissant'` without the final `?` for exactly that reason. Then nothing ever +arrives "after", and a strict timed wait hangs for its full timeout. Requiring it +cost 4 runs in a 111-run cell, `medical_barge_in_twice` taking 46s to fail on +three turns it had already committed correctly. + +So a wait resolves on either: something arriving after the speech, or the session +going quiet with something already in hand. The fallback covers the +early commit, and also covers a final `Say` that correctly produces nothing — +which previously only avoided a timeout by accident, because a stale event +satisfied it. `food_backchannel`'s workaround of dropping the step entirely is no +longer load-bearing, though it is left in place. + +Two holes survived that fix, both the original bug wearing the new clock. +"Something arriving after the speech" does not say *whose* speech: the late +commit for an earlier fragment — the very event the watermark fell for — also +lands after the last `Say` ends whenever the provider's commit lag exceeds the +fragment gap, and satisfied the timed wait just as instantly. And the quiet +fallback floored at 1.0s on holdless detectors while ElevenLabs' shipped +endpointer waits 1.2s of silence before committing, so "quiet with something in +hand" could describe a commit 200ms from landing, with only an older utterance's +commit in hand. Both are closed by reading what the provider is still doing +rather than guessing from timestamps: a partial newer than everything in hand +means speech is still being transcribed — a commit or a deliberate suppression +is pending — so the wait stays open until that commit lands or the partial goes +stale for `settle_secs`; and `settle_secs` itself now also derives from the +STT's own endpointing delay (`vad_silence_threshold_secs` and friends), for the +same reason it derives from the detector's holds: the failure mode of choosing +it by hand is invisible and reads as the provider dropping a turn. + +Under the fix `medical_filler_midway` on `elevenlabs/provider` reports the same +thing 3/3: a split, no errors, nothing lost. `food_long_pause` on +`elevenlabs/lexical` goes 1/3 to 4/4 with content intact, and its known failure — +which blamed the provider for dropping audio — is gone. Two things worth +knowing: latency percentiles for affected cells were computed on a biased subset, +since a turn that never commits also never reports `eou→audio` (the repro went +from 3 samples to 6), so the ElevenLabs latency figures recorded before this are +drawn from a biased subset of turns. + +**The re-baseline then caught ElevenLabs dropping barge-in commits outright.** +`medical_barge_in`, `support_barge_in`, `support_barge_in_instant` and +`support_barge_in_late` under `elevenlabs/lexical` went from 12/12 passing to +5/12 in the space of four hours, on cells whose only code change in between was +the harness. The interrupting turn interrupts — `interrupted=True`, heard text +present — and then never commits: + +``` +[+ 4.84s] say "Sorry, one more thing." +[+ 5.02s] partial "I don't know." <- invented; this is what barges in +[+ 7.02s] partial "Sorry, one more thing." <- the real text, as a partial +[+ 8.23s] partial "Yeah." <- invented +[+ 9.24s] partial "Yeah." <- no commit, ever +``` + +Not the harness, and not the hold change: forcing the waits back to resolving +immediately reproduces it identically, 0/3, and the same scenario passes 3/3 on +`deepgram-nova/lexical` while ElevenLabs passes 4/4 on non-barge-in scenarios. The +mechanism is visible in the trace — ElevenLabs hallucinates on the trailing +silence, and with `commit_strategy="vad"` each invented partial restarts the 1.2s +silence timer that would otherwise commit, so the real utterance is held open +indefinitely. That is the same threshold the merge rates depend on, which makes it +a poor thing to be load-bearing twice. + +**But calling it provider-side was letting Timbal off too easily.** The session +already has a watchdog for precisely this — a partial the provider never commits, +whose docstring describes the words hanging "as a '…' caption forever" and which is +meant to fire once "the provider clearly won't". It measured staleness from the +*last partial arrival*, so partials landing 1.0–1.2s apart against its 2.5s +threshold kept it permanently disarmed: the churn that stops ElevenLabs committing +is the same churn that stops the safety net noticing. Logging the anchor shows it +firing at `stale_secs` of 0.0, 0.4 and 0.9 once mic silence counts too — three +rescues that could never have happened on transcript staleness alone. + +Anchoring on mic silence instead (`_mic_quiet_for`: Silero heard ≤0.1s of speech in +the window, so the user demonstrably stopped, whatever the provider is still +emitting) takes the four barge-in scenarios from 4/12 to **15/24**, no session +errors, with the three remaining failures flaky rather than hard and +`support_barge_in_instant` passing outright. It is deliberately not gated on +`_vad_evidence`: that flag guards inferences which can *suppress* something the +user did, and this one only rescues speech that would otherwise be lost. Confirmed +free of collateral damage across 156 runs on the four baselined Deepgram cells — no +unexpected failures, no errors, and the single ghost is `banking_confirmation` on +`deepgram-flux/local`, which produced the identical ghost in 1 of 3 repeats before +the change. + +**Most of the remainder turned out to be this harness, not the provider.** Two further +changes took the same four scenarios from 15/24 to **24/24** with zero ghost turns, and +the second is the one that mattered. + +The first keeps the churn from destroying what the watchdog exists to rescue. A +hallucinated `"Yeah."` overwrote `_latest_partial_text` and refreshed the staleness +anchor, so the real utterance was both gone and un-rescuable; a partial is now ignored +for those two purposes when a stranded partial is already waiting, the new text is not +that one refined, and Silero heard nothing recent. All three conditions are needed — +silence alone would break the watchdog's founding case, where the user speaks quietly +under playback and neither the provider VAD nor Silero registers it. Worth 19/24 alone. + +The second: `settle_secs` was a flat 1.0s, and the shortest HOLD tier is 1.2s. A hold +emits nothing while it debounces — that is the entire point of it — so a quiescence rule +shorter than the hold cannot distinguish "finished" from "deliberately waiting", and the +harness tore down mid-hold. The trace is unambiguous: `'Sorry, one more thing.'` commits +as a 1.5s `lexical_hold` and the run ends before it can fire, scoring as the product +losing a turn. It is now derived from the detector's own hold timeouts (`local` 3.5s, +`lexical` 2.0s, the holdless detectors 1.0s) rather than chosen, because the failure +mode of getting it wrong is invisible and looks like someone else's bug. + +That correction is worth stating plainly: the five failures written off above as +provider variance were substantially mine. The A/B that appeared to exonerate the wait +changes only compared *whether* a wait resolved, not whether it resolved early, and a +settle that fires mid-hold resolves perfectly happily. Full suite after both, previously +unbaselinable: `elevenlabs/local` 61/62, `elevenlabs/lexical` 63/66, no session errors on +either. Deepgram is unmoved — `flux/lexical` 34/34, `flux/provider` at its usual 26/27, +and the pause family that stood to lose most from a longer settle is 16/16 across +`support_pause`, `support_pause_short`, `food_long_pause` and `coding_double_pause`. + +Two fixes that looked obvious and measured worse, both worth not re-trying: + +| Hypothesis | Result | +|---|---| +| `commit_strategy="manual"`, so Timbal drives commits instead of ElevenLabs' VAD | **0/12**, and every run also errored `timed out waiting for assistant audio`. The first turn still commits via the endpointer, but the reply never arrives in time. Only viable at all where the endpointer arms — under `heuristic` or `provider` nothing would ever commit. | +| Dropping the synthesis fallback's requirement that the partial text hold still across its 400ms grace, since a hallucination landing inside that window skips the rescue | **4/12** and it synthesized a hallucinated `"Yeah."` as a turn of its own. The stability check is not merely guarding against a racing commit: it is the only thing separating real speech from churn. A stranded turn beats an invented one. | + +Consequence for the baseline: 5 of 12 cells carry a current one, all Deepgram, all +at `--jobs 6`. The two ElevenLabs entries are stale — `--jobs 1`, ~31 runs from a +`--quick` subset — and still cannot be refreshed, since a cell with unmarked +failures is refused and the barge-ins remain flaky at 62%. Latency there is +consequently ungated on a concurrency mismatch as well. + +Two incidental findings from the same session. `sweep.py` never configured +structlog, so every sweep — including the tier retune above — buried its own +result table under a few hundred thousand DEBUG lines, far enough to be dropped +outright by anything that caps captured output. And `python/tests/voice/` had four +red tests: one asserting the pre-retune `0.35`, and three that set +`session._endpointer` without `_vad_evidence`, which the endpointer-arming fix +split into separate flags — so the tests covering VAD veto behaviour were all +silently exercising the no-evidence path. Both are fixed. The tier assertion also +turned out to be unsatisfiable rather than merely stale: the retune moved the +complete tier to 1.2, which is what `TEXT_INCOMPLETE_HOLD_TIMEOUT_SECS` already +was, so the two confidence tiers now differ only in the logged reason and the +inverse tier has never been re-swept against its new counterpart. + +Baselines may also now be taken in parallel. `--update-baseline` used to refuse +`--jobs > 1`, on the grounds that contention would bake the machine's load into +the latency the gate compares against. Measured on `deepgram-nova/local`, +`--quick`, 3 repeats, it does not: p50 278ms and p95 407ms at `--jobs 6` against +280ms and 453ms serial. `eou→audio` is mostly spent waiting on STT and TTS +sockets rather than competing for CPU, so the sessions overlap their waiting; the +only outlier in either run is a 962ms first-run model load, in the *serial* one. +The refusal bought nothing a busy machine at `--jobs 1` would not also spoil, and +it priced a full baseline at roughly six times what it needs to cost, which is +the kind of price that stops anyone taking one. What makes it sound is unchanged +and unrelated: a baseline records its own concurrency, and latency comparison is +declined across a mismatch. + +### The provider's patience was hiding a sentence we could have read + +The 1.2s conclusion above was measured against a harness that tore down mid-hold, +which penalises exactly the configurations relying on our holds rather than the +provider's. Re-swept once that was fixed, the aggressive end is nowhere near as bad +as it looked — 0.3 goes from 47% to 77% — and at `--repeat 3` the shipped value +looked barely defensible at all: + +| `vad_silence_threshold_secs` | pause merges | dead air p50 | p95 | +|---|---:|---:|---:| +| 0.3 | 77% | 775ms | 2602ms | +| 0.5 | 83% | 765ms | **2388ms** | +| 0.8 | 73% | 962ms | 2924ms | +| 1.2 (ships) | 85% | 1409ms | 2882ms | + +Two points for 642ms is a trade anyone would take. It is also not real: 0.8 scoring +below both neighbours is the signature of an underpowered measurement, n=78 makes +two points about 1.5 runs, and at `--repeat 6` on `local` alone the gap reopens to +94% against 83%. **1.2 stays.** Worth stating plainly because the tempting version +of this exercise — sweep, take the winner, ship it — would have shipped a 6% merge +regression on the strength of noise, twice, in opposite directions. + +What the tighter sweep bought instead was the *shape* of the disagreement. Nothing +here is a curve; it is two families pulling opposite ways, each pinned at 0% or 100% +across all six repeats: + +| scenario | 0.5 | 0.6 | 0.8 | 1.2 | +|---|---:|---:|---:|---:| +| `medical_self_correction` | 0% | 0% | 0% | 100% | +| `banking_correction` | 0% | 0% | 0% | 100% | +| `food_list_pause` | 100% | 100% | 100% | 17% | + +The two that collapse are the two the hold-tier sweep could never fix at any value, +and the trace says why — with `holding=False`, no hold is armed at all, so there was +never a tier to blame: + +``` +commit 'Send it to account 447. Sorry.' holding=False → new_turn +commit '448.' → new_turn, 2 turns, FAIL +``` + +Both EOU signals score that finished and both are right: it *is* a complete +sentence. What makes it incomplete is neither acoustic nor syntactic but discourse +— "Sorry." and "No, wait." announce a retraction, the way a hedge announces a pause. +So `local` now holds on a finished sentence plus a trailing correction marker +regardless of either score (`trailing_correction_marker`, short tier, since the +correction follows within ~300ms by nature and a false positive should not cost 3s). +The sentence-then-marker *shape* is the entire safety margin: a bare "Sorry." is +somebody's whole turn, and "I'm sorry." is an apology. + +Measured at `--repeat 6`, at the shipped 1.2, against cells where both scenarios +were marked known failures: + +| | `flux/local` | `nova/local` | `elevenlabs/local` | +|---|---:|---:|---:| +| `banking_correction` | **6/6** | 4/6 | 5/6 | +| `medical_self_correction` | 0/6 | **6/6** | 5/6 | + +Two markers retired outright and three downgraded to a race the hold either wins or +loses. Both Deepgram cells gate at 100%; ElevenLabs holds its 96.8% baseline, and +the three scenarios that dipped to 2/3 there never trigger the rule once across 9 +runs, so they are the provider variance documented above. It reaches only `local` — +`LexicalTurnDetector` is a sibling of `LocalAudioTurnDetector`, not a subclass — so +the improvements visible on `lexical` cells in the same runs are not this change, +which is its own hint that some markers there are stale. + +The threshold question is now a different one. ElevenLabs' 1.2s is not buying +merges in general, it is buying *these* merges, by absorbing corrections before any +detector sees them — which is why the defect surfaced on Flux and Nova instead. Each +such case read at the detector is 630ms of dead air per turn that stops needing to +be bought. + +### Flux's own threshold was never ours to leave alone + +`provider` has no hold, so on Flux `eot_threshold` *is* the turn policy — it is the +only setting in the pipeline that can merge a pause there, and `provider` is the +pairing Flux exists for. We shipped Deepgram's default and overrode only +`eot_timeout_ms`, which is the ElevenLabs situation again: a number governing +turn-taking, in our config, with nobody's measurement behind it. + +Unlike the ElevenLabs sweep this one is a curve, monotonic in both columns: + +| `eot_threshold` | pause merges | dead air p50 | p95 | +|---|---:|---:|---:| +| 0.4 | *rejected: HTTP 400* | — | — | +| 0.6 | 46% | 448ms | 1409ms | +| 0.7 (Deepgram's default) | 68% | 635ms | 1034ms | +| **0.8 (now ships)** | **82%** | 859ms | 1538ms | +| 0.9 | 91% | 1944ms | 2386ms | + +0.8 buys fourteen points for 214ms; 0.9 buys nine more for a further second and lands +at worse dead air than ElevenLabs, which is the point at which you have stopped tuning +and started choosing to be slow. 0.7→0.8 is what carries `coding_double_pause` +(17%→100%), `banking_digits_pause` (83%→100%) and `medical_hesitant_pause` (17%→67%). +All four Flux cells now gate at 100% with zero ghost turns, dead air p50 677–885ms. + +`banking_correction` is 0% at *every* threshold, which is worth more than the ones +that moved: the lever cannot reach it, so its marker on `flux/provider` is structural +rather than untuned. And `medical_self_correction` fails on Flux because Flux +transcribes "no wait" as **"no weight"** — no correction-marker rule can fire on a +word the STT never produces, which explains that cell without reference to holds at +all. + +Two things fell out of confirming it. Deepgram rejects `eot_threshold=0.4` with a +400, and the harness passed it through and scored 78 refused sessions as a 0% row — +indistinguishable, in the table above, from a setting that merely merges nothing. +`SWEEPABLE_STT_KEYS` validates key *names*, and encoding each provider's accepted +ranges beside them would go stale the first time one changed theirs; noticing the +refusal does not. `config_rejection` now matches a 4xx on the handshake — and only +4xx, since a rejected handshake is deterministic for a given config while a timeout +or a 5xx is the transient the repeats exist to average over. The sweep stops +scheduling that value and prints the provider's message where the row would go, with +`-` rather than `0%` in the per-scenario table; `cli.py` reports the cell as refused +before scoring, so it cannot be baselined or gated against, and exits 2. The cost was +never the six seconds of wasted runtime — it was a number in a results table that +looked like a measurement. And at 0.8 a stray `'I'` began committing its own turn after a +finished one on `flux/heuristic`: a more patient threshold holds the turn open long +enough for Flux to transcribe something in the trailing audio, and a holdless +detector takes it. That is a ghost turn, the worst class the suite has, and it slipped +between both existing filters — `_is_garbage_commit` requires non-alphanumeric, and +`_is_unvoiced_hallucination` requires Silero to have heard nothing, whereas here the +user genuinely had just spoken. A one-word commit needs neither test: +`_is_contentless_single_token` drops a closed class of function words that cannot be +an utterance ("I", "a", "the", "of", "to", "my"), and deliberately not the one-word +turns that carry intent — `banking_short_reject` *is* the single word "No." + +Unexploited, and the obvious way to stop paying the 214ms: Flux emits +`EagerEndOfTurn` and `TurnResumed`, and `deepgram.py` only logs them. A patient +threshold plus starting the reply on the eager event would let the LLM work during +the window Flux spends confirming, which is dead air currently buying nothing but +confidence. + +### The suite had been grading the default configuration all along without noticing + +Every finding above is about a detector someone chose. The interesting question is +what happens to someone who chooses nothing, and the matrix already answered it — +the answer was just sitting in a column nobody read as a default. + +`timbal.server.voice` only overrode the detector for Flux, where `stt_is_flux` +forces `provider`. For every other STT, an unset `turn_detector` fell through to +`resolve_turn_detector(None)`, which is `HeuristicTurnDetector` — the one detector +that cannot hold. Nova and ElevenLabs commit each fragment of a paused utterance +separately, so a detector that cannot hold cannot put them back together, and the +`heuristic` column is where the pause family goes to die. Counting scenarios +carrying a `known_failure` marker, out of 39: + +| STT | `heuristic` | `lexical` | `local` | +| --- | --- | --- | --- | +| `deepgram-nova` | 15 broken | 12 | **7** | +| `elevenlabs` | 14 broken | **6** | 8 | + +Fifteen of 39 broken was the shipping default for Nova. It reads as 100% in +`baseline.json` only because every one of those failures is marked — the markers +made the default's cost invisible in exactly the metric we gate on. The nine +scenarios `local` fixes on Nova are all one shape: `coding_pause`, +`coding_double_pause`, `food_pause`, `food_trailing_preposition`, +`medical_filler_midway`, `medical_long_utterance`, `support_pause`, +`support_pause_short`, `support_trailing_conjunction`. + +Worse, the two code paths already disagreed. `_warm_voice_models` resolves +`"local"` when `voice_config` names no detector, so a server with no configuration +was loading Smart Turn, Namo and Silero at startup and then handing the session a +detector that used none of them. The cost was being paid and the benefit discarded. + +`select_turn_detector_spec` now makes the choice explicitly, by STT, and +`test_server_voice.py` pins it — including the pre-existing Flux rule, which had no +test at all. The fallback is `lexical` rather than `local` when `timbal[voice]` is +absent, because `local` without an audio EOU model returns the heuristic decision +verbatim (its punctuation fallback sits *behind* the audio branch at +`turn_detection.py`), so it would have bought nothing. + +Left open deliberately: on ElevenLabs, `lexical` is the better of the two by this +count, and the four scenarios `local` breaks there are all one family — +`banking_correction`, `banking_digits_pause`, `food_trailing_preposition`, +`medical_self_correction`. That is the same correction-and-digits family whose +merges ElevenLabs' 1.2s VAD window performs for us, which suggests Smart Turn +scores the audio complete and cuts the hold short *before* the provider's window +merges the correction. Two scenarios is too thin a margin to special-case a +provider on, and it trades against latency the marker count cannot see: `local` is +what arms the VAD endpointing fast path, and ElevenLabs already has the worst dead +air in the matrix. Worth measuring, not guessing. + +### Every number above came from a recording booth + +The standing caveat on this whole document was that the audio is studio-clean: TTS +at 16 kHz, no noise floor, full bandwidth, handed straight to the STT. That is what +makes runs reproducible, and it is also why none of the endpointing values here +could be called right for a real caller. Both of the thresholds tuned in this +document — ElevenLabs' `vad_silence_threshold_secs` and Flux's `eot_threshold` — +decide when speech has *stopped*, which is precisely the judgement a noise floor +makes harder. + +`degrade.py` adds the missing path, applied in `ScriptFeeder.stream` in the order a +real one applies: + + user speech (+ echo leak) -> + noise bed -> telephone band -> STT + +Two details are load-bearing. The noise runs continuously, *through the pauses*: +noise gated to the clips would be a cue rather than a distractor, telling the +endpointer exactly where the utterance ended, and would test something easier than +clean audio rather than harder. And it is levelled per scenario against that +scenario's own speech (`active_rms`, which ignores silence), so `snr=15` means the +same thing in a scenario that is 60% deliberate pause as in one that is +wall-to-wall talking. `snr` is therefore the acoustic SNR at the microphone, not +the ratio the STT receives — speech and noise lose different amounts to the +telephone band. + +`--verify` earned its keep immediately by failing a check that encoded a wrong +belief of mine: that pink noise would survive the phone band better than white. It +is the reverse. At matched broadband level pink puts most of its energy below +300 Hz, exactly where the codec's high-pass throws it away, so white delivers more +in-band energy and is the harsher of the two. Pink is still the default because +rooms are pink; `white` is there to stress the STT. + +Not modelled, and worth knowing before quoting any number from this axis: packet +loss and jitter — a dropped 20 ms frame mid-word is a different failure and +plausibly the more dangerous one for turn-taking — and accent, which is a TTS voice +axis rather than a DSP one. This is still English in one voice. + +First measurement, `deepgram-flux/provider`, the primary production cell, whole +suite. The clean row is a control run at the same concurrency, not the stored +baseline, because parallel runs distort latency and comparing against a serial +baseline would have attributed that distortion to the noise: + +| mic path | passed | ghost turns | dead air p50 | dead air p95 | eou→audio p95 | +| --- | --- | --- | --- | --- | --- | +| clean | 27/27 | 0 | 652ms | 1442ms | 312ms | +| `snr=15,phone` | 25/27 | 0 | 860ms | 2282ms | 690ms | +| `snr=5,phone` | 25/27 | 0 | 845ms | 2007ms | 465ms | + +The reassuring half: turn-taking degrades gracefully in *correctness*. No ghost +turns at either level, no session errors, and the failures are two scenarios rather +than a collapse — a 10 dB change in noise barely moves the pass rate, which says +the cliff is not nearby. + +The half that matters more: the cost shows up as latency, and it is large. Dead air +p50 rises ~200ms and p95 nearly doubles, while `eou→audio` p50 barely moves +(217 -> 223ms) and its p95 more than doubles. That shape — median flat, tail +blowing out — is Flux's end-of-turn confidence taking longer to clear a noisy +signal. Which lands directly on a decision made earlier in this document: +`eot_threshold=0.8` was chosen on clean audio, on the argument that +214ms of dead +air bought fourteen points of pause merges. A patient threshold is exactly the +setting whose cost noise inflates, so that trade needs re-measuring here before it +can be called right for real callers. Sweeping `eot_threshold` *under* `--mic-path` +is the obvious next move and is now a one-liner. + +`food_trailing_preposition` is the one scenario that fails under noise having +passed clean, at both levels. It is intermittent rather than deterministic — a +single repeat merged correctly — and the partials show the mechanism: Flux hears +"Can you deliver it to" as "Can you do", then emits a stray `--` marker mid-stream, +so the fragment it endpoints is not the fragment the scenario is about. + +Deliberately not done yet: no markers and no baseline for this axis. The leak axis +earned its marker table only after a first run showed which failures recurred, and +inventing one here before knowing whether a scenario fails at 15 dB, at 5 dB, or +only in parallel would be bookkeeping ahead of evidence. + +## Running + +```bash +export ELEVENLABS_API_KEY=... +export DEEPGRAM_API_KEY=... + +uv run python benchmarks/voice/cli.py # all scenarios +uv run python benchmarks/voice/cli.py --list # what exists +uv run python benchmarks/voice/cli.py -s barge_in -s pause # a subset +uv run python benchmarks/voice/cli.py --detector lexical # Timbal's HOLD path +uv run python benchmarks/voice/cli.py --stt elevenlabs +uv run python benchmarks/voice/cli.py --quick # ~30s subset, one per domain +uv run python benchmarks/voice/cli.py --repeat 3 # variance + flaky detection +uv run python benchmarks/voice/cli.py --quiet # results only +uv run python benchmarks/voice/cli.py --dump # WAVs to results/dumps/ +uv run python benchmarks/voice/cli.py --update-baseline # accept current behavior + +# the matrix: --stt and --detector are crossed, one scorecard per cell +uv run python benchmarks/voice/cli.py --detector local,lexical,provider --jobs 4 + +# reproduce one cell of a sweep with the event stream visible — the step between +# "this value scores worse" and knowing why +uv run python benchmarks/voice/cli.py -s coding_double_pause --stt elevenlabs \ + --detector local --stt-param vad_silence_threshold_secs=0.3 --verbose +``` + +`--stt-param` and `--detector-param` take `KEY=VALUE`. STT keys are checked against +a per-backend allowlist (`harness.SWEEPABLE_STT_KEYS`) because providers ignore +unknown query params in silence: a typo would otherwise sweep one value under four +labels and every number in the table would agree with every other. + +Exit code is non-zero when any expectation fails or a regression is gated. + +## The matrix + +`--stt` and `--detector` take comma-separated lists and are crossed, producing one +scorecard, one baseline entry and one gate per cell, plus a scenario × cell grid at +the end. The grid is the point: a row tells you whether a scenario fails everywhere +(Timbal's problem) or in one column (that provider's problem), which no amount of +staring at a single cell will tell you. + +``` + 1 2 3 + banking_digits_pause ✓ ✓ x + banking_hesitation ✓ ✓ · + medical_hesitant_pause x x x + + 1 deepgram-flux/local 19/19 p50 214ms + 2 deepgram-flux/lexical 19/19 p50 213ms + 3 deepgram-flux/provider 16/16 p50 222ms + + ✓ pass ✗ FAIL x known failure ! XPASS ~ flaky · not run +``` + +`x` and `✗` are deliberately different glyphs: a row of `x` is a documented +limitation, a single `✗` is a regression. `·` means the scenario opted out of that +detector (see *Per-detector expectations*). + +`known_failure` and `expect_by_detector` keys are matched most-specific-first: +`"deepgram-nova/lexical"`, then `"deepgram-nova/*"`, then `"lexical"`, then `"*"`. +Scope findings to a cell whenever they came from one — a bare detector key silently +applies a Flux observation to every other backend, which is how `food_long_pause` +came to fail Nova for behaving reasonably. The `"deepgram-nova/*"` rung is for +failures that are the STT's alone and land identically under all four detectors, +like Nova committing a "mm-hmm"; writing those per cell states one fact four times +and hides that it is one fact. + +`--jobs N` overlaps runs across the whole queue, cells included. One rule keeps it +from corrupting the thing it accelerates: latency is never compared across differing +concurrency — it is printed with a note instead. Baselines may be taken in parallel +(see the measurement above), so what a parallel run records is gated against later +runs at the same `--jobs`. Pass rates and ghost turns gate regardless. + +## Results and the regression gate + +Every run appends one JSON line per scenario to `results/run-.jsonl`, +so a run stays analyzable after the fact rather than only pass/fail in the moment. + +Gating compares against `baseline.json` (committed — deliberately not under the +gitignored `results/`), one entry per `stt/detector` label so a single file covers +the whole matrix. It fails on **movement**, never on absolute thresholds, since +STT providers drift under us and yesterday's ceiling is tomorrow's false alarm: + +| Gated | Not gated | +|---|---| +| a scenario's pass rate drops | speedups | +| ghost turns increase | brand-new scenarios (nothing to compare) | +| median `eou→audio` worsens >15% | either timing with fewer than 20 samples | +| median dead air worsens >15% | either timing measured at a different `--jobs` | + +Both timings gate through the same `_compare_timing` helper. That is deliberate +rather than tidiness: the latency block used to `return` early on a `--jobs` +mismatch, and a dead-air check bolted on after it would have been skipped +silently on exactly the parallel runs people actually use — the same shape of +blind spot the dead-air metric exists to close. + +**Latency gates on p50, not p95.** With ~8 latency samples, p95 is the maximum in +disguise: one slow turn moved it 394ms → 477ms on an unchanged tree while all five +scenarios passed. The scorecard prints `n=` and says "ungated: too few samples" +outright, so nobody reads stability into a number that has none. + +Measured on this suite, p50 is worth gating and the tail is not — two independent +`--repeat 3` runs on an unchanged tree: + +| Statistic | Run A | Run B | Drift | +|---|---:|---:|---:| +| p50 | 298ms | 299ms | 0.3% | +| p95 | 383ms | 375ms | 2% | +| max | 462ms | 378ms | 18% | + +At 38 scenarios a single pass produces well over 40 latency samples, so +`--repeat 1` clears the floor on its own. Repeats still buy flaky detection: any +scenario that passes some repeats and fails others is reported by name as +**FLAKY** — usually a real race rather than a bad test. This is not optional +rigour: `coding_pause` sat in the baseline as a clean pass until `--repeat 3` +caught it splitting once in three under `provider`. + +Flaky detection deliberately looks *within* a cell, not across cells. Two detectors +disagreeing is the matrix working as designed; the same detector disagreeing with +itself is a race. It also covers known failures, which the per-cell scorecard +cannot see — those are excluded from `per_scenario` by design, which is how a +marked scenario passing 5 runs in 6 stays invisible until you ask for repeats. + +## Known failures + +A scenario can declare `known_failure={detector_or_star: reason}`. It still runs +and still reports, but it does not gate, and its *desired* expectations stay in +the file rather than being rewritten to match broken behavior. Known failures are +excluded from `per_scenario`, so marking one cannot quietly bake a defect into the +baseline. If one starts passing, that's reported as an unexpected pass telling you +to drop the marker — unless it's also `intermittent=True`, where passing sometimes +means nothing. + +`--update-baseline` refuses to save while any ungated run is failing. Otherwise +the first bad run silently becomes the accepted status quo. Fix it, or name it a +known failure with a written reason. + +Audio tooling, standalone: + +```bash +uv run python benchmarks/voice/synth.py "I'd like to order a large" # listen to a clip +uv run python benchmarks/voice/synth.py --verify # reproducibility check +``` + +## Writing a scenario + +A scenario is a script plus expectations. Silence is explicit and exact, because +gap duration is what turn detection actually keys on: + +```python +Scenario( + id="pause", + domain="food_ordering", + replies=["Sure — one large pepperoni. Anything else?"], + script=[ + *fluent("I'd like to order a large", 1.6, "pepperoni pizza please."), + Silence(3.0), + ], + expect=[Merged("I'd like to order a large pepperoni pizza please."), NoErrors()], +) +``` + +**Use `fluent()` for any pause *inside* a sentence.** It renders the whole sentence +in one TTS call and slices it at character timestamps, so each fragment keeps the +intonation it actually has mid-sentence. Writing that example as three separate +steps — `Say(...)`, `Silence(1.6)`, `Say(...)` — is the trap described above: it +hands the detector a fragment that sounds finished and tests TTS phrasing instead +of turn-taking. Plain `Say` is for genuinely separate utterances, like the second +half of a barge-in. + +`AwaitAssistantAudio(offset_ms=600)` is the reactive step that makes barge-in +testable — the interruption lands at a known offset into the reply rather than +wherever a fixed sleep happens to fall. + +Expectations: `UserTurns`, `Merged`, `NoGhostTurns`, `Interrupted`, `HeardPrefix`, +`NoAgentReply`, `MaxLatency`, `NoErrors`. Text comparisons use normalized +similarity — **never assert verbatim transcripts**, STT wobbles between runs. + +## Per-detector expectations + +The same observable behavior can be correct for different reasons, so scenarios +can override expectations per detector (`expect_by_detector`) or declare which +detectors they're meaningful for at all (`detectors`). + +This matters: with `--stt deepgram-flux --detector provider`, Flux holds through +the `pause` scenario's 1.6s gap and merges inside its own turn machine, so that +run exercises none of Timbal's HOLD path. The same scenario under `--detector +lexical` does. `hesitation` is skipped entirely for `provider`, where trusting +the provider's commit means a bare "Um..." legitimately starts a turn. + +## What the current library measures + +Scenarios come in matched sets, because a single data point about turn-taking is +rarely interpretable on its own. + +Pause length is swept at 0.6s, 1.2–1.5s and 3.0s (`support_pause_short`, the +`*_pause` family, `food_long_pause`), which brackets each backend's turn window +and separates "merges anything" from "merges what a speaker would". Barge-in is +swept by offset at 150ms, 600ms and 2500ms against the same reply, yielding heard +prefixes from nothing to ~77 characters — which is how you check the truncation +path scales with real playback position — and by shape: one word, twice in a +session, echoing the assistant's own words, and a backchannel that must not +interrupt at all. + +Each family also carries its own inverse, so a detector cannot score well by +always guessing the same way. Against the pause merges sit `food_rapid_fire` (two +finished sentences 0.9s apart, must split) and `medical_long_utterance` (12 +unbroken seconds, must not). Against the barge-ins sits `food_backchannel`. +Against every commit sits `support_silence_only`, which says an open mic with no +speech produces no turn at all. + +## Voices + +Two distinct ElevenLabs voices — replaying the user in the assistant's voice +would trip the session's echo heuristics on legitimate speech. + +| Env var | Default | Role | +|---|---|---| +| `ELEVENLABS_VOICE_ID` | `1SM7GgM6IMuvQlz2BwM3` | assistant (TTS) | +| `TIMBAL_BENCH_USER_VOICE_ID` | `21m00Tcm4TlvDq8ikWAM` | user (replayed) | + +Both must exist on your account; cloned and custom voices are account-specific, +so override if the defaults 404. + +## Cache + +Synthesized clips land in `cache/`, keyed by a hash of +`(text, voice_id, tts_model)` — nothing is generated twice unless the script text +changes. Both `cache/` and `results/` are gitignored: audio does not belong in +git, and the generation script plus a content-addressed cache is reproducible +enough. + +## Reading the output + +Real-time pacing is mandatory, so a run takes as long as the conversation. + +``` +=== barge_in (deepgram-flux/provider) === + [+ 0.00s] session_started + [+ 0.00s] say "Tell me about your return policy." + [+ 1.63s] partial "Tell me about your return policy." + [+ 1.99s] await_audio 600ms into reply + [+ 2.46s] committed "Tell me about your return policy." + [+ 3.08s] metrics eou→audio 373.3ms segments 1 acks True vad_eou False + [+ 3.45s] say "Actually, cancel that." + [+ 4.92s] interrupted heard: 'Our return policy allows returns within' +``` + +`acks True` confirms the harness is exercising the client-truth playback path +rather than the wall-clock estimate fallback — production behavior, not a +degraded fallback. diff --git a/benchmarks/voice/baseline.json b/benchmarks/voice/baseline.json new file mode 100644 index 00000000..dd8a3c9f --- /dev/null +++ b/benchmarks/voice/baseline.json @@ -0,0 +1,865 @@ +{ + "deepgram-flux/heuristic": { + "label": "deepgram-flux/heuristic", + "stt": "deepgram-flux", + "detector": "heuristic", + "runs": 96, + "passed": 96, + "pass_rate": 1.0, + "ghost_turns": 0, + "errors": 0, + "latency_p50": 222.6, + "latency_p95": 317.1, + "latency_min": 166.4, + "latency_max": 428.0, + "latency_samples": 146, + "dead_air_p50": 677.2, + "dead_air_p95": 1530.7, + "dead_air_samples": 143, + "per_scenario": { + "banking_digits": 1.0, + "banking_digits_pause": 1.0, + "banking_hesitation": 1.0, + "banking_short_reject": 1.0, + "coding_barge_in": 1.0, + "coding_barge_in_echo": 1.0, + "coding_followup_after_reply": 1.0, + "coding_simple": 1.0, + "food_barge_in": 1.0, + "food_list_pause": 1.0, + "food_long_pause": 1.0, + "food_pause": 1.0, + "food_rapid_fire": 1.0, + "food_simple": 1.0, + "food_trailing_preposition": 1.0, + "medical_barge_in": 1.0, + "medical_barge_in_twice": 1.0, + "medical_filler_midway": 1.0, + "medical_hesitation": 1.0, + "medical_long_utterance": 1.0, + "medical_simple": 1.0, + "support_barge_in": 1.0, + "support_barge_in_instant": 1.0, + "support_barge_in_late": 1.0, + "support_barge_in_one_word": 1.0, + "support_closer": 1.0, + "support_echo_silence": 1.0, + "support_pause": 1.0, + "support_pause_short": 1.0, + "support_silence_only": 1.0, + "support_simple": 1.0, + "support_trailing_conjunction": 1.0 + }, + "flaky": [], + "known_failures": [ + "banking_confirmation", + "banking_correction", + "coding_double_pause", + "coding_pause", + "food_backchannel", + "medical_hesitant_pause", + "medical_self_correction" + ], + "unexpected_passes": [], + "wall_secs": 823.0, + "jobs": 6 + }, + "deepgram-flux/lexical": { + "label": "deepgram-flux/lexical", + "stt": "deepgram-flux", + "detector": "lexical", + "runs": 102, + "passed": 102, + "pass_rate": 1.0, + "ghost_turns": 0, + "errors": 0, + "latency_p50": 219.1, + "latency_p95": 311.0, + "latency_min": 169.4, + "latency_max": 397.6, + "latency_samples": 148, + "dead_air_p50": 859.0, + "dead_air_p95": 2345.1, + "dead_air_samples": 144, + "per_scenario": { + "banking_correction": 1.0, + "banking_digits": 1.0, + "banking_digits_pause": 1.0, + "banking_hesitation": 1.0, + "banking_short_reject": 1.0, + "coding_barge_in": 1.0, + "coding_barge_in_echo": 1.0, + "coding_followup_after_reply": 1.0, + "coding_pause": 1.0, + "coding_simple": 1.0, + "food_barge_in": 1.0, + "food_list_pause": 1.0, + "food_long_pause": 1.0, + "food_pause": 1.0, + "food_rapid_fire": 1.0, + "food_simple": 1.0, + "food_trailing_preposition": 1.0, + "medical_barge_in": 1.0, + "medical_barge_in_twice": 1.0, + "medical_filler_midway": 1.0, + "medical_hesitation": 1.0, + "medical_long_utterance": 1.0, + "medical_simple": 1.0, + "support_barge_in": 1.0, + "support_barge_in_instant": 1.0, + "support_barge_in_late": 1.0, + "support_barge_in_one_word": 1.0, + "support_closer": 1.0, + "support_echo_silence": 1.0, + "support_pause": 1.0, + "support_pause_short": 1.0, + "support_silence_only": 1.0, + "support_simple": 1.0, + "support_trailing_conjunction": 1.0 + }, + "flaky": [], + "known_failures": [ + "banking_confirmation", + "coding_double_pause", + "food_backchannel", + "medical_hesitant_pause", + "medical_self_correction" + ], + "unexpected_passes": [], + "wall_secs": 861.2, + "jobs": 6 + }, + "deepgram-flux/local": { + "label": "deepgram-flux/local", + "stt": "deepgram-flux", + "detector": "local", + "runs": 102, + "passed": 102, + "pass_rate": 1.0, + "ghost_turns": 0, + "errors": 0, + "latency_p50": 226.4, + "latency_p95": 310.6, + "latency_min": 173.0, + "latency_max": 394.0, + "latency_samples": 146, + "dead_air_p50": 884.7, + "dead_air_p95": 2146.4, + "dead_air_samples": 143, + "per_scenario": { + "banking_correction": 1.0, + "banking_digits": 1.0, + "banking_digits_pause": 1.0, + "banking_hesitation": 1.0, + "banking_short_reject": 1.0, + "coding_barge_in": 1.0, + "coding_barge_in_echo": 1.0, + "coding_followup_after_reply": 1.0, + "coding_pause": 1.0, + "coding_simple": 1.0, + "food_barge_in": 1.0, + "food_list_pause": 1.0, + "food_long_pause": 1.0, + "food_pause": 1.0, + "food_rapid_fire": 1.0, + "food_simple": 1.0, + "food_trailing_preposition": 1.0, + "medical_barge_in": 1.0, + "medical_barge_in_twice": 1.0, + "medical_filler_midway": 1.0, + "medical_hesitation": 1.0, + "medical_long_utterance": 1.0, + "medical_simple": 1.0, + "support_barge_in": 1.0, + "support_barge_in_instant": 1.0, + "support_barge_in_late": 1.0, + "support_barge_in_one_word": 1.0, + "support_closer": 1.0, + "support_echo_silence": 1.0, + "support_pause": 1.0, + "support_pause_short": 1.0, + "support_silence_only": 1.0, + "support_simple": 1.0, + "support_trailing_conjunction": 1.0 + }, + "flaky": [], + "known_failures": [ + "banking_confirmation", + "coding_double_pause", + "food_backchannel", + "medical_hesitant_pause", + "medical_self_correction" + ], + "unexpected_passes": [], + "wall_secs": 869.6, + "jobs": 6 + }, + "deepgram-flux/local[leak=0.15]": { + "label": "deepgram-flux/local[leak=0.15]", + "stt": "deepgram-flux", + "detector": "local", + "runs": 120, + "passed": 102, + "pass_rate": 0.85, + "ghost_turns": 10, + "errors": 4, + "latency_p50": 228.4, + "latency_p95": 314.9, + "latency_min": 169.6, + "latency_max": 440.3, + "latency_samples": 207, + "dead_air_p50": 728.2, + "dead_air_p95": 1974.2, + "dead_air_samples": 195, + "per_scenario": { + "banking_hesitation": 1.0, + "banking_short_reject": 0.75, + "coding_barge_in": 1.0, + "coding_barge_in_echo": 0.5, + "coding_followup_after_reply": 0.75, + "coding_pause": 1.0, + "coding_simple": 0.75, + "food_barge_in": 0.75, + "food_list_pause": 1.0, + "food_long_pause": 1.0, + "food_pause": 0.5, + "food_rapid_fire": 1.0, + "food_simple": 0.25, + "food_trailing_preposition": 1.0, + "medical_barge_in": 1.0, + "medical_barge_in_twice": 0.25, + "medical_filler_midway": 1.0, + "medical_hesitation": 1.0, + "medical_long_utterance": 1.0, + "medical_simple": 1.0, + "support_barge_in": 1.0, + "support_barge_in_instant": 1.0, + "support_barge_in_late": 0.5, + "support_closer": 1.0, + "support_echo_silence": 0.75, + "support_pause": 1.0, + "support_pause_short": 1.0, + "support_silence_only": 1.0, + "support_simple": 0.75, + "support_trailing_conjunction": 1.0 + }, + "flaky": [ + "banking_short_reject", + "coding_barge_in_echo", + "coding_followup_after_reply", + "coding_simple", + "food_barge_in", + "food_pause", + "food_simple", + "medical_barge_in_twice", + "support_barge_in_late", + "support_echo_silence", + "support_simple" + ], + "known_failures": [ + "banking_confirmation", + "banking_correction", + "banking_digits", + "banking_digits_pause", + "coding_double_pause", + "food_backchannel", + "medical_hesitant_pause", + "medical_self_correction", + "support_barge_in_one_word" + ], + "unexpected_passes": [], + "wall_secs": 1212.7, + "jobs": 6 + }, + "deepgram-flux/provider": { + "label": "deepgram-flux/provider", + "stt": "deepgram-flux", + "detector": "provider", + "runs": 81, + "passed": 81, + "pass_rate": 1.0, + "ghost_turns": 0, + "errors": 0, + "latency_p50": 218.2, + "latency_p95": 336.6, + "latency_min": 174.5, + "latency_max": 655.3, + "latency_samples": 146, + "dead_air_p50": 685.9, + "dead_air_p95": 1837.1, + "dead_air_samples": 143, + "per_scenario": { + "banking_digits": 1.0, + "banking_short_reject": 1.0, + "coding_barge_in": 1.0, + "coding_barge_in_echo": 1.0, + "coding_followup_after_reply": 1.0, + "coding_simple": 1.0, + "food_barge_in": 1.0, + "food_list_pause": 1.0, + "food_long_pause": 1.0, + "food_pause": 1.0, + "food_rapid_fire": 1.0, + "food_simple": 1.0, + "food_trailing_preposition": 1.0, + "medical_barge_in": 1.0, + "medical_barge_in_twice": 1.0, + "medical_long_utterance": 1.0, + "medical_simple": 1.0, + "support_barge_in": 1.0, + "support_barge_in_instant": 1.0, + "support_barge_in_late": 1.0, + "support_barge_in_one_word": 1.0, + "support_closer": 1.0, + "support_echo_silence": 1.0, + "support_pause": 1.0, + "support_silence_only": 1.0, + "support_simple": 1.0, + "support_trailing_conjunction": 1.0 + }, + "flaky": [], + "known_failures": [ + "banking_confirmation", + "banking_correction", + "banking_digits_pause", + "coding_double_pause", + "coding_pause", + "food_backchannel", + "medical_filler_midway", + "medical_hesitant_pause", + "medical_self_correction", + "support_pause_short" + ], + "unexpected_passes": [], + "wall_secs": 799.7, + "jobs": 6 + }, + "deepgram-nova/heuristic": { + "label": "deepgram-nova/heuristic", + "stt": "deepgram-nova", + "detector": "heuristic", + "runs": 72, + "passed": 72, + "pass_rate": 1.0, + "ghost_turns": 0, + "errors": 0, + "latency_p50": 213.5, + "latency_p95": 299.8, + "latency_min": 164.7, + "latency_max": 412.6, + "latency_samples": 189, + "dead_air_p50": 523.9, + "dead_air_p95": 735.2, + "dead_air_samples": 180, + "per_scenario": { + "banking_digits": 1.0, + "banking_digits_pause": 1.0, + "banking_hesitation": 1.0, + "banking_short_reject": 1.0, + "coding_barge_in": 1.0, + "coding_barge_in_echo": 1.0, + "coding_followup_after_reply": 1.0, + "coding_simple": 1.0, + "food_barge_in": 1.0, + "food_long_pause": 1.0, + "food_rapid_fire": 1.0, + "food_simple": 1.0, + "medical_barge_in": 1.0, + "medical_barge_in_twice": 1.0, + "medical_hesitation": 1.0, + "medical_simple": 1.0, + "support_barge_in": 1.0, + "support_barge_in_instant": 1.0, + "support_barge_in_late": 1.0, + "support_barge_in_one_word": 1.0, + "support_closer": 1.0, + "support_echo_silence": 1.0, + "support_silence_only": 1.0, + "support_simple": 1.0 + }, + "flaky": [], + "known_failures": [ + "banking_confirmation", + "banking_correction", + "coding_double_pause", + "coding_pause", + "food_backchannel", + "food_list_pause", + "food_pause", + "food_trailing_preposition", + "medical_filler_midway", + "medical_hesitant_pause", + "medical_long_utterance", + "medical_self_correction", + "support_pause", + "support_pause_short", + "support_trailing_conjunction" + ], + "unexpected_passes": [], + "wall_secs": 817.6, + "jobs": 6 + }, + "deepgram-nova/lexical": { + "label": "deepgram-nova/lexical", + "stt": "deepgram-nova", + "detector": "lexical", + "runs": 81, + "passed": 80, + "pass_rate": 0.9876543209876543, + "ghost_turns": 0, + "errors": 0, + "latency_p50": 220.0, + "latency_p95": 307.6, + "latency_min": 165.5, + "latency_max": 428.6, + "latency_samples": 163, + "dead_air_p50": 548.5, + "dead_air_p95": 2067.2, + "dead_air_samples": 154, + "per_scenario": { + "banking_digits": 1.0, + "banking_hesitation": 1.0, + "banking_short_reject": 1.0, + "coding_barge_in": 1.0, + "coding_barge_in_echo": 1.0, + "coding_double_pause": 0.6666666666666666, + "coding_followup_after_reply": 1.0, + "coding_simple": 1.0, + "food_barge_in": 1.0, + "food_long_pause": 1.0, + "food_rapid_fire": 1.0, + "food_simple": 1.0, + "medical_barge_in": 1.0, + "medical_barge_in_twice": 1.0, + "medical_hesitation": 1.0, + "medical_self_correction": 1.0, + "medical_simple": 1.0, + "support_barge_in": 1.0, + "support_barge_in_instant": 1.0, + "support_barge_in_late": 1.0, + "support_barge_in_one_word": 1.0, + "support_closer": 1.0, + "support_echo_silence": 1.0, + "support_pause": 1.0, + "support_pause_short": 1.0, + "support_silence_only": 1.0, + "support_simple": 1.0 + }, + "flaky": [ + "coding_double_pause" + ], + "known_failures": [ + "banking_confirmation", + "banking_correction", + "banking_digits_pause", + "coding_pause", + "food_backchannel", + "food_list_pause", + "food_pause", + "food_trailing_preposition", + "medical_filler_midway", + "medical_hesitant_pause", + "medical_long_utterance", + "support_trailing_conjunction" + ], + "unexpected_passes": [], + "wall_secs": 830.8, + "jobs": 6 + }, + "deepgram-nova/local": { + "label": "deepgram-nova/local", + "stt": "deepgram-nova", + "detector": "local", + "runs": 96, + "passed": 92, + "pass_rate": 0.9583333333333334, + "ghost_turns": 0, + "errors": 0, + "latency_p50": 224.2, + "latency_p95": 300.5, + "latency_min": 167.0, + "latency_max": 482.6, + "latency_samples": 152, + "dead_air_p50": 645.5, + "dead_air_p95": 2040.2, + "dead_air_samples": 149, + "per_scenario": { + "banking_digits": 1.0, + "banking_hesitation": 1.0, + "banking_short_reject": 1.0, + "coding_barge_in": 1.0, + "coding_barge_in_echo": 1.0, + "coding_double_pause": 0.3333333333333333, + "coding_followup_after_reply": 1.0, + "coding_pause": 0.6666666666666666, + "coding_simple": 1.0, + "food_barge_in": 1.0, + "food_long_pause": 0.6666666666666666, + "food_pause": 1.0, + "food_rapid_fire": 1.0, + "food_simple": 1.0, + "food_trailing_preposition": 1.0, + "medical_barge_in": 1.0, + "medical_barge_in_twice": 1.0, + "medical_filler_midway": 1.0, + "medical_hesitation": 1.0, + "medical_long_utterance": 1.0, + "medical_simple": 1.0, + "support_barge_in": 1.0, + "support_barge_in_instant": 1.0, + "support_barge_in_late": 1.0, + "support_barge_in_one_word": 1.0, + "support_closer": 1.0, + "support_echo_silence": 1.0, + "support_pause": 1.0, + "support_pause_short": 1.0, + "support_silence_only": 1.0, + "support_simple": 1.0, + "support_trailing_conjunction": 1.0 + }, + "flaky": [ + "coding_double_pause", + "coding_pause", + "food_long_pause" + ], + "known_failures": [ + "banking_confirmation", + "banking_correction", + "banking_digits_pause", + "food_backchannel", + "food_list_pause", + "medical_hesitant_pause", + "medical_self_correction" + ], + "unexpected_passes": [], + "wall_secs": 894.1, + "jobs": 6 + }, + "deepgram-nova/provider": { + "label": "deepgram-nova/provider", + "stt": "deepgram-nova", + "detector": "provider", + "runs": 66, + "passed": 66, + "pass_rate": 1.0, + "ghost_turns": 0, + "errors": 0, + "latency_p50": 208.4, + "latency_p95": 288.8, + "latency_min": 158.9, + "latency_max": 379.9, + "latency_samples": 189, + "dead_air_p50": 517.1, + "dead_air_p95": 691.5, + "dead_air_samples": 180, + "per_scenario": { + "banking_digits": 1.0, + "banking_digits_pause": 1.0, + "banking_short_reject": 1.0, + "coding_barge_in": 1.0, + "coding_barge_in_echo": 1.0, + "coding_followup_after_reply": 1.0, + "coding_simple": 1.0, + "food_barge_in": 1.0, + "food_long_pause": 1.0, + "food_rapid_fire": 1.0, + "food_simple": 1.0, + "medical_barge_in": 1.0, + "medical_barge_in_twice": 1.0, + "medical_simple": 1.0, + "support_barge_in": 1.0, + "support_barge_in_instant": 1.0, + "support_barge_in_late": 1.0, + "support_barge_in_one_word": 1.0, + "support_closer": 1.0, + "support_echo_silence": 1.0, + "support_silence_only": 1.0, + "support_simple": 1.0 + }, + "flaky": [], + "known_failures": [ + "banking_confirmation", + "banking_correction", + "coding_double_pause", + "coding_pause", + "food_backchannel", + "food_list_pause", + "food_pause", + "food_trailing_preposition", + "medical_filler_midway", + "medical_hesitant_pause", + "medical_long_utterance", + "medical_self_correction", + "support_pause", + "support_pause_short", + "support_trailing_conjunction" + ], + "unexpected_passes": [], + "wall_secs": 794.5, + "jobs": 6 + }, + "elevenlabs/heuristic": { + "label": "elevenlabs/heuristic", + "stt": "elevenlabs", + "detector": "heuristic", + "runs": 75, + "passed": 75, + "pass_rate": 1.0, + "ghost_turns": 0, + "errors": 0, + "latency_p50": 217.3, + "latency_p95": 313.5, + "latency_min": 164.6, + "latency_max": 429.8, + "latency_samples": 171, + "dead_air_p50": 1477.2, + "dead_air_p95": 1735.0, + "dead_air_samples": 168, + "per_scenario": { + "banking_digits": 1.0, + "banking_hesitation": 1.0, + "banking_short_reject": 1.0, + "coding_barge_in": 1.0, + "coding_barge_in_echo": 1.0, + "coding_followup_after_reply": 1.0, + "coding_simple": 1.0, + "food_backchannel": 1.0, + "food_barge_in": 1.0, + "food_long_pause": 1.0, + "food_simple": 1.0, + "medical_barge_in": 1.0, + "medical_barge_in_twice": 1.0, + "medical_hesitation": 1.0, + "medical_long_utterance": 1.0, + "medical_simple": 1.0, + "support_barge_in": 1.0, + "support_barge_in_instant": 1.0, + "support_barge_in_late": 1.0, + "support_barge_in_one_word": 1.0, + "support_closer": 1.0, + "support_echo_silence": 1.0, + "support_pause_short": 1.0, + "support_silence_only": 1.0, + "support_simple": 1.0 + }, + "flaky": [], + "known_failures": [ + "banking_confirmation", + "banking_correction", + "banking_digits_pause", + "coding_double_pause", + "coding_pause", + "food_list_pause", + "food_pause", + "food_rapid_fire", + "food_trailing_preposition", + "medical_filler_midway", + "medical_hesitant_pause", + "medical_self_correction", + "support_pause", + "support_trailing_conjunction" + ], + "unexpected_passes": [], + "wall_secs": 899.4, + "jobs": 6 + }, + "elevenlabs/lexical": { + "label": "elevenlabs/lexical", + "stt": "elevenlabs", + "detector": "lexical", + "runs": 99, + "passed": 97, + "pass_rate": 0.9797979797979798, + "ghost_turns": 0, + "errors": 0, + "latency_p50": 220.6, + "latency_p95": 309.6, + "latency_min": 169.3, + "latency_max": 491.2, + "latency_samples": 145, + "dead_air_p50": 1478.4, + "dead_air_p95": 2999.4, + "dead_air_samples": 144, + "per_scenario": { + "banking_correction": 1.0, + "banking_digits": 1.0, + "banking_digits_pause": 0.6666666666666666, + "banking_hesitation": 1.0, + "banking_short_reject": 1.0, + "coding_barge_in": 1.0, + "coding_barge_in_echo": 1.0, + "coding_double_pause": 1.0, + "coding_followup_after_reply": 1.0, + "coding_pause": 1.0, + "coding_simple": 1.0, + "food_backchannel": 1.0, + "food_barge_in": 1.0, + "food_long_pause": 1.0, + "food_simple": 1.0, + "food_trailing_preposition": 1.0, + "medical_barge_in": 0.6666666666666666, + "medical_barge_in_twice": 1.0, + "medical_filler_midway": 1.0, + "medical_hesitation": 1.0, + "medical_long_utterance": 1.0, + "medical_self_correction": 1.0, + "medical_simple": 1.0, + "support_barge_in": 1.0, + "support_barge_in_instant": 1.0, + "support_barge_in_late": 1.0, + "support_barge_in_one_word": 1.0, + "support_closer": 1.0, + "support_echo_silence": 1.0, + "support_pause": 1.0, + "support_pause_short": 1.0, + "support_silence_only": 1.0, + "support_simple": 1.0 + }, + "flaky": [ + "banking_digits_pause", + "medical_barge_in" + ], + "known_failures": [ + "banking_confirmation", + "food_list_pause", + "food_pause", + "food_rapid_fire", + "medical_hesitant_pause", + "support_trailing_conjunction" + ], + "unexpected_passes": [], + "wall_secs": 943.9, + "jobs": 6 + }, + "elevenlabs/local": { + "label": "elevenlabs/local", + "stt": "elevenlabs", + "detector": "local", + "runs": 93, + "passed": 90, + "pass_rate": 0.967741935483871, + "ghost_turns": 0, + "errors": 0, + "latency_p50": 234.8, + "latency_p95": 328.8, + "latency_min": 169.5, + "latency_max": 482.6, + "latency_samples": 144, + "dead_air_p50": 1525.4, + "dead_air_p95": 2833.0, + "dead_air_samples": 141, + "per_scenario": { + "banking_digits": 1.0, + "banking_hesitation": 1.0, + "banking_short_reject": 1.0, + "coding_barge_in": 1.0, + "coding_barge_in_echo": 1.0, + "coding_double_pause": 0.6666666666666666, + "coding_followup_after_reply": 1.0, + "coding_pause": 1.0, + "coding_simple": 1.0, + "food_backchannel": 1.0, + "food_barge_in": 1.0, + "food_long_pause": 1.0, + "food_pause": 1.0, + "food_simple": 1.0, + "medical_barge_in": 1.0, + "medical_barge_in_twice": 0.3333333333333333, + "medical_filler_midway": 1.0, + "medical_hesitation": 1.0, + "medical_long_utterance": 1.0, + "medical_simple": 1.0, + "support_barge_in": 1.0, + "support_barge_in_instant": 1.0, + "support_barge_in_late": 1.0, + "support_barge_in_one_word": 1.0, + "support_closer": 1.0, + "support_echo_silence": 1.0, + "support_pause": 1.0, + "support_pause_short": 1.0, + "support_silence_only": 1.0, + "support_simple": 1.0, + "support_trailing_conjunction": 1.0 + }, + "flaky": [ + "coding_double_pause", + "medical_barge_in_twice" + ], + "known_failures": [ + "banking_confirmation", + "banking_correction", + "banking_digits_pause", + "food_list_pause", + "food_rapid_fire", + "food_trailing_preposition", + "medical_hesitant_pause", + "medical_self_correction" + ], + "unexpected_passes": [], + "wall_secs": 933.8, + "jobs": 6 + }, + "elevenlabs/provider": { + "label": "elevenlabs/provider", + "stt": "elevenlabs", + "detector": "provider", + "runs": 69, + "passed": 69, + "pass_rate": 1.0, + "ghost_turns": 0, + "errors": 0, + "latency_p50": 216.4, + "latency_p95": 302.1, + "latency_min": 166.6, + "latency_max": 340.3, + "latency_samples": 172, + "dead_air_p50": 1480.9, + "dead_air_p95": 1722.1, + "dead_air_samples": 169, + "per_scenario": { + "banking_digits": 1.0, + "banking_short_reject": 1.0, + "coding_barge_in": 1.0, + "coding_barge_in_echo": 1.0, + "coding_followup_after_reply": 1.0, + "coding_simple": 1.0, + "food_backchannel": 1.0, + "food_barge_in": 1.0, + "food_long_pause": 1.0, + "food_simple": 1.0, + "medical_barge_in": 1.0, + "medical_barge_in_twice": 1.0, + "medical_long_utterance": 1.0, + "medical_simple": 1.0, + "support_barge_in": 1.0, + "support_barge_in_instant": 1.0, + "support_barge_in_late": 1.0, + "support_barge_in_one_word": 1.0, + "support_closer": 1.0, + "support_echo_silence": 1.0, + "support_pause_short": 1.0, + "support_silence_only": 1.0, + "support_simple": 1.0 + }, + "flaky": [], + "known_failures": [ + "banking_confirmation", + "banking_correction", + "banking_digits_pause", + "coding_double_pause", + "coding_pause", + "food_list_pause", + "food_pause", + "food_rapid_fire", + "food_trailing_preposition", + "medical_filler_midway", + "medical_hesitant_pause", + "medical_self_correction", + "support_pause", + "support_trailing_conjunction" + ], + "unexpected_passes": [], + "wall_secs": 878.2, + "jobs": 6 + } +} diff --git a/benchmarks/voice/cli.py b/benchmarks/voice/cli.py new file mode 100644 index 00000000..efecb2ba --- /dev/null +++ b/benchmarks/voice/cli.py @@ -0,0 +1,383 @@ +"""Runner for the voice replay harness. + +Usage (from repo root):: + + export ELEVENLABS_API_KEY=... + export DEEPGRAM_API_KEY=... + + uv run python benchmarks/voice/cli.py # all scenarios + uv run python benchmarks/voice/cli.py -s barge_in -s pause # a subset + uv run python benchmarks/voice/cli.py --detector lexical # Timbal's HOLD path + uv run python benchmarks/voice/cli.py --detector local,lexical,provider --jobs 4 + uv run python benchmarks/voice/cli.py --repeat 3 # variance + flaky detection + uv run python benchmarks/voice/cli.py --update-baseline # accept current behavior + uv run python benchmarks/voice/cli.py --quiet # results only + uv run python benchmarks/voice/cli.py --dump # WAVs per run + +``--stt`` and ``--detector`` take comma-separated lists and are crossed into a +matrix, one scorecard and one baseline entry per cell. A row of the resulting grid +answers the question a single cell cannot: whether a scenario fails because Timbal +is wrong or because one provider is. + +Replay runs in real time — the STT provider keeps its own wall clock and Silero +needs an unbroken stream — so a scenario costs roughly what the conversation +would cost a human. ``--jobs`` overlaps runs to claw that back, at the cost of +latency fidelity: concurrent sessions contend for the same ONNX inference, so +those numbers are reported but never gated (§ ``score.compare``). + +Exit code is non-zero when an expectation fails or a regression is gated. +""" + +from __future__ import annotations + +import argparse +import asyncio +import logging +import sys +import time +from dataclasses import dataclass + +import structlog +from degrade import parse_mic_path +from dotenv import load_dotenv +from harness import HarnessConfig, RunResult, coerce_param, config_rejection, run_scenario +from scenario import SCENARIOS, Scenario, select +from score import ( + RunRecord, + build_scorecard, + compare, + cross_cell_flaky, + format_grid, + format_scorecard, + load_baseline, + record, + save_baseline, + write_jsonl, +) +from synth import synthesize_clips, synthesize_fluent + + +def _report(result: RunResult, known_failure: str | None, intermittent: bool = False) -> list[str]: + lines = ["", f" turns: {result.committed}"] + if result.interrupted: + lines.append(f" heard: {result.heard_text!r}") + if result.latencies_ms: + lines.append(f" eou→audio: {', '.join(f'{v:.0f}ms' for v in result.latencies_ms)}") + lines.append(f" audio: {result.audio_chunks} chunks, {result.audio_bytes} bytes") + lines.append(f" wall: {result.wall_secs:.1f}s") + lines.extend(f" ✗ {failure}" for failure in result.failures) + if known_failure and not result.passed: + lines.append(f" KNOWN FAIL (not gated): {known_failure}") + elif known_failure and intermittent: + lines.append(" PASS (known intermittent failure — passing proves nothing)") + elif known_failure: + lines.append(" XPASS — known failure now passes, drop the known_failure marker") + else: + lines.append(f" {'PASS' if result.passed else 'FAIL'}") + return lines + + +def _values(flags: list[str] | None, default: str) -> list[str]: + """Flatten repeated and comma-separated flags, order preserved, deduplicated.""" + if not flags: + return [default] + out = [part.strip() for flag in flags for part in flag.split(",") if part.strip()] + return list(dict.fromkeys(out)) + + +def _parse_params(flags: list[str] | None) -> dict[str, object]: + """``KEY=VALUE`` flags to a coerced mapping.""" + params: dict[str, object] = {} + for flag in flags or []: + key, _, raw = flag.partition("=") + if not _ or not key.strip(): + raise ValueError(f"expected KEY=VALUE, got {flag!r}") + params[key.strip()] = coerce_param(raw.strip()) + return params + + +@dataclass +class _Job: + config: HarnessConfig + scenario: Scenario + repeat: int + repeats: int + + @property + def header(self) -> str: + suffix = f" repeat {self.repeat + 1}/{self.repeats}" if self.repeats > 1 else "" + return f"\n=== {self.scenario.id} ({self.config.label}){suffix} ===" + + +def _list_scenarios() -> int: + domain = "" + for s in SCENARIOS: + if s.domain != domain: + domain = s.domain + print(f"\n{domain}") + scope = "" if s.detectors is None else f" detectors={','.join(sorted(s.detectors))}" + print(f" {'*' if s.quick else ' '} {s.id:<24}{scope}") + for detector, reason in s.known_failure.items(): + kind = "intermittent" if s.intermittent else "known failure" + print(f" [{kind}, {detector}] {reason}") + for detector, reason in s.known_failure_under_leak.items(): + print(f" [under --aec-leak, {detector}] {reason}") + if s.note: + print(f" {s.note}") + known = sum(1 for s in SCENARIOS if s.known_failure) + leak_known = sum(1 for s in SCENARIOS if s.known_failure_under_leak) + print( + f"\n{len(SCENARIOS)} scenarios, {known} known failures, " + f"{leak_known} more under --aec-leak; * = --quick subset" + ) + return 0 + + +async def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("-s", "--scenario", action="append", help="scenario id (repeatable)") + parser.add_argument("--stt", action="append", help="elevenlabs | deepgram-flux | deepgram-nova (comma-separated)") + parser.add_argument("--detector", action="append", help="heuristic | provider | lexical | local (comma-separated)") + parser.add_argument("--language", default="en") + parser.add_argument("--repeat", type=int, default=1, help="runs per scenario (variance / flaky detection)") + parser.add_argument( + "--jobs", type=int, default=1, help="concurrent runs; latency gates whenever this matches the baseline's" + ) + parser.add_argument("--dump", action="store_true", help="write input/output WAVs per run") + parser.add_argument( + "--aec-leak", + type=float, + default=0.0, + metavar="GAIN", + help="mix the assistant's own output back into the mic at this gain (0.1-0.3 is a " + "realistic imperfect echo canceller); exercises the echo suppressor, which clean " + "user-only audio never does", + ) + parser.add_argument( + "--mic-path", + metavar="SPEC", + help="degrade the mic path: comma-separated 'snr=' (noise floor, levelled " + "against this scenario's speech), 'pink'|'white', and 'phone' (300-3400Hz + " + "G.711). Every baseline in the repo was measured without this, on studio-clean " + "16kHz TTS; e.g. --mic-path snr=15,phone", + ) + # Reproducing one cell of a sweep with the event stream visible was the missing + # step between "this value scores worse" and knowing why. + parser.add_argument( + "--stt-param", + action="append", + metavar="KEY=VALUE", + help="provider STT knob, e.g. vad_silence_threshold_secs=0.3 (see harness.SWEEPABLE_STT_KEYS)", + ) + parser.add_argument( + "--detector-param", + action="append", + metavar="KEY=VALUE", + help="detector attribute, e.g. text_complete_hold_timeout_secs=1.2", + ) + parser.add_argument("--quick", action="store_true", help="representative subset (one per domain)") + parser.add_argument("--quiet", action="store_true", help="hide the per-event stream") + parser.add_argument("--list", action="store_true", help="list scenarios and exit") + parser.add_argument("--update-baseline", action="store_true", help="accept these results as the baseline") + parser.add_argument("--no-gate", action="store_true", help="report regressions without failing") + parser.add_argument("--verbose", action="store_true", help="keep timbal DEBUG logs") + args = parser.parse_args() + + if args.list: + return _list_scenarios() + + load_dotenv(override=True) + # Quiet by default because session/agent INFO logs bury the event stream; DEBUG + # under --verbose because that is where provider internals like Flux's + # end_of_turn_confidence are reported. + structlog.configure( + wrapper_class=structlog.make_filtering_bound_logger(logging.DEBUG if args.verbose else logging.WARNING) + ) + + jobs = max(1, args.jobs) + repeats = max(1, args.repeat) + try: + stt_extra = _parse_params(args.stt_param) + detector_params = _parse_params(args.detector_param) + except ValueError as e: + print(e) + return 2 + cells = [ + HarnessConfig( + stt=stt, + detector=detector, + language=args.language, + dump=args.dump, + aec_leak=args.aec_leak, + mic_path=parse_mic_path(args.mic_path), + stt_extra=stt_extra, + detector_params=detector_params, + ) + for stt in _values(args.stt, "deepgram-flux") + for detector in _values(args.detector, "provider") + ] + + if args.update_baseline and (args.scenario or args.quick): + # A baseline entry is replaced wholesale, so accepting a filtered run would + # silently drop every scenario it didn't cover and disarm their gates. + print("refusing to update the baseline from a filtered run: it would drop the other scenarios") + return 2 + # A parallel baseline used to be refused here, on the grounds that contention + # would bake the machine's load into the latency the gate compares against. + # Measured, that does not happen: deepgram-nova/local, --quick, 3 repeats, is + # p50 278ms / p95 407ms at --jobs 6 against 280ms / 453ms serial — no worse, + # because eou→audio is mostly waiting on STT and TTS sockets rather than + # competing for CPU. What keeps this sound is that the baseline records its own + # concurrency and `_compare_latency` declines to compare across a mismatch, so + # a parallel baseline is only ever read by equally parallel runs. The refusal + # bought no protection against a busy machine at --jobs 1 either, and cost a + # ~6x slower baseline, which is the kind of price that stops people taking one. + + queue: list[_Job] = [] + skipped: list[tuple[HarnessConfig, Scenario]] = [] + for config in cells: + selected, cell_skipped = select(args.scenario, config.detector, quick=args.quick) + skipped.extend((config, s) for s in cell_skipped) + queue.extend(_Job(config, scenario, repeat, repeats) for repeat in range(repeats) for scenario in selected) + if not queue: + detectors = ", ".join(sorted({c.detector for c in cells})) + print(f"nothing to run for detector(s) {detectors}; scenarios: {[s.id for s in SCENARIOS]}") + return 2 + + wanted = {job.scenario.id: job.scenario for job in queue}.values() + clips = await synthesize_clips([text for s in wanted for text in s.standalone_texts()]) + clips |= await synthesize_fluent([g for s in wanted for g in s.fluent_groups()]) + + if len(cells) > 1 or jobs > 1: + print(f"\n{len(queue)} run(s) across {len(cells)} cell(s) at --jobs {jobs}") + + # Serial keeps the live event stream, which is the whole debugging story for a + # single scenario. Concurrent runs buffer it, because interleaved streams from + # four sessions are unreadable and worse than none. + live = jobs == 1 and not args.quiet + semaphore = asyncio.Semaphore(jobs) + started = time.monotonic() + + async def run(job: _Job) -> RunRecord: + async with semaphore: + if live: + print(job.header) + t0 = time.monotonic() + buffer: list[str] = [] + + def log(kind: str, detail: str = "") -> None: + if args.quiet: + return + line = f" [+{time.monotonic() - t0:6.2f}s] {kind:<18}{detail}" + print(line) if live else buffer.append(line) + + result = await run_scenario(job.scenario, clips, job.config, log=log) + leak = job.config.aec_leak + known = job.scenario.known_failure_reason(job.config.detector, job.config.stt, leak) + report = _report(result, known, job.scenario.is_intermittent(job.config.detector, job.config.stt, leak)) + if live: + print("\n".join(report)) + else: + print("\n".join([job.header, *buffer, *report])) + return record( + job.scenario, result, repeat=job.repeat, jobs=jobs, aec_leak=leak, label=job.config.label + ) + + records = list(await asyncio.gather(*(run(job) for job in queue))) + + for config, scenario in skipped: + print(f"\n=== {scenario.id} ({config.label}) SKIPPED (not meaningful for {config.detector}) ===") + if scenario.note: + print(f" {scenario.note}") + + path = write_jsonl(records) + baseline = load_baseline() + cards = [] + exit_code = 0 + + for config in cells: + cell_records = [r for r in records if r.label == config.label] + if not cell_records: + continue + # A cell whose every session was refused measured nothing, and a scorecard + # would present that as behaviour: `--stt-param eot_threshold=0.4` is outside + # Flux's range and reads as a plain 0% cell. Reported before scoring so it + # cannot be baselined or gated against. + refused = [why for r in cell_records if (why := config_rejection(r.errors))] + if refused: + print(f"\n{'─' * 72}") + every = len(refused) == len(cell_records) + scope = "every run" if every else f"{len(refused)}/{len(cell_records)} runs" + print(f"{config.label}: {scope} refused by the provider") + for why in sorted(set(refused)): + print(f" {why}") + exit_code = max(exit_code, 2) + # Partial refusals still say something about the runs that connected; + # a cell where none did has no behaviour to score. + if every: + continue + card = build_scorecard(cell_records) + cards.append(card) + + print(f"\n{'─' * 72}") + print(format_scorecard(card)) + for name in card.known_failures: + reason = next(r.known_failure for r in cell_records if r.scenario == name) + print(f" known: {name} — {reason}") + + if args.update_baseline: + # Only a scenario that failed *every* repeat blocks. Saving one would + # record broken behavior as the accepted status quo and disarm its gate, + # which is what this guard is for — but it used to refuse on any failing + # run at all, and that is how a stale baseline outlives the truth. The + # ElevenLabs cells kept asserting a pass rate of 1.0 taken before a + # regression, gating every later run against a number only luck could + # meet, because one flaky repeat in forty was enough to block the + # refresh. A partial rate is not a broken baseline: `per_scenario` + # records 0.75 and the gate fires when it drops, which is strictly more + # honest than an unreachable 1.0 or a `known_failure` marker claiming a + # scenario cannot pass when it passes three times in four. + hard = sorted(name for name, rate in card.per_scenario.items() if rate == 0.0) + if hard: + print( + "\n refusing to update the baseline: " + f"{', '.join(hard)} failed every repeat and are not marked known_failure" + ) + exit_code = 1 + continue + if card.flaky: + print(f"\n recording flaky scenarios at their measured rate: {', '.join(card.flaky)}") + save_baseline(card) + print(f" baseline updated for {card.label}") + continue + + comparison = compare(card, baseline, partial=bool(args.scenario or args.quick)) + for line in comparison.notes: + print(f"\n note: {line}") + if comparison.improvements: + print("\n improvements vs baseline:") + for line in comparison.improvements: + print(f" + {line}") + if comparison.regressions: + print("\n REGRESSIONS vs baseline:") + for line in comparison.regressions: + print(f" - {line}") + if card.pass_rate < 1.0 or (comparison.regressions and not args.no_gate): + exit_code = 1 + + if len(cards) > 1: + print(f"\n{'─' * 72}") + print(format_grid(cards, records)) + + flaky = cross_cell_flaky(records) + if flaky: + print("\n FLAKY (passed some repeats, not others):") + for line in flaky: + print(f" ~ {line}") + + print(f"\n elapsed: {time.monotonic() - started:.0f}s results: {path}") + return exit_code + + +if __name__ == "__main__": + sys.exit(asyncio.run(main())) diff --git a/benchmarks/voice/degrade.py b/benchmarks/voice/degrade.py new file mode 100644 index 00000000..3a9303a2 --- /dev/null +++ b/benchmarks/voice/degrade.py @@ -0,0 +1,507 @@ +"""Microphone-path degradation for the voice replay harness. + +Every number this suite has produced came from clean 16 kHz TTS speech handed +straight to the STT: no room noise, no telephone band, no companding. That makes +the measurements reproducible, and it is also the standing reason not to trust +them for real callers. Endpointing thresholds in particular are tuned against +synthetic prosody in silence — `vad_silence_threshold_secs` on ElevenLabs and +`eot_threshold` on Flux both decide when speech has *stopped*, which is precisely +the judgement a noise floor makes harder. + +This module builds the missing path, in the order a real one applies: + + user speech (+ echo leak) -> + noise bed -> telephone band -> STT + +Noise is continuous, including through silence. That matters more than the level: +noise that starts and stops with the speech is a *cue* — it tells the endpointer +exactly where the utterance ended — so gating it to the clips would test +something easier than clean audio rather than harder. + +Deliberately not modelled, and worth knowing before reading any result from here: +packet loss and jitter. A dropped 20 ms frame mid-word is a different failure from +a noisy one and is likely the more dangerous of the two for turn-taking. + +Everything is deterministic given a seed, because a degradation axis that cannot +be replayed cannot be bisected. + +Standalone use:: + + uv run python benchmarks/voice/degrade.py --verify + uv run python benchmarks/voice/degrade.py --wav 15 --telephone "some text" +""" + +from __future__ import annotations + +import argparse +import array +import asyncio +import math +import random +import sys +from dataclasses import dataclass +from pathlib import Path + +from synth import ( + BYTES_PER_SECOND, + FRAME_BYTES, + SAMPLE_RATE, + duration_secs, + synthesize_clips, + write_wav, +) + +# --------------------------------------------------------------------------- +# G.711 mu-law companding +# --------------------------------------------------------------------------- +# +# 8-bit logarithmic quantisation: ~38 dB SNR on speech, and coarse enough at low +# amplitude to matter for a VAD deciding whether near-silence is silence. + +_ULAW_BIAS = 0x84 +_ULAW_CLIP = 32635 +_ULAW_EXPONENT = [ + 0, 0, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, + *([4] * 16), *([5] * 32), *([6] * 64), *([7] * 128), +] + + +def _encode_ulaw(sample: int) -> int: + sign = 0x80 if sample < 0 else 0x00 + magnitude = min(-sample if sample < 0 else sample, _ULAW_CLIP) + _ULAW_BIAS + exponent = _ULAW_EXPONENT[(magnitude >> 7) & 0xFF] + mantissa = (magnitude >> (exponent + 3)) & 0x0F + return ~(sign | (exponent << 4) | mantissa) & 0xFF + + +def _decode_ulaw(byte: int) -> int: + byte = ~byte & 0xFF + magnitude = (((byte & 0x0F) << 3) + _ULAW_BIAS) << ((byte >> 4) & 0x07) + magnitude -= _ULAW_BIAS + return -magnitude if byte & 0x80 else magnitude + + +# One table lookup per sample beats two function calls: this runs on every frame +# of every run, inside the loop that also has to hit a 20ms deadline. +_COMPAND = [_decode_ulaw(_encode_ulaw(s)) for s in range(-32768, 32768)] + + +def compand(pcm: bytes) -> bytes: + """Round-trip PCM16 through G.711 mu-law, as a phone network would.""" + samples = array.array("h", pcm) + for i, s in enumerate(samples): + samples[i] = _COMPAND[s + 32768] + return samples.tobytes() + + +# --------------------------------------------------------------------------- +# Biquad filtering (RBJ cookbook, coefficients derived not tabulated) +# --------------------------------------------------------------------------- + +Coeffs = tuple[float, float, float, float, float] +BiquadState = list[float] + + +def _lowpass(fc: float, fs: int = SAMPLE_RATE, q: float = 0.7071) -> Coeffs: + w0 = 2 * math.pi * fc / fs + alpha = math.sin(w0) / (2 * q) + cos_w0 = math.cos(w0) + b0, b1, b2 = (1 - cos_w0) / 2, 1 - cos_w0, (1 - cos_w0) / 2 + a0, a1, a2 = 1 + alpha, -2 * cos_w0, 1 - alpha + return (b0 / a0, b1 / a0, b2 / a0, a1 / a0, a2 / a0) + + +def _highpass(fc: float, fs: int = SAMPLE_RATE, q: float = 0.7071) -> Coeffs: + w0 = 2 * math.pi * fc / fs + alpha = math.sin(w0) / (2 * q) + cos_w0 = math.cos(w0) + b0, b1, b2 = (1 + cos_w0) / 2, -(1 + cos_w0), (1 + cos_w0) / 2 + a0, a1, a2 = 1 + alpha, -2 * cos_w0, 1 - alpha + return (b0 / a0, b1 / a0, b2 / a0, a1 / a0, a2 / a0) + + +def new_state() -> BiquadState: + return [0.0, 0.0, 0.0, 0.0] + + +def _run_biquad(samples: list[float], c: Coeffs, state: BiquadState) -> None: + """In-place Direct Form I. ``state`` carries across calls. + + Filter state must survive frame boundaries: resetting per frame would stamp a + transient every 20ms, which is a periodic click the STT would hear and this + module would not be measuring. + """ + b0, b1, b2, a1, a2 = c + x1, x2, y1, y2 = state + for i, x0 in enumerate(samples): + y0 = b0 * x0 + b1 * x1 + b2 * x2 - a1 * y1 - a2 * y2 + samples[i] = y0 + x2, x1 = x1, x0 + y2, y1 = y1, y0 + state[0], state[1], state[2], state[3] = x1, x2, y1, y2 + + +# The G.711 passband. Two low-pass stages, because one 12 dB/octave skirt still +# leaks enough 5-6 kHz energy that the result sounds like a muffled wideband +# recording rather than a phone call. +_TELEPHONE_STAGES: tuple[Coeffs, ...] = ( + _highpass(300.0), + _lowpass(3400.0), + _lowpass(3400.0), +) + + +def telephone_state() -> list[BiquadState]: + return [new_state() for _ in _TELEPHONE_STAGES] + + +def band_limit(pcm: bytes, state: list[BiquadState] | None = None) -> bytes: + """Restrict to the 300-3400 Hz telephone band.""" + if state is None: + state = telephone_state() + samples = [float(s) for s in array.array("h", pcm)] + for stage, st in zip(_TELEPHONE_STAGES, state, strict=True): + _run_biquad(samples, stage, st) + out = array.array("h", bytes(len(pcm))) + for i, s in enumerate(samples): + out[i] = 32767 if s > 32767 else (-32768 if s < -32768 else int(s)) + return out.tobytes() + + +def resample_8k(pcm: bytes) -> bytes: + """Decimate to 8 kHz and interpolate back, as an 8 kHz link would. + + Only meaningful after :func:`band_limit`; on its own it would alias. The band + limit is what changes the sound, but the round trip is cheap and it is the + part that is literally true of a phone call. + """ + samples = array.array("h", pcm) + if not samples: + return pcm + out = array.array("h", bytes(len(pcm))) + for i in range(len(samples)): + if i % 2 == 0: + out[i] = samples[i] + else: + prev = samples[i - 1] + nxt = samples[i + 1] if i + 1 < len(samples) else prev + out[i] = (prev + nxt) // 2 + return out.tobytes() + + +# --------------------------------------------------------------------------- +# Noise +# --------------------------------------------------------------------------- + + +def rms(pcm: bytes) -> float: + samples = array.array("h", pcm) + if not samples: + return 0.0 + return math.sqrt(sum(float(s) * s for s in samples) / len(samples)) + + +def active_rms(pcm: bytes, floor_ratio: float = 0.1) -> float: + """RMS of the speech, ignoring the silence around and inside it. + + SNR has to be stated against the level of the speech, not the level of the + recording: a scenario that is 60% deliberate pause would otherwise get a much + louder noise bed than one that is not, and the two would stop being + comparable at the same nominal SNR. + """ + frame_rms = [rms(pcm[i : i + FRAME_BYTES]) for i in range(0, len(pcm), FRAME_BYTES)] + if not frame_rms: + return 0.0 + threshold = max(frame_rms) * floor_ratio + active = [r for r in frame_rms if r >= threshold] + if not active: + return 0.0 + return math.sqrt(sum(r * r for r in active) / len(active)) + + +def noise_rms_for_snr(speech_rms: float, snr_db: float) -> float: + return speech_rms / (10 ** (snr_db / 20)) + + +def white_noise(n: int, rng: random.Random) -> list[float]: + return [rng.gauss(0.0, 1.0) for _ in range(n)] + + +def pink_noise(n: int, rng: random.Random) -> list[float]: + """1/f noise (Kellet's economy method). + + Pink is the default because room, traffic and HVAC noise is pink, so it is the + honest stand-in for a caller who is not in a recording booth. + + Note it is not the harsher of the two in the telephone band: at matched + broadband level, pink puts most of its energy below 300 Hz where the codec's + high-pass removes it, so `white` delivers *more* in-band energy (measured in + ``--verify``). Use `white` to stress the STT, `pink` to be realistic. + """ + b = [0.0] * 7 + out: list[float] = [] + for _ in range(n): + w = rng.gauss(0.0, 1.0) + b[0] = 0.99886 * b[0] + w * 0.0555179 + b[1] = 0.99332 * b[1] + w * 0.0750759 + b[2] = 0.96900 * b[2] + w * 0.1538520 + b[3] = 0.86650 * b[3] + w * 0.3104856 + b[4] = 0.55000 * b[4] + w * 0.5329522 + b[5] = -0.7616 * b[5] - w * 0.0168980 + out.append(b[0] + b[1] + b[2] + b[3] + b[4] + b[5] + b[6] + w * 0.5362) + b[6] = w * 0.115926 + return out + + +NOISE_KINDS = ("pink", "white") + + +def noise_bed(secs: float, target_rms: float, *, kind: str = "pink", seed: int = 0) -> bytes: + """A loopable bed of noise at ``target_rms``, deterministic for ``seed``.""" + if kind not in NOISE_KINDS: + raise ValueError(f"unknown noise kind {kind!r}; expected one of {NOISE_KINDS}") + n = max(1, int(secs * SAMPLE_RATE)) + rng = random.Random(seed) + raw = pink_noise(n, rng) if kind == "pink" else white_noise(n, rng) + scale = 0.0 + current = math.sqrt(sum(s * s for s in raw) / len(raw)) + if current > 0: + scale = target_rms / current + out = array.array("h", bytes(n * 2)) + for i, s in enumerate(raw): + v = s * scale + out[i] = 32767 if v > 32767 else (-32768 if v < -32768 else int(v)) + return out.tobytes() + + +# --------------------------------------------------------------------------- +# The axis +# --------------------------------------------------------------------------- + +# One minute, looped. Long enough that no scenario hears a repeat, short enough +# that generating it costs a fraction of a second of the run's startup. +BED_SECS = 60.0 + + +@dataclass(frozen=True) +class MicPath: + """What happens to audio between the speaker's mouth and the STT. + + The default is the identity: every run before this axis existed. + """ + + snr_db: float | None = None + telephone: bool = False + noise: str = "pink" + seed: int = 0 + + @property + def active(self) -> bool: + return self.snr_db is not None or self.telephone + + @property + def label(self) -> str: + """Cell-label suffix, matching the `[leak=...]` axis convention.""" + parts = [] + if self.snr_db is not None: + kind = "" if self.noise == "pink" else f",{self.noise}" + parts.append(f"snr={self.snr_db:g}{kind}") + if self.telephone: + parts.append("phone") + return f"[{','.join(parts)}]" if parts else "" + + def bed(self, speech_rms: float) -> bytes: + """The noise to mix in, scaled against the measured speech level. + + Returned *clean*: the caller mixes it with the speech and then runs the + codec over the sum, which is the order a real microphone and line apply. + So ``snr_db`` is the acoustic SNR at the microphone, not the SNR the STT + receives — speech and noise lose different amounts of energy to the + telephone band, so the ratio downstream of it is not the same number. + """ + if self.snr_db is None or speech_rms <= 0: + return b"" + target = noise_rms_for_snr(speech_rms, self.snr_db) + return noise_bed(BED_SECS, target, kind=self.noise, seed=self.seed) + + def apply(self, pcm: bytes, state: list[BiquadState] | None = None) -> bytes: + """Run the codec over one frame or clip. Pass ``state`` to keep it continuous.""" + if not self.telephone: + return pcm + return compand(resample_8k(band_limit(pcm, state))) + + +def parse_mic_path(spec: str | None, *, seed: int = 0) -> MicPath: + """Parse ``--mic-path`` : comma-separated ``snr=15``, ``white``, ``phone``. + + Examples: ``snr=15``, ``phone``, ``snr=10,white,phone``. + """ + if not spec: + return MicPath(seed=seed) + snr_db: float | None = None + telephone = False + noise = "pink" + for token in (t.strip().lower() for t in spec.split(",") if t.strip()): + if token in ("phone", "telephone", "g711"): + telephone = True + elif token in NOISE_KINDS: + noise = token + elif token.startswith("snr="): + snr_db = float(token[4:]) + else: + raise ValueError( + f"unknown --mic-path token {token!r}; expected snr=, " + f"one of {NOISE_KINDS}, or 'phone'" + ) + return MicPath(snr_db=snr_db, telephone=telephone, noise=noise, seed=seed) + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def _tone(freq: float, secs: float, amplitude: int = 8000) -> bytes: + n = int(secs * SAMPLE_RATE) + out = array.array("h", bytes(n * 2)) + for i in range(n): + out[i] = int(amplitude * math.sin(2 * math.pi * freq * i / SAMPLE_RATE)) + return out.tobytes() + + +def _db(ratio: float) -> float: + return 20 * math.log10(ratio) if ratio > 0 else -math.inf + + +def _verify() -> int: + """Offline checks: no API key, no network.""" + checks: list[tuple[str, bool, str]] = [] + + # Companding must be idempotent, or repeated application in a chain would + # keep degrading and the axis would not mean one pass through a codec. + speech_like = _tone(220, 0.5, amplitude=12000) + once = compand(speech_like) + twice = compand(once) + checks.append(("compand is idempotent", once == twice, f"{len(once)} bytes")) + + err = rms(bytes(array.array("h", [a - b for a, b in + zip(array.array("h", speech_like), array.array("h", once), strict=True)]).tobytes())) + snr = _db(rms(speech_like) / err) if err else math.inf + checks.append(("compand SNR is G.711-like", 30 <= snr <= 45, f"{snr:.1f} dB")) + + # The band: 1kHz through, 6kHz gone, rumble gone. + for freq, lo, hi, name in ( + (1000.0, -2.0, 1.0, "1kHz passes"), + (6000.0, -math.inf, -25.0, "6kHz is rejected"), + (100.0, -math.inf, -12.0, "100Hz is rejected"), + ): + tone = _tone(freq, 0.5) + # Skip the filter's settling transient before measuring. + got = _db(rms(band_limit(tone)[FRAME_BYTES * 5 :]) / rms(tone[FRAME_BYTES * 5 :])) + checks.append((name, lo <= got <= hi, f"{got:+.1f} dB")) + + path = MicPath(snr_db=15.0, telephone=True) + checks.append(("telephone is deterministic", path.apply(speech_like) == path.apply(speech_like), "byte-exact")) + checks.append(("length is preserved", len(path.apply(speech_like)) == len(speech_like), f"{len(speech_like)} bytes")) + + # SNR must come out at the number that was asked for, or every result on this + # axis is labelled with a level it does not have. + speech = _tone(300, 1.0, amplitude=10000) + bytes(BYTES_PER_SECOND) # half speech, half silence + level = active_rms(speech) + checks.append(("active_rms ignores silence", abs(_db(level / rms(speech)) - 3.0) < 0.6, f"{_db(level / rms(speech)):+.1f} dB vs whole")) + bed = noise_bed(1.0, noise_rms_for_snr(level, 15.0), seed=1) + got_snr = _db(level / rms(bed)) + checks.append(("bed hits the requested SNR", abs(got_snr - 15.0) < 0.5, f"{got_snr:.2f} dB")) + + checks.append(("bed is deterministic", noise_bed(0.2, 500, seed=7) == noise_bed(0.2, 500, seed=7), "seed 7")) + checks.append(("seeds differ", noise_bed(0.2, 500, seed=7) != noise_bed(0.2, 500, seed=8), "7 vs 8")) + # What makes pink pink: energy rising towards DC. Checked below 300 Hz, which + # is also exactly the region the telephone high-pass throws away — so at + # matched broadband level, white ends up the harsher of the two in-band. + def _sub_300(pcm: bytes) -> float: + samples = [float(s) for s in array.array("h", pcm)] + state = new_state() + stage = _lowpass(300.0) + _run_biquad(samples, stage, state) + return math.sqrt(sum(s * s for s in samples) / len(samples)) + + pink_bed = noise_bed(1.0, 2000, kind="pink", seed=3) + white_bed = noise_bed(1.0, 2000, kind="white", seed=3) + checks.append(( + "pink is low-frequency weighted", + _sub_300(pink_bed) > _sub_300(white_bed) * 2, + f"{_sub_300(pink_bed):.0f} vs {_sub_300(white_bed):.0f} below 300Hz", + )) + checks.append(( + "so white is harsher in the phone band", + rms(band_limit(white_bed)) > rms(band_limit(pink_bed)), + f"{rms(band_limit(white_bed)):.0f} vs {rms(band_limit(pink_bed)):.0f} in band", + )) + + checks.append(("labels round-trip", parse_mic_path("snr=10,white,phone").label == "[snr=10,white,phone]", parse_mic_path("snr=10,white,phone").label)) + checks.append(("identity by default", not MicPath().active and MicPath().label == "", "no suffix")) + + # This runs inside the loop that must hand the session a frame every 20ms, so + # the cost is part of the contract, not an implementation detail. + frame = _tone(440, 0.02) + state = telephone_state() + import time + + t0 = time.perf_counter() + reps = 200 + for _ in range(reps): + MicPath(telephone=True).apply(frame, state) + per_frame_ms = (time.perf_counter() - t0) / reps * 1000 + checks.append(( + "cost fits the frame budget", + per_frame_ms < 4.0, + f"{per_frame_ms:.2f} ms per 20 ms frame ({per_frame_ms / 20 * 100:.0f}%)", + )) + + for name, ok, detail in checks: + print(f" {'ok ' if ok else 'FAIL'} {name:<44} {detail}") + failed = [name for name, ok, _ in checks if not ok] + print(f"\n{len(checks) - len(failed)}/{len(checks)} checks passed") + return 0 if not failed else 1 + + +async def _write_sample(text: str, spec: str, out: Path | None) -> int: + path = parse_mic_path(spec) + clips = await synthesize_clips([text]) + clean = clips[text] + bed = path.bed(active_rms(clean)) + # Same order as ScriptFeeder.stream, or this writes a file that sounds like + # something the harness never feeds: noise into the mic, then the line. + degraded = clean + if bed: + from harness import mix_pcm16 + + degraded = mix_pcm16(degraded, bed[: len(degraded)], 1.0) + degraded = path.apply(degraded) + out = out or Path(__file__).parent / "results" / "synth" / f"degraded{path.label}.wav" + write_wav(out, degraded) + print(f"{duration_secs(degraded):.2f}s {path.label or '[clean]'} -> {out}") + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("text", nargs="*", help="text to synthesize and degrade") + parser.add_argument("--verify", action="store_true", help="offline self-checks") + parser.add_argument("--mic-path", default="snr=15,phone", help="e.g. 'snr=15,phone'") + parser.add_argument("-o", "--out", type=Path) + args = parser.parse_args() + + if args.verify: + return _verify() + if not args.text: + parser.print_help() + return 2 + + from dotenv import load_dotenv + + load_dotenv(override=True) + return asyncio.run(_write_sample(" ".join(args.text), args.mic_path, args.out)) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/benchmarks/voice/harness.py b/benchmarks/voice/harness.py new file mode 100644 index 00000000..d22d0eac --- /dev/null +++ b/benchmarks/voice/harness.py @@ -0,0 +1,699 @@ +"""Replay harness: drives a real ``VoiceSession`` from a scripted scenario. + +The harness is a **fake browser**. No session changes are needed because the +three seams already exist: + +=============== ========================================== ================== +Seam Session API Harness role +=============== ========================================== ================== +audio in ``session.run(audio_in)`` paced PCM feeder +events out yields ``VoiceSessionEvent`` script driver +playback ``session.playback.on_playback_ack(ms)`` ack pump +=============== ========================================== ================== + +The ack pump is not optional: without acks ``playback_acks_received`` is False +and interruption truncation runs on the wall-clock estimate, which is a +*different code path* than production. +""" + +from __future__ import annotations + +import array +import asyncio +import re +import time +from collections import deque +from collections.abc import AsyncIterator, Callable, Mapping, Sequence +from contextlib import aclosing +from dataclasses import dataclass, field +from typing import Any + +from degrade import MicPath, active_rms, telephone_state +from scenario import ( + AwaitAssistantAudio, + AwaitAssistantDone, + AwaitCommit, + Say, + Scenario, + Silence, +) +from synth import ( + ASSISTANT_VOICE_ID, + BYTES_PER_SECOND, + FRAME_BYTES, + FRAME_SECS, + HERE, + SAMPLE_RATE, + SILENCE_FRAME, + TTS_MODEL, + frames, + write_wav, +) +from timbal import Agent +from timbal.core.test_model import TestModel +from timbal.state.tracing.providers import InMemoryTracingProvider +from timbal.voice import ( + AgentTextDone, + AudioInputConfig, + AudioOutput, + AudioOutputConfig, + SessionError, + SessionInterrupted, + SessionStarted, + TranscriptCommitted, + TranscriptPartial, + TurnMetrics, + TurnMetricsEvent, + VoiceSession, + VoiceSessionEvent, + resolve_stt, + resolve_turn_detector, +) +from timbal.voice.elevenlabs import ElevenLabsStreamTTS + +DUMP_DIR = HERE / "results" / "dumps" + +Logger = Callable[[str, str], None] + + +@dataclass(frozen=True) +class HarnessConfig: + stt: str = "deepgram-flux" + detector: str = "provider" + language: str = "en" + dump: bool = False + # Detector attributes to override per run, e.g. + # {"text_complete_hold_timeout_secs": 1.0}. Exists so a parameter sweep can + # ask "what is the right value" instead of editing product constants and + # re-running by hand, which is how the hold tier came to be tested at 0.35 + # and 3.0 and nowhere in between. Applied after construction, so only + # instance attributes — class-level constants must have a matching + # instance attribute to be reachable. + detector_params: Mapping[str, Any] = field(default_factory=dict) + # Provider STT knobs to override per run, merged over the defaults in + # `stt_config`. The detector is not the only endpointer in the pipeline: + # ElevenLabs ships `vad_silence_threshold_secs=1.2` and Nova ships + # `endpointing=300`, and until these were sweepable the 4x asymmetry + # between them read as a provider property rather than a setting. + stt_extra: Mapping[str, Any] = field(default_factory=dict) + # Fraction of the assistant's own output mixed back into the mic, standing + # in for imperfect echo cancellation. 0.0 is every run before this one. + aec_leak: float = 0.0 + # Noise floor and telephone band between the speaker and the STT. The default + # is the identity — studio-clean 16kHz TTS, which is what every number in the + # README was measured on and the main reason to be careful quoting them for + # real callers. + mic_path: MicPath = field(default_factory=MicPath) + + @property + def label(self) -> str: + suffix = self.mic_path.label + if self.aec_leak: + suffix += f"[leak={self.aec_leak}]" + for params in (self.detector_params, self.stt_extra): + if params: + suffix += "[" + ",".join(f"{k}={v}" for k, v in sorted(params.items())) + "]" + return f"{self.stt}/{self.detector}{suffix}" + + +@dataclass +class RunResult: + scenario_id: str + stt: str + detector: str + committed: list[str] = field(default_factory=list) + # Arrival times, parallel to `committed` / `replies_spoken`. A wait for "one + # more commit" cannot tell a commit for the speech just fed from one for + # earlier speech that arrived late, and where a provider's commit lags the + # audio by more than the gap between fragments, that is every pause + # scenario it has. See `AwaitCommit` in `drive`. + committed_at: list[float] = field(default_factory=list) + replies_spoken: list[str] = field(default_factory=list) + replied_at: list[float] = field(default_factory=list) + # Last time the session said anything at all — partial, commit or reply. What + # "the session has finished with the audio I fed it" is actually made of, + # since neither a count nor a single timestamp can express it. + last_event_at: float = 0.0 + interrupted: bool = False + heard_text: str | None = None + latencies_ms: list[float] = field(default_factory=list) + metrics: list[TurnMetrics] = field(default_factory=list) + errors: list[str] = field(default_factory=list) + audio_chunks: int = 0 + audio_bytes: int = 0 + wall_secs: float = 0.0 + failures: list[str] = field(default_factory=list) + # Wall time from the user falling silent to the turn being accepted — the + # dead air a caller actually sits through, and the half of the picture + # `latencies_ms` cannot see. `eou→first audio` starts at the *accepted* + # commit, so every second a hold spends deciding is invisible to it. That + # blindness let a tier-removal experiment read as 11 fixes and 0 + # regressions while adding 2.6s to six barge-in cells. + dead_air_ms: list[float] = field(default_factory=list) + # Monotonic timestamp of the most recent speech end, consumed by the next + # commit. Not an output; internal to the measurement. + speech_ended_at: float | None = None + # Arrival of the newest partial. A partial newer than every commit means the + # provider is still transcribing something — a commit (or a deliberate + # suppression) is pending — and a wait must not read the session as settled. + # See `_settled` in `run_scenario`. Internal to the measurement. + last_partial_at: float = 0.0 + + @property + def passed(self) -> bool: + return not self.failures + + +# --------------------------------------------------------------------------- +# Fake browser internals +# --------------------------------------------------------------------------- + + +class PlaybackSim: + """Playhead over a gapless client-side queue, acked back at browser cadence. + + Caveat: this re-derives the same wall-clock schedule + ``BufferedPlaybackTracker`` computes internally, so it validates the *ack code + path* (``ack_received`` true, extrapolation branch live) rather than the + accuracy of the estimate. Real accuracy needs real playback hardware. + """ + + def __init__(self, bytes_per_second: int = BYTES_PER_SECOND) -> None: + self._bps = bytes_per_second + self._scheduled = 0 + self._playing_until = 0.0 + + def on_emit(self, num_bytes: int) -> None: + now = time.monotonic() + self._playing_until = max(now, self._playing_until) + num_bytes / self._bps + self._scheduled += num_bytes + + def on_interrupted(self) -> None: + self._scheduled = self.played_bytes + self._playing_until = time.monotonic() + + @property + def played_bytes(self) -> int: + remaining = max(0.0, self._playing_until - time.monotonic()) * self._bps + return max(0, int(self._scheduled - remaining)) + + @property + def played_ms(self) -> float: + return self.played_bytes / self._bps * 1000 + + +def mix_pcm16(base: bytes, leak: bytes, gain: float) -> bytes: + """Sum two PCM16 frames with ``leak`` attenuated, clipping at int16 bounds.""" + out = array.array("h", base) + other = array.array("h", leak) + for i in range(min(len(out), len(other))): + out[i] = max(-32768, min(32767, int(out[i] + other[i] * gain))) + return out.tobytes() + + +class ScriptFeeder: + """Paced PCM source: clip audio when speaking, silence the rest of the time. + + With ``leak_gain`` above zero it also mixes the assistant's own output back + into the mic, standing in for an echo canceller that does not fully cancel. + Every run before this fed clean user-only audio, which means + ``_likely_stt_echo`` — the guard that stops the assistant interrupting + itself on speaker bleed — had never once been exercised. + + ``mic_path`` adds a noise floor and the telephone band. It applies here, at the + last moment before the frame leaves, rather than to the clips: a real mic path + degrades *everything* it carries — the user, the silence between them, and the + echo — and the noise has to keep running through the pauses, since noise that + stops when the speaker does would tell the endpointer where the turn ended. + """ + + # Cap on buffered assistant audio. TTS generates faster than realtime, so an + # uncapped buffer would drift seconds behind and leak the assistant's voice + # into silence long after it stopped speaking — a harsher and less realistic + # test than the thing being simulated. + MAX_LEAK_SECS = 1.0 + + def __init__(self, leak_gain: float = 0.0, mic_path: MicPath | None = None) -> None: + self._pending: deque[bytes] = deque() + self._leak = bytearray() + self._leak_gain = leak_gain + self._max_leak_bytes = int(BYTES_PER_SECOND * self.MAX_LEAK_SECS) + self._stopped = False + self._mic_path = mic_path or MicPath() + # One filter state for the whole session, so the codec has no per-frame + # transient, and one read position so the bed plays as continuous noise + # rather than restarting every 20ms. + self._codec_state = telephone_state() + self._bed = b"" + self._bed_pos = 0 + + def set_noise_bed(self, bed: bytes) -> None: + self._bed = bed + + def _next_noise(self) -> bytes | None: + if not self._bed: + return None + if self._bed_pos + FRAME_BYTES > len(self._bed): + self._bed_pos = 0 + frame = self._bed[self._bed_pos : self._bed_pos + FRAME_BYTES] + self._bed_pos += FRAME_BYTES + return frame + + def push(self, pcm: bytes) -> None: + self._pending.extend(frames(pcm)) + + def push_leak(self, pcm: bytes) -> None: + """Queue assistant output to bleed into the mic.""" + if self._leak_gain <= 0: + return + self._leak.extend(pcm) + if len(self._leak) > self._max_leak_bytes: + del self._leak[: len(self._leak) - self._max_leak_bytes] + + def _next_leak(self) -> bytes | None: + if self._leak_gain <= 0 or len(self._leak) < FRAME_BYTES: + return None + frame = bytes(self._leak[:FRAME_BYTES]) + del self._leak[:FRAME_BYTES] + return frame + + async def drain(self) -> None: + while self._pending: + await asyncio.sleep(FRAME_SECS) + + def stop(self) -> None: + self._stopped = True + + async def stream(self) -> AsyncIterator[bytes]: + """20ms frames, continuously. + + The stream never gaps: STT sockets and Silero cannot tell a pause in the + script from a stalled connection. Pacing follows an absolute schedule + because ``sleep(FRAME_SECS)`` per iteration accumulates drift and slowly + desynchronizes the replay from real time. + """ + next_at = time.monotonic() + while not self._stopped: + frame = self._pending.popleft() if self._pending else SILENCE_FRAME + if (leak := self._next_leak()) is not None: + frame = mix_pcm16(frame, leak, self._leak_gain) + # Speech and echo first, then the room, then the line: the noise is + # picked up by the microphone, so it goes through the codec too. + if (noise := self._next_noise()) is not None: + frame = mix_pcm16(frame, noise, 1.0) + if self._mic_path.telephone: + frame = self._mic_path.apply(frame, self._codec_state) + yield frame + next_at += FRAME_SECS + await asyncio.sleep(max(0.0, next_at - time.monotonic())) + + +# STT knobs `--stt-param` may vary, per backend. An allowlist because providers +# ignore unknown query params silently: a typo would sweep one value under +# several labels and every number in the table would agree, convincingly. +SWEEPABLE_STT_KEYS: dict[str, frozenset[str]] = { + "elevenlabs": frozenset( + {"commit_strategy", "min_speech_duration_ms", "vad_silence_threshold_secs", "vad_threshold"} + ), + "deepgram-nova": frozenset({"endpointing", "utterance_end_ms", "smart_format", "punctuate", "interim_results"}), + "deepgram-flux": frozenset({"eot_timeout_ms", "eot_threshold", "eager_eot_threshold"}), +} + + +_HANDSHAKE_REJECTION = re.compile(r"server rejected WebSocket connection: HTTP (4\d\d)") + + +def config_rejection(errors: Sequence[str]) -> str | None: + """The provider refused the configuration, as opposed to the run going badly. + + ``SWEEPABLE_STT_KEYS`` validates key *names*, so a value the provider rejects + still runs: `eot_threshold=0.4` is outside Flux's accepted range and every + session dies on the handshake, which the sweep then reported as a 0% row + indistinguishable from a setting that merely merges nothing. Encoding each + provider's accepted ranges here would go stale the first time one changed + theirs; noticing the refusal does not. + + Only 4xx. A rejected handshake is deterministic for a given config, so + repeating it 77 more times cannot learn anything — whereas a timeout or a 5xx + is exactly the transient the repeats exist to average over. + """ + for e in errors: + if m := _HANDSHAKE_REJECTION.search(e): + return f"provider rejected the connection (HTTP {m.group(1)}): {e}" + return None + + +def coerce_param(raw: str) -> float | int | bool | str: + """Best-effort literal from a command-line value (`0.3` -> float, `true` -> bool).""" + for cast in (int, float): + try: + return cast(raw) + except ValueError: + continue + if raw.lower() in ("true", "false"): + return raw.lower() == "true" + return raw + + +def stt_config(stt: str, language: str, extra: Mapping[str, Any] | None = None) -> AudioInputConfig: + """Provider defaults for a cell, with `extra` merged over them. + + The ElevenLabs values mirror ``timbal.voice.server.default_voice_config_from_env`` + rather than inventing benchmark-only ones — measuring a configuration the + product does not ship would report dead air nobody experiences. + """ + overrides = dict(extra or {}) + if unknown := set(overrides) - SWEEPABLE_STT_KEYS.get(stt, frozenset()): + raise ValueError(f"{stt} has no sweepable STT param(s) {sorted(unknown)}") + if stt.startswith("deepgram"): + return AudioInputConfig(language=language, sample_rate=SAMPLE_RATE, extra=overrides) + return AudioInputConfig( + model="scribe_v2_realtime", + language=language, + sample_rate=SAMPLE_RATE, + extra={ + "commit_strategy": "vad", + "min_speech_duration_ms": 100, + "vad_silence_threshold_secs": 1.2, + "vad_threshold": 0.4, + **overrides, + }, + ) + + +# --------------------------------------------------------------------------- +# Runner +# --------------------------------------------------------------------------- + + +def _build_detector(config: HarnessConfig) -> Any: + """The detector name, or a configured instance when params are overridden. + + Rejects unknown attribute names rather than silently setting them: a typo in + a sweep would otherwise run the default configuration under a label claiming + it was something else, and every number in the sweep would be wrong in a way + nothing could detect afterwards. + """ + if not config.detector_params: + return config.detector + detector = resolve_turn_detector(config.detector) + for name, value in config.detector_params.items(): + if not hasattr(detector, name): + raise ValueError( + f"{type(detector).__name__} has no attribute {name!r}; cannot override it for {config.label}" + ) + setattr(detector, name, value) + return detector + + +def _settle_secs(config: HarnessConfig) -> float: + """How long the session must say nothing before a wait accepts what it has. + + Must exceed the longest HOLD the detector can arm, and this is the whole reason + the value is derived rather than chosen. A hold emits nothing while it debounces + — that is what it is for — so a quiescence rule shorter than the hold cannot tell + "the session is finished" from "the session is deliberately waiting", and the + harness tears down mid-hold. It reads as the *product* dropping a turn. + + Measured on the ElevenLabs barge-ins, where the interrupting utterance commits as + a 1.5s `lexical_hold` against a flat 1.0s settle: 19/24 at 1.0s, 24/24 once the + settle clears the hold. Those five were previously written off as provider + flakiness, which is what a harness bug looks like from the outside when it only + bites the slowest provider. + + Floors at 1.0s to cover the gap between speech ending and the first partial for + it (~0.4s on ElevenLabs, the slowest measured here) on detectors that never hold. + """ + detector = resolve_turn_detector(_build_detector(config)) + holds = [ + getattr(detector, name, None) + # DEFAULT_ included because LexicalTurnDetector passes its class constant + # straight into the decision without ever binding an instance attribute. + for name in ( + "hold_timeout_secs", + "DEFAULT_HOLD_TIMEOUT_SECS", + "text_complete_hold_timeout_secs", + "text_incomplete_hold_timeout_secs", + ) + ] + longest = max((h for h in holds if isinstance(h, int | float)), default=0.0) + # The provider's own endpointer is a hold too, and the floor has to clear it: + # ElevenLabs ships vad_silence_threshold_secs=1.2, so on a holdless detector a + # 1.0s quiescence rule declared "settled, with something in hand" while the + # commit for the newest speech was ~0.2s from landing — and the something in + # hand was an *older* utterance's commit. Same reasoning as the detector + # holds above: derived rather than chosen, because the failure mode is + # invisible and reads as the provider dropping a turn. + extra = stt_config(config.stt, config.language, config.stt_extra).extra + provider_secs = max( + float(extra.get("vad_silence_threshold_secs", 0.0)), + float(extra.get("endpointing", 0.0)) / 1000, + float(extra.get("utterance_end_ms", 0.0)) / 1000, + float(extra.get("eot_timeout_ms", 0.0)) / 1000, + ) + return max(1.0, longest + 0.5, provider_secs + 0.5) + + +async def run_scenario( + scenario: Scenario, + clips: dict[str, bytes], + config: HarnessConfig, + *, + log: Logger | None = None, +) -> RunResult: + """Replay one scenario against a live session and check its expectations.""" + result = RunResult(scenario_id=scenario.id, stt=config.stt, detector=config.detector) + + # Tracing must stay on: the session chains parent_id across turns for memory, + # and disabling it makes every turn after the first log "Parent trace not + # found". Pinned to in-memory so a platform config in the environment can't + # drag the bench into a real tracing backend. + agent = Agent( + name="bench", + model=TestModel(responses=scenario.replies), + tools=[], + tracing_provider=InMemoryTracingProvider, + ) + session = VoiceSession( + agent=agent, + stt=resolve_stt(config.stt), + tts=ElevenLabsStreamTTS(), + audio_input=stt_config(config.stt, config.language, config.stt_extra), + audio_output=AudioOutputConfig(model=TTS_MODEL, voice=ASSISTANT_VOICE_ID, sample_rate=SAMPLE_RATE), + turn_detector=_build_detector(config), + record_audio=config.dump, + ) + + feeder = ScriptFeeder(leak_gain=config.aec_leak, mic_path=config.mic_path) + if config.mic_path.snr_db is not None: + # Level the noise against this scenario's own speech, so the same nominal + # SNR means the same thing in a scenario that is mostly pause as in one + # that is wall-to-wall talking. + spoken = b"".join( + clips[s.clip_key] for s in scenario.script if isinstance(s, Say) and s.clip_key in clips + ) + feeder.set_noise_bed(config.mic_path.bed(active_rms(spoken))) + playback = PlaybackSim() + assistant_speaking = asyncio.Event() + # session.close() interrupts any reply still playing. That teardown interrupt + # is not a barge-in and must not be observed as one. + closing = asyncio.Event() + started = time.monotonic() + + def emit(kind: str, detail: str = "") -> None: + if log is not None: + log(kind, detail) + + async def ack_loop() -> None: + """Report the playhead the way the playground does — every 250ms.""" + while True: + await asyncio.sleep(0.25) + session.playback.on_playback_ack(playback.played_ms) + + async def wait_for(predicate, timeout: float, what: str) -> None: + deadline = time.monotonic() + timeout + while not predicate(): + if time.monotonic() > deadline: + result.errors.append(f"timed out waiting for {what}") + emit("timeout", what) + return + await asyncio.sleep(FRAME_SECS) + + settle_secs = _settle_secs(config) + + def _settled(events: list[float], after: float) -> bool: + """Whether the session is done with the speech fed up to ``after``. + + Anything arriving after that instant is *usually* the answer, since mic + audio is paced at realtime and the provider cannot have been sent a clip + it has not received yet. Usually, because the converse event looks + identical: a commit for *earlier* speech can land that late too — + ElevenLabs commits ~1.6s after speech ends, longer than the fragment gap + in every pause scenario — and resolving on it tears the session down + with the newest speech's commit still in flight, the same + phantom-content-loss bias the timed wait replaced the watermark to fix. + In flight is observable: the provider streams partials for speech it has + not committed yet, so a partial newer than everything in hand keeps the + wait open until the commit lands or the partial goes stale for + ``settle_secs`` — past that it is a stranded partial, the product's + problem to report rather than the harness's to wait on. + + Otherwise the provider may simply have answered early — it can have all + the audio it needs before the harness finishes handing the clip over, and + `food_simple` commits without the final "?" for exactly that reason. Then + nothing will ever arrive "after", so a wait for one hangs for its whole + timeout. Falling back to a quiet session with something already in hand + covers that, and covers a last `Say` that correctly produces nothing at + all, which used to depend on a stale event to avoid timing out. + """ + now = time.monotonic() + if result.last_partial_at > max([after, *events]) and now - result.last_partial_at < settle_secs: + return False + if any(t > after for t in events): + return True + if not events: + return False + return now - max(after, result.last_event_at) >= settle_secs + + async def drive() -> None: + last_say_ended = 0.0 + + for step in scenario.script: + if isinstance(step, Say): + emit("say", f'"{step.text}"') + feeder.push(clips[step.clip_key]) + await feeder.drain() + # drain() returns once the last frame is pushed, so this is the + # instant the speaker fell silent. A later part of the same + # fluent utterance overwrites it, leaving the final speech end. + result.speech_ended_at = time.monotonic() + last_say_ended = result.speech_ended_at + elif isinstance(step, Silence): + emit("silence", f"{step.secs:.1f}s") + await asyncio.sleep(step.secs) + elif isinstance(step, AwaitAssistantAudio): + emit("await_audio", f"{step.offset_ms:.0f}ms into reply") + await wait_for(assistant_speaking.is_set, step.timeout, "assistant audio") + start = playback.played_ms + await wait_for( + lambda start=start, offset=step.offset_ms: playback.played_ms - start >= offset, + step.timeout, + f"{step.offset_ms:.0f}ms of playback", + ) + elif isinstance(step, AwaitCommit): + emit("await_commit") + await wait_for( + lambda after=last_say_ended: _settled(result.committed_at, after), + step.timeout, + "commit", + ) + elif isinstance(step, AwaitAssistantDone): + emit("await_reply") + await wait_for( + lambda after=last_say_ended: _settled(result.replied_at, after), + step.timeout, + "reply", + ) + await asyncio.sleep(0.5) # let trailing events land before teardown + closing.set() + feeder.stop() + await session.close() + + async with aclosing(session.run(feeder.stream())) as stream: + driver: asyncio.Task[None] | None = None + acker: asyncio.Task[None] | None = None + try: + async for event in stream: + _observe(event, result, playback, feeder, assistant_speaking, closing, emit) + if isinstance(event, SessionStarted): + started = time.monotonic() + acker = asyncio.create_task(ack_loop()) + driver = asyncio.create_task(drive()) + finally: + for task in (driver, acker): + if task is not None and not task.done(): + task.cancel() + for task in (driver, acker): + if task is not None: + await asyncio.gather(task, return_exceptions=True) + + result.wall_secs = time.monotonic() - started + + if config.dump: + for label, pcm in (("in", session.input_audio), ("out", session.output_audio)): + if pcm: + path = DUMP_DIR / f"{scenario.id}-{config.stt}-{config.detector}-{label}.wav" + write_wav(path, pcm) + emit("dump", str(path)) + + result.failures = [ + message + for expectation in scenario.expectations_for(config.detector, config.stt) + if (message := expectation.check(result, scenario)) is not None + ] + return result + + +def _observe( + event: VoiceSessionEvent, + result: RunResult, + playback: PlaybackSim, + feeder: ScriptFeeder, + assistant_speaking: asyncio.Event, + closing: asyncio.Event, + emit: Logger, +) -> None: + if isinstance(event, AudioOutput): + # Individual chunks are far too chatty to log; the playhead is what matters. + playback.on_emit(len(event.data)) + feeder.push_leak(event.data) + assistant_speaking.set() + result.audio_chunks += 1 + result.audio_bytes += len(event.data) + elif isinstance(event, SessionStarted): + emit("session_started") + elif isinstance(event, TranscriptPartial): + result.last_event_at = result.last_partial_at = time.monotonic() + emit("partial", f'"{event.text}"') + elif isinstance(event, TranscriptCommitted): + result.last_event_at = time.monotonic() + detail = f'"{event.text}"{" (replace)" if event.replace else ""}' + if result.speech_ended_at is not None: + dead_air = (time.monotonic() - result.speech_ended_at) * 1000 + result.dead_air_ms.append(dead_air) + result.speech_ended_at = None + detail += f" dead air {dead_air:.0f}ms" + emit("committed", detail) + if event.replace and result.committed: + result.committed[-1] = event.text + result.committed_at[-1] = time.monotonic() + else: + result.committed.append(event.text) + result.committed_at.append(time.monotonic()) + elif isinstance(event, AgentTextDone): + result.last_event_at = time.monotonic() + emit("agent_done", f'"{event.text}"') + result.replies_spoken.append(event.text) + result.replied_at.append(time.monotonic()) + assistant_speaking.clear() + elif isinstance(event, SessionInterrupted): + playback.on_interrupted() + if closing.is_set(): + emit("teardown", "interrupt from close(), not counted") + else: + result.interrupted = True + result.heard_text = event.heard_text + emit("interrupted", f"heard: {event.heard_text!r}") + elif isinstance(event, TurnMetricsEvent): + metrics = event.metrics + result.metrics.append(metrics) + if metrics.eou_to_first_audio_ms is not None: + result.latencies_ms.append(metrics.eou_to_first_audio_ms) + emit( + "metrics", + f"eou→audio {metrics.eou_to_first_audio_ms}ms segments {metrics.tts_segments} " + f"acks {metrics.playback_acks_received} vad_eou {metrics.vad_endpointed}", + ) + elif isinstance(event, SessionError): + result.errors.append(event.message) + emit("error", event.message) diff --git a/benchmarks/voice/scenario.py b/benchmarks/voice/scenario.py new file mode 100644 index 00000000..8ffeb997 --- /dev/null +++ b/benchmarks/voice/scenario.py @@ -0,0 +1,1533 @@ +"""Scenario DSL for the voice replay harness. + +A scenario is a *script* (what the user does, including silences and reactive +barge-ins) plus *expectations* (behavioral labels checked after the run). + +Two rules learned the hard way, both encoded here: + +* **Silence is the real input.** Turn detection keys on gap duration, so every + pause is an explicit ``Silence`` with a duration we control rather than + something we hope the TTS produces. +* **Never assert verbatim transcripts.** STT wobbles between runs. Text + expectations compare normalized similarity against a threshold. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from difflib import SequenceMatcher +from typing import TYPE_CHECKING, Any, Protocol + +if TYPE_CHECKING: + from harness import RunResult + + +# --------------------------------------------------------------------------- +# Script primitives +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class Say: + """Inject one synthesized clip. + + ``utterance`` marks this as slice ``part`` of a longer sentence rendered in + one piece — see :func:`fluent`. + """ + + text: str + utterance: str | None = None + part: int = 0 + + @property + def clip_key(self) -> str: + return self.text if self.utterance is None else f"{self.utterance}#{self.part}" + + +@dataclass(frozen=True) +class Silence: + """Inject exactly N seconds of silence.""" + + secs: float + + +@dataclass(frozen=True) +class AwaitAssistantAudio: + """Block until N ms of the assistant's reply has *played*, then continue. + + The seam that makes barge-in testable: the interruption lands at a known + offset into the reply instead of wherever a fixed sleep happens to fall. + """ + + offset_ms: float + timeout: float = 20.0 + + +@dataclass(frozen=True) +class AwaitCommit: + """Block until the session commits another user transcript.""" + + timeout: float = 15.0 + + +@dataclass(frozen=True) +class AwaitAssistantDone: + """Block until the agent finishes generating a reply.""" + + timeout: float = 20.0 + + +Step = Say | Silence | AwaitAssistantAudio | AwaitCommit | AwaitAssistantDone + + +def fluent(*parts: str | float) -> list[Step]: + """Script one sentence the speaker pauses inside, keeping mid-sentence prosody. + + Takes alternating text and gap seconds — ``fluent("The pain started", 1.5, + "maybe on Tuesday.")`` — and renders the joined sentence in a single TTS call, + slicing it at the character timestamps. + + Synthesizing the fragments separately is what a naive harness does, and it is + wrong in a way that silently changes the answer: ElevenLabs renders each + fragment as its own sentence with a falling contour. Measured with Smart Turn + v3 on identical words, "I want to return an item I bought" scores 0.986 + (finished) standalone and 0.019 (mid-thought) cut from the fluent render. Use + this for any pause *inside* a sentence; use bare ``Say`` for genuinely separate + utterances. + """ + texts = [p for p in parts if isinstance(p, str)] + utterance = " ".join(texts) + steps: list[Step] = [] + index = 0 + for part in parts: + if isinstance(part, str): + steps.append(Say(part, utterance=utterance, part=index)) + index += 1 + else: + steps.append(Silence(part)) + return steps + + +# --------------------------------------------------------------------------- +# Text comparison +# --------------------------------------------------------------------------- + + +_DIGIT_WORDS = { + "0": "zero", "1": "one", "2": "two", "3": "three", "4": "four", + "5": "five", "6": "six", "7": "seven", "8": "eight", "9": "nine", +} # fmt: skip + + +def _spell_digits(match: re.Match[str]) -> str: + return " ".join(_DIGIT_WORDS[d] for d in match.group()) + + +def normalize(text: str) -> str: + """Lowercase, strip punctuation, and render digit runs as spoken words. + + The digit pass exists because providers disagree about a spoken digit string + with no bearing on turn-taking: Flux writes "four four seven two nine one" + and Nova writes "447291" for the same audio. Without it, `banking_digits` + fails in seven of twelve cells having committed exactly one turn in all + twelve — a pure formatting difference reported as a turn-taking defect. + + Digits are spelled out one by one rather than parsed as numbers, matching how + an account number is read aloud. "30 days" would normalize to "three zero + days", which is wrong, but nothing in the suite says a number that way. + """ + spelled = re.sub(r"\d+", _spell_digits, text.lower()) + return re.sub(r"\s+", " ", re.sub(r"[^\w\s]", "", spelled)).strip() + + +def similarity(a: str, b: str) -> float: + return SequenceMatcher(None, normalize(a), normalize(b)).ratio() + + +def best_window(haystack: str, needle: str) -> float: + """Similarity between ``needle`` and the closest same-length word window of ``haystack``. + + Whole-string similarity cannot see a dropped fragment inside a long utterance; + this can, because a missing part matches no window. + """ + hay, ned = normalize(haystack).split(), normalize(needle).split() + if not ned: + return 1.0 + if not hay: + return 0.0 + joined = " ".join(ned) + windows = (" ".join(hay[i : i + len(ned)]) for i in range(max(1, len(hay) - len(ned) + 1))) + return max(SequenceMatcher(None, joined, w).ratio() for w in windows) + + +# Below this, window matching is not evidence: a one- or two-word needle finds a +# passable window in almost any script, and a spurious lone "I" turn (Flux emits +# them) is a real defect that has to stay countable. +MIN_GHOST_WINDOW_WORDS = 3 + + +def attributable(text: str, spoken: list[str], min_similarity: float) -> bool: + """Whether a committed turn's content is something the script actually said. + + Matched against a *window* of the whole script rather than any single spoken + entry, because turn boundaries are the thing under test and rarely line up + with script boundaries: a correct three-fragment merge is ~2.5x longer than + any fragment of `coding_double_pause` and resembles none of them, while a + run-on split into three commits fails the same way inverted. + + The one attribution rule, shared by :class:`NoGhostTurns` and + ``count_ghost_turns`` in score.py. They used to disagree — the expectation + compared per entry while the gate used a window — so the same run could fail + `NoGhostTurns` on a correct merge yet gate zero ghost turns, or the reverse + on a split. + """ + if len(normalize(text).split()) >= MIN_GHOST_WINDOW_WORDS: + return best_window(" ".join(spoken), text) >= min_similarity + return any(similarity(text, said) >= min_similarity for said in spoken) + + +# --------------------------------------------------------------------------- +# Expectations +# --------------------------------------------------------------------------- + + +class Expectation(Protocol): + def check(self, result: RunResult, scenario: Scenario) -> str | None: + """Return ``None`` when satisfied, else a human-readable failure.""" + + +@dataclass(frozen=True) +class UserTurns: + """Exact number of committed user turns. + + The single most valuable assertion in the suite: a fragment wrongly split + into two turns, or two utterances wrongly merged into one, both show up here. + """ + + count: int + + def check(self, result: RunResult, scenario: Scenario) -> str | None: + if len(result.committed) != self.count: + return f"expected {self.count} user turn(s), got {len(result.committed)}: {result.committed}" + return None + + +@dataclass(frozen=True) +class Merged: + """All spoken fragments arrived as one turn resembling ``text``.""" + + text: str + min_similarity: float = 0.85 + + def check(self, result: RunResult, scenario: Scenario) -> str | None: + if len(result.committed) != 1: + return f"expected a single merged turn, got {len(result.committed)}: {result.committed}" + # Whole-string similarity alone lets a dropped fragment through: ElevenLabs + # committed coding_double_pause without "inside a test" and still scored 0.84 + # against the full sentence, so the scenario reported a clean merge on a third + # of the utterance being lost. A merge that loses a part is not a merge. + if (fault := AllPartsHeard().check(result, scenario)) is not None: + return fault + score = similarity(self.text, result.committed[0]) + if score < self.min_similarity: + return f"merged text similarity {score:.2f} < {self.min_similarity}: {result.committed[0]!r}" + return None + + +@dataclass(frozen=True) +class ContentPreserved: + """Everything spoken reached the agent, across however many turns it took. + + The turn-count-agnostic counterpart to :class:`Merged`. Use it when *whether* + an utterance splits is a defensible product choice but losing half of it is + not — at a long pause, holding and splitting are both reasonable, and a + scenario that demands one of them is asserting a policy it cannot justify. + This still catches the failure that actually hurts: ElevenLabs committing + "I'd like to order a large" and silently dropping the pizza. + """ + + min_similarity: float = 0.85 + + def check(self, result: RunResult, scenario: Scenario) -> str | None: + if (fault := AllPartsHeard().check(result, scenario)) is not None: + return fault + spoken = " ".join(scenario.texts()) + heard = " ".join(result.committed) + score = similarity(spoken, heard) + if score < self.min_similarity: + return f"content similarity {score:.2f} < {self.min_similarity}: {heard!r} vs {spoken!r}" + return None + + +@dataclass(frozen=True) +class NoGhostTurns: + """Every committed turn resembles something the script actually said. + + Catches transcripts invented from echo, noise or a mis-fired watchdog. + Attribution is :func:`attributable` — window-matched against the whole + script, identically to the gated ghost-turn count in score.py. + """ + + min_similarity: float = 0.6 + + def check(self, result: RunResult, scenario: Scenario) -> str | None: + spoken = scenario.texts() + ghosts = [text for text in result.committed if not attributable(text, spoken, self.min_similarity)] + if ghosts: + return f"turns not present in the script: {ghosts}" + return None + + +@dataclass(frozen=True) +class AllPartsHeard: + """Every scripted fragment reached the agent, wherever the turns fell. + + Similarity over a whole utterance is blind to a dropped tail, and the longer + the utterance the blinder it gets. ElevenLabs committed `coding_double_pause` + as "The build fails when I import the module" — "inside a test" simply gone — + and that still scores 0.84 against the full sentence, clearing the 0.8 bar on + `Merged`. The scenario reported a clean merge while a third of it was lost. + Matching each fragment against its best window catches the drop no matter how + much correct text surrounds it. + """ + + min_similarity: float = 0.75 + + def check(self, result: RunResult, scenario: Scenario) -> str | None: + heard = " ".join(result.committed) + missing = [t for t in scenario.texts() if best_window(heard, t) < self.min_similarity] + if missing: + return f"fragments never heard: {missing} — got {result.committed}" + return None + + +@dataclass(frozen=True) +class Interrupted: + """Whether a barge-in cancelled the assistant. + + Teardown interrupts from ``session.close()`` are excluded by the harness, so + this only ever reflects a real barge-in. + """ + + expected: bool = True + + def check(self, result: RunResult, scenario: Scenario) -> str | None: + if result.interrupted != self.expected: + return f"expected interrupted={self.expected}, got {result.interrupted}" + return None + + +@dataclass(frozen=True) +class HeardPrefix: + """``heard_text`` is a genuine prefix of the reply, optionally short enough. + + Guards the truncation path: memory and transcript must match what the user + actually heard, not the whole reply the agent generated. + """ + + max_chars: int | None = None + + def check(self, result: RunResult, scenario: Scenario) -> str | None: + heard = result.heard_text + if not heard: + return f"expected heard text, got {heard!r}" + if not any(normalize(reply).startswith(normalize(heard)) for reply in scenario.replies): + return f"heard text is not a prefix of any reply: {heard!r}" + if self.max_chars is not None and len(heard) > self.max_chars: + return f"heard {len(heard)} chars > {self.max_chars}: {heard!r}" + return None + + +@dataclass(frozen=True) +class NoAgentReply: + """The agent never spoke — the utterance should not have started a turn.""" + + def check(self, result: RunResult, scenario: Scenario) -> str | None: + if result.replies_spoken: + return f"expected no reply, agent said: {result.replies_spoken}" + return None + + +@dataclass(frozen=True) +class MaxLatency: + """Every turn's ``eou_to_first_audio_ms`` stayed under a ceiling. + + Measures the *pipeline* after a turn is accepted. It cannot see a hold, + because its clock starts at the accepted commit — use :class:`MaxDeadAir` + for anything that changes turn-taking timing. + """ + + ms: float + + def check(self, result: RunResult, scenario: Scenario) -> str | None: + over = [v for v in result.latencies_ms if v > self.ms] + if over: + return f"eou→audio over {self.ms}ms: {[round(v) for v in over]}" + return None + + +@dataclass(frozen=True) +class MaxDeadAir: + """Time from the user falling silent to the turn being accepted. + + The assertion that makes hold policy falsifiable. Every other metric here is + blind to it: `eou→audio` starts at the accepted commit, so a hold that sits + for three seconds before committing costs nothing measurable, and pass/fail + only ever saw whether the turn eventually merged. + + That blindness had teeth. Disabling the text-complete hold tier reads as 11 + scenario-cells fixed and zero regressions, and on that evidence it looks + like an obvious win — while adding 2.6s before the assistant answers an + ordinary question in six barge-in cells, 2.6s to three closers, and 8.2s to + a long run-on it "fixed". Use this on any scenario where the speaker has + genuinely finished, so buying a merge with dead air has to be declared. + """ + + ms: float + + def check(self, result: RunResult, scenario: Scenario) -> str | None: + over = [v for v in result.dead_air_ms if v > self.ms] + if over: + return f"dead air over {self.ms}ms: {[round(v) for v in over]}" + return None + + +@dataclass(frozen=True) +class NoErrors: + def check(self, result: RunResult, scenario: Scenario) -> str | None: + if result.errors: + return f"session errors: {result.errors}" + return None + + +# --------------------------------------------------------------------------- +# Scenario +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class Scenario: + id: str + domain: str + replies: list[str] + """Fixed agent replies (``TestModel``). Reply *length* decides what a + barge-in can land inside, so these are part of the test, not scaffolding.""" + script: list[Step] + expect: list[Expectation] + expect_by_detector: dict[str, list[Expectation]] = field(default_factory=dict) + """Overrides keyed by ``"detector"``, ``"stt/detector"`` or ``"*"``, most + specific first. Necessary because the same observable behavior can be correct + for different reasons: Deepgram Flux merges a mid-thought pause inside its own + turn machine, while ``lexical`` merges it via Timbal's HOLD.""" + detectors: frozenset[str] | None = None + """Detectors this scenario is meaningful for (``None`` = all).""" + quick: bool = False + """Part of the representative ``--quick`` subset (one per domain, plus a barge-in).""" + known_failure: dict[str, str] = field(default_factory=dict) + """``"detector"`` / ``"stt/detector"`` / ``"*"`` -> why this cannot pass today. + + The scenario still runs and still reports, but it does not gate, and its + *desired* expectations stay in the file rather than being rewritten to match + broken behavior. If it starts passing, that's reported as an unexpected pass. + """ + known_failure_under_leak: dict[str, str] = field(default_factory=dict) + """Same key grammar, but only when the run injects echo (``--aec-leak`` > 0). + + Echo is a separate axis, not a harder version of the clean one: a scenario can + be sound under user-only audio and fail purely because the assistant hears + itself. Folding the two into one table would either excuse a clean-run failure + or make every leak run report nine regressions it already knows about. + + Always treated as intermittent, whatever :attr:`intermittent` says, because + under-suppression depends on how the STT happens to mis-hear a given phrase + rather than on the gain — so a pass at any single gain proves nothing. + """ + intermittent: bool = False + """The known failure is nondeterministic, so passing sometimes means nothing. + + Suppresses the unexpected-pass report, which would otherwise fire on roughly + half of all runs and train everyone to ignore it. + """ + note: str = "" + + def _scoped(self, table: dict[str, Any], detector: str, stt: str | None) -> Any | None: + """Look up ``table`` by cell, then STT, then detector, then ``"*"``. + + Both overrides and known failures need this. A finding is usually specific + to one *cell*, not to a detector everywhere: `food_list_pause` merges under + `lexical` on Flux and splits under `lexical` on Nova, because on Flux the + provider merged it before any detector saw it. Keying by detector alone + would let a Flux observation silently excuse a Nova failure. + + The ``"deepgram-nova/*"`` rung exists because some failures are the STT's + alone and land identically under all four detectors — Nova committing a + "mm-hmm", ElevenLabs merging two finished sentences. Writing those out + per cell repeats one fact four times and hides that it is one fact. + """ + rungs = [f"{stt}/{detector}", f"{stt}/*"] if stt is not None else [] + for key in (*rungs, detector, "*"): + if (hit := table.get(key)) is not None: + return hit + return None + + def expectations_for(self, detector: str, stt: str | None = None) -> list[Expectation]: + return self._scoped(self.expect_by_detector, detector, stt) or self.expect + + def known_failure_reason(self, detector: str, stt: str | None = None, aec_leak: float = 0.0) -> str | None: + """The leak table wins when echo is injected: it is the more specific claim, + and a scenario marked for both is failing for two different reasons.""" + if aec_leak and (hit := self._scoped(self.known_failure_under_leak, detector, stt)) is not None: + return hit + return self._scoped(self.known_failure, detector, stt) + + def is_intermittent(self, detector: str, stt: str | None = None, aec_leak: float = 0.0) -> bool: + if aec_leak and self._scoped(self.known_failure_under_leak, detector, stt) is not None: + return True + return self.intermittent + + def applies_to(self, detector: str) -> bool: + return self.detectors is None or detector in self.detectors + + def texts(self) -> list[str]: + """Everything the user says, fluent parts included — what was *spoken*.""" + return [s.text for s in self.script if isinstance(s, Say)] + + def standalone_texts(self) -> list[str]: + """Clips synthesized on their own; fluent parts are sliced from one render.""" + return [s.text for s in self.script if isinstance(s, Say) and s.utterance is None] + + def fluent_groups(self) -> list[tuple[str, tuple[str, ...]]]: + """``(whole utterance, parts)`` for each sentence spoken across pauses.""" + groups: dict[str, list[str]] = {} + for step in self.script: + if isinstance(step, Say) and step.utterance is not None: + groups.setdefault(step.utterance, []).append(step.text) + return [(utterance, tuple(parts)) for utterance, parts in groups.items()] + + +# --------------------------------------------------------------------------- +# Library +# --------------------------------------------------------------------------- + +# Barge-in needs a reply long enough to interrupt *inside*, so reply length is +# part of the test rather than scaffolding. +_RETURN_POLICY_REPLY = ( + "Our return policy allows returns within thirty days of purchase, provided " + "the item is unused and you still have the original receipt or a valid proof " + "of purchase from one of our authorized retailers." +) +_MENU_REPLY = ( + "Today we have a roasted vegetable soup, a grilled chicken sandwich with " + "house slaw, a mushroom risotto, and for dessert a lemon tart or a chocolate " + "brownie served warm with vanilla ice cream." +) +_FEVER_REPLY = ( + "For a mild fever, rest and steady fluids are usually enough, and you can " + "take paracetamol to bring your temperature down, but if it stays above " + "thirty nine degrees for more than three days you should see a doctor." +) +_IMPORTS_REPLY = ( + "When Python imports a module it first checks the module cache in sys dot " + "modules, then searches each entry on the path in order, compiles the source " + "to bytecode, caches that bytecode on disk, and finally executes the module " + "body exactly once." +) + +# Bare fillers must not start a turn — except under `provider`, where trusting +# the provider's commit makes a commit the correct behavior. +_FILLER_DETECTORS = frozenset({"heuristic", "lexical", "local"}) +_FILLER_NOTE = "Skipped for `provider`: trusting the provider's commit means a bare filler legitimately starts a turn." + +# Measured, not assumed. The fragment Flux commits mid-pause is, in isolation, a +# grammatically complete sentence — "The pain started." before "maybe on Tuesday" — +# and Namo, the text EOU that `lexical` resolves to, scores it 1.00 complete with or +# without the trailing period. Only prosody marks the continuation, and while the +# audio EOU does hear it (Smart Turn scores this fragment 0.054 on fluent audio), a +# text-confidence tier shrinks the resulting hold to 0.35s against a 1.5s pause, so +# `local` splits it too — intermittently; it merges roughly one run in five. Flux's +# own `end_of_turn_confidence` is no help either: sampled across the suite, +# fragments (0.690-0.920) and true ends (0.701-0.904) overlap almost entirely. +# +# Every other scenario in this class stopped failing once fluent synthesis landed +# (see `fluent`), which is the measure of how much the old per-`Say` fixture was +# testing TTS phrasing rather than turn-taking. +_TRAILING_MODIFIER_FAILURE = ( + "the committed fragment is a complete sentence in isolation, so text EOU scores it " + "complete; the audio EOU does hear the continuation but the text-complete tier cuts " + "its hold to 0.35s" +) + +# The announced half of the family above: "...447. Sorry." and "...blue pill. No, wait." +# say outright that a correction is coming, and `local` now holds on that marker alone +# (trailing_correction_marker). What remains here is timing, not judgement — the hold is +# armed, and the correction either arrives inside it or does not. Retired the markers on +# flux/local (banking, 6/6) and nova/local (medical, 6/6) outright. +_CORRECTION_MARKER_HELD = ( + "the correction marker now arms a hold, so this is a race rather than a misjudgement: " + "the retraction has to arrive inside the short tier" +) + +# Off Flux, Namo alone is not enough to hold a mid-sentence pause. Nova commits a +# fragment that reads as a finished sentence and `lexical` has no second opinion to +# contradict it — the audio EOU is exactly what `local` adds, and the gap between +# them on this family is 20% vs 60% (Nova) and 70% vs 80% (ElevenLabs). Marked per +# cell rather than per detector because the same detector merges these on Flux, +# where the provider held the fragment before any detector saw it. +_TEXT_ONLY_PAUSE_SHORTFALL = ( + "text EOU alone cannot hold this pause off Flux: the committed fragment reads finished and nothing contradicts it" +) + +# Even with Smart Turn, some pauses are past what either signal resolves. Kept +# separate from the text-only reason so a fix to one is not credited to the other. +_AUDIO_PAUSE_SHORTFALL = "audio EOU hears the continuation but the hold does not survive this gap" + +# `heuristic` and `provider` have no HOLD at all, so a pause the STT commits at is a +# turn boundary and nothing downstream can merge it. Not a defect but the control +# group: it measures what the hold path is worth, which is 65-69% against 98-100% on +# the same STT. +# +# Marked per cell rather than per detector, because whether it bites is an STT × +# detector interaction and not a property of either. `deepgram-flux/heuristic` scores +# 90% on the identical detector, since Flux holds the fragment provider-side before +# any detector sees it — so a blanket "holdless detector cannot merge" rule would +# falsely mark nine scenarios it passes and hide regressions there. Nova and +# ElevenLabs commit each fragment, and their two holdless cells fail the same family. +_NO_HOLD_TO_MERGE = ( + "`heuristic` and `provider` have no hold, so each fragment the STT commits becomes its own turn" +) + +# ElevenLabs' endpointer waits longer than Flux's, which helps mid-sentence pauses +# and hurts here: two finished sentences 0.9s apart come back as one turn, in all +# four cells, with no detector given a chance to disagree. +_EL_OVERMERGE = "ElevenLabs merges the two sentences into one turn before any detector sees them" + +# Under `provider` the session defers to the STT's endpointing entirely, so none of +# Timbal's hold logic runs and whatever Flux decides stands. +# Under `provider` there is no Timbal hold to override the STT, so any pause Flux +# chooses to end its turn at becomes a split. Flux usually holds through these two +# and occasionally does not; a single run reads as a clean pass, and only --repeat +# shows the 2-in-3. Both fragments here end at the "--" Flux writes for a hesitation. +_FLUX_PROVIDER_SPLIT = ( + "Flux intermittently ends its turn at the pause rather than holding through it, and under " + "`provider` nothing in Timbal runs to override that — merged 2/3 at --repeat 3" +) + +# A spurious single-token "I" turn committed just after a correctly merged utterance. +# Long attributed to `banking_digits_pause` and assumed to be something about digit +# strings; `medical_filler_midway` reproduced it verbatim on ordinary words, so it is +# the Flux + `provider` path, not the content. Roughly one run in four to six, which is +# why it took a wider suite to catch twice. +_FLUX_PROVIDER_GHOST_I = ( + "Flux + `provider` intermittently commits a spurious single-token 'I' turn after the real " + "one; not specific to this scenario" +) + +# What survives echo suppression once the suppressor actually works. +# +# Thirteen scenarios carried this marker when the leak axis first ran, all failing the +# same way: the assistant heard its own bleed, interrupted itself, and committed a +# fragment of its own reply. Eleven were retired by fixing `_likely_stt_echo` to score +# against a same-length window, and the twelfth by refusing to let the stale-partial +# watchdog resurrect a commit that had already been suppressed as echo. +# +# This one is different in kind and will not yield to a better threshold. The STT hands +# over a *single* commit containing both the echo tail and the user's real word — +# "retailers. Stop." out of "...authorized retailers." plus "Stop." — so suppression and +# admission are the same decision applied to two utterances. Dropping it loses the +# interruption the scenario exists to test; keeping it scores a ghost turn. Splitting +# the commit is the only real fix, and that needs the echo span located within it +# rather than a verdict on the whole string. +_ECHO_MIXED_COMMIT = ( + "the STT merges the echo tail and the user's word into one commit " + "('retailers. Stop.'), so it cannot be suppressed without losing the barge-in" +) +_ECHO_LEAK_CELL = "deepgram-flux/local" + +# Digit readback is the worst case for echo, from both directions at once. +# +# Garbled echo is only suppressed once a session has proved it leaks, by producing one +# verbatim echo first (`TurnDetector.echo_verdict`). Digits never get there: the leak +# is quiet, so the STT garbles every copy of it, and a run of digits garbles into other +# digits that still resemble the reply. Nothing verbatim ever arrives to arm the filter. +# +# And the obvious repair — trust resemblance here, since a readback obviously echoes — +# is exactly backwards. The reply *is* the user's digits, so a genuine correction +# ("Four four eight." against "…account four four seven.") scores 0.75, inside the range +# real echo occupies. Digit readback is where resemblance is least informative and most +# tempting, which is why these two are marked rather than tuned around. +# +# Both are gated on measurement, not marked: everything else that fails under leak is +# intermittent and the baseline now records it at its true rate. Only these two fail +# every repeat. +_ECHO_GARBLED_DIGITS = ( + "garbled echo of a digit readback: the session never produces verbatim echo to arm " + "the resemblance check, and resemblance cannot be trusted when the reply is the digits" +) + +_PROVIDER_ENDPOINTING_FAILURE = ( + "`provider` defers to Flux's endpointing, which splits here; `lexical` merges it " + "correctly because Namo scores the fragment 0.00 incomplete" +) + +# Standard tail: wait for the turn to finish rather than sleeping a fixed guess, +# then leave a moment for any stray extra commit to show up. +_SETTLE = [AwaitCommit(), AwaitAssistantDone(), Silence(0.8)] + +SCENARIOS: list[Scenario] = [ + # -- support: long replies (barge-in surface), policy jargon --------------- + Scenario( + id="support_simple", + domain="support", + replies=["Returns are accepted within thirty days of purchase."], + script=[Say("What's your return policy?"), *_SETTLE], + expect=[UserTurns(1), NoGhostTurns(), Interrupted(False), NoErrors()], + quick=True, + note=( + "One question, one reply, no pauses — and at a 0.15 echo leak the assistant " + "interrupts itself on 3 of 4 repeats. The simplest scenario in the suite is enough " + "to show that a mediocre echo canceller does not degrade turn-taking at the " + "margins, it breaks the ordinary case." + ), + ), + Scenario( + id="support_pause", + domain="support", + replies=["I can help with that. Do you have the order number?"], + script=[ + *fluent("I want to return an item I bought", 1.5, "about three weeks ago."), + *_SETTLE, + ], + expect=[ + Merged("I want to return an item I bought about three weeks ago."), + NoGhostTurns(), + Interrupted(False), + NoErrors(), + ], + known_failure={ + "elevenlabs/heuristic": _NO_HOLD_TO_MERGE, + "elevenlabs/provider": _NO_HOLD_TO_MERGE, + "deepgram-nova/heuristic": _NO_HOLD_TO_MERGE, + "deepgram-nova/provider": _NO_HOLD_TO_MERGE, + }, + note=( + "Used to split under `provider` and `local` and was marked a known failure for " + "both. Fluent synthesis fixed it outright: rendered standalone this fragment " + "scores 0.986 finished, and cut from the whole sentence it scores 0.019.\n\n" + "The last marker, `deepgram-nova/lexical`, went the same way once holds could be " + "extended on mic energy instead of only on new partials: a text-only detector has " + "no prosody to hold on, so it was expiring mid-pause. Passes 3/3 there now, and " + "the marker was never flagged intermittent, so it had been failing reliably." + ), + ), + Scenario( + id="support_closer", + domain="support", + replies=["Happy to help. Have a good day."], + script=[Say("That's all."), *_SETTLE], + expect=[UserTurns(1), NoGhostTurns(), Interrupted(False), MaxDeadAir(3500), NoErrors()], + note=( + "The case the text-complete hold tier exists for, and the only one in the suite: " + "Smart Turn under-scores this closer (0.115 — 13 of 14 closers measured score " + "above 0.5 and never reach the tier) while the text reads finished, so the hold " + "is armed on an utterance that is actually over. Disabling the tier costs 2.7s of " + "dead air here (speech end to reply: 0.91s with it, 3.59s without), which is why " + "the tier survives even though it is what splits medical_hesitant_pause. " + "Carries MaxDeadAir rather than MaxLatency, because eou→audio is measured from " + "the *accepted* commit and reads ~200ms whether the hold ran or not. Dead air " + "sits at 1.9s median / 2.8s worst with the tier; removing it puts this at ~4.5s, " + "which is what the 3500ms ceiling is placed to catch. It is a tripwire on a " + "6-sample distribution, not a tight bound — the per-cell dead-air gate in " + "score.py is the real instrument." + ), + ), + Scenario( + id="support_barge_in", + domain="support", + replies=[_RETURN_POLICY_REPLY], + script=[ + Say("Tell me about your return policy."), + AwaitAssistantAudio(offset_ms=600), + Say("Actually, cancel that."), + *_SETTLE, + ], + expect=[UserTurns(2), Interrupted(True), HeardPrefix(max_chars=100), NoGhostTurns(), NoErrors()], + quick=True, + note=( + "The ceiling was 80 and sat inside the harness's own resolution: playback acks " + "arrive every 250ms, several characters of speech per tick, so runs landed at 82 " + "and 85 with the interrupt working perfectly and a genuine prefix heard. It read " + "as a barge-in regression on a run that had none. 100 leaves room for ack " + "granularity while still separating this from support_barge_in_late (~77+ chars " + "at a 2500ms offset is the contrast being drawn, and that one carries no ceiling)." + ), + ), + Scenario( + id="support_barge_in_late", + domain="support", + replies=[_RETURN_POLICY_REPLY], + script=[ + Say("Tell me about your return policy."), + AwaitAssistantAudio(offset_ms=2500), + Say("Okay, thanks."), + *_SETTLE, + ], + expect=[ + UserTurns(2), + Interrupted(True), + HeardPrefix(), # later barge-in -> longer prefix, so no ceiling + NoGhostTurns(), + NoErrors(), + ], + note="Pairs with support_barge_in: same reply, later offset, measurably longer heard text.", + ), + Scenario( + id="support_barge_in_instant", + domain="support", + replies=[_RETURN_POLICY_REPLY], + script=[ + Say("Tell me about your return policy."), + AwaitAssistantAudio(offset_ms=150), + Say("Actually, cancel that."), + *_SETTLE, + ], + expect=[UserTurns(2), Interrupted(True), NoGhostTurns(), NoErrors()], + note=( + "Barge-in before the assistant is really speaking, racing playback start against " + "the interrupt. Deliberately asserts no HeardPrefix: at 150ms there may honestly " + "be nothing heard yet, and failing on that would be testing the expectation." + ), + ), + Scenario( + id="support_barge_in_one_word", + domain="support", + replies=[_RETURN_POLICY_REPLY], + script=[ + Say("Tell me about your return policy."), + AwaitAssistantAudio(offset_ms=900), + Say("Stop."), + *_SETTLE, + ], + expect=[UserTurns(2), Interrupted(True), NoGhostTurns(), NoErrors()], + known_failure_under_leak={_ECHO_LEAK_CELL: _ECHO_MIXED_COMMIT}, + note=( + "The most important barge-in in voice UX and the one the code is built to ignore: " + "both HeuristicTurnDetector and ProviderTurnDetector drop partials shorter than " + "MIN_BARGE_IN_PARTIAL_WORDS (2) as mic blips, and 'Stop.' is one word. Asserts the " + "desired behavior rather than the implemented one — if this fails everywhere, the " + "gate is the finding." + ), + ), + Scenario( + id="support_pause_short", + domain="support", + replies=["Of course. What's the order number?"], + script=[ + *fluent("I need help with", 0.6, "my recent order."), + *_SETTLE, + ], + expect=[ + Merged("I need help with my recent order."), + NoGhostTurns(), + Interrupted(False), + NoErrors(), + ], + known_failure={ + "deepgram-flux/provider": _FLUX_PROVIDER_SPLIT, + "deepgram-nova/heuristic": _NO_HOLD_TO_MERGE, + "deepgram-nova/provider": _NO_HOLD_TO_MERGE, + }, + intermittent=True, + note=( + "The floor of the pause family: 0.6s is shorter than any endpointer's silence " + "window, so every configuration should merge it. Exists to prove the harder pause " + "scenarios are measuring difficulty rather than a broken fixture — and it earns " + "that, merging in eleven of twelve cells. The exception is instructive: `provider` " + "on Flux splits even 0.6s about one run in three, so the floor is not the " + "detector's, it is whatever the STT decided." + ), + ), + Scenario( + id="support_trailing_conjunction", + domain="support", + replies=["I'm sorry to hear that. Let me check the tracking."], + script=[ + *fluent("I ordered a lamp last week and", 1.5, "it still hasn't arrived."), + *_SETTLE, + ], + expect=[ + Merged("I ordered a lamp last week and it still hasn't arrived."), + NoGhostTurns(), + NoErrors(), + ], + known_failure={ + "deepgram-nova/lexical": _TEXT_ONLY_PAUSE_SHORTFALL, + "elevenlabs/lexical": _TEXT_ONLY_PAUSE_SHORTFALL + " — merged 3/4", + "elevenlabs/heuristic": _NO_HOLD_TO_MERGE, + "elevenlabs/provider": _NO_HOLD_TO_MERGE, + "deepgram-nova/heuristic": _NO_HOLD_TO_MERGE, + "deepgram-nova/provider": _NO_HOLD_TO_MERGE, + }, + intermittent=True, + note=( + "A dangling coordinating conjunction is the strongest continuation cue in text, so " + "this is the pause case text EOU should get right without any help from prosody — " + "the complement to medical_hesitant_pause, where the fragment reads complete and " + "only prosody says otherwise." + ), + ), + Scenario( + id="support_echo_silence", + domain="support", + replies=[_RETURN_POLICY_REPLY], + script=[ + Say("Tell me about your return policy."), + # The user says nothing for the whole reply. Anything committed here + # came out of the assistant's own mouth. + AwaitAssistantDone(), + Silence(4.0), + ], + expect=[UserTurns(1), Interrupted(False), NoGhostTurns(), NoErrors()], + note=( + "Run with --aec-leak to bleed the assistant's output back into the mic. " + "`_likely_stt_echo` exists so speaker bleed does not make the assistant " + "interrupt itself, and until this scenario nothing had ever fed it echo: every " + "run was clean user-only audio. coding_barge_in_echo proves the suppressor does " + "not fire on genuine speech; this is the other half.\n\n" + "Measured on deepgram-flux/local: clean at 0.0, intermittent from 0.20, and 4-5 " + "failures in 6 at 0.30 — where the assistant cuts itself off mid-reply and commits " + "its own words as a user turn. Real barge-in still works at 0.30, so the suppressor " + "is letting echo through rather than over-suppressing.\n\n" + "0.15 was recorded as clean here and is not: it fails ~1 in 3. The original reading " + "came from too few repeats on the one scenario the leak axis was ever run against. " + "The axis now carries its own baseline, so the whole suite's leak behaviour is " + "recorded at measured rates instead of being marked scenario by scenario.\n\n" + "The mechanism is visible in what gets committed: 'Our retailers.', 'Arise " + "retailers.', 'has aroused retailers.' — all manglings of the reply's own tail, " + "'...authorized retailers.' The suppressor is a text-similarity check against " + "what the assistant said, so echo that the STT transcribes *badly* stops " + "resembling its source and passes the very filter built to catch it. Louder, " + "cleaner echo would be caught; garbled echo is not." + ), + ), + Scenario( + id="support_silence_only", + domain="support", + replies=["Hello?"], + script=[Silence(4.0)], + expect=[UserTurns(0), NoAgentReply(), NoErrors()], + note=( + "Four seconds of nothing. No scenario previously asserted that an open mic with no " + "speech produces no turn, which is the purest form of the ghost-turn bug and the " + "one a VAD or watchdog misfire would cause." + ), + ), + # -- food_ordering: enumerations, mid-list pauses -------------------------- + Scenario( + id="food_simple", + domain="food_ordering", + replies=["One large coffee and a croissant. Anything else?"], + script=[Say("Can I get a large coffee and a croissant?"), *_SETTLE], + expect=[UserTurns(1), NoGhostTurns(), Interrupted(False), NoErrors()], + ), + Scenario( + id="food_pause", + domain="food_ordering", + replies=["Sure — one large pepperoni. Anything else?"], + script=[ + *fluent("I'd like to order a large", 1.5, "pepperoni pizza please."), + *_SETTLE, + ], + expect=[ + Merged("I'd like to order a large pepperoni pizza please."), + NoGhostTurns(), + Interrupted(False), + NoErrors(), + ], + known_failure={ + "deepgram-nova/lexical": _TEXT_ONLY_PAUSE_SHORTFALL, + "elevenlabs/lexical": _TEXT_ONLY_PAUSE_SHORTFALL + " — merged 2/3", + "elevenlabs/heuristic": _NO_HOLD_TO_MERGE, + "elevenlabs/provider": _NO_HOLD_TO_MERGE, + "deepgram-nova/heuristic": _NO_HOLD_TO_MERGE, + "deepgram-nova/provider": _NO_HOLD_TO_MERGE, + }, + intermittent=True, + quick=True, + note=( + "Under deepgram-flux + provider, Flux holds through the gap and merges inside " + "its own turn machine — no Timbal HOLD involved. Under lexical it is our HOLD. " + "Same expectation, different mechanism." + ), + ), + Scenario( + id="food_long_pause", + domain="food_ordering", + replies=["Got it — a large pepperoni pizza."], + script=[ + # long enough that a split is the correct answer + *fluent("I'd like to order a large", 3.0, "pepperoni pizza please."), + *_SETTLE, + ], + expect=[ContentPreserved(), NoGhostTurns(), NoErrors()], + note=( + "The one scenario whose desired outcome is genuinely ambiguous, so it asserts " + "content rather than turn count. At a 3.0s gap both answers are defensible — " + "holding costs 3s of dead air, splitting sends a fragment to the LLM — and the " + "suite has no basis for calling either wrong. Observed: Flux merges under " + "`lexical`/`local` and splits under `heuristic`/`provider`, Nova splits under all " + "four, ElevenLabs splits under `lexical`. Flux's merge is not Timbal's: " + "the trace shows Flux streaming partials through the whole gap and emitting one " + "commit, so no fragment ever reaches a detector. It previously demanded a merge " + "wherever one had been seen, which quietly turned observations into requirements " + "and failed Nova for behaving reasonably.\n\n" + "It also carried a known failure blaming ElevenLabs for dropping the second " + "fragment instead of splitting. That was the harness: `AwaitCommit` resolved on " + "the *first* fragment's commit, which on ElevenLabs lands ~1.6s late and so " + "arrived after the wait was armed, leaving only the 0.8s tail for a commit that " + "needed ~1.0s. The turn was lost to teardown, not by the provider — and one " + "observed turn instead of two is also why this note used to say ElevenLabs merged " + "here. It splits, 4/4, content intact." + ), + ), + Scenario( + id="food_list_pause", + domain="food_ordering", + replies=["A burger, fries and a chocolate milkshake. Coming up."], + script=[ + # mid-list breath, not an end of turn + *fluent("I'll take a burger, some fries,", 1.4, "and a chocolate milkshake."), + *_SETTLE, + ], + expect=[ + Merged("I'll take a burger, some fries, and a chocolate milkshake."), + NoGhostTurns(), + NoErrors(), + ], + known_failure={ + "deepgram-nova/local": _AUDIO_PAUSE_SHORTFALL, + "deepgram-nova/lexical": _TEXT_ONLY_PAUSE_SHORTFALL, + "elevenlabs/local": _AUDIO_PAUSE_SHORTFALL + " — merged 2/3", + "elevenlabs/lexical": _TEXT_ONLY_PAUSE_SHORTFALL, + "elevenlabs/heuristic": _NO_HOLD_TO_MERGE, + "elevenlabs/provider": _NO_HOLD_TO_MERGE, + "deepgram-nova/heuristic": _NO_HOLD_TO_MERGE, + "deepgram-nova/provider": _NO_HOLD_TO_MERGE, + }, + intermittent=True, + note=( + "A pause after a comma mid-enumeration: prosody says continue even though the " + "gap is long. Failed on every detector until fluent synthesis; now passes on all " + "three." + ), + ), + Scenario( + id="food_barge_in", + domain="food_ordering", + replies=[_MENU_REPLY], + script=[ + Say("Tell me what's on the menu today."), + AwaitAssistantAudio(offset_ms=800), + Say("Just the specials, please."), + *_SETTLE, + ], + expect=[UserTurns(2), Interrupted(True), HeardPrefix(), NoGhostTurns(), NoErrors()], + ), + Scenario( + id="food_backchannel", + domain="food_ordering", + replies=[_MENU_REPLY], + script=[ + Say("Tell me what's on the menu today."), + AwaitAssistantAudio(offset_ms=1200), + Say("Mm-hmm."), + # No AwaitAssistantDone: the reply to turn 1 has already been generated and a + # correct run produces no second one, so waiting for one always times out. + Silence(3.0), + ], + expect=[UserTurns(1), Interrupted(False), NoGhostTurns(), NoErrors()], + known_failure={ + "deepgram-nova/*": "Nova transcribes the backchannel as 'Mhmm.' and commits it, " + "which cuts the assistant off; no detector sees anything to distinguish it from a " + "one-word barge-in", + "deepgram-flux/*": "same commit, intermittently — Flux usually swallows the sound " + "but lets it through often enough to fail roughly one run in three, under both " + "`local` and `lexical`", + }, + intermittent=True, + note=( + "Listeners say 'mm-hmm' while you talk and do not mean stop. Nothing in the suite " + "distinguished a backchannel from a barge-in before this, and the two are " + "acoustically similar and semantically opposite. The split is by STT, not " + "detector: Flux and ElevenLabs swallow the sound, Nova transcribes it and all four " + "detectors then interrupt on it. Timbal has no notion of a backchannel, so on any " + "STT that transcribes one, an acknowledgement silences the assistant. Nova fails " + "it every run; Flux leaks it roughly one run in three, which is how it passed " + "three straight repeats under `lexical` before failing a re-baseline." + ), + ), + Scenario( + id="food_trailing_preposition", + domain="food_ordering", + replies=["Sure, I can deliver there."], + script=[ + *fluent("Can you deliver it to", 1.5, "the office on Main Street?"), + *_SETTLE, + ], + expect=[ + Merged("Can you deliver it to the office on Main Street?"), + NoGhostTurns(), + NoErrors(), + ], + known_failure={ + "deepgram-nova/lexical": _TEXT_ONLY_PAUSE_SHORTFALL, + "elevenlabs/local": _AUDIO_PAUSE_SHORTFALL + " — merged 2/3", + "elevenlabs/heuristic": _NO_HOLD_TO_MERGE, + "elevenlabs/provider": _NO_HOLD_TO_MERGE, + "deepgram-nova/heuristic": _NO_HOLD_TO_MERGE, + "deepgram-nova/provider": _NO_HOLD_TO_MERGE, + }, + intermittent=True, + note=( + "A stranded preposition: syntactically incomplete, so text EOU should hold without " + "prosody. It does on Flux and under `local` on Nova — the off-Flux `lexical` failure " + "is the one that should be fixable on text alone, and is not." + ), + ), + Scenario( + id="food_rapid_fire", + domain="food_ordering", + replies=["Two coffees, got it."], + script=[ + Say("I'll have a coffee."), + Silence(0.9), + Say("Actually, make it two."), + *_SETTLE, + ], + expect=[UserTurns(2), NoGhostTurns(), NoErrors()], + known_failure={"elevenlabs/*": _EL_OVERMERGE}, + note=( + "The inverse of every pause scenario: two genuinely finished sentences separated by " + "a short gap, which must *not* merge. The suite is full of cases punishing a split " + "and had none punishing a merge, so a detector could have scored well by simply " + "holding everything." + ), + ), + # -- medical_intake: hesitation-heavy ------------------------------------- + Scenario( + id="medical_simple", + domain="medical_intake", + replies=["How long has the headache lasted?"], + script=[Say("I've had a headache since yesterday."), *_SETTLE], + expect=[UserTurns(1), NoGhostTurns(), Interrupted(False), NoErrors()], + ), + Scenario( + id="medical_hesitation", + domain="medical_intake", + replies=["Take your time."], + script=[Say("Um..."), Silence(2.5)], + expect=[NoAgentReply(), NoErrors()], + detectors=_FILLER_DETECTORS, + quick=True, + note=_FILLER_NOTE, + ), + Scenario( + id="medical_hesitant_pause", + domain="medical_intake", + replies=["Thanks. Has it changed since then?"], + script=[ + *fluent("The pain started", 1.5, "maybe on Tuesday."), + *_SETTLE, + ], + expect=[Merged("The pain started maybe on Tuesday."), NoGhostTurns(), NoErrors()], + known_failure={"*": _TRAILING_MODIFIER_FAILURE}, + intermittent=True, + note=( + "Trailing off mid-recall, the archetypal case for holding instead of " + "endpointing, and the last of its class still failing after fluent synthesis. " + "`local` merges it about one run in five, so passing proves nothing." + ), + ), + Scenario( + id="medical_barge_in", + domain="medical_intake", + replies=[_FEVER_REPLY], + script=[ + Say("What should I do about a fever?"), + AwaitAssistantAudio(offset_ms=700), + Say("Sorry, one more thing."), + *_SETTLE, + ], + expect=[UserTurns(2), Interrupted(True), HeardPrefix(), NoGhostTurns(), NoErrors()], + ), + Scenario( + id="medical_barge_in_twice", + domain="medical_intake", + replies=[_FEVER_REPLY], + script=[ + Say("What should I do about a fever?"), + AwaitAssistantAudio(offset_ms=600), + Say("Wait, hold on."), + AwaitAssistantAudio(offset_ms=600), + Say("Okay, never mind."), + *_SETTLE, + ], + expect=[UserTurns(3), Interrupted(True), NoGhostTurns(), NoErrors()], + note=( + "Two interruptions in one session. Every other barge-in scenario stops after the " + "first, so nothing checked that the session recovers enough to be interrupted " + "again — a stuck playback or un-cleared interrupt flag would pass all of them." + ), + ), + Scenario( + id="medical_self_correction", + domain="medical_intake", + replies=["Noted, the white one."], + script=[ + *fluent("I take the blue pill, no wait,", 1.2, "the white one."), + *_SETTLE, + ], + expect=[ + Merged("I take the blue pill, no wait, the white one.", min_similarity=0.8), + NoGhostTurns(min_similarity=0.5), + NoErrors(), + ], + known_failure={ + "deepgram-flux/*": _TRAILING_MODIFIER_FAILURE, + "elevenlabs/local": _CORRECTION_MARKER_HELD + " — merged 5/6", + "elevenlabs/heuristic": _NO_HOLD_TO_MERGE, + "elevenlabs/provider": _NO_HOLD_TO_MERGE, + "deepgram-nova/heuristic": _NO_HOLD_TO_MERGE, + "deepgram-nova/provider": _NO_HOLD_TO_MERGE, + }, + intermittent=True, + note=( + "Self-repair mid-utterance. 'No wait,' is a complete clause boundary that reads " + "finished to text EOU while guaranteeing more speech follows — and committing here " + "sends the LLM the retracted answer, which is worse than a split usually is. Fails " + "0/3 in every Flux cell and passes on nova/lexical and three ElevenLabs cells, so " + "the ceiling is the STT's segmentation rather than the detector's judgement." + ), + ), + Scenario( + id="medical_filler_midway", + domain="medical_intake", + replies=["Thanks. Anything else?"], + script=[ + *fluent("The pain is, um,", 1.3, "mostly at night."), + *_SETTLE, + ], + expect=[ + Merged("The pain is, um, mostly at night.", min_similarity=0.8), + NoGhostTurns(min_similarity=0.5), + NoErrors(), + ], + known_failure={ + "deepgram-flux/provider": _FLUX_PROVIDER_GHOST_I, + "deepgram-nova/lexical": _TEXT_ONLY_PAUSE_SHORTFALL, + "elevenlabs/heuristic": _NO_HOLD_TO_MERGE, + "elevenlabs/provider": _NO_HOLD_TO_MERGE, + "deepgram-nova/heuristic": _NO_HOLD_TO_MERGE, + "deepgram-nova/provider": _NO_HOLD_TO_MERGE, + }, + intermittent=True, + note=( + "A filler immediately before the gap, which is where hedges actually occur in " + "speech. medical_hesitation covers a bare 'Um...' at idle; this covers one " + "mid-sentence, where the hedge logic has to hold rather than suppress. It merges " + "the pause correctly everywhere and is marked only for the ghost 'I' — which it " + "was the first scenario other than banking_digits_pause to produce, retiring the " + "theory that the ghost had anything to do with digits." + ), + ), + Scenario( + id="medical_long_utterance", + domain="medical_intake", + replies=["That's helpful, thank you."], + script=[ + Say( + "I've been having headaches every morning for about two weeks and they " + "usually fade after breakfast but yesterday it lasted the whole day and " + "I felt dizzy whenever I stood up too quickly." + ), + *_SETTLE, + ], + expect=[UserTurns(1), ContentPreserved(min_similarity=0.8), NoGhostTurns(), NoErrors()], + known_failure={ + "deepgram-nova/lexical": "Nova endpoints at the clause boundaries inside this run-on " + "and text EOU scores each piece finished, so it commits three turns", + "deepgram-nova/heuristic": _NO_HOLD_TO_MERGE, + "deepgram-nova/provider": _NO_HOLD_TO_MERGE, + }, + note=( + "Twelve seconds of unbroken speech with several clause boundaries an endpointer " + "could mistake for an ending. The longest utterance in the suite by a wide margin; " + "everything else is short enough that a premature commit has nowhere to happen." + ), + ), + # -- banking: digit strings and short confirmations ------------------------ + Scenario( + id="banking_digits", + domain="banking", + replies=["Thank you. I've found the account."], + # Spelled out, because that is what comes back: ElevenLabs speaks "447291" + # digit by digit and Deepgram transcribes the words, so a numeral here would + # look like a hallucinated turn to every text comparison. + script=[Say("My account number is four four seven two nine one."), *_SETTLE], + expect=[UserTurns(1), NoGhostTurns(), Interrupted(False), NoErrors()], + known_failure_under_leak={_ECHO_LEAK_CELL: _ECHO_GARBLED_DIGITS}, + note="Digit strings are the STT-hostile case: no lexical context to fall back on.", + ), + Scenario( + id="banking_digits_pause", + domain="banking", + replies=["Got it, account 447291."], + script=[ + *fluent("My account number is four four seven", 1.4, "two nine one."), + *_SETTLE, + ], + expect=[ + Merged("My account number is four four seven two nine one.", min_similarity=0.75), + NoGhostTurns(min_similarity=0.4), + NoErrors(), + ], + known_failure={ + "deepgram-flux/provider": _FLUX_PROVIDER_GHOST_I, + "deepgram-nova/local": _AUDIO_PAUSE_SHORTFALL + " — merged 2/3", + "deepgram-nova/lexical": _TEXT_ONLY_PAUSE_SHORTFALL, + "elevenlabs/local": _AUDIO_PAUSE_SHORTFALL + " — merged 2/3", + "elevenlabs/heuristic": _NO_HOLD_TO_MERGE, + "elevenlabs/provider": _NO_HOLD_TO_MERGE, + }, + known_failure_under_leak={_ECHO_LEAK_CELL: _ECHO_GARBLED_DIGITS}, + intermittent=True, + note=( + "The hardest merge in the suite: a digit string broken across a pause, with no " + "words to hint continuation. `lexical` and `local` merge it once the fragment " + "carries mid-sentence prosody. `provider` mostly merges it too, but drops to a " + "split plus a ghost 'I' about one run in six, visible only under --repeat. That " + "ghost was assumed to be about digit strings until medical_filler_midway produced " + "the identical turn on ordinary words.\n\n" + "On deepgram-nova/lexical this used to merge and now splits, which is a real " + "change and the right one: the merge was an artifact of Nova taking 7.5s to " + "finalize, long enough to swallow the 1.4s gap whole. Once the endpointer armed " + "for text-only detectors and commits landed at ~1.0s, the pause became visible " + "and that cell splits it — consistent with its 20% pause-merge rate everywhere " + "else. A merge bought with 6.5s of dead air was never worth having." + ), + ), + Scenario( + id="banking_confirmation", + domain="banking", + replies=["Great, the transfer is scheduled."], + script=[Say("Yes, that's correct."), *_SETTLE], + expect=[UserTurns(1), NoGhostTurns(), Interrupted(False), NoErrors()], + quick=True, + known_failure={ + "*": "intermittently splits into 'Yes.' + \"That's correct.\", and the second " + "fragment barges in on the reply to the first" + }, + intermittent=True, + note=( + "Observed passing and failing on consecutive runs of an unchanged tree. Short " + 'utterances have a history here: ElevenLabs once transcribed "yes." / "work." ' + "as partials that never committed at all, hanging the session." + ), + ), + Scenario( + id="banking_hesitation", + domain="banking", + replies=["No problem, take your time."], + script=[Say("Uh..."), Silence(2.5)], + expect=[NoAgentReply(), NoErrors()], + detectors=_FILLER_DETECTORS, + note=_FILLER_NOTE, + ), + Scenario( + id="banking_short_reject", + domain="banking", + replies=["Understood, I've cancelled it."], + script=[Say("No."), *_SETTLE], + expect=[UserTurns(1), NoGhostTurns(min_similarity=0.3), Interrupted(False), NoErrors()], + note=( + "The shortest possible real turn. Pairs with banking_confirmation, which is " + "intermittent for exactly this reason: ElevenLabs has been seen transcribing " + "one-word answers as partials that never commit, hanging the session outright. A " + "'No.' that goes unheard is a worse failure than a split." + ), + ), + Scenario( + id="banking_correction", + domain="banking", + replies=["Transferring to account four four eight."], + script=[ + *fluent("Send it to account four four seven, sorry,", 1.3, "four four eight."), + *_SETTLE, + ], + expect=[ + Merged("Send it to account four four seven, sorry, four four eight.", min_similarity=0.75), + NoGhostTurns(min_similarity=0.4), + NoErrors(), + ], + known_failure={ + "deepgram-flux/provider": _TRAILING_MODIFIER_FAILURE, + "deepgram-nova/local": _CORRECTION_MARKER_HELD + " — merged 4/6", + "deepgram-nova/lexical": _TRAILING_MODIFIER_FAILURE, + "elevenlabs/local": _CORRECTION_MARKER_HELD + " — merged 5/6", + "deepgram-flux/heuristic": _NO_HOLD_TO_MERGE, + "elevenlabs/heuristic": _NO_HOLD_TO_MERGE, + "elevenlabs/provider": _NO_HOLD_TO_MERGE, + "deepgram-nova/heuristic": _NO_HOLD_TO_MERGE, + "deepgram-nova/provider": _NO_HOLD_TO_MERGE, + }, + intermittent=True, + note=( + "Self-correction on a digit string: no lexical context, and committing early sends " + "the LLM the wrong account number rather than merely a fragment. The one scenario " + "where a split has a concrete, wrong consequence — the agent acts on 447 when the " + "caller said 448. `lexical` merges it 3/3 on Flux where `local` and `provider` " + "never do, the clearest case in the suite of Namo beating the audio EOU outright." + ), + ), + # -- coding_help: technical tokens --------------------------------------- + Scenario( + id="coding_simple", + domain="coding_help", + replies=["It means you indexed something that doesn't support indexing."], + script=[Say("What does TypeError not subscriptable mean?"), *_SETTLE], + expect=[UserTurns(1), NoGhostTurns(min_similarity=0.5), Interrupted(False), NoErrors()], + quick=True, + note="Technical tokens have no conversational prior, so STT leans harder on acoustics.", + ), + Scenario( + id="coding_pause", + domain="coding_help", + replies=["Can you paste the traceback?"], + script=[ + *fluent("I'm getting an error in my", 1.5, "async generator function."), + *_SETTLE, + ], + expect=[ + Merged("I'm getting an error in my async generator function.", min_similarity=0.8), + NoGhostTurns(min_similarity=0.5), + NoErrors(), + ], + known_failure={ + "deepgram-flux/provider": _FLUX_PROVIDER_SPLIT, + "deepgram-nova/lexical": _TEXT_ONLY_PAUSE_SHORTFALL, + "deepgram-flux/heuristic": _NO_HOLD_TO_MERGE, + "elevenlabs/heuristic": _NO_HOLD_TO_MERGE, + "elevenlabs/provider": _NO_HOLD_TO_MERGE, + "deepgram-nova/heuristic": _NO_HOLD_TO_MERGE, + "deepgram-nova/provider": _NO_HOLD_TO_MERGE, + }, + intermittent=True, + note=( + "Baselined as a clean pass until --repeat 3 caught `provider` splitting it once in " + "three. Nothing changed in the product; the suite had only ever run it once per " + "cell. A worthwhile reminder that a green single run is weak evidence." + ), + ), + Scenario( + id="coding_barge_in", + domain="coding_help", + replies=[_IMPORTS_REPLY], + script=[ + Say("Explain how Python handles imports."), + AwaitAssistantAudio(offset_ms=900), + Say("Actually, never mind."), + *_SETTLE, + ], + expect=[UserTurns(2), Interrupted(True), HeardPrefix(), NoGhostTurns(), NoErrors()], + ), + Scenario( + id="coding_barge_in_echo", + domain="coding_help", + replies=[_IMPORTS_REPLY], + script=[ + Say("Explain how Python handles imports."), + AwaitAssistantAudio(offset_ms=1400), + Say("Sorry, the module cache?"), + *_SETTLE, + ], + expect=[UserTurns(2), Interrupted(True), NoGhostTurns(min_similarity=0.5), NoErrors()], + note=( + "A real interruption made of words the assistant is in the middle of saying, which " + "is what asking someone to repeat themselves sounds like. `_likely_stt_echo` " + "suppresses partials resembling the assistant's own text to survive a leaky AEC, " + "and cannot tell this apart from the echo it exists to drop. Measures that " + "suppressor's false-positive rate on genuine speech." + ), + ), + Scenario( + id="coding_double_pause", + domain="coding_help", + replies=["Sounds like an import cycle. Can you share the traceback?"], + script=[ + *fluent("The build fails when I", 1.2, "import the module", 1.2, "inside a test."), + *_SETTLE, + ], + expect=[ + Merged("The build fails when I import the module inside a test.", min_similarity=0.8), + NoGhostTurns(min_similarity=0.5), + NoErrors(), + ], + known_failure={ + "deepgram-flux/*": "holds through the first gap and splits at the second; the " + "fragment it commits reads as a finished sentence, so nothing argues for holding", + "elevenlabs/heuristic": _NO_HOLD_TO_MERGE, + "elevenlabs/provider": _NO_HOLD_TO_MERGE, + "deepgram-nova/heuristic": _NO_HOLD_TO_MERGE, + "deepgram-nova/provider": _NO_HOLD_TO_MERGE, + }, + intermittent=True, + note=( + "Two gaps in one sentence, and the first is the one that gets held: every Flux cell " + "merges 'The build fails when I' with 'import the module' and then commits, leaving " + "'inside a test' as its own turn. A hold that fires once and latches would look " + "identical, which is why one-gap scenarios could not have found this. ElevenLabs " + "merges all three — but see AllPartsHeard: it used to drop 'inside a test' entirely " + "and still score 0.84 similarity, and this scenario is what exposed that." + ), + ), + Scenario( + id="coding_followup_after_reply", + domain="coding_help", + replies=["A segfault means bad memory access."], + script=[ + Say("What does a segmentation fault mean?"), + AwaitCommit(), + AwaitAssistantDone(), + # AwaitAssistantDone fires when generation ends, not when playback drains, so + # speaking here would barge in on the tail. This waits out a ~2.5s reply. + Silence(5.0), + Say("And how do I debug it?"), + *_SETTLE, + ], + expect=[UserTurns(2), Interrupted(False), NoGhostTurns(), NoErrors()], + note=( + "A second turn taken after the assistant has finished speaking, uninterrupted. " + "This is a regression test for a bug we shipped: STT not resuming once the " + "assistant stopped, so the follow-up was never heard. Every other multi-turn " + "scenario barges in, which exercises the opposite path and would keep passing." + ), + ), +] + + +def select( + ids: list[str] | None, + detector: str, + *, + quick: bool = False, +) -> tuple[list[Scenario], list[Scenario]]: + """Return ``(applicable, skipped)`` for a detector, filtered by ids or ``quick``.""" + chosen = SCENARIOS + if ids: + chosen = [s for s in chosen if s.id in set(ids)] + if quick: + chosen = [s for s in chosen if s.quick] + return ( + [s for s in chosen if s.applies_to(detector)], + [s for s in chosen if not s.applies_to(detector)], + ) diff --git a/benchmarks/voice/score.py b/benchmarks/voice/score.py new file mode 100644 index 00000000..4373c7c6 --- /dev/null +++ b/benchmarks/voice/score.py @@ -0,0 +1,483 @@ +"""Scoring, result persistence and regression gating for the voice replay harness. + +Three ideas do the work here: + +* **Records, not assertions.** Every run serializes to one JSONL line, so a run + is analyzable after the fact rather than only pass/fail at the time. +* **Deltas, not absolutes.** Gating compares against a committed baseline. + Absolute thresholds rot within a week because STT providers drift under us — + yesterday's 400ms ceiling is tomorrow's false alarm in both directions. +* **Flaky is a finding, not noise.** STT is nondeterministic, so a scenario that + passes some repeats and fails others is surfaced by name. Those are usually a + real race, not a bad test. +""" + +from __future__ import annotations + +import json +import time +from dataclasses import asdict, dataclass, field +from pathlib import Path + +from harness import RunResult +from scenario import Scenario, attributable + +HERE = Path(__file__).parent +RESULTS_DIR = HERE / "results" +# Not under results/ (which is gitignored) — the baseline is meant to be committed. +BASELINE_PATH = HERE / "baseline.json" + +# Gate thresholds. Deliberately loose: this catches direction changes, not noise. +LATENCY_REGRESSION_RATIO = 1.15 +# Latency gating needs a distribution, not a handful of samples. At suite sizes +# below this, p95 is effectively the maximum and a single slow turn trips the +# gate while every scenario passes (observed: 394ms -> 477ms on an unchanged +# tree). Below the floor, latency is reported but never gated, and the gate uses +# p50 — robust to one outlier — rather than p95. +MIN_LATENCY_SAMPLES = 20 + + +# --------------------------------------------------------------------------- +# Records +# --------------------------------------------------------------------------- + + +@dataclass +class RunRecord: + """One scenario × config × repeat.""" + + scenario: str + domain: str + stt: str + detector: str + repeat: int + passed: bool + failures: list[str] + user_turns: int + ghost_turns: int + interrupted: bool + heard_chars: int | None + latencies_ms: list[float] + dead_air_ms: list[float] + vad_endpointed: list[bool] + errors: list[str] + wall_secs: float + known_failure: str = "" + """Non-empty when this scenario is expected to fail for this detector.""" + intermittent: bool = False + jobs: int = 1 + """Concurrency this run was measured under. Latency is only comparable at equal + concurrency, so it is recorded per run rather than inferred later.""" + aec_leak: float = 0.0 + """Echo gain fed back into the mic. Recorded because it changes what the run means + — a leak run and a clean one were previously indistinguishable in the JSONL, so a + file could only be read correctly by whoever remembered the command.""" + label: str = "" + """Full cell label, axis suffixes included (``[leak=0.15]``, swept params). + + Scoring and baselining key off this rather than rebuilding ``stt/detector``, which + silently dropped every non-default axis: a leak or sweep run matched no cell at + all, so it printed no scorecard, gated nothing, and its ``--update-baseline`` was + a no-op. Once matched, the same stripping would have let a leak cell overwrite the + clean baseline of the same stt/detector.""" + + @property + def xfail(self) -> bool: + return bool(self.known_failure) + + @property + def xpass(self) -> bool: + return self.xfail and self.passed and not self.intermittent + + def to_json(self) -> str: + return json.dumps(asdict(self)) + + +def count_ghost_turns(result: RunResult, scenario: Scenario, min_similarity: float = 0.6) -> int: + """Committed turns whose content the script never said. + + Tracked as a metric independent of whether a scenario declared the matching + expectation — a hallucinated transcript is worth knowing about everywhere. + + Attribution is `scenario.attributable`, the same rule `NoGhostTurns` checks: + the two used to diverge (per-entry there, windowed here), so a run could fail + the expectation while gating zero ghost turns, or the reverse. The rationale + for window matching lives on the helper. + """ + spoken = scenario.texts() + return sum(not attributable(text, spoken, min_similarity) for text in result.committed) + + +def record( + scenario: Scenario, + result: RunResult, + repeat: int = 0, + jobs: int = 1, + aec_leak: float = 0.0, + label: str = "", +) -> RunRecord: + return RunRecord( + scenario=scenario.id, + domain=scenario.domain, + stt=result.stt, + detector=result.detector, + repeat=repeat, + passed=result.passed, + failures=list(result.failures), + user_turns=len(result.committed), + ghost_turns=count_ghost_turns(result, scenario), + interrupted=result.interrupted, + heard_chars=None if result.heard_text is None else len(result.heard_text), + latencies_ms=[round(v, 1) for v in result.latencies_ms], + dead_air_ms=[round(v, 1) for v in result.dead_air_ms], + vad_endpointed=[m.vad_endpointed for m in result.metrics], + errors=list(result.errors), + wall_secs=round(result.wall_secs, 2), + known_failure=scenario.known_failure_reason(result.detector, result.stt, aec_leak) or "", + intermittent=scenario.is_intermittent(result.detector, result.stt, aec_leak), + jobs=jobs, + aec_leak=aec_leak, + label=label or f"{result.stt}/{result.detector}", + ) + + +def write_jsonl(records: list[RunRecord], path: Path | None = None) -> Path: + path = path or RESULTS_DIR / f"run-{time.strftime('%Y%m%d-%H%M%S')}.jsonl" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("".join(f"{r.to_json()}\n" for r in records)) + return path + + +# --------------------------------------------------------------------------- +# Aggregation +# --------------------------------------------------------------------------- + + +def percentile(values: list[float], q: float) -> float | None: + """Linear-interpolated percentile. ``q`` in [0, 1]. + + Rounded: these land in a committed baseline, and float noise like + ``382.97999999999996`` makes review diffs unreadable. + """ + if not values: + return None + ordered = sorted(values) + if len(ordered) == 1: + return round(ordered[0], 1) + pos = q * (len(ordered) - 1) + low = int(pos) + high = min(low + 1, len(ordered) - 1) + return round(ordered[low] + (ordered[high] - ordered[low]) * (pos - low), 1) + + +@dataclass +class Scorecard: + label: str + stt: str + detector: str + runs: int + """Gated runs only — known failures are counted separately.""" + passed: int + pass_rate: float + ghost_turns: int + errors: int + latency_p50: float | None + latency_p95: float | None + latency_min: float | None = None + latency_max: float | None = None + latency_samples: int = 0 + # Speech end -> turn accepted. Gated alongside latency because it is the only + # metric that can see a hold: `eou→audio` starts counting at the accepted + # commit, so hold policy is invisible to it by construction. + dead_air_p50: float | None = None + dead_air_p95: float | None = None + dead_air_samples: int = 0 + per_scenario: dict[str, float] = field(default_factory=dict) + """Gated scenario id -> pass rate across repeats. Known failures are excluded, so + marking one cannot quietly bake broken behavior into the baseline.""" + flaky: list[str] = field(default_factory=list) + known_failures: list[str] = field(default_factory=list) + unexpected_passes: list[str] = field(default_factory=list) + wall_secs: float = 0.0 + jobs: int = 1 + """Concurrency the cell ran at. Latency gating is suppressed unless this matches + the baseline's, so a baseline and its runs must agree; the concurrency itself is + not the problem — see the note in `_compare_latency`.""" + + +def build_scorecard(records: list[RunRecord]) -> Scorecard: + if not records: + raise ValueError("no records to score") + + gated = [r for r in records if not r.xfail] + by_scenario: dict[str, list[RunRecord]] = {} + for r in gated: + by_scenario.setdefault(r.scenario, []).append(r) + + per_scenario = {name: sum(r.passed for r in runs) / len(runs) for name, runs in by_scenario.items()} + # Latency describes the stack, not an expectation, so known failures still count. + latencies = [v for r in records for v in r.latencies_ms] + dead_air = [v for r in records for v in r.dead_air_ms] + passed = sum(r.passed for r in gated) + + return Scorecard( + # Fallback keeps result files written before labels were recorded scoreable. + label=records[0].label or f"{records[0].stt}/{records[0].detector}", + stt=records[0].stt, + detector=records[0].detector, + runs=len(gated), + passed=passed, + pass_rate=passed / len(gated) if gated else 1.0, + ghost_turns=sum(r.ghost_turns for r in gated), + errors=sum(len(r.errors) for r in gated), + latency_p50=percentile(latencies, 0.5), + latency_p95=percentile(latencies, 0.95), + latency_min=min(latencies) if latencies else None, + latency_max=max(latencies) if latencies else None, + latency_samples=len(latencies), + dead_air_p50=percentile(dead_air, 0.5), + dead_air_p95=percentile(dead_air, 0.95), + dead_air_samples=len(dead_air), + per_scenario=dict(sorted(per_scenario.items())), + flaky=sorted(name for name, rate in per_scenario.items() if 0.0 < rate < 1.0), + known_failures=sorted({r.scenario for r in records if r.xfail}), + unexpected_passes=sorted({r.scenario for r in records if r.xpass}), + wall_secs=round(sum(r.wall_secs for r in records), 1), + jobs=max(r.jobs for r in records), + ) + + +def format_scorecard(card: Scorecard) -> str: + lines = [ + f"{card.label}: {card.passed}/{card.runs} passed ({card.pass_rate:.0%})", + f" ghost turns: {card.ghost_turns} session errors: {card.errors}", + ] + if card.latency_p50 is not None: + gated = "" if card.latency_samples >= MIN_LATENCY_SAMPLES else " (ungated: too few samples)" + lines.append( + f" eou→audio: p50 {card.latency_p50:.0f}ms p95 {card.latency_p95:.0f}ms " + f"range {card.latency_min:.0f}-{card.latency_max:.0f}ms " + f"n={card.latency_samples}{gated}" + ) + if card.dead_air_p50 is not None: + gated = "" if card.dead_air_samples >= MIN_LATENCY_SAMPLES else " (ungated: too few samples)" + lines.append( + f" dead air: p50 {card.dead_air_p50:.0f}ms p95 {card.dead_air_p95:.0f}ms " + f"n={card.dead_air_samples}{gated} (speech end → turn accepted)" + ) + failing = [name for name, rate in card.per_scenario.items() if rate < 1.0] + if failing: + lines.append(f" failing: {', '.join(failing)}") + if card.flaky: + lines.append(f" FLAKY: {', '.join(card.flaky)} (passed some repeats, not others)") + if card.known_failures: + lines.append(f" known fail: {', '.join(card.known_failures)} (reported, not gated)") + if card.unexpected_passes: + lines.append( + f" XPASS: {', '.join(card.unexpected_passes)} — now passing, drop the known_failure marker" + ) + return "\n".join(lines) + + +# --------------------------------------------------------------------------- +# Matrix +# --------------------------------------------------------------------------- + +# Grid glyphs. Deliberately distinguishes "failed" from "expected to fail": a cell +# full of x is a documented limitation, a single ✗ is a regression. +_PASS, _FAIL, _XFAIL, _XPASS, _FLAKY, _ABSENT = "✓", "✗", "x", "!", "~", "·" + + +def format_grid(cards: list[Scorecard], records: list[RunRecord]) -> str: + """Scenario × cell grid — the question the matrix exists to answer. + + Reading down a column compares scenarios within one configuration; reading + across a row shows which configurations handle a given situation, which is the + only way to tell a Timbal bug from a provider quirk. + """ + labels = [c.label for c in cards] + by_cell: dict[str, dict[str, list[RunRecord]]] = {label: {} for label in labels} + for r in records: + # Match on the full cell label, not a rebuilt stt/detector: the label carries + # axis suffixes ([leak=0.15], swept params), and scorecards are keyed by it — + # stripping them here left every axis run matching no column, a matrix of `·` + # under correct per-cell scorecards. Same fallback as build_scorecard for + # result files written before labels were recorded. + cell = by_cell.get(r.label or f"{r.stt}/{r.detector}") + if cell is not None: + cell.setdefault(r.scenario, []).append(r) + + scenarios = sorted({r.scenario for r in records}) + width = max((len(s) for s in scenarios), default=0) + columns = [str(i + 1) for i in range(len(labels))] + + def glyph(runs: list[RunRecord]) -> str: + if not runs: + return _ABSENT + rate = sum(r.passed for r in runs) / len(runs) + if runs[0].xfail: + return _XPASS if all(r.xpass for r in runs) else _XFAIL + if rate == 1.0: + return _PASS + return _FAIL if rate == 0.0 else _FLAKY + + lines = ["", "matrix (rows: scenarios, columns: cells)", ""] + lines.append(f" {'':<{width}} " + " ".join(f"{c:>3}" for c in columns)) + for name in scenarios: + row = " ".join(f"{glyph(by_cell[label].get(name, [])):>3}" for label in labels) + lines.append(f" {name:<{width}} {row}") + + lines.append("") + for i, card in enumerate(cards): + rate = f"{card.passed}/{card.runs}" + p50 = f"{card.latency_p50:.0f}ms" if card.latency_p50 is not None else "-" + lines.append(f" {i + 1:>3} {card.label:<28} {rate:>7} p50 {p50:>7}") + lines.append("") + lines.append( + f" {_PASS} pass {_FAIL} FAIL {_XFAIL} known failure {_XPASS} XPASS {_FLAKY} flaky {_ABSENT} not run" + ) + return "\n".join(lines) + + +def cross_cell_flaky(records: list[RunRecord]) -> list[str]: + """Scenarios that are flaky *within* at least one cell. + + Deliberately not "differs across cells" — that is the matrix working as + intended, since detectors are supposed to behave differently. Flakiness inside + a single cell is the one that means a race. + """ + by_cell_scenario: dict[tuple[str, str], list[RunRecord]] = {} + for r in records: + # Keyed by full label so distinct axis runs (a leak cell and its clean + # counterpart, two sweep values) are never pooled — pooled, a scenario that + # deterministically passes one and fails the other would read as flaky. + by_cell_scenario.setdefault((r.label or f"{r.stt}/{r.detector}", r.scenario), []).append(r) + flaky = set() + for (label, scenario), runs in by_cell_scenario.items(): + if len(runs) < 2: + continue + rate = sum(r.passed for r in runs) / len(runs) + if 0.0 < rate < 1.0: + flaky.add(f"{scenario} [{label}] {sum(r.passed for r in runs)}/{len(runs)}") + return sorted(flaky) + + +# --------------------------------------------------------------------------- +# Baseline + gating +# --------------------------------------------------------------------------- + + +def load_baseline(path: Path = BASELINE_PATH) -> dict[str, dict]: + if not path.exists(): + return {} + return json.loads(path.read_text()) + + +def save_baseline(card: Scorecard, path: Path = BASELINE_PATH) -> None: + """Merge one config's scorecard into the baseline, leaving the others alone.""" + baseline = load_baseline(path) + baseline[card.label] = asdict(card) + path.write_text(json.dumps(dict(sorted(baseline.items())), indent=2) + "\n") + + +@dataclass +class Comparison: + regressions: list[str] = field(default_factory=list) + improvements: list[str] = field(default_factory=list) + notes: list[str] = field(default_factory=list) + """Informational: movement seen but deliberately not gated.""" + + @property + def ok(self) -> bool: + return not self.regressions + + +def compare(card: Scorecard, baseline: dict[str, dict], partial: bool = False) -> Comparison: + """Gate on movement away from the baseline, not on absolute thresholds. + + ``partial`` marks a filtered run (``-s`` / ``--quick``). Per-scenario rates stay + comparable because they are matched by name, but every *aggregate* is computed over + a different population than the baseline's, so comparing them invents movement that + is only a change of subject. Measured: the barge-in subset alone carries most of the + suite's ghost turns, so running just those reported "ghost turns 1 → 3" against a + full-suite baseline while being clean — and the reverse is worse, since a subset + that excludes them looks like an improvement. Latency and dead air are the same + story, a p50 over eight scenarios against a p50 over thirty-nine. + """ + out = Comparison() + previous = baseline.get(card.label) + if previous is None: + out.notes.append(f"no baseline for {card.label} yet — run with --update-baseline") + return out + + was: dict[str, float] = previous.get("per_scenario", {}) + for name, rate in card.per_scenario.items(): + before = was.get(name) + if before is None: + continue + if rate < before: + out.regressions.append(f"{name}: pass rate {before:.0%} → {rate:.0%}") + elif rate > before: + out.improvements.append(f"{name}: pass rate {before:.0%} → {rate:.0%}") + + new_scenarios = sorted(set(card.per_scenario) - set(was)) + if new_scenarios: + out.notes.append(f"new scenarios, nothing to compare: {', '.join(new_scenarios)}") + + for name in card.unexpected_passes: + out.improvements.append(f"{name}: known failure now passes — drop the known_failure marker") + + if partial: + out.notes.append( + "filtered run: per-scenario rates gated, aggregates (ghost turns, latency, dead air) not comparable" + ) + return out + + if card.ghost_turns > previous.get("ghost_turns", 0): + out.regressions.append(f"ghost turns {previous.get('ghost_turns', 0)} → {card.ghost_turns}") + + _compare_timing(out, card, previous, "latency", "eou→audio p50", card.latency_p50, card.latency_samples) + _compare_timing(out, card, previous, "dead_air", "dead air p50", card.dead_air_p50, card.dead_air_samples) + + return out + + +def _compare_timing( + out: Comparison, + card: Scorecard, + previous: dict, + key: str, + label: str, + now: float | None, + samples: int, +) -> None: + """Gate one timing percentile against its baseline entry. + + Both timings gate identically, but they answer different questions and a + change can move one without the other: removing the text-complete hold tier + left `eou→audio` flat — it starts at the accepted commit — while adding + 2.6s of dead air to six barge-in cells. + """ + before = previous.get(f"{key}_p50") + if not before or not now: + return + delta = f"{label} {before:.0f}ms → {now:.0f}ms" + # Only compare like with like. Measured on deepgram-nova/local, --quick, + # 3 repeats, contention is not detectable at --jobs 6: p50 278ms vs 280ms + # serial, p95 407ms vs 453ms — better, because eou→audio is mostly waiting + # on STT and TTS sockets rather than competing for CPU, and the serial run's + # one 962ms sample is first-run model warmup weighing on a smaller n. The + # guard stays because it is free and the equality is what makes it sound. + before_jobs = previous.get("jobs", 1) + if card.jobs != before_jobs: + out.notes.append(f"{delta} — not gated, measured at --jobs {card.jobs} against a --jobs {before_jobs} baseline") + return + moved = now / before + if moved > LATENCY_REGRESSION_RATIO: + worse = f"{delta} (>{(LATENCY_REGRESSION_RATIO - 1) * 100:.0f}% worse)" + if min(samples, previous.get(f"{key}_samples", 0)) >= MIN_LATENCY_SAMPLES: + out.regressions.append(worse) + else: + out.notes.append(f"{worse} — not gated, fewer than {MIN_LATENCY_SAMPLES} samples") + elif moved < 1 / LATENCY_REGRESSION_RATIO: + out.improvements.append(delta) diff --git a/benchmarks/voice/sweep.py b/benchmarks/voice/sweep.py new file mode 100644 index 00000000..11600bae --- /dev/null +++ b/benchmarks/voice/sweep.py @@ -0,0 +1,199 @@ +"""Parameter sweep: score one setting on merges gained against dead air added. + +The hold tier is the worked example. ``TEXT_COMPLETE_HOLD_TIMEOUT_SECS`` shipped at +0.35s having been measured at exactly two values: 0.35 (splits four +trailing-modifier scenarios) and 3.0 (fixes 11 scenario-cells and adds 2.6s of +dead air to six barge-in cells). Nobody had tried 0.8, or 1.2 — which is what it +ships at now. The interesting question — is there a setting that buys most of the +merges for a fraction of the silence — was unanswerable until dead air became +measurable, because a configuration that merged everything by holding forever +looked free. + +Two metrics, deliberately. Optimizing pass rate alone reproduces the mistake this +harness was built to catch:: + + uv run python benchmarks/voice/sweep.py \\ + --param text_complete_hold_timeout_secs --values 0.35,0.8,1.2,2.0,3.0 \\ + --stt deepgram-nova,elevenlabs --detector local --repeat 3 + +``stt.``-prefixed params vary the provider's own endpointing instead, which is the +other half of the pipeline and the half nobody had tuned:: + + uv run python benchmarks/voice/sweep.py \\ + --param stt.vad_silence_threshold_secs --values 0.4,0.6,0.8,1.2 \\ + --stt elevenlabs --detector local,lexical --repeat 3 + +Defaults to the pause-merge family (every scenario asserting ``Merged``), since +that is the only family where detectors differ at all — barge-in, plain turns and +silence sit at 100% in all twelve cells and would only add runtime and noise. + +Results are indicative, not conclusive: 39 English scenarios on three backends, +18 of them carrying a known failure somewhere, and real intermittency. Treat a +winner as a candidate to confirm against the full grid and the Flux gate, never as +a decision. +""" + +from __future__ import annotations + +import argparse +import asyncio +import logging +import statistics as st +import sys +import time +from dataclasses import dataclass, field + +import structlog +from dotenv import load_dotenv +from harness import HarnessConfig, coerce_param, config_rejection, run_scenario +from scenario import SCENARIOS, Merged, Scenario +from synth import synthesize_clips, synthesize_fluent + + +def pause_family() -> list[Scenario]: + """Scenarios that assert a merge — the only discriminating family.""" + return [s for s in SCENARIOS if any(isinstance(e, Merged) for e in s.expect)] + + +@dataclass +class Outcome: + """What one parameter value scored across every cell and repeat.""" + + value: str + passed: int = 0 + total: int = 0 + dead_air: list[float] = field(default_factory=list) + per_scenario: dict[str, list[bool]] = field(default_factory=dict) + wall_secs: float = 0.0 + + @property + def rate(self) -> float: + return self.passed / self.total if self.total else 0.0 + + def summary(self) -> str: + air = f"{st.median(self.dead_air):>6.0f}" if self.dead_air else " -" + p95 = f"{sorted(self.dead_air)[int(0.95 * len(self.dead_air))]:>6.0f}" if self.dead_air else " -" + return f"{self.value:>10} {self.passed:>3}/{self.total:<3} {self.rate:>5.0%} {air}ms {p95}ms" + + +async def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + "--param", + required=True, + help="attribute to vary: a detector attribute, or 'stt.' for a provider STT knob " + "(see harness.SWEEPABLE_STT_KEYS)", + ) + parser.add_argument("--values", required=True, help="comma-separated values to try") + parser.add_argument("--stt", default="deepgram-nova,elevenlabs") + parser.add_argument("--detector", default="local") + parser.add_argument("-s", "--scenario", action="append", help="override the default pause family") + parser.add_argument( + "--all", + action="store_true", + help="every scenario, not just the pause family — needed to price a winner, since the " + "cost of holding longer lands on closers and barge-ins that the pause family excludes", + ) + parser.add_argument("--repeat", type=int, default=3, help="runs per scenario per cell (default 3)") + parser.add_argument("--jobs", type=int, default=6) + args = parser.parse_args() + + values = [coerce_param(v.strip()) for v in args.values.split(",") if v.strip()] + param_field, param_key = ( + ("stt_extra", args.param.removeprefix("stt.")) + if args.param.startswith("stt.") + else ("detector_params", args.param) + ) + stts = [v.strip() for v in args.stt.split(",") if v.strip()] + detectors = [v.strip() for v in args.detector.split(",") if v.strip()] + + scenarios = list(SCENARIOS) if args.all else pause_family() + if args.scenario: + wanted = set(args.scenario) + scenarios = [s for s in SCENARIOS if s.id in wanted] + if not scenarios: + print("no scenarios selected") + return 2 + + clips = await synthesize_clips({t for s in scenarios for t in s.standalone_texts()}) + clips.update(await synthesize_fluent([g for s in scenarios for g in s.fluent_groups()])) + + cells = [(stt, det) for stt in stts for det in detectors] + runs = len(values) * len(cells) * len(scenarios) * args.repeat + print(f"\n{runs} runs: {len(values)} values x {len(cells)} cell(s) x {len(scenarios)} scenarios x {args.repeat}") + print(f"varying {args.param} over {values}") + print(f"scenarios: {', '.join(s.id for s in scenarios)}\n") + + semaphore = asyncio.Semaphore(max(1, args.jobs)) + outcomes = {str(v): Outcome(value=str(v)) for v in values} + started = time.monotonic() + + rejected: dict[str, str] = {} + + async def one(value: object, stt: str, det: str, scenario: Scenario) -> None: + if str(value) in rejected: + return + config = HarnessConfig(stt=stt, detector=det, **{param_field: {param_key: value}}) + async with semaphore: + result = await run_scenario(scenario, clips, config) + if why := config_rejection(result.errors): + rejected.setdefault(str(value), why) + return + out = outcomes[str(value)] + out.total += 1 + out.passed += result.passed + # Only the final commit. Pooling every dead-air sample rewards splitting: + # a merge yields one long wait, a split yields two short ones, so the + # configuration that fragments the utterance scores *better* on silence. + # Final speech end -> final commit is the number a caller actually feels + # and is comparable across both outcomes. + if result.dead_air_ms: + out.dead_air.append(result.dead_air_ms[-1]) + out.wall_secs += result.wall_secs + out.per_scenario.setdefault(scenario.id, []).append(result.passed) + + await asyncio.gather( + *( + one(value, stt, det, scenario) + for value in values + for stt, det in cells + for scenario in scenarios + for _ in range(args.repeat) + ) + ) + + print(f"{args.param:>10} {'merges':<10} {'':>4} {'dead air p50':>10} {'p95':>6}") + for value in values: + if why := rejected.get(str(value)): + print(f" {value!s:>10} {why}") + continue + print(" " + outcomes[str(value)].summary()) + + print("\nper scenario (pass rate across cells and repeats)\n") + header = "".join(f"{v!s:>9}" for v in values) + print(f" {'scenario':<30}{header}") + for scenario in scenarios: + row = "" + for value in values: + if str(value) in rejected: + row += f"{'-':>8} " + continue + runs_ = outcomes[str(value)].per_scenario.get(scenario.id, []) + row += f"{(sum(runs_) / len(runs_) if runs_ else 0):>8.0%} " + print(f" {scenario.id:<30}{row}") + + print(f"\n elapsed: {time.monotonic() - started:.0f}s") + print("\n A winner here is a candidate, not a decision: confirm it against the full") + print(" 12-cell grid and the Flux gate before believing it.") + # Non-zero on a rejected value: the sweep did not measure what it was asked to. + return 1 if rejected else 0 + + +if __name__ == "__main__": + load_dotenv() + # A sweep is hundreds of sessions, and at DEBUG each one emits a few hundred + # lines: the result table lands tens of megabytes below the scroll and gets + # dropped outright by anything that caps captured output. Same default as + # `cli.py`, which had it from the start. + structlog.configure(wrapper_class=structlog.make_filtering_bound_logger(logging.WARNING)) + sys.exit(asyncio.run(main())) diff --git a/benchmarks/voice/synth.py b/benchmarks/voice/synth.py new file mode 100644 index 00000000..111c0899 --- /dev/null +++ b/benchmarks/voice/synth.py @@ -0,0 +1,341 @@ +"""Audio synthesis, caching and PCM helpers for the voice replay harness. + +ElevenLabs -> raw PCM16 16k mono, cached by a hash of ``(text, voice, model)``. +Synthesis is the only expensive part of the harness, so nothing is ever generated +twice for unchanged script text. + +Everything here is deterministic once the cache is warm: the same script always +composes to the same bytes, which is what makes replay runs comparable. + +Standalone use — hear what the harness will actually feed the session:: + + uv run python benchmarks/voice/synth.py "I'd like to order a large" + uv run python benchmarks/voice/synth.py --verify +""" + +from __future__ import annotations + +import argparse +import asyncio +import base64 +import hashlib +import json +import logging +import os +import sys +import wave +from collections.abc import Iterable, Iterator, Sequence +from pathlib import Path + +import httpx +import structlog +from dotenv import load_dotenv +from timbal.voice import AudioOutputConfig +from timbal.voice.elevenlabs import ElevenLabsStreamTTS + +SAMPLE_RATE = 16_000 +BYTES_PER_SECOND = SAMPLE_RATE * 2 # PCM16 mono +FRAME_SECS = 0.02 +FRAME_BYTES = int(BYTES_PER_SECOND * FRAME_SECS) +SILENCE_FRAME = b"\x00" * FRAME_BYTES + +TTS_MODEL = os.environ.get("TIMBAL_TTS_MODEL", "eleven_flash_v2_5") +# Distinct voices for the two sides: replaying the user in the assistant's voice +# would trip the session's echo heuristics on legitimate speech. +ASSISTANT_VOICE_ID = os.environ.get("ELEVENLABS_VOICE_ID", "1SM7GgM6IMuvQlz2BwM3") +USER_VOICE_ID = os.environ.get("TIMBAL_BENCH_USER_VOICE_ID", "21m00Tcm4TlvDq8ikWAM") + +HERE = Path(__file__).parent +CACHE_DIR = HERE / "cache" + + +# --------------------------------------------------------------------------- +# PCM helpers +# --------------------------------------------------------------------------- + + +def silence(secs: float) -> bytes: + """Exactly ``secs`` of silence, rounded to whole 20ms frames.""" + return SILENCE_FRAME * max(0, round(secs / FRAME_SECS)) + + +def duration_secs(pcm: bytes) -> float: + return len(pcm) / BYTES_PER_SECOND + + +def frame_pad(pcm: bytes) -> bytes: + """Zero-pad to a whole number of frames. + + Clips rarely end on a frame boundary. Padding each part keeps the composed + stream frame-aligned at the cost of <20ms of silence at a seam, which is + below anything turn detection can resolve. + """ + remainder = len(pcm) % FRAME_BYTES + return pcm if remainder == 0 else pcm + b"\x00" * (FRAME_BYTES - remainder) + + +def frames(pcm: bytes) -> Iterator[bytes]: + """Split into 20ms frames, padding the tail.""" + padded = frame_pad(pcm) + for i in range(0, len(padded), FRAME_BYTES): + yield padded[i : i + FRAME_BYTES] + + +def compose(parts: Iterable[bytes | float]) -> bytes: + """Concatenate clips (``bytes``) and silences (``float`` seconds). + + The static half of a script: everything except reactive steps, which need the + session's own audio timing and therefore live in the harness. + """ + return b"".join(frame_pad(p) if isinstance(p, bytes) else silence(p) for p in parts) + + +def write_wav(path: Path, pcm: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with wave.open(str(path), "wb") as wav: + wav.setnchannels(1) + wav.setsampwidth(2) + wav.setframerate(SAMPLE_RATE) + wav.writeframes(pcm) + + +# --------------------------------------------------------------------------- +# Synthesis + cache +# --------------------------------------------------------------------------- + + +def clip_path(text: str, voice_id: str = USER_VOICE_ID, model: str = TTS_MODEL) -> Path: + digest = hashlib.sha256(f"{model}|{voice_id}|{text}".encode()).hexdigest()[:16] + return CACHE_DIR / f"{digest}.pcm" + + +async def synthesize_clips( + texts: Iterable[str], + *, + voice_id: str = USER_VOICE_ID, + model: str = TTS_MODEL, + quiet: bool = False, +) -> dict[str, bytes]: + """Return ``{text: pcm}``, synthesizing only what the cache is missing.""" + wanted = list(dict.fromkeys(texts)) + CACHE_DIR.mkdir(parents=True, exist_ok=True) + missing = [t for t in wanted if not clip_path(t, voice_id, model).exists()] + + if missing: + if not quiet: + print(f"synthesizing {len(missing)} clip(s) with voice {voice_id}...") + tts = ElevenLabsStreamTTS() + await tts.connect(AudioOutputConfig(model=model, voice=voice_id, sample_rate=SAMPLE_RATE)) + try: + for text in missing: + buf = bytearray() + async for chunk in tts.synthesize(text): + buf.extend(chunk) + if not buf: + raise RuntimeError(f"ElevenLabs returned no audio for {text!r}") + clip_path(text, voice_id, model).write_bytes(bytes(buf)) + finally: + await tts.close() + + return {t: clip_path(t, voice_id, model).read_bytes() for t in wanted} + + +# --------------------------------------------------------------------------- +# Fluent utterances +# --------------------------------------------------------------------------- +# +# A speaker who pauses mid-sentence keeps the intonation of a sentence still in +# progress. Synthesizing the fragments separately does not: ElevenLabs renders +# each one as its own sentence and gives it a falling, finished contour. +# +# Measured with Smart Turn v3 on identical words: "I want to return an item I +# bought" scores 0.986 (finished) rendered standalone and 0.019 (mid-thought) +# when cut out of the fluent whole-sentence render. That is the difference +# between a scenario testing turn-taking and a scenario testing TTS phrasing. +# +# So fluent utterances are rendered once, whole, and sliced at the character +# timestamps ElevenLabs returns alongside the audio. + +ALIGNMENT_ENDPOINT = "https://api.elevenlabs.io/v1/text-to-speech/{voice_id}/with-timestamps" + + +def alignment_path(text: str, voice_id: str = USER_VOICE_ID, model: str = TTS_MODEL) -> Path: + return clip_path(text, voice_id, model).with_suffix(".json") + + +def part_key(utterance: str, index: int) -> str: + """Cache/clip key for one slice of a fluent utterance.""" + return f"{utterance}#{index}" + + +def _part_end_chars(parts: Sequence[str]) -> list[int]: + """Exclusive character offset of each part within ``" ".join(parts)``.""" + ends: list[int] = [] + pos = 0 + for i, part in enumerate(parts): + if i: + pos += 1 # the joining space + pos += len(part) + ends.append(pos) + return ends + + +def slice_fluent(pcm: bytes, char_end_times: Sequence[float], parts: Sequence[str]) -> list[bytes]: + """Cut ``pcm`` at the boundary where each part's last character ends. + + The final part runs to the end of the audio so no trailing samples are lost. + """ + slices: list[bytes] = [] + start = 0 + for i, end_char in enumerate(_part_end_chars(parts)): + if i == len(parts) - 1: + stop = len(pcm) + else: + # Byte offsets must stay sample-aligned or the PCM16 frames shear. + stop = min(int(char_end_times[end_char - 1] * SAMPLE_RATE) * 2, len(pcm)) + slices.append(pcm[start:stop]) + start = stop + return slices + + +async def _synthesize_aligned(utterance: str, voice_id: str, model: str) -> tuple[bytes, list[float]]: + """Whole-utterance PCM plus per-character end times, cached on disk. + + Uses the REST ``with-timestamps`` endpoint; the streaming websocket used for + ordinary clips does not return alignment. + """ + pcm_file = clip_path(utterance, voice_id, model) + align_file = alignment_path(utterance, voice_id, model) + if pcm_file.exists() and align_file.exists(): + return pcm_file.read_bytes(), json.loads(align_file.read_text()) + + api_key = os.environ.get("ELEVENLABS_API_KEY") + if not api_key: + raise ValueError("Set ELEVENLABS_API_KEY to synthesize fluent utterances.") + async with httpx.AsyncClient(timeout=60) as client: + response = await client.post( + ALIGNMENT_ENDPOINT.format(voice_id=voice_id), + params={"output_format": f"pcm_{SAMPLE_RATE}"}, + headers={"xi-api-key": api_key}, + json={"text": utterance, "model_id": model}, + ) + response.raise_for_status() + payload = response.json() + + pcm = base64.b64decode(payload["audio_base64"]) + ends = payload["alignment"]["character_end_times_seconds"] + CACHE_DIR.mkdir(parents=True, exist_ok=True) + pcm_file.write_bytes(pcm) + align_file.write_text(json.dumps(ends)) + return pcm, ends + + +async def synthesize_fluent( + groups: Iterable[tuple[str, Sequence[str]]], + *, + voice_id: str = USER_VOICE_ID, + model: str = TTS_MODEL, + quiet: bool = False, +) -> dict[str, bytes]: + """Return ``{part_key: pcm}`` for each ``(utterance, parts)`` group.""" + wanted = {utterance: tuple(parts) for utterance, parts in groups} + if not wanted: + return {} + missing = [u for u in wanted if not alignment_path(u, voice_id, model).exists()] + if missing and not quiet: + print(f"synthesizing {len(missing)} fluent utterance(s) with alignment...") + + clips: dict[str, bytes] = {} + for utterance, parts in wanted.items(): + pcm, ends = await _synthesize_aligned(utterance, voice_id, model) + for i, chunk in enumerate(slice_fluent(pcm, ends, parts)): + clips[part_key(utterance, i)] = chunk + return clips + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +_VERIFY_PARTS: list[str | float] = [ + "Verification clip one.", + 0.5, + "Verification clip two.", +] + + +async def _verify() -> int: + """Phase 1 exit criterion: clips and silences compose byte-exactly.""" + texts = [p for p in _VERIFY_PARTS if isinstance(p, str)] + checks: list[tuple[str, bool, str]] = [] + + first = await synthesize_clips(texts) + second = await synthesize_clips(texts) # must be a pure cache read + checks.append(("cache read is stable", first == second, f"{len(first)} clip(s)")) + + def _build(clips: dict[str, bytes]) -> bytes: + return compose([clips[p] if isinstance(p, str) else p for p in _VERIFY_PARTS]) + + a, b = _build(first), _build(second) + digest = hashlib.sha256(a).hexdigest()[:16] + checks.append(("compose is byte-exact", a == b, f"sha256:{digest}")) + checks.append(("frame-aligned", len(a) % FRAME_BYTES == 0, f"{len(a)} bytes")) + checks.append(("non-empty", duration_secs(a) > 1.0, f"{duration_secs(a):.2f}s")) + + gap = silence(0.5) + checks.append(("silence is exact", duration_secs(gap) == 0.5, f"{len(gap)} bytes")) + checks.append(("frames round-trip", b"".join(frames(a)) == a, f"{len(a) // FRAME_BYTES} frames")) + + # Slicing a fluent utterance must lose no audio and stay on PCM16 sample + # boundaries, or every downstream fragment shears by half a sample. + parts = ("Verification clip one,", "sliced in two.") + utterance = " ".join(parts) + pcm, ends = await _synthesize_aligned(utterance, USER_VOICE_ID, TTS_MODEL) + pieces = slice_fluent(pcm, ends, parts) + checks.append(("fluent slice is lossless", b"".join(pieces) == pcm, f"{len(pcm)} bytes")) + checks.append( + ( + "fluent slice is sample-aligned", + all(len(p) % 2 == 0 for p in pieces), + " + ".join(f"{duration_secs(p):.2f}s" for p in pieces), + ) + ) + + for name, ok, detail in checks: + print(f" {'ok ' if ok else 'FAIL'} {name:<24} {detail}") + failed = [name for name, ok, _ in checks if not ok] + print(f"\n{len(checks) - len(failed)}/{len(checks)} checks passed") + return 0 if not failed else 1 + + +async def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("text", nargs="*", help="text to synthesize") + parser.add_argument("-o", "--out", type=Path, help="write a WAV here (default: results/synth/)") + parser.add_argument("--voice", default=USER_VOICE_ID) + parser.add_argument("--model", default=TTS_MODEL) + parser.add_argument("--verify", action="store_true", help="check byte-exact reproducibility") + parser.add_argument("--verbose", action="store_true", help="keep timbal debug logs") + args = parser.parse_args() + + load_dotenv(override=True) + if not args.verbose: + structlog.configure(wrapper_class=structlog.make_filtering_bound_logger(logging.WARNING)) + + if args.verify: + return await _verify() + if not args.text: + parser.print_help() + return 2 + + text = " ".join(args.text) + clips = await synthesize_clips([text], voice_id=args.voice, model=args.model) + pcm = clips[text] + out = args.out or HERE / "results" / "synth" / f"{clip_path(text, args.voice, args.model).stem}.wav" + write_wav(out, pcm) + print(f"{duration_secs(pcm):.2f}s {len(pcm)} bytes -> {out}") + return 0 + + +if __name__ == "__main__": + sys.exit(asyncio.run(main())) diff --git a/pyproject.toml b/pyproject.toml index aacc1e22..e1444a34 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,6 +61,7 @@ server = [ "websockets>=15.0.1", ] voice = [ + "aiortc>=1.9.0", "huggingface-hub>=0.28.0", "onnxruntime>=1.20.0", "transformers>=4.40.0", diff --git a/python/tests/server/test_recording_upload.py b/python/tests/server/test_recording_upload.py new file mode 100644 index 00000000..2be7fece --- /dev/null +++ b/python/tests/server/test_recording_upload.py @@ -0,0 +1,193 @@ +"""Platform push for call recordings (option C of the recording handoff). + +Contract under test: PUT multipart via ``platform.utils._request`` (auth and +host from ``resolve_platform_config``) to +``orgs/{org}/projects/{project}/voice-recordings/{session_id}``, delete +local files only on 2xx, keep them on 4xx (no retry) and on exhausted +5xx/network retries. +""" + +# ruff: noqa: ARG001 — mock handlers/uploaders must match real signatures +from __future__ import annotations + +import asyncio +from pathlib import Path + +import httpx +import pytest +import timbal.state.config_loader as config_loader +from timbal.server.recording_upload import ( + _upload_tasks, + platform_recording_upload_hook, + upload_recording, +) +from timbal.state.config_loader import resolve_platform_config + +PLATFORM_ENV = { + "TIMBAL_API_HOST": "api.timbal.test", + "TIMBAL_ORG_ID": "org1", + "TIMBAL_PROJECT_ID": "proj1", + "TIMBAL_API_TOKEN": "tok-123", +} + +PATH = "orgs/org1/projects/proj1/sessions/sess1" +URL = f"https://api.timbal.test/{PATH}" + + +@pytest.fixture(autouse=True) +def _platform_env(monkeypatch: pytest.MonkeyPatch): + """Point platform config at the test platform; restore the module cache after.""" + saved = (config_loader._cached_default_config, config_loader._default_config_resolved) + for k in ("TIMBAL_API_KEY", "TIMBAL_APP_ID"): + monkeypatch.delenv(k, raising=False) + for k, v in PLATFORM_ENV.items(): + monkeypatch.setenv(k, v) + resolve_platform_config(force_refresh=True) + yield + config_loader._cached_default_config, config_loader._default_config_resolved = saved + + +def _files(tmp_path: Path, session_id: str = "sess1") -> tuple[Path, Path]: + audio = tmp_path / f"{session_id}.mp3" + manifest = tmp_path / f"{session_id}.json" + audio.write_bytes(b"\xff\xfbmp3data") + manifest.write_text('{"session_id": "sess1"}') + return audio, manifest + + +def _mock_http(monkeypatch: pytest.MonkeyPatch, statuses: list[int]) -> list[httpx.Request]: + """Route all httpx clients through a MockTransport replying with *statuses* in order.""" + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + status = statuses[min(len(requests) - 1, len(statuses) - 1)] + if isinstance(status, int): + return httpx.Response(status) + raise status # an exception instance → simulate a network error + + transport = httpx.MockTransport(handler) + real_client = httpx.AsyncClient + monkeypatch.setattr(httpx, "AsyncClient", lambda **kw: real_client(**{**kw, "transport": transport})) + return requests + + +class TestUploadRecording: + async def test_2xx_uploads_multipart_and_deletes_files( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + audio, manifest = _files(tmp_path) + requests = _mock_http(monkeypatch, [200]) + ok = await upload_recording(audio, manifest, path=PATH) + assert ok + assert not audio.exists() and not manifest.exists() + + (req,) = requests + assert req.method == "PUT" + assert str(req.url) == URL + assert req.headers["authorization"] == "Bearer tok-123" # from resolve_platform_config + assert req.headers["content-type"].startswith("multipart/form-data") + body = req.read() + assert b'name="manifest"' in body and b"application/json" in body + assert b'name="audio"' in body and b"audio/mpeg" in body + assert b"\xff\xfbmp3data" in body + + @pytest.mark.parametrize("status", [400, 401, 403, 404, 409, 413]) + async def test_4xx_keeps_files_and_never_retries( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, status: int + ) -> None: + """The contract's permanent failures: bad parts / auth / project / ownership / size.""" + audio, manifest = _files(tmp_path) + requests = _mock_http(monkeypatch, [status]) + ok = await upload_recording(audio, manifest, path=PATH) + assert not ok + assert audio.exists() and manifest.exists() + assert len(requests) == 1 + + async def test_5xx_retries_with_backoff_then_succeeds( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + audio, manifest = _files(tmp_path) + requests = _mock_http(monkeypatch, [500, 502, 201]) + ok = await upload_recording(audio, manifest, path=PATH, backoff=lambda _: 0.01) + assert ok + assert len(requests) == 3 + assert not audio.exists() and not manifest.exists() + + async def test_network_errors_are_retryable( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + audio, manifest = _files(tmp_path) + requests = _mock_http(monkeypatch, [httpx.ConnectError("boom"), 200]) # type: ignore[list-item] + ok = await upload_recording(audio, manifest, path=PATH, backoff=lambda _: 0.01) + assert ok + assert len(requests) == 2 + + async def test_exhausted_retries_give_up_and_keep_files( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + audio, manifest = _files(tmp_path) + requests = _mock_http(monkeypatch, [500]) + ok = await upload_recording(audio, manifest, path=PATH, max_retries=2, backoff=lambda _: 0.01) + assert not ok + assert audio.exists() and manifest.exists() + assert len(requests) == 3 # initial + 2 retries + + async def test_missing_files_abort_without_request( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + requests = _mock_http(monkeypatch, [200]) + ok = await upload_recording(tmp_path / "gone.mp3", tmp_path / "gone.json", path=PATH) + assert not ok and not requests + + +class TestPlatformHookFactory: + def test_incomplete_platform_config_returns_none(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("TIMBAL_PROJECT_ID", raising=False) # org present, project missing + assert platform_recording_upload_hook() is None + + async def test_hook_builds_path_from_platform_config_and_schedules_upload( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + captured: dict = {} + + async def _fake_upload(audio_path, manifest_path, *, path, **kwargs): + captured.update(audio=audio_path, manifest=manifest_path, path=path) + return True + + monkeypatch.setattr("timbal.server.recording_upload.upload_recording", _fake_upload) + hook = platform_recording_upload_hook() + assert hook is not None + + audio, manifest = _files(tmp_path, session_id="abc42") + + class _Result: + audio_path = audio + manifest_path = manifest + + await hook(_Result()) # returns immediately — the upload is a background task + await asyncio.gather(*_upload_tasks) + + assert captured["path"] == "orgs/org1/projects/proj1/sessions/abc42" + assert captured["audio"] == audio and captured["manifest"] == manifest + + async def test_hook_skips_when_manifest_missing( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + """mp3-without-json = crashed call: leave it for the sweeper, don't push.""" + called = False + + async def _fake_upload(*args, **kwargs): + nonlocal called + called = True + + monkeypatch.setattr("timbal.server.recording_upload.upload_recording", _fake_upload) + hook = platform_recording_upload_hook() + + class _Result: + audio_path = tmp_path / "x.mp3" + manifest_path = None + + await hook(_Result()) + await asyncio.gather(*_upload_tasks) + assert not called diff --git a/python/tests/server/test_voice_detector_choice.py b/python/tests/server/test_voice_detector_choice.py new file mode 100644 index 00000000..043ee3ce --- /dev/null +++ b/python/tests/server/test_voice_detector_choice.py @@ -0,0 +1,99 @@ +"""Turn detector selection for served voice sessions. + +The rules live in :func:`timbal.server.voice.select_turn_detector_spec` rather +than in the websocket handler precisely so they can be pinned here: they decide +what every deployment that doesn't configure a detector actually runs. +""" + +from timbal.server.voice import select_turn_detector_spec +from timbal.voice.turn_detection import HeuristicTurnDetector, LocalAudioTurnDetector + + +class _FakeLocal: + """Stand-in for what ``resolve_turn_detector("local")`` returns.""" + + def __init__(self, audio_eou: object | None) -> None: + self.audio_eou = audio_eou + + +def _with_extra(monkeypatch, *, available: bool) -> None: + """Fake the presence of timbal[voice] via the resolver's degradation signal.""" + import timbal.voice.turn_detection as td + + monkeypatch.setattr( + td, + "resolve_turn_detector", + lambda _spec=None: _FakeLocal(object() if available else None), + ) + + +class TestFluxOwnsEndpointing: + def test_nothing_chosen_gets_provider(self): + assert select_turn_detector_spec(None, None, stt_is_flux=True) == "provider" + + def test_local_is_overridden(self): + assert select_turn_detector_spec("local", None, stt_is_flux=True) == "provider" + + def test_lexical_is_overridden(self): + assert select_turn_detector_spec("lexical", None, stt_is_flux=True) == "provider" + + def test_an_instance_is_overridden_too(self): + # Documents current behaviour: Flux wins even over an explicit instance + # from voice_config, which is why the override is logged. + assert select_turn_detector_spec(LocalAudioTurnDetector(), None, stt_is_flux=True) == "provider" + + def test_explicit_heuristic_is_respected(self): + assert select_turn_detector_spec("heuristic", None, stt_is_flux=True) == "heuristic" + + def test_explicit_raw_is_respected(self): + assert select_turn_detector_spec("raw", None, stt_is_flux=True) == "raw" + + +class TestSilenceEndpointingDefault: + """Nova / ElevenLabs / anything that commits on a silence timeout.""" + + def test_defaults_to_local_with_the_extra(self, monkeypatch): + _with_extra(monkeypatch, available=True) + assert select_turn_detector_spec(None, None, stt_is_flux=False) == "local" + + def test_falls_back_to_lexical_without_the_extra(self, monkeypatch): + # `local` without an audio EOU model returns the heuristic decision + # verbatim, so it would not hold; `lexical` does, with no extra deps. + _with_extra(monkeypatch, available=False) + assert select_turn_detector_spec(None, None, stt_is_flux=False) == "lexical" + + def test_never_defaults_to_the_holdless_heuristic(self, monkeypatch): + for available in (True, False): + _with_extra(monkeypatch, available=available) + assert select_turn_detector_spec(None, None, stt_is_flux=False) != "heuristic" + + def test_explicit_heuristic_is_still_honoured(self): + assert select_turn_detector_spec("heuristic", None, stt_is_flux=False) == "heuristic" + + def test_an_instance_is_passed_through(self): + detector = HeuristicTurnDetector() + assert select_turn_detector_spec(detector, None, stt_is_flux=False) is detector + + def test_a_factory_is_passed_through(self): + def factory() -> HeuristicTurnDetector: + return HeuristicTurnDetector() + + assert select_turn_detector_spec(factory, None, stt_is_flux=False) is factory + + +class TestClientOverride: + def test_client_mode_beats_the_server_spec(self): + assert select_turn_detector_spec("local", "heuristic", stt_is_flux=False) == "heuristic" + + def test_client_mode_beats_a_server_instance(self): + assert select_turn_detector_spec(LocalAudioTurnDetector(), "lexical", stt_is_flux=False) == "lexical" + + def test_blank_client_mode_is_ignored(self): + assert select_turn_detector_spec("lexical", " ", stt_is_flux=False) == "lexical" + + def test_non_string_client_spec_is_refused(self): + # A browser must not be able to hand the server a callable. + assert select_turn_detector_spec("lexical", {"mode": "local"}, stt_is_flux=False) == "lexical" + + def test_client_cannot_escape_the_flux_override(self): + assert select_turn_detector_spec(None, "local", stt_is_flux=True) == "provider" diff --git a/python/tests/server/test_voice_rtc.py b/python/tests/server/test_voice_rtc.py new file mode 100644 index 00000000..47c69653 --- /dev/null +++ b/python/tests/server/test_voice_rtc.py @@ -0,0 +1,253 @@ +"""End-to-end tests for ``POST /voice/rtc`` — a real aiortc loopback in one process. + +The test plays the browser: its own ``RTCPeerConnection``, a silent mic +track, and a client-created data channel, with SDP exchanged through the +actual FastAPI endpoint. Media and SCTP flow over localhost UDP between the +test's event loop and the TestClient portal loop. STT/TTS are mocked at the +module boundary exactly like ``test_voice_ws.py`` — everything from the +signaling down through ICE, DTLS, Opus, and the session is real. +""" + +# ruff: noqa: ARG002 +from __future__ import annotations + +import asyncio +import contextlib +import json +from pathlib import Path +from typing import Any + +import pytest + +pytest.importorskip("aiortc", reason="timbal[voice] extra (aiortc) not installed") + +from aiortc import RTCConfiguration, RTCPeerConnection, RTCSessionDescription # noqa: E402 +from aiortc.mediastreams import AudioStreamTrack # noqa: E402 +from fastapi.testclient import TestClient # noqa: E402 +from timbal.server.http import create_app # noqa: E402 +from timbal.voice import ( # noqa: E402 + AudioInputConfig, + SpeechToText, + TranscriptEvent, +) + +from .test_voice_ws import _make_tts_class, _write_agent_module # noqa: E402 +from .voice_env import VOICE_ENV_KEYS # noqa: E402 + + +def _make_delayed_stt_class(script: list[tuple[float, TranscriptEvent]], *, end_after: float = 1.0): + """An STT class replaying ``(delay_secs, event)`` pairs, then ending. + + The WS mocks replay instantly on connect; over RTC the session must + outlive the ICE/DTLS/SCTP handshake for anything to reach the client, so + events are spaced on a real clock. + """ + _script = list(script) + + class _STT(SpeechToText): + def __init__(self, api_key: Any = None) -> None: + self._queue: asyncio.Queue[TranscriptEvent | None] = asyncio.Queue() + self._feeder: asyncio.Task | None = None + + async def connect(self, config: AudioInputConfig) -> None: + async def _feed() -> None: + for delay, ev in _script: + await asyncio.sleep(delay) + await self._queue.put(ev) + await asyncio.sleep(end_after) + await self._queue.put(None) + + self._feeder = asyncio.create_task(_feed()) + + async def push_audio(self, chunk: bytes) -> None: + pass + + async def commit(self) -> None: + pass + + async def events(self): + while True: + item = await self._queue.get() + if item is None: + break + if item.text: + yield item + + async def close(self) -> None: + if self._feeder is not None and not self._feeder.done(): + self._feeder.cancel() + + return _STT + + +async def _rtc_call( + client: TestClient, + *, + config: dict | None = None, + timeout: float = 20.0, +) -> tuple[list[dict], list[Any]]: + """Run one full call as the browser would; returns (messages, downlink frames).""" + pc = RTCPeerConnection(configuration=RTCConfiguration(iceServers=[])) + pc.addTrack(AudioStreamTrack()) # silent mic + channel = pc.createDataChannel("events") + + messages: list[dict] = [] + frames: list[Any] = [] + ended = asyncio.Event() + + @channel.on("message") + def on_message(msg: Any) -> None: + data = json.loads(msg) + messages.append(data) + if data.get("type") == "session_ended": + ended.set() + + @pc.on("track") + def on_track(track: Any) -> None: + async def _pull() -> None: + with contextlib.suppress(Exception): + while True: + frames.append(await track.recv()) + + asyncio.ensure_future(_pull()) + + offer = await pc.createOffer() + await pc.setLocalDescription(offer) + resp = await asyncio.to_thread( + client.post, + "/voice/rtc", + json={"sdp": pc.localDescription.sdp, "type": "offer", "config": config or {}}, + ) + assert resp.status_code == 200, resp.text + await pc.setRemoteDescription(RTCSessionDescription(**resp.json())) + try: + await asyncio.wait_for(ended.wait(), timeout=timeout) + finally: + await pc.close() + return messages, frames + + +def _setup_env(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, *, responses: list[str] | None = None) -> None: + spec = _write_agent_module(tmp_path, responses=responses) + monkeypatch.setenv("TIMBAL_RUNNABLE", spec) + for k in VOICE_ENV_KEYS: + monkeypatch.delenv(k, raising=False) + monkeypatch.setenv("TIMBAL_STUN_URL", "") # loopback: host candidates only + + +class TestVoiceRtcRoundTrip: + async def test_full_call_over_webrtc(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + _setup_env(monkeypatch, tmp_path, responses=["Hi there!"]) + monkeypatch.setattr( + "timbal.voice.elevenlabs.ElevenLabsRealtimeSTT", + _make_delayed_stt_class( + [(0.2, TranscriptEvent(type="committed", text="Hello"))], + end_after=1.5, + ), + ) + monkeypatch.setattr( + "timbal.voice.elevenlabs.ElevenLabsStreamTTS", + _make_tts_class(chunk=b"\x01\x02" * 800, num_chunks=2), + ) + + app = create_app() + with TestClient(app) as client: + messages, frames = await _rtc_call(client, config={"turn_detector": "heuristic"}) + + types = [m["type"] for m in messages] + started = next(m for m in messages if m["type"] == "session_started") + assert started["transport"] == "webrtc" + assert started["playback_acks"] == "native" + assert "transcript_committed" in types + assert "agent_text_done" in types + assert types[-1] == "session_ended" + # TTS rides the audio track, never the data channel. + assert "audio" not in types + # Media actually flowed: the downlink track delivered paced frames. + assert len(frames) > 10 + + async def test_error_before_handshake_still_reaches_the_client( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + """A session that dies instantly must not vanish into a closed pc. + + The driver waits for the data channel before starting the session, so + even an immediate failure (here: STT connect raising, live: a bad API + key) is delivered as an error payload followed by session_ended. + """ + + class _BrokenSTT(SpeechToText): + def __init__(self, api_key: Any = None) -> None: + pass + + async def connect(self, config: AudioInputConfig) -> None: + raise RuntimeError("bad credentials") + + async def push_audio(self, chunk: bytes) -> None: + pass + + async def commit(self) -> None: + pass + + async def events(self): + return + yield + + async def close(self) -> None: + pass + + _setup_env(monkeypatch, tmp_path) + monkeypatch.setattr("timbal.voice.elevenlabs.ElevenLabsRealtimeSTT", _BrokenSTT) + monkeypatch.setattr("timbal.voice.elevenlabs.ElevenLabsStreamTTS", _make_tts_class()) + + app = create_app() + with TestClient(app) as client: + messages, _ = await _rtc_call(client) + + types = [m["type"] for m in messages] + assert "error" in types + assert types[-1] == "session_ended" + + +class TestVoiceRtcSignalingErrors: + def test_rejects_body_without_an_offer(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + _setup_env(monkeypatch, tmp_path) + app = create_app() + with TestClient(app) as client: + resp = client.post("/voice/rtc", json={"type": "offer"}) + assert resp.status_code == 400 + + async def test_rejects_offer_without_an_audio_track( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + _setup_env(monkeypatch, tmp_path) + app = create_app() + + pc = RTCPeerConnection(configuration=RTCConfiguration(iceServers=[])) + pc.createDataChannel("events") # SCTP only, no mic + offer = await pc.createOffer() + await pc.setLocalDescription(offer) + try: + with TestClient(app) as client: + resp = await asyncio.to_thread( + client.post, + "/voice/rtc", + json={"sdp": pc.localDescription.sdp, "type": "offer"}, + ) + finally: + await pc.close() + assert resp.status_code == 400 + assert "audio track" in resp.json()["error"] + + def test_501_without_aiortc(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + import sys + + _setup_env(monkeypatch, tmp_path) + # Simulate a missing extra: None in sys.modules makes `from aiortc + # import ...` raise ImportError inside the route. + monkeypatch.setitem(sys.modules, "aiortc", None) + app = create_app() + with TestClient(app) as client: + resp = client.post("/voice/rtc", json={"sdp": "v=0", "type": "offer"}) + assert resp.status_code == 501 + assert "timbal[voice]" in resp.json()["error"] diff --git a/python/tests/server/test_voice_ws.py b/python/tests/server/test_voice_ws.py index c327710a..bbd424f0 100644 --- a/python/tests/server/test_voice_ws.py +++ b/python/tests/server/test_voice_ws.py @@ -10,13 +10,14 @@ import asyncio import base64 +import json from collections.abc import AsyncIterator from pathlib import Path import pytest from fastapi.testclient import TestClient from timbal.server.http import create_app -from timbal.voice.session import ( +from timbal.voice import ( AudioInputConfig, AudioOutputConfig, SpeechToText, @@ -94,7 +95,7 @@ async def close(self) -> None: # --------------------------------------------------------------------------- -def _write_agent_module(tmp_path: Path, *, responses: list[str] | None = None) -> str: +def _write_agent_module(tmp_path: Path, *, responses: list[str] | None = None, extra: str = "") -> str: """Write a temp module with a TestModel Agent and return its import spec.""" resp_repr = repr(responses or ["Hello from agent!"]) mod = tmp_path / "voice_agent.py" @@ -102,6 +103,7 @@ def _write_agent_module(tmp_path: Path, *, responses: list[str] | None = None) - "from timbal import Agent\n" "from timbal.core.test_model import TestModel\n" f"agent = Agent(name='voice_test', model=TestModel(responses={resp_repr}), tools=[])\n" + + extra ) return f"{mod.resolve()}::agent" @@ -289,12 +291,14 @@ def test_session_started_advertises_playback_acks( app = create_app() with TestClient(app) as client: with client.websocket_connect("/voice/ws") as ws: - ws.send_json({}) + # Pinned, not defaulted: this asserts what a detector *without* an + # audio EOU advertises, so it must not follow the server default. + ws.send_json({"turn_detector": "heuristic"}) messages = _collect_ws_messages(ws) started = next(m for m in messages if m["type"] == "session_started") assert started["playback_acks"] == "recommended" - # Default heuristic detector has no audio EOU model → the local VAD + # The heuristic detector has no audio EOU model → the local VAD # endpointing fast path never arms, and session_started must say so. assert started["vad_endpointing"] is False @@ -530,13 +534,20 @@ def test_client_mode_overrides_server_default(self, monkeypatch, tmp_path: Path) ) assert started["turn_detector"] == "LexicalTurnDetector" - def test_default_is_heuristic_and_advertised(self, monkeypatch, tmp_path: Path) -> None: + # A holding detector: which one depends on whether timbal[voice] is installed + # (see test_voice_detector_choice.py, which pins that branch directly). The + # contract asserted here is that an unconfigured session gets a detector that + # *can* hold — the holdless heuristic splits paused utterances into several + # turns on any STT that endpoints on silence. + _HOLDS = ("LocalAudioTurnDetector", "LexicalTurnDetector") + + def test_default_holds_and_is_advertised(self, monkeypatch, tmp_path: Path) -> None: started = self._run_session(monkeypatch, tmp_path, {}) - assert started["turn_detector"] == "HeuristicTurnDetector" + assert started["turn_detector"] in self._HOLDS def test_non_string_client_value_is_ignored(self, monkeypatch, tmp_path: Path) -> None: started = self._run_session(monkeypatch, tmp_path, {"turn_detector": {"evil": True}}) - assert started["turn_detector"] == "HeuristicTurnDetector" + assert started["turn_detector"] in self._HOLDS def test_unknown_mode_name_falls_back_to_default(self, monkeypatch, tmp_path: Path) -> None: started = self._run_session(monkeypatch, tmp_path, {"turn_detector": "quantum"}) @@ -616,6 +627,71 @@ def test_malformed_early_frames_do_not_end_handshake( assert started["turn_detector"] == "ProviderTurnDetector" +class TestVoiceWsTurnTimeoutConfig: + """``turn_timeout_secs`` / ``turn_timeout_fallback`` must reach the session.""" + + def _capture_session_kwargs(self, monkeypatch: pytest.MonkeyPatch) -> dict: + import timbal.voice as voice_pkg + + real = voice_pkg.VoiceSession + captured: dict = {} + + class _CapturingSession(real): # type: ignore[misc, valid-type] + def __init__(self, *args, **kwargs): + captured.update(kwargs) + super().__init__(*args, **kwargs) + + monkeypatch.setattr(voice_pkg, "VoiceSession", _CapturingSession, raising=False) + return captured + + def test_turn_timeout_keys_are_plumbed_into_the_session( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ) -> None: + spec = _write_agent_module(tmp_path) + monkeypatch.setenv("TIMBAL_RUNNABLE", spec) + for k in VOICE_ENV_KEYS: + monkeypatch.delenv(k, raising=False) + + monkeypatch.setattr("timbal.voice.elevenlabs.ElevenLabsRealtimeSTT", _make_stt_class([])) + monkeypatch.setattr("timbal.voice.elevenlabs.ElevenLabsStreamTTS", _make_tts_class()) + captured = self._capture_session_kwargs(monkeypatch) + + app = create_app() + with TestClient(app) as client: + with client.websocket_connect("/voice/ws") as ws: + ws.send_json({"turn_timeout_secs": 12, "turn_timeout_fallback": "hold on"}) + _collect_ws_messages(ws) + + assert captured["turn_timeout_secs"] == 12.0 + assert captured["turn_timeout_fallback"] == "hold on" + + def test_bad_turn_timeout_value_keeps_the_session_default( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ) -> None: + """A non-numeric value must be dropped, not zero the watchdog out.""" + spec = _write_agent_module(tmp_path) + monkeypatch.setenv("TIMBAL_RUNNABLE", spec) + for k in VOICE_ENV_KEYS: + monkeypatch.delenv(k, raising=False) + + monkeypatch.setattr("timbal.voice.elevenlabs.ElevenLabsRealtimeSTT", _make_stt_class([])) + monkeypatch.setattr("timbal.voice.elevenlabs.ElevenLabsStreamTTS", _make_tts_class()) + captured = self._capture_session_kwargs(monkeypatch) + + app = create_app() + with TestClient(app) as client: + with client.websocket_connect("/voice/ws") as ws: + ws.send_json({"turn_timeout_secs": "soon"}) + messages = _collect_ws_messages(ws) + + assert "turn_timeout_secs" not in captured + assert any(m["type"] == "session_started" for m in messages) + + class TestVoiceWsAudioTransport: """Verify audio bytes survive the base64 round-trip over WS.""" @@ -642,10 +718,196 @@ def test_audio_chunks_are_valid_base64_pcm( app = create_app() with TestClient(app) as client: with client.websocket_connect("/voice/ws") as ws: - ws.send_json({}) + # This is a transport test: it needs the injected commit to start a + # turn immediately. A holding detector (the server default) would + # correctly park unpunctuated "test" for seconds and produce no + # audio at all. + ws.send_json({"turn_detector": "heuristic"}) messages = _collect_ws_messages(ws) audio_msgs = [m for m in messages if m["type"] == "audio"] assert len(audio_msgs) == 1 decoded = base64.b64decode(audio_msgs[0]["data"]) assert decoded == pcm_chunk + + +class TestVoiceWsRecording: + """Call recording: server-defaults-only config → MP3 + manifest per session.""" + + def _run_session(self, monkeypatch, spec: str, hello: dict, extra_env: dict | None = None) -> list[dict]: + monkeypatch.setenv("TIMBAL_RUNNABLE", spec) + for k in VOICE_ENV_KEYS: + monkeypatch.delenv(k, raising=False) + for k, v in (extra_env or {}).items(): + monkeypatch.setenv(k, v) + monkeypatch.setattr( + "timbal.voice.elevenlabs.ElevenLabsRealtimeSTT", + _make_stt_class([TranscriptEvent(type="committed", text="Hello")]), + ) + monkeypatch.setattr("timbal.voice.elevenlabs.ElevenLabsStreamTTS", _make_tts_class()) + # Holding detectors (server default) park unpunctuated "Hello" and + # can end the session with zero TTS — empty MP3 close then flakes on + # Windows. Pin heuristic like the audio transport tests. + hello = {"turn_detector": "heuristic", **hello} + app = create_app() + with TestClient(app) as client: + with client.websocket_connect("/voice/ws") as ws: + ws.send_json(hello) + return _collect_ws_messages(ws) + + def test_server_configured_recording_writes_audio_and_manifest( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ) -> None: + pytest.importorskip("av", reason="timbal[voice] extra (av) not installed") + rec_dir = tmp_path / "recordings" + spec = _write_agent_module( + tmp_path, + responses=["Hi there!"], + extra=f"agent.voice_config = {{'recording': {{'dir': {str(rec_dir)!r}}}}}\n", + ) + messages = self._run_session(monkeypatch, spec, {"language": "en"}) + + started = next(m for m in messages if m["type"] == "session_started") + session_id = started["session_id"] + assert session_id + + # Files are finalized before session_ended reaches the client. + assert (rec_dir / f"{session_id}.mp3").exists() + manifest = json.loads((rec_dir / f"{session_id}.json").read_text()) + assert manifest["session_id"] == session_id + assert manifest["meta"]["transport"] == "websocket" + assert [e["role"] for e in manifest["transcript"]] == ["user", "assistant"] + assert all(e["offset_ms"] >= 0 for e in manifest["transcript"]) + + # The wire transcript carries the same timing info. + transcript = next(m for m in messages if m["type"] == "session_transcript") + assert transcript["started_at"] == pytest.approx(manifest["started_at"]) + assert all("offset_ms" in e for e in transcript["entries"]) + + def test_client_hello_cannot_switch_recording_on( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ) -> None: + rec_dir = tmp_path / "recordings" + spec = _write_agent_module(tmp_path) # no recording in server defaults + messages = self._run_session( + monkeypatch, spec, {"recording": {"dir": str(rec_dir)}} + ) + assert any(m["type"] == "session_ended" for m in messages) + assert not rec_dir.exists() + + def test_env_var_enables_recording( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ) -> None: + pytest.importorskip("av", reason="timbal[voice] extra (av) not installed") + rec_dir = tmp_path / "recordings" + spec = _write_agent_module(tmp_path) + monkeypatch.setenv("TIMBAL_RUNNABLE", spec) + for k in VOICE_ENV_KEYS: + monkeypatch.delenv(k, raising=False) + monkeypatch.setenv("TIMBAL_VOICE_RECORDING_DIR", str(rec_dir)) + monkeypatch.setattr( + "timbal.voice.elevenlabs.ElevenLabsRealtimeSTT", + _make_stt_class([TranscriptEvent(type="committed", text="Hello")]), + ) + monkeypatch.setattr("timbal.voice.elevenlabs.ElevenLabsStreamTTS", _make_tts_class()) + app = create_app() + with TestClient(app) as client: + with client.websocket_connect("/voice/ws") as ws: + ws.send_json({"language": "en", "turn_detector": "heuristic"}) + messages = _collect_ws_messages(ws) + + started = next(m for m in messages if m["type"] == "session_started") + assert (rec_dir / f"{started['session_id']}.mp3").exists() + + def test_env_knobs_set_layout_and_bitrate( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ) -> None: + pytest.importorskip("av", reason="timbal[voice] extra (av) not installed") + rec_dir = tmp_path / "recordings" + spec = _write_agent_module(tmp_path) + messages = self._run_session( + monkeypatch, spec, {"language": "en"}, + extra_env={ + "TIMBAL_VOICE_RECORDING_DIR": str(rec_dir), + "TIMBAL_VOICE_RECORDING_LAYOUT": "split", + "TIMBAL_VOICE_RECORDING_BITRATE_KBPS": "64", + }, + ) + started = next(m for m in messages if m["type"] == "session_started") + manifest = json.loads((rec_dir / f"{started['session_id']}.json").read_text()) + assert manifest["audio"]["layout"] == "split" + assert manifest["audio"]["bitrate_kbps"] == 64 + + def test_user_voice_config_wins_over_env_knobs( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ) -> None: + pytest.importorskip("av", reason="timbal[voice] extra (av) not installed") + rec_dir = tmp_path / "recordings" + spec = _write_agent_module( + tmp_path, + extra=f"agent.voice_config = {{'recording': {{'dir': {str(rec_dir)!r}, 'layout': 'mixed'}}}}\n", + ) + messages = self._run_session( + monkeypatch, spec, {"language": "en"}, + extra_env={"TIMBAL_VOICE_RECORDING_LAYOUT": "split"}, + ) + started = next(m for m in messages if m["type"] == "session_started") + manifest = json.loads((rec_dir / f"{started['session_id']}.json").read_text()) + assert manifest["audio"]["layout"] == "mixed" + + def test_platform_identity_env_is_stamped_into_manifest_meta( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ) -> None: + pytest.importorskip("av", reason="timbal[voice] extra (av) not installed") + rec_dir = tmp_path / "recordings" + spec = _write_agent_module(tmp_path) + messages = self._run_session( + monkeypatch, spec, {"language": "en"}, + extra_env={ + "TIMBAL_VOICE_RECORDING_DIR": str(rec_dir), + "TIMBAL_ORG_ID": "org-9", + "TIMBAL_PROJECT_ID": "proj-7", + "TIMBAL_PROJECT_ENV_ID": "env-3", + "TIMBAL_APP_ID": "app-1", + "TIMBAL_PROJECT_REV": "rev-42", + }, + ) + started = next(m for m in messages if m["type"] == "session_started") + # Identity lands in the manifest (self-describing files for sweeper + # ingest), not in the wire payload. + assert "org_id" not in started + manifest = json.loads((rec_dir / f"{started['session_id']}.json").read_text()) + assert manifest["meta"]["org_id"] == "org-9" + assert manifest["meta"]["project_id"] == "proj-7" + assert manifest["meta"]["project_env_id"] == "env-3" + assert manifest["meta"]["app_id"] == "app-1" + assert manifest["meta"]["project_rev"] == "rev-42" + assert manifest["meta"]["transport"] == "websocket" # session meta still wins/merges + + def test_no_tmp_manifest_left_behind( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ) -> None: + """The manifest write is atomic (tmp + rename) — sweepers key on json presence.""" + pytest.importorskip("av", reason="timbal[voice] extra (av) not installed") + rec_dir = tmp_path / "recordings" + spec = _write_agent_module(tmp_path) + self._run_session( + monkeypatch, spec, {"language": "en"}, + extra_env={"TIMBAL_VOICE_RECORDING_DIR": str(rec_dir)}, + ) + assert not list(rec_dir.glob("*.tmp")) + assert len(list(rec_dir.glob("*.json"))) == 1 diff --git a/python/tests/server/voice_env.py b/python/tests/server/voice_env.py index fce58c97..f06c22a7 100644 --- a/python/tests/server/voice_env.py +++ b/python/tests/server/voice_env.py @@ -10,4 +10,8 @@ "ELEVENLABS_VOICE_ID", "TIMBAL_VOICE_ID", "TIMBAL_VOICE_LANGUAGE", + "TIMBAL_VOICE_RECORDING_DIR", + "TIMBAL_VOICE_RECORDING_LAYOUT", + "TIMBAL_VOICE_RECORDING_BITRATE_KBPS", + "TIMBAL_VOICE_RECORDING_UPLOAD", ) diff --git a/python/tests/voice/test_deepgram.py b/python/tests/voice/test_deepgram.py index 2409709e..240974ef 100644 --- a/python/tests/voice/test_deepgram.py +++ b/python/tests/voice/test_deepgram.py @@ -11,19 +11,19 @@ from urllib.parse import parse_qs, urlparse import pytest +from timbal.voice import AudioInputConfig from timbal.voice.deepgram import ( + _AUDIO_FLUSH_BYTES, DEFAULT_FLUX_MODEL, DEFAULT_NOVA_MODEL, DeepgramFluxSTT, DeepgramNovaSTT, - _AUDIO_FLUSH_BYTES, effective_stt_model, is_flux_model, resolve_stt, stt_provider_id, ) from timbal.voice.elevenlabs import ElevenLabsRealtimeSTT -from timbal.voice.session import AudioInputConfig def _drain(stt) -> list: diff --git a/python/tests/voice/test_endpointing.py b/python/tests/voice/test_endpointing.py index 21954d29..7474a5d4 100644 --- a/python/tests/voice/test_endpointing.py +++ b/python/tests/voice/test_endpointing.py @@ -17,19 +17,17 @@ from timbal.core.test_model import TestModel from timbal.voice import ( AudioInputConfig, + AudioOutputConfig, LocalAudioTurnDetector, + SpeechToText, + TextToSpeech, + TranscriptEvent, VadEndpointer, VoiceSession, endpointing_delay, ) from timbal.voice.eou import AudioEouModel from timbal.voice.metrics import TurnMetricsEvent -from timbal.voice.session import ( - AudioOutputConfig, - SpeechToText, - TextToSpeech, - TranscriptEvent, -) # --------------------------------------------------------------------------- # Fakes diff --git a/python/tests/voice/test_namo.py b/python/tests/voice/test_namo.py index cfa95094..65da251f 100644 --- a/python/tests/voice/test_namo.py +++ b/python/tests/voice/test_namo.py @@ -16,10 +16,10 @@ from timbal.voice import turn_detection as turn_detection_module # noqa: E402 from timbal.voice.eou import PunctuationEouPredictor # noqa: E402 from timbal.voice.namo import ( # noqa: E402 + _ACK_FORCE_P, DEFAULT_REPO_ID, MULTILINGUAL_REPO_ID, NamoTextEouPredictor, - _ACK_FORCE_P, _prepare_text_for_namo, ) diff --git a/python/tests/voice/test_provider_connection_loss.py b/python/tests/voice/test_provider_connection_loss.py new file mode 100644 index 00000000..dbc8393b --- /dev/null +++ b/python/tests/voice/test_provider_connection_loss.py @@ -0,0 +1,174 @@ +"""What happens to a call when a provider socket dies mid-conversation. + +Neither STT adapter reconnects, so a dropped socket ends the call. Two +invariants make that ending honest instead of silent, and both are pinned here +for Deepgram and ElevenLabs alike: + +* The receive loop must enqueue its end-of-stream sentinel on *every* exit + path. ``VoiceSession._process_stt_events`` closes the session when + ``stt.events()`` completes, and nothing else notices a dead STT. Drop the + sentinel and the caller keeps talking to a session that can no longer hear + them, indefinitely, with no error. + +* An *unrequested* close — any code, 1000/1001 included — must surface an + error event. The provider hanging up mid-call is not the user hanging up; + before this was pinned, a normal-code close ended the session with a + debug-level log as its only trace. Requested closes (``close()`` sets + ``_stop`` before touching the socket) stay silent. + +A dead-but-open TCP connection is the websockets library's problem, not ours: +both adapters keep the library's default ping keepalive (20s interval, 20s +timeout), so a peer that stops answering pings surfaces in the receive loop as +``ConnectionClosedError`` 1011 and flows through the same paths pinned here. +""" + +from __future__ import annotations + +import asyncio +from typing import Any + +import pytest +from timbal.voice.deepgram import _DeepgramSTTBase +from timbal.voice.elevenlabs import ElevenLabsRealtimeSTT +from websockets.exceptions import ConnectionClosedError, ConnectionClosedOK +from websockets.frames import Close + + +class _ClosingWs: + """A socket that yields nothing and reports itself closed on first use.""" + + def __init__(self, error: BaseException) -> None: + self._error = error + self.sent: list[Any] = [] + + def __aiter__(self) -> _ClosingWs: + return self + + async def __anext__(self) -> str: + raise self._error + + async def send(self, _data: Any) -> None: + # Real sockets raise here too once closed; the adapters swallow it. + raise self._error + + +class _StubDeepgramSTT(_DeepgramSTTBase): + """Concrete subclass: the base leaves ``_handle_message`` abstract.""" + + def _build_uri(self, _config: Any) -> str: + return "wss://example.invalid/listen" + + async def _handle_message(self, _msg: dict[str, Any]) -> None: + return None + + async def commit(self) -> None: + await self._flush_audio() + await self._send_json({"type": "Finalize"}) + + +def _deepgram(error: BaseException) -> _StubDeepgramSTT: + stt = _StubDeepgramSTT(api_key="test-key") + stt._ws = _ClosingWs(error) + return stt + + +def _elevenlabs(error: BaseException) -> ElevenLabsRealtimeSTT: + stt = ElevenLabsRealtimeSTT(api_key="test-key") + stt._ws = _ClosingWs(error) + return stt + + +PROVIDERS = ( + pytest.param(_deepgram, id="deepgram"), + pytest.param(_elevenlabs, id="elevenlabs"), +) +CLOSES = ( + pytest.param(ConnectionClosedOK(Close(1000, ""), None), id="1000-normal"), + pytest.param(ConnectionClosedOK(Close(1001, "going away"), None), id="1001-going-away"), + pytest.param(ConnectionClosedError(Close(1011, "internal error"), None), id="1011-internal"), + pytest.param(ConnectionClosedError(Close(1006, "abnormal"), None), id="1006-abnormal"), +) + + +class TestSttStreamAlwaysTerminates: + """``events()`` must complete when the socket dies, whatever the close code.""" + + @pytest.mark.parametrize("make_stt", PROVIDERS) + @pytest.mark.parametrize("error", CLOSES) + async def test_receive_loop_enqueues_the_sentinel(self, make_stt, error: BaseException) -> None: + stt = make_stt(error) + await stt._receive_loop() + + drained = _drain_queue(stt) + missing = ( + f"no end-of-stream sentinel after {type(error).__name__}: events() would await " + f"forever and the session would run on with a dead STT (queued: {drained})" + ) + assert drained and drained[-1] is None, missing + + @pytest.mark.parametrize("make_stt", PROVIDERS) + @pytest.mark.parametrize("error", CLOSES) + async def test_events_completes_and_raises_on_unrequested_close( + self, make_stt, error: BaseException + ) -> None: + """Every unrequested close — normal codes included — is an error, not a shrug.""" + stt = make_stt(error) + receiver = asyncio.create_task(stt._receive_loop()) + + async def _drain() -> None: + with pytest.raises(RuntimeError, match="STT connection closed"): + async for _ in stt.events(): + pass + + # A hang here is the bug this file exists to catch, so bound the wait. + await asyncio.wait_for(_drain(), timeout=2.0) + await asyncio.wait_for(receiver, timeout=2.0) + + +class TestRequestedCloseStaysSilent: + """Our own ``close()`` must not manufacture an error out of its side effects. + + ``close()`` sets ``_stop`` before touching the socket; the receive loop + reads that flag to tell a teardown we asked for apart from a provider + hanging up on us. An error event here races session close and can surface + a spurious SessionError on a perfectly normal hangup. + """ + + @pytest.mark.parametrize("make_stt", PROVIDERS) + async def test_no_error_event_when_stop_was_requested(self, make_stt) -> None: + stt = make_stt(ConnectionClosedOK(Close(1000, ""), None)) + stt._stop.set() + await stt._receive_loop() + drained = _drain_queue(stt) + assert [e for e in drained if e is not None] == [] + assert drained[-1] is None # the sentinel still terminates events() + + +class TestSendsSurviveADeadSocket: + """Audio/control writes to a closed socket must not raise into their callers. + + Silent dropping is only acceptable because the receive loop's sentinel is + already tearing the session down. The callers that must not see a raise: + the session's audio forwarder (``push_audio``), the adapters' periodic + flushers, and the VAD endpointing fast path (``commit()``). + """ + + @pytest.mark.parametrize("make_stt", PROVIDERS) + async def test_push_audio_and_flush(self, make_stt) -> None: + stt = make_stt(ConnectionClosedOK(Close(1000, ""), None)) + # Enough PCM to cross Deepgram's eager-send threshold (2560 bytes). + await stt.push_audio(b"\x00\x01" * 4096) + await stt._flush_audio() + + @pytest.mark.parametrize("make_stt", PROVIDERS) + async def test_commit(self, make_stt) -> None: + stt = make_stt(ConnectionClosedOK(Close(1000, ""), None)) + await stt.push_audio(b"\x00\x01" * 64) + await stt.commit() + + +def _drain_queue(stt: Any) -> list[Any]: + out = [] + while not stt._queue.empty(): + out.append(stt._queue.get_nowait()) + return out diff --git a/python/tests/voice/test_recording.py b/python/tests/voice/test_recording.py new file mode 100644 index 00000000..7c706fbf --- /dev/null +++ b/python/tests/voice/test_recording.py @@ -0,0 +1,324 @@ +"""CallRecorder: timeline placement, mixing, truncation, and decode round-trips. + +Every assertion decodes the actual MP3 back and checks *where* energy sits on +the timeline — that's the recorder's whole job. Tolerances are generous +(100ms guard bands) because lame adds encoder delay/padding. +""" + +from __future__ import annotations + +import json +import math +from pathlib import Path + +import pytest + +pytest.importorskip("av", reason="timbal[voice] extra (av) not installed") +np = pytest.importorskip("numpy", reason="timbal[voice] extra (numpy) not installed") + +import av # noqa: E402 +from timbal.voice.recording import CallRecorder, build_manifest # noqa: E402 + +SR = 16_000 + + +def _tone(secs: float, freq: float = 440.0, amp: int = 12000) -> bytes: + n = int(secs * SR) + buf = bytearray() + for i in range(n): + v = int(amp * math.sin(2 * math.pi * freq * i / SR)) + buf += v.to_bytes(2, "little", signed=True) + return bytes(buf) + + +def _silence(secs: float) -> bytes: + return b"\x00" * (int(secs * SR) * 2) + + +def _decode(path: Path) -> np.ndarray: + """Decode to float32, shape (channels, samples).""" + chunks = [] + with av.open(str(path)) as container: + for frame in container.decode(audio=0): + arr = frame.to_ndarray() + if arr.dtype == np.int16: + arr = arr.astype(np.float32) / 32768.0 + if not frame.format.is_planar and frame.layout.nb_channels > 1: + arr = arr.reshape(-1, frame.layout.nb_channels).T + chunks.append(arr.astype(np.float32)) + return np.concatenate(chunks, axis=1) + + +def _rms(x: np.ndarray) -> float: + return float(np.sqrt(np.mean(np.square(x)))) if x.size else 0.0 + + +def _span(audio: np.ndarray, ch: int, t0: float, t1: float) -> np.ndarray: + return audio[ch, int(t0 * SR) : int(t1 * SR)] + + +class TestTimeline: + def test_agent_audio_lands_where_the_mic_clock_said(self, tmp_path: Path) -> None: + rec = CallRecorder(tmp_path / "call.mp3", sample_rate=SR) + # 1s of quiet call, then the agent speaks 0.5s while the caller is silent. + rec.add_mic(_silence(1.0)) + rec.add_agent(_tone(0.5)) + rec.add_mic(_silence(1.0)) + assert rec.close(manifest=None) is not None + + audio = _decode(tmp_path / "call.mp3") + assert audio.shape[0] == 1 # mixed => mono + assert abs(audio.shape[1] / SR - 2.0) < 0.15 + assert _rms(_span(audio, 0, 0.1, 0.9)) < 0.01 # before: silence + assert _rms(_span(audio, 0, 1.1, 1.4)) > 0.1 # agent speaking + assert _rms(_span(audio, 0, 1.6, 1.9)) < 0.01 # after: silence again + + def test_mixed_layout_sums_both_voices(self, tmp_path: Path) -> None: + rec = CallRecorder(tmp_path / "call.mp3", sample_rate=SR) + rec.add_agent(_tone(1.0, freq=300)) + rec.add_mic(_tone(1.0, freq=800)) # both talk over the same second + rec.close() + + audio = _decode(tmp_path / "call.mp3") + assert audio.shape[0] == 1 + assert _rms(audio[0]) > 0.3 # both voices present, no channel lost + + def test_split_layout_keeps_sides_separate(self, tmp_path: Path) -> None: + rec = CallRecorder(tmp_path / "call.mp3", sample_rate=SR, layout="split") + rec.add_agent(_tone(1.0)) + rec.add_mic(_silence(1.0)) # caller silent while the agent talks + rec.close() + + audio = _decode(tmp_path / "call.mp3") + assert audio.shape[0] == 2 + assert _rms(audio[0]) < 0.02 # left = caller: silent + assert _rms(audio[1]) > 0.1 # right = agent: tone + + def test_close_drains_agent_audio_past_the_mic_timeline(self, tmp_path: Path) -> None: + """Call ends while TTS is still playing: the tail belongs in the file.""" + rec = CallRecorder(tmp_path / "call.mp3", sample_rate=SR) + rec.add_mic(_silence(0.2)) + rec.add_agent(_tone(1.0)) + rec.close() + + audio = _decode(tmp_path / "call.mp3") + assert abs(audio.shape[1] / SR - 1.2) < 0.15 + + +class TestBargeInTruncation: + def test_dropped_tail_never_reaches_the_file(self, tmp_path: Path) -> None: + rec = CallRecorder(tmp_path / "call.mp3", sample_rate=SR) + rec.add_agent(_tone(2.0)) # a long reply, synthesized instantly + rec.add_mic(_silence(0.5)) # 0.5s heard... + emitted = int(2.0 * SR) * 2 + heard = int(0.5 * SR) * 2 + dropped = rec.drop_agent_tail(emitted - heard) # ...then barge-in + assert dropped == emitted - heard + rec.add_mic(_silence(1.0)) # call continues + rec.close() + + audio = _decode(tmp_path / "call.mp3") + assert abs(audio.shape[1] / SR - 1.5) < 0.15 + assert _rms(_span(audio, 0, 0.1, 0.4)) > 0.1 # heard prefix present + assert _rms(_span(audio, 0, 0.7, 1.4)) < 0.01 # unheard tail gone + + def test_drop_clamps_to_the_queue(self, tmp_path: Path) -> None: + rec = CallRecorder(tmp_path / "call.mp3", sample_rate=SR) + rec.add_agent(_tone(0.1)) + assert rec.drop_agent_tail(10**9) == int(0.1 * SR) * 2 + assert rec.agent_pending_bytes == 0 + rec.close() + + +class TestRobustness: + def test_odd_sized_chunks_stay_sample_aligned(self, tmp_path: Path) -> None: + rec = CallRecorder(tmp_path / "call.mp3", sample_rate=SR) + tone = _tone(0.5) + rec.add_mic(tone[:333]) + rec.add_mic(tone[333:]) + rec.close() + audio = _decode(tmp_path / "call.mp3") + assert abs(audio.shape[1] / SR - 0.5) < 0.1 + + def test_close_is_idempotent_and_feeds_after_close_are_noops(self, tmp_path: Path) -> None: + rec = CallRecorder(tmp_path / "call.mp3", sample_rate=SR) + rec.add_mic(_silence(0.2)) + first = rec.close() + rec.add_mic(_silence(1.0)) + rec.add_agent(_tone(1.0)) + assert rec.close() is first + + def test_empty_close_still_leaves_a_playable_file(self, tmp_path: Path) -> None: + """Zero-packet MP3 close deletes the file on Windows; pad silence.""" + path = tmp_path / "empty.mp3" + rec = CallRecorder(path, sample_rate=SR) + result = rec.close() + assert result is not None + assert path.exists() + assert path.stat().st_size > 0 + assert result.duration_secs > 0 + + def test_rejects_bad_layout(self, tmp_path: Path) -> None: + with pytest.raises(ValueError, match="layout"): + CallRecorder(tmp_path / "call.mp3", layout="both") # type: ignore[arg-type] + + +class TestSessionIntegration: + async def test_session_writes_recording_manifest_and_fires_on_saved(self, tmp_path: Path) -> None: + import asyncio + from contextlib import aclosing + + from timbal import Agent + from timbal.core.test_model import TestModel + from timbal.voice import ( + AgentTextDone, + SessionStarted, + TranscriptEvent, + VoiceSession, + VoiceSessionEvent, + ) + + from .test_session import DelayedMockSTT, MockTTS + + saved: list = [] + + async def _on_saved(result) -> None: + saved.append(result) + + recorder = CallRecorder(tmp_path / "call.mp3", sample_rate=SR, on_saved=_on_saved) + stt = DelayedMockSTT() + session = VoiceSession( + agent=Agent(name="rec", model=TestModel(responses=["Hi there!"]), tools=[]), + stt=stt, + tts=MockTTS(chunk=b"\x01\x02" * 800, num_chunks=2), + recorder=recorder, + session_id="fixed-session-id", + ) + session.recording_meta = {"transport": "test", "model": "test/model"} + + events: list[VoiceSessionEvent] = [] + + async def _mic() -> object: + # A short burst of real mic PCM so the caller side has timeline. + for _ in range(5): + yield b"\x00" * 640 + await asyncio.sleep(0.01) + while not any(isinstance(e, AgentTextDone) for e in events): + await asyncio.sleep(0.01) + await stt.finish() + + async def _drive() -> None: + while not any(isinstance(e, SessionStarted) for e in events): + await asyncio.sleep(0.01) + await stt.inject(TranscriptEvent(type="committed", text="Hello")) + + async def _run() -> None: + async with aclosing(session.run(_mic())) as stream: + driver = asyncio.create_task(_drive()) + async for ev in stream: + events.append(ev) + await driver + + await asyncio.wait_for(_run(), timeout=10) + + audio_path = tmp_path / "call.mp3" + manifest_path = tmp_path / "call.json" + assert audio_path.exists() and manifest_path.exists() + assert saved and saved[0].audio_path == audio_path + + manifest = json.loads(manifest_path.read_text()) + assert manifest["session_id"] == "fixed-session-id" + assert manifest["meta"] == {"transport": "test", "model": "test/model"} + roles = [e["role"] for e in manifest["transcript"]] + assert roles == ["user", "assistant"] + assert all(e["offset_ms"] >= 0 for e in manifest["transcript"]) + assert manifest["audio"]["duration_secs"] > 0 + assert manifest["turns"] # per-turn latency metrics for the UI chips + + decoded = _decode(audio_path) + assert decoded.shape[1] > 0 + + +class TestSessionBargeIn: + async def test_interrupt_drops_the_unheard_tail_from_the_recording(self, tmp_path: Path) -> None: + import asyncio + from contextlib import aclosing + + from timbal import Agent + from timbal.core.test_model import TestModel + from timbal.voice import SessionStarted, TranscriptEvent, VoiceSession + + from .test_session import DelayedMockSTT, MockTTS + + recorder = CallRecorder(tmp_path / "call.mp3", sample_rate=SR) + stt = DelayedMockSTT() + # 4s of agent audio emitted near-instantly (TTS is faster than real time). + emitted_secs = 4.0 + session = VoiceSession( + agent=Agent(name="rec", model=TestModel(responses=["A very long reply"]), tools=[]), + stt=stt, + tts=MockTTS(chunk=b"\x01\x02" * (SR // 2), num_chunks=int(emitted_secs)), + recorder=recorder, + ) + + started = asyncio.Event() + + async def _mic() -> object: + await started.wait() + yield b"\x00" * 640 + + async def _drive() -> None: + while recorder.agent_pending_bytes == 0: + await asyncio.sleep(0.01) + # Barge-in almost immediately: nearly nothing was heard yet. + await session.interrupt() + assert recorder.agent_pending_bytes < int(1.0 * SR) * 2 + await stt.finish() + + async def _run() -> None: + async with aclosing(session.run(_mic())) as stream: + driver: asyncio.Task | None = None + async for ev in stream: + if isinstance(ev, SessionStarted): + started.set() + await stt.inject(TranscriptEvent(type="committed", text="Hello")) + driver = asyncio.create_task(_drive()) + if driver is not None: + await driver + + await asyncio.wait_for(_run(), timeout=10) + + audio = _decode(tmp_path / "call.mp3") + duration = audio.shape[1] / SR + assert 0 < duration < emitted_secs / 2 # the unheard tail never landed + + +class TestManifest: + def test_manifest_written_next_to_the_audio_with_refreshed_duration(self, tmp_path: Path) -> None: + from timbal.voice import TranscriptEntry + + rec = CallRecorder(tmp_path / "call.mp3", sample_rate=SR) + rec.add_mic(_silence(0.5)) + rec.add_agent(_tone(0.5)) # queued; drained by close() after manifest build + t0 = 1_000_000.0 + manifest = build_manifest( + session_id="abc123", + started_at=t0, + meta={"transport": "webrtc", "model": "test/model"}, + transcript=[ + TranscriptEntry(role="user", text="Hello", timestamp=t0 + 1.2), + TranscriptEntry(role="assistant", text="Hi!", timestamp=t0 + 2.5), + ], + turns=[], + recorder=rec, + ) + result = rec.close(manifest=manifest) + assert result is not None and result.manifest_path is not None + + saved = json.loads(result.manifest_path.read_text()) + assert saved["session_id"] == "abc123" + assert saved["meta"]["transport"] == "webrtc" + assert [e["offset_ms"] for e in saved["transcript"]] == [1200, 2500] + # 0.5s mic + 0.5s tail drained at close — not the pre-drain 0.5s. + assert abs(saved["audio"]["duration_secs"] - 1.0) < 0.1 + assert abs(result.duration_secs - 1.0) < 0.1 diff --git a/python/tests/voice/test_session.py b/python/tests/voice/test_session.py index 741f1b83..f8d14103 100644 --- a/python/tests/voice/test_session.py +++ b/python/tests/voice/test_session.py @@ -10,9 +10,7 @@ import pytest from timbal import Agent from timbal.core.test_model import TestModel -from timbal.voice.metrics import TurnMetricsEvent -from timbal.voice.playback import PlaybackTracker -from timbal.voice.session import ( +from timbal.voice import ( AgentTextDelta, AgentTextDone, AudioInputConfig, @@ -29,8 +27,10 @@ TTSStream, VoiceSession, VoiceSessionEvent, - _strip_markdown, ) +from timbal.voice.metrics import TurnMetricsEvent +from timbal.voice.playback import PlaybackTracker +from timbal.voice.session import _strip_markdown from timbal.voice.turn_detection import ( CommitAction, CommitDecision, @@ -535,6 +535,175 @@ async def test_stt_error_event_becomes_session_error(self) -> None: assert "STT" in errors[0].message +# --------------------------------------------------------------------------- +# Tests: agent-turn timeout +# --------------------------------------------------------------------------- + + +class _HungAgent: + """Duck-typed agent whose event stream never yields — models a hung LLM.""" + + def __call__(self, **_kwargs: object) -> AsyncIterator[object]: + return self._gen() + + async def _gen(self) -> AsyncIterator[object]: + await asyncio.Event().wait() + if False: # pragma: no cover — keep this an async generator + yield None + + +class _HungTTS(TextToSpeech): + """TTS whose synthesize never yields — models a hung synthesis provider.""" + + def __init__(self) -> None: + self.synthesize_calls: list[str] = [] + + async def connect(self, config: AudioOutputConfig) -> None: + pass + + async def synthesize(self, text: str) -> AsyncIterator[bytes]: + self.synthesize_calls.append(text) + await asyncio.Event().wait() + if False: # pragma: no cover — keep this an async generator + yield b"" + + async def close(self) -> None: + pass + + +class TestTurnTimeout: + async def test_hung_agent_times_out_speaks_fallback_and_stays_open(self) -> None: + """A silent hung turn must not leave the caller waiting forever. + + Expect: SessionError, spoken fallback, session accepts a second turn. + """ + stt = DelayedMockSTT() + tts = MockTTS() + session = VoiceSession( + agent=_HungAgent(), # type: ignore[arg-type] + stt=stt, + tts=tts, + turn_timeout_secs=0.15, + turn_timeout_fallback="Sorry, try again.", + ) + # Swap in a real agent for the *second* turn after the timeout. + live_agent = Agent(name="t", model=TestModel(responses=["Recovered."]), tools=[]) + + events: list[VoiceSessionEvent] = [] + + async def _empty_audio() -> AsyncIterator[bytes]: + return + yield # noqa: RET504 + + async def _drive() -> None: + while not any(isinstance(e, SessionStarted) for e in events): + await asyncio.sleep(0.01) + + await stt.inject(TranscriptEvent(type="committed", text="Are you there?")) + while not any(isinstance(e, SessionError) and "timed out" in e.message for e in events): + await asyncio.sleep(0.01) + while not any(isinstance(e, AgentTextDone) for e in events): + await asyncio.sleep(0.01) + # AgentTextDone is emitted while the turn's ``finally`` is still + # retiring it. A commit landing in that window sees an active + # assistant and gets dismissed as a trailing crumb of "Are you + # there?" — correct product behavior, wrong test timing. Wait for + # the turn task itself. + while session._current_turn_task is not None and not session._current_turn_task.done(): + await asyncio.sleep(0.01) + + session.agent = live_agent + await stt.inject(TranscriptEvent(type="committed", text="Hello again")) + while sum(1 for e in events if isinstance(e, AgentTextDone)) < 2: + await asyncio.sleep(0.01) + await stt.finish() + + async def _run() -> None: + async with aclosing(session.run(_empty_audio())) as stream: + driver = asyncio.create_task(_drive()) + async for ev in stream: + events.append(ev) + await driver + + await asyncio.wait_for(_run(), timeout=5) + + errors = [e for e in events if isinstance(e, SessionError)] + assert any("timed out" in e.message for e in errors) + assert "Sorry, try again." in tts.synthesized_texts + assert any(isinstance(e, AgentTextDone) and e.text == "Sorry, try again." for e in events) + assert session.transcript[-1].role == "assistant" + assert "Recovered" in session.transcript[-1].text + + async def test_hung_tts_cannot_hang_the_timeout_rescue(self) -> None: + """When the TTS is the hung component, the fallback speak is bounded too. + + An unbounded rescue would hang exactly like the turn it rescues: agent + times out, the apology goes to the same dead TTS, and the caller waits + forever anyway. Expect: rescue attempted, given up, session closable. + """ + stt = DelayedMockSTT() + tts = _HungTTS() + session = VoiceSession( + agent=_HungAgent(), # type: ignore[arg-type] + stt=stt, + tts=tts, + turn_timeout_secs=0.15, + ) + events: list[VoiceSessionEvent] = [] + + async def _empty_audio() -> AsyncIterator[bytes]: + return + yield # noqa: RET504 + + async def _drive() -> None: + while not any(isinstance(e, SessionStarted) for e in events): + await asyncio.sleep(0.01) + await stt.inject(TranscriptEvent(type="committed", text="Hello?")) + while not any(isinstance(e, SessionError) for e in events): + await asyncio.sleep(0.01) + # The bounded rescue expires after min(turn_timeout_secs, 10s). + while not tts.synthesize_calls: + await asyncio.sleep(0.01) + await asyncio.sleep(0.3) + await stt.finish() + + async def _run() -> None: + async with aclosing(session.run(_empty_audio())) as stream: + driver = asyncio.create_task(_drive()) + async for ev in stream: + events.append(ev) + await driver + + await asyncio.wait_for(_run(), timeout=5) + + assert any(isinstance(e, SessionError) and "timed out" in e.message for e in events) + assert tts.synthesize_calls # rescue was attempted against the dead TTS + assert not any(isinstance(e, AgentTextDone) for e in events) # and abandoned + assert not session.transcript or session.transcript[-1].role != "assistant" + + async def test_timeout_disabled_when_secs_non_positive(self) -> None: + """``turn_timeout_secs <= 0`` must not arm a deadline.""" + session, _, _ = _make_session( + stt_script=[TranscriptEvent(type="committed", text="Hi")], + responses=["Hello!"], + ) + session.turn_timeout_secs = 0 + events = await _collect_events(session) + assert not any(isinstance(e, SessionError) for e in events) + assert any(isinstance(e, AgentTextDone) for e in events) + + async def test_fast_turn_unaffected_by_default_timeout(self) -> None: + session, _, tts = _make_session( + stt_script=[TranscriptEvent(type="committed", text="Hi")], + responses=["Hello!"], + ) + assert session.turn_timeout_secs > 0 + events = await _collect_events(session) + assert not any(isinstance(e, SessionError) for e in events) + assert any(isinstance(e, AgentTextDone) for e in events) + assert tts.synthesized_texts # real reply, not the timeout fallback + + # --------------------------------------------------------------------------- # Tests: _strip_markdown # --------------------------------------------------------------------------- @@ -726,6 +895,7 @@ async def test_final_metrics_persisted_by_serializing_provider(self, tmp_path) - assert len(records) == 1 persisted = records[0]["spans"][0]["metadata"]["voice_turn_metrics"] assert persisted == live.model_dump() + assert live.run_id == records[0]["run_id"] async def test_metrics_emitted_after_agent_text_done(self) -> None: session, _, _ = _make_session( @@ -1559,6 +1729,19 @@ async def close(self) -> None: pass +def _arm_vad(session: VoiceSession, endpointer: object | None) -> None: + """Attach a fake endpointer *and* the evidence flag that gates its use. + + ``_vad_evidence`` is a separate flag set by ``_maybe_start_endpointer`` + only when the detector carries an audio EOU model, because the endpointer + now also arms for text-only detectors that must not inherit the veto + behaviours. Assigning ``_endpointer`` alone therefore exercises the + no-evidence path — which is how three tests here went red unnoticed. + """ + session._endpointer = endpointer # type: ignore[assignment] + session._vad_evidence = endpointer is not None + + class TestVadBargeInVeto: """STT hallucinates plausible phrases from silence (observed live: 'Not, not too bad, mate.' while nobody spoke) — they pass every text gate @@ -1568,16 +1751,16 @@ class TestVadBargeInVeto: def test_veto_matrix(self) -> None: session, _, _ = _make_session() # No endpointer (heuristic mode, no Silero) → never veto. - session._endpointer = None + _arm_vad(session, None) assert session._vad_vetoes_barge_in("no stop that") is False # VAD armed but unhealthy/starved (None) → no evidence, never veto. - session._endpointer = _FakeEndpointer(None) + _arm_vad(session, _FakeEndpointer(None)) assert session._vad_vetoes_barge_in("no stop that") is False # Real speech energy present → allow the barge-in. - session._endpointer = _FakeEndpointer(1.0) + _arm_vad(session, _FakeEndpointer(1.0)) assert session._vad_vetoes_barge_in("no stop that") is False # Armed, healthy, and the mic was silent → hallucination, veto. - session._endpointer = _FakeEndpointer(0.0) + _arm_vad(session, _FakeEndpointer(0.0)) assert session._vad_vetoes_barge_in("no stop that") is True async def _run_barge_in_scenario(self, speech_secs: float | None) -> tuple[VoiceSession, list[VoiceSessionEvent]]: @@ -1590,7 +1773,7 @@ async def _run_barge_in_scenario(self, speech_secs: float | None) -> tuple[Voice tts = SlowMockTTS(delay=0.05, num_chunks=6) tracker = FakePlaybackTracker() session = VoiceSession(agent=agent, stt=stt, tts=tts, playback_tracker=tracker) - session._endpointer = _FakeEndpointer(speech_secs) + _arm_vad(session, _FakeEndpointer(speech_secs)) events: list[VoiceSessionEvent] = [] @@ -1763,7 +1946,7 @@ async def _run_hold_scenario( ) session._hold_partial_grace_secs = grace_secs if endpointer is not None: - session._endpointer = endpointer # type: ignore[assignment] + _arm_vad(session, endpointer) events: list[VoiceSessionEvent] = [] elapsed = 0.0 @@ -1853,6 +2036,39 @@ async def close(self) -> None: f"hold expiry took {elapsed:.2f}s — pre-commit VAD speech stretched the grace" ) + async def test_live_mic_extends_hold_without_any_partial(self) -> None: + """Measured on ElevenLabs at a 0.3s VAD threshold: an interior pause + commits the fragment, then the next fragment's audio is in flight for + over a second before the STT emits a partial for it. Extending only on + partials loses that race and the hold fires mid-sentence, splitting the + utterance — so mic energy alone has to be enough. + """ + + class _LiveSpeechEP: + """Mic is hot *now*; the STT has produced nothing for it yet.""" + + def speech_secs_in_window(self, window_secs: float) -> float: # noqa: ARG002 + return 0.5 + + def notify_committed(self) -> None: + return None + + async def close(self) -> None: + return None + + elapsed, _ = await self._run_hold_scenario( + grace_secs=2.0, + timeout_secs=0.2, + partial_after_commit_delay=None, + endpointer=_LiveSpeechEP(), + ) + # Speech never stops, so the extension runs to its cap and no further: + # unbounded, the assistant's own echo could hold a turn open forever. + assert elapsed > 1.0, f"hold expired after {elapsed:.2f}s while the mic was live" + assert elapsed < VoiceSession.HOLD_VAD_MAX_EXTENSION_SECS + 1.5, ( + f"hold ran {elapsed:.2f}s — the extension cap is not holding" + ) + async def test_dangling_token_hold_is_dropped_on_expiry(self) -> None: """Live bug: stale-force-committed \"I\" HOLDs for 3s then becomes a user turn and the LLM invents a reply. Dangling tokens must die here.""" @@ -2018,3 +2234,60 @@ async def _run() -> None: assert session.transcript[0].role == "user" assert session.transcript[0].text == "What are you doing?" + + async def _run_churn_scenario(self, endpointer: object) -> tuple[_StuckPartialSTT, VoiceSession]: + """Stranded partial while the provider keeps emitting *new* partials + faster than the stale threshold, so transcript staleness never accrues. + """ + agent = Agent(name="t", model=TestModel(responses=["ok"]), tools=[]) + stt = _StuckPartialSTT() + session = VoiceSession(agent=agent, stt=stt, tts=MockTTS()) + session._stale_partial_poll_secs = 0.05 + session._stale_partial_commit_secs = 0.4 + _arm_vad(session, endpointer) + + events: list[VoiceSessionEvent] = [] + + async def _empty_audio() -> AsyncIterator[bytes]: + return + yield # noqa: RET504 + + async def _drive() -> None: + while not any(isinstance(e, SessionStarted) for e in events): + await asyncio.sleep(0.01) + stt.pending_text = "Sorry, one more thing." + # 0.1s apart against a 0.4s threshold: every hallucination resets the + # anchor well before it can expire. + for text in ("Sorry, one more thing.", "Yeah.", "I don't know.", "Yeah.", "Mm.", "Yeah."): + await stt.inject(TranscriptEvent(type="partial", text=text)) + await asyncio.sleep(0.1) + await stt.finish() + + async def _run() -> None: + async with aclosing(session.run(_empty_audio())) as stream: + driver = asyncio.create_task(_drive()) + async for ev in stream: + events.append(ev) + await driver + + await asyncio.wait_for(_run(), timeout=10) + return stt, session + + async def test_partial_churn_does_not_disarm_sweeper(self) -> None: + """Measured on ElevenLabs barge-ins: it hallucinates on trailing silence, + and each invented partial both restarts its own VAD silence timer (so it + never commits) and refreshes this sweeper's anchor (so the rescue never + fires). The user's interruption was lost outright in 8 of 12 runs. A quiet + mic has to be enough to conclude the provider is not going to commit. + """ + stt, session = await self._run_churn_scenario(_FakeEndpointer(0.0)) + assert stt.commit_calls >= 1, "churning partials kept the sweeper disarmed" + assert any(e.role == "user" and e.text == "Sorry, one more thing." for e in session.transcript) + + async def test_live_mic_still_defers_the_sweeper(self) -> None: + """The mirror of the above: partials arriving while the mic is genuinely + hot mean the user is still talking, and force-committing there would cut + them off mid-utterance. Only *silence* may substitute for staleness. + """ + stt, _ = await self._run_churn_scenario(_FakeEndpointer(0.5)) + assert stt.commit_calls == 0, "sweeper force-committed while the user was still speaking" diff --git a/python/tests/voice/test_turn_detection.py b/python/tests/voice/test_turn_detection.py index daa643b8..667c4e05 100644 --- a/python/tests/voice/test_turn_detection.py +++ b/python/tests/voice/test_turn_detection.py @@ -10,10 +10,11 @@ from contextlib import aclosing +import pytest from timbal import Agent from timbal.core.test_model import TestModel +from timbal.voice import TranscriptEvent, VoiceSession, VoiceSessionEvent from timbal.voice.eou import AudioEouModel, PunctuationEouPredictor, TextEouPredictor -from timbal.voice.session import TranscriptEvent, VoiceSession, VoiceSessionEvent from timbal.voice.turn_detection import ( CommitAction, CommitDecision, @@ -26,13 +27,218 @@ SemanticTurnDetector, TurnDetector, TurnState, + _ends_with_correction_marker, + _is_contentless_single_token, _is_garbage_commit, _is_same_user_utterance_refinement, + _is_unvoiced_hallucination, + _likely_stt_echo, resolve_turn_detector, ) + +class TestContentlessSingleToken: + """A lone function word is a fragment; a lone "No." is an answer. + + The negatives are the point. `banking_short_reject` is a turn consisting of the + single word "No.", so a filter that keys on length alone would delete a scenario. + """ + + def test_bare_function_words_are_dropped(self) -> None: + for text in ("I", "I.", "the", "A.", "my", "To."): + assert _is_contentless_single_token(text), text + + def test_meaningful_one_word_turns_survive(self) -> None: + for text in ("No.", "Yes.", "Stop.", "Wait.", "And?", "So?", "Okay.", "You?"): + assert not _is_contentless_single_token(text), text + + def test_a_function_word_inside_an_utterance_survives(self) -> None: + for text in ("I need help.", "The blue one.", "To account 448."): + assert not _is_contentless_single_token(text), text + + async def test_provider_ignores_a_lone_pronoun(self) -> None: + """Observed on flux/heuristic: 'I' committing its own turn after a finished one, + which is a ghost turn — the assistant answers a word the user did not say.""" + det = ProviderTurnDetector() + decision = await det.on_committed("I", _state()) + assert decision.action is CommitAction.IGNORE + assert decision.reason == "contentless" + + +class TestTrailingCorrectionMarker: + """A finished sentence plus "Sorry." is the one incompleteness no EOU model sees. + + Both signals score these complete and both are right about the sentence — the + speaker is simply not done. The shape requirement is the whole safety margin, so + the negatives matter more than the positives here. + """ + + def test_correction_after_a_finished_sentence(self) -> None: + for text in ( + "Send it to account 447. Sorry.", + "I take the blue pill. No, wait.", + "Book it for Tuesday. Actually.", + "The total is 40. Hold on.", + ): + assert _ends_with_correction_marker(text), text + + def test_a_bare_marker_is_a_whole_turn(self) -> None: + """Holding these buys nothing and costs dead air: there is no sentence in front + of the marker, so nothing is being corrected.""" + for text in ("Sorry.", "Wait, hold on.", "Actually, cancel that.", "Hold on."): + assert not _ends_with_correction_marker(text), text + + def test_marker_words_inside_a_sentence_do_not_count(self) -> None: + """An apology is not a retraction, and "Actually, Wednesday." is the correction + itself rather than the announcement of one.""" + for text in ( + "I'm sorry.", + "Book it for Tuesday. Actually, Wednesday.", + "I need help with my recent order.", + "What is your return policy?", + ): + assert not _ends_with_correction_marker(text), text + from .test_session import MockSTT, MockTTS + +class TestUnvoicedHallucination: + """A silent mic may veto ElevenLabs' inventions, and nothing else. + + The filter runs without the ``_vad_evidence`` gate, so what stops it suppressing + real speech is only its narrowness. These pin that: the phrases are the ones + actually observed committed on trailing silence, and the negatives are the short + barge-ins nearby in length that must survive. + """ + + @staticmethod + def _state(*, mic_speech: bool | None) -> TurnState: + return TurnState( + assistant_active=False, + audio_playing=False, + assistant_text="", + active_user_text="", + seconds_since_turn_start=1.0, + seconds_since_last_commit=1.4, + partials_since_last_commit=2, + mic_speech_since_last_commit=mic_speech, + ) + + @pytest.mark.parametrize("text", ["Yes.", "Yeah.", "Okay.", "Bye.", "I don't."]) + def test_silent_mic_vetoes_stock_phrases(self, text: str) -> None: + assert _is_unvoiced_hallucination(text, self._state(mic_speech=False)) + + @pytest.mark.parametrize("text", ["Stop.", "Yes.", "Wait, hold on.", "Actually, cancel that."]) + def test_mic_energy_admits_everything(self, text: str) -> None: + """A real interruption carries energy, which is what makes the filter safe: + even a one-word "Stop." is untouchable once Silero has heard it.""" + assert not _is_unvoiced_hallucination(text, self._state(mic_speech=True)) + + @pytest.mark.parametrize("text", ["Wait, hold on.", "Actually, cancel that.", "No, the white one."]) + def test_longer_barge_ins_survive_a_silent_mic(self, text: str) -> None: + """The second belt: over two words, VAD alone cannot drop it. Silero misses + quiet speech, and losing a real instruction costs more than a ghost turn.""" + assert not _is_unvoiced_hallucination(text, self._state(mic_speech=False)) + + def test_no_opinion_never_suppresses(self) -> None: + """``None`` is no endpointer or too long a gap to judge — distinct from False, + and reading it as False would drop turns on every session without Silero.""" + assert not _is_unvoiced_hallucination("Yeah.", self._state(mic_speech=None)) + + +class TestLikelySttEcho: + """Garbled echo must be suppressed, but only in a session shown to leak. + + The transcripts are real, from ``--aec-leak`` runs. What they pin is the split + between the two branches: verbatim echo is evidence and always counts, garbled + echo is a guess that stays disarmed until the verbatim branch has fired. + """ + + REPLY = ( + "Our return policy allows returns within thirty days of purchase, provided " + "the item is unused and you still have the original receipt or a valid proof " + "of purchase from one of our authorized retailers." + ) + SEGFAULT_REPLY = "A segfault means bad memory access." + GARBLED = ( + "Our retailers.", + "Arise retailers.", + "Advised retailers.", + "advertised retailers.", + "Archives to retailers.", + "has aroused retailers.", + ) + + @staticmethod + def _state(reply: str) -> TurnState: + return TurnState( + assistant_active=True, + audio_playing=True, + assistant_text=reply, + active_user_text="", + seconds_since_turn_start=2.0, + seconds_since_last_commit=2.0, + partials_since_last_commit=1, + ) + + def test_garbled_echo_needs_a_leaking_session(self) -> None: + """Unlatched, none of these may be suppressed: the same scores are produced by + a user repeating the assistant, and dropping their turn is the worse error.""" + detector = HeuristicTurnDetector() + for echo in self.GARBLED: + assert not detector.echo_verdict(echo, self._state(self.REPLY)), echo + + def test_verbatim_echo_arms_the_garbled_branch(self) -> None: + """One verbatim leak is proof the mic hears the speaker, and from then on + resemblance is admissible. This is the whole mechanism in three lines.""" + detector = HeuristicTurnDetector() + state = self._state(self.REPLY) + assert detector.echo_verdict("authorized retailers.", state, arm=True) + for echo in self.GARBLED: + assert detector.echo_verdict(echo, state), echo + + def test_a_partial_cannot_arm_the_latch(self) -> None: + """Regression: `food_long_pause` on two cells, 100% -> 0%. + + In a confirmation flow the user's own partials are verbatim substrings of the + reply by construction, so letting a partial count as proof hands the fuzzy + branch a loaded gun and it shoots the commit that follows. Suppressing the + partial itself is fine and predates this; arming on it is not. + """ + detector = HeuristicTurnDetector() + state = self._state("Got it — a large pepperoni pizza.") + assert detector.echo_verdict("pepperoni pizza", state) + assert not detector.echo_verdict("Pepperoni pizza, please.", state) + + def test_the_latch_does_not_leak_across_sessions(self) -> None: + """Detectors are cloned per session, so proof from one call must not arm another.""" + leaking = HeuristicTurnDetector() + assert leaking.echo_verdict("authorized retailers.", self._state(self.REPLY)) + assert not HeuristicTurnDetector().echo_verdict("Our retailers.", self._state(self.REPLY)) + + def test_genuine_barge_ins_survive_even_when_latched(self) -> None: + detector = HeuristicTurnDetector() + state = self._state(self.REPLY) + detector.echo_verdict("authorized retailers.", state) + for real in ("Actually, cancel that.", "Wait, hold on.", "Okay, never mind.", "Okay, thanks."): + assert not detector.echo_verdict(real, state), real + + def test_assistant_repeating_the_user_is_not_echo(self) -> None: + """The regression that produced the latch. `food_long_pause` lost the user's + order to a confirmation containing it, on a clean run where no echo existed — + and it scores 0.67, inside the range real echo occupies, so only the absence + of proof can save it.""" + reply = "Got it — a large pepperoni pizza." + assert not HeuristicTurnDetector().echo_verdict("Pepperoni pizza, please.", self._state(reply)) + + def test_verbatim_branch_is_exact_only(self) -> None: + assert _likely_stt_echo("memory access.", self.SEGFAULT_REPLY) + assert not _likely_stt_echo("and memory access.", self.SEGFAULT_REPLY) + + def test_no_assistant_text_is_never_echo(self) -> None: + assert not HeuristicTurnDetector().echo_verdict("Our retailers.", self._state("")) + + # --------------------------------------------------------------------------- # Moved heuristic functions (ported from test_voice_session_stt_refinement.py) # --------------------------------------------------------------------------- @@ -478,11 +684,12 @@ async def test_late_barge_in_on_finished_turn_not_held(self) -> None: async def test_incomplete_audio_complete_text_short_hold(self) -> None: """Confidence tier: Smart Turn under-scores short closers ("Thank you." - p=0.036 live) — terminal punctuation disagrees, so the hold shrinks to - the short tier instead of eating the full budget as dead air. + p=0.036 live) — terminal punctuation disagrees, so the hold shrinks + below the full budget instead of eating all of it as dead air. - Keep this well under the incomplete-text HOLD (1.2s): VAD has usually - already waited, and a second 1.2s tax is the cold-start "slow" feel. + The tier was 0.35s until the sweep retuned it to 1.2s, which is also + what the incomplete-text tier costs: the two now differ only in the + logged reason, and "short" means short of ``hold_timeout_secs`` alone. """ det = LocalAudioTurnDetector(audio_eou=_FixedAudioEou(0.1)) await det.start(type("C", (), {"sample_rate": 16000})()) @@ -491,8 +698,8 @@ async def test_incomplete_audio_complete_text_short_hold(self) -> None: assert decision.action is CommitAction.HOLD assert decision.reason == "audio_hold_text_complete" assert decision.hold_timeout_secs == det.text_complete_hold_timeout_secs - assert det.text_complete_hold_timeout_secs == 0.35 - assert det.text_complete_hold_timeout_secs < det.text_incomplete_hold_timeout_secs + assert det.text_complete_hold_timeout_secs == 1.2 + assert det.text_complete_hold_timeout_secs <= det.text_incomplete_hold_timeout_secs assert det.text_complete_hold_timeout_secs < det.hold_timeout_secs # Per-session clones keep the knob. assert det.clone().text_complete_hold_timeout_secs == det.text_complete_hold_timeout_secs @@ -582,6 +789,20 @@ async def test_complete_audio_new_turn(self) -> None: assert decision.action is CommitAction.NEW_TURN await det.close() + async def test_trailing_correction_holds_despite_both_signals_complete(self) -> None: + """`banking_correction` and `medical_self_correction`, marked on five and four + cells: audio scores 0.9, the text is a finished sentence, and the correction that + follows becomes a second turn. Nothing else in the detector can catch it, because + nothing else is wrong — the sentence really is complete.""" + det = LocalAudioTurnDetector(audio_eou=_FixedAudioEou(0.9)) + await det.start(type("C", (), {"sample_rate": 16000})()) + det.push_audio(b"\x00\x01" * 8000) + decision = await det.on_committed("Send it to account 447. Sorry.", _state()) + assert decision.action is CommitAction.HOLD + assert decision.reason == "trailing_correction_marker" + assert decision.hold_timeout_secs == det.text_incomplete_hold_timeout_secs + await det.close() + async def test_short_audio_incomplete_text_holds(self) -> None: """< 0.5s of buffered PCM must not pass an incomplete utterance through as NEW_TURN — the lexical fallback holds it (audio model never called).""" diff --git a/python/tests/voice/test_turn_detection_adversarial.py b/python/tests/voice/test_turn_detection_adversarial.py index 6dc9f4a3..18544b85 100644 --- a/python/tests/voice/test_turn_detection_adversarial.py +++ b/python/tests/voice/test_turn_detection_adversarial.py @@ -12,8 +12,8 @@ from timbal import Agent from timbal.core.test_model import TestModel +from timbal.voice import TranscriptEvent, VoiceSession, VoiceSessionEvent from timbal.voice.eou import AudioEouModel -from timbal.voice.session import TranscriptEvent, VoiceSession, VoiceSessionEvent from timbal.voice.turn_detection import ( CommitAction, CommitDecision, diff --git a/python/tests/voice/test_webrtc.py b/python/tests/voice/test_webrtc.py new file mode 100644 index 00000000..1beeacd4 --- /dev/null +++ b/python/tests/voice/test_webrtc.py @@ -0,0 +1,157 @@ +"""WebRTC transport primitives: pacing, played-axis accounting, resampling. + +The riskiest part of the WebRTC transport is the downlink pacing math — +``recv()`` timestamps drive both what the caller hears and what +interruption truncation believes was heard. These tests pin it in isolation, +before any signaling exists. +""" + +from __future__ import annotations + +import asyncio +import fractions +import time + +import pytest + +pytest.importorskip("aiortc", reason="timbal[voice] extra (aiortc) not installed") + +from aiortc.mediastreams import MediaStreamError # noqa: E402 +from av import AudioFrame # noqa: E402 +from timbal.voice.webrtc import PacedPlaybackTracker, PcmQueueTrack, track_to_pcm # noqa: E402 + +_SR = 16_000 +_FRAME_BYTES = int(_SR * 0.02) * 2 # 20ms of PCM16 mono + + +class TestPcmQueueTrack: + async def test_first_frame_is_immediate_then_paced_at_real_time(self) -> None: + track = PcmQueueTrack(sample_rate=_SR) + t0 = time.monotonic() + for _ in range(6): + await track.recv() + elapsed = time.monotonic() - t0 + # 6 frames = first immediate + 5 paced gaps of 20ms. + assert elapsed >= 0.5 * 5 * 0.02 # generous lower bound for CI jitter + assert elapsed < 1.0 + + async def test_empty_queue_yields_silence_and_no_played_bytes(self) -> None: + track = PcmQueueTrack(sample_rate=_SR) + frame = await track.recv() + assert bytes(frame.planes[0])[: _FRAME_BYTES] == b"\x00" * _FRAME_BYTES + assert frame.samples == int(_SR * 0.02) + assert frame.sample_rate == _SR + assert frame.time_base == fractions.Fraction(1, _SR) + assert track.played_bytes == 0 + + async def test_played_bytes_counts_real_pcm_only(self) -> None: + track = PcmQueueTrack(sample_rate=_SR) + track.write(b"\x01\x02" * (_FRAME_BYTES // 2)) # exactly one frame + track.write(b"\x03\x04" * 10) # 20 bytes: a partial second frame + f1 = await track.recv() + assert bytes(f1.planes[0])[:_FRAME_BYTES] == b"\x01\x02" * (_FRAME_BYTES // 2) + assert track.played_bytes == _FRAME_BYTES + f2 = await track.recv() + data = bytes(f2.planes[0])[:_FRAME_BYTES] + assert data[:20] == b"\x03\x04" * 10 + assert data[20:] == b"\x00" * (_FRAME_BYTES - 20) # silence pad + assert track.played_bytes == _FRAME_BYTES + 20 # pad not counted + + async def test_pts_advances_by_frame_samples(self) -> None: + track = PcmQueueTrack(sample_rate=_SR) + f1 = await track.recv() + f2 = await track.recv() + f3 = await track.recv() + step = int(_SR * 0.02) + assert (f1.pts, f2.pts, f3.pts) == (0, step, 2 * step) + + async def test_flush_drops_unsent_audio_and_freezes_played_axis(self) -> None: + track = PcmQueueTrack(sample_rate=_SR) + track.write(b"\x01\x02" * (_FRAME_BYTES // 2) * 3) + await track.recv() + assert track.played_bytes == _FRAME_BYTES + dropped = track.flush() + assert dropped == 2 * _FRAME_BYTES + assert track.buffered_bytes == 0 + assert track.played_bytes == _FRAME_BYTES # dropped audio was never heard + frame = await track.recv() # back to silence, still paced + assert bytes(frame.planes[0])[:_FRAME_BYTES] == b"\x00" * _FRAME_BYTES + + async def test_stopped_track_raises_media_stream_error(self) -> None: + track = PcmQueueTrack(sample_rate=_SR) + track.stop() + with pytest.raises(MediaStreamError): + await track.recv() + + +class TestPacedPlaybackTracker: + def test_position_is_native_truth(self) -> None: + track = PcmQueueTrack(sample_rate=_SR) + tracker = PacedPlaybackTracker(track) + assert tracker.ack_received is True + assert tracker.played_bytes == 0 + tracker.on_playback_ack(9999.0) # acks carry no information here + assert tracker.played_bytes == 0 + + async def test_reads_through_to_the_track(self) -> None: + track = PcmQueueTrack(sample_rate=_SR) + tracker = PacedPlaybackTracker(track) + track.write(b"\x01\x02" * _FRAME_BYTES) # two frames + tracker.on_audio_emitted(2 * _FRAME_BYTES) # no-op: counting is on dequeue + assert tracker.played_bytes == 0 + assert tracker.is_playing is True + await track.recv() + assert tracker.played_bytes == _FRAME_BYTES + + async def test_interruption_flushes_the_queue(self) -> None: + track = PcmQueueTrack(sample_rate=_SR) + tracker = PacedPlaybackTracker(track) + track.write(b"\x01\x02" * _FRAME_BYTES * 4) + await track.recv() + tracker.on_interrupted() + assert tracker.is_playing is False + assert track.buffered_bytes == 0 + assert tracker.played_bytes == _FRAME_BYTES + + +class _FrameSourceTrack: + """Duck-typed uplink track: replays canned AudioFrames, then ends.""" + + def __init__(self, frames: list[AudioFrame]) -> None: + self._frames = list(frames) + + async def recv(self) -> AudioFrame: + if not self._frames: + raise MediaStreamError + await asyncio.sleep(0) + return self._frames.pop(0) + + +def _mic_frame(pts: int, *, rate: int = 48_000, samples: int = 960, fill: bytes = b"\x11\x22") -> AudioFrame: + frame = AudioFrame(format="s16", layout="mono", samples=samples) + frame.planes[0].update(fill * samples) + frame.pts = pts + frame.sample_rate = rate + frame.time_base = fractions.Fraction(1, rate) + return frame + + +class TestTrackToPcm: + async def test_resamples_48k_mic_to_16k_session_pcm(self) -> None: + n_frames = 25 # 500ms of 48k audio + frames = [_mic_frame(i * 960) for i in range(n_frames)] + out = b"".join([chunk async for chunk in track_to_pcm(_FrameSourceTrack(frames), sample_rate=_SR)]) + expected = int(n_frames * 0.02 * _SR) * 2 # 500ms at 16k PCM16 + # The resampler's filter delay holds back a tail; allow one frame. + assert abs(len(out) - expected) <= _FRAME_BYTES + assert len(out) % 2 == 0 + + async def test_ends_cleanly_when_the_track_ends(self) -> None: + agen = track_to_pcm(_FrameSourceTrack([]), sample_rate=_SR) + chunks = [c async for c in agen] + assert chunks == [] + + async def test_passthrough_at_native_rate(self) -> None: + frames = [_mic_frame(i * 320, rate=_SR, samples=320) for i in range(10)] + out = b"".join([chunk async for chunk in track_to_pcm(_FrameSourceTrack(frames), sample_rate=_SR)]) + assert abs(len(out) - 10 * 320 * 2) <= _FRAME_BYTES diff --git a/python/timbal/platform/utils.py b/python/timbal/platform/utils.py index e972c4c1..87243ef2 100644 --- a/python/timbal/platform/utils.py +++ b/python/timbal/platform/utils.py @@ -1,7 +1,7 @@ import asyncio import json as json_lib import os -from collections.abc import AsyncGenerator +from collections.abc import AsyncGenerator, Callable from typing import Any, Literal import httpx @@ -97,7 +97,7 @@ def _resolve_url_and_headers( async def _request( - method: Literal["GET", "POST", "PATCH", "DELETE"], + method: Literal["GET", "POST", "PUT", "PATCH", "DELETE"], path: str, headers: dict[str, str] = {}, params: dict[str, Any] | None = None, @@ -105,10 +105,21 @@ async def _request( content: bytes | None = None, files: dict[str, tuple[str, bytes, str]] | None = None, max_retries: int = 3, + backoff: Callable[[int], float] | None = None, + timeout: httpx.Timeout | float | None = None, service: str | None = None, ) -> Any: - """Utility function for making HTTP requests.""" + """Utility function for making HTTP requests. + + ``backoff`` overrides the retry delay: called with the 0-based attempt + index, returns seconds to wait (default: 0.1s doubling). A ``Retry-After`` + on 429 still wins when longer. ``timeout`` overrides the default + ``Timeout(10.0, read=None)`` — e.g. large multipart uploads need an + unbounded write timeout. + """ url, headers = _resolve_url_and_headers(service, path, headers) + if timeout is None: + timeout = httpx.Timeout(10.0, read=None) payload_kwargs = {} # `is not None` so an empty dict still sends a JSON body + Content-Type # (parameterless POSTs would otherwise 415 Unsupported Media Type). @@ -121,7 +132,7 @@ async def _request( for attempt in range(max_retries + 1): try: - async with httpx.AsyncClient(timeout=httpx.Timeout(10.0, read=None)) as client: + async with httpx.AsyncClient(timeout=timeout) as client: res = await client.request( method, url, @@ -154,7 +165,8 @@ async def _request( f" Response body: {error_body or None}", status_code=exc.response.status_code, ) from exc - wait_time = 0.1 * (2**attempt) # Exponential backoff: 100ms, 200ms, 400ms + # Default exponential backoff: 100ms, 200ms, 400ms + wait_time = backoff(attempt) if backoff is not None else 0.1 * (2**attempt) if exc.response.status_code == 429: retry_after = exc.response.headers.get("Retry-After") if retry_after is not None: @@ -173,7 +185,7 @@ async def _request( # Retry on any other error (network, timeout, etc.) if attempt == max_retries: raise - wait_time = 0.1 * (2**attempt) + wait_time = backoff(attempt) if backoff is not None else 0.1 * (2**attempt) logger.warning( f"Request failed, retrying in {wait_time:.1f}s", attempt=attempt + 1, diff --git a/python/timbal/server/README.md b/python/timbal/server/README.md index b07cd40b..a8b9d481 100644 --- a/python/timbal/server/README.md +++ b/python/timbal/server/README.md @@ -86,7 +86,7 @@ All downlink messages are **text JSON** with a **`type`** field. | `type` | Fields | Meaning | |-------------------------|---------------|--------| -| `session_started` | `playback_acks`, `stt_provider`, `stt_model`, `model`, `turn_detector`, `vad_endpointing` | Voice session is live; safe to show “listening”. `playback_acks: "recommended"` advertises the [playback ack](#playback-acks-client--server) protocol. `stt_provider` is the config id actually in effect (`elevenlabs` / `deepgram-flux` / `deepgram-nova`), not the Python class name. `turn_detector` is the class name of the detector actually in effect (e.g. `LocalAudioTurnDetector`). `vad_endpointing` is a bool: whether the local [VAD endpointing](#config-overrides) fast path actually armed for this session (not merely what was requested). | +| `session_started` | `session_id`, `playback_acks`, `stt_provider`, `stt_model`, `model`, `turn_detector`, `vad_endpointing` | Voice session is live; safe to show “listening”. `session_id` keys the session's [call recording](#call-recording) files and any platform-side conversation record. `playback_acks: "recommended"` advertises the [playback ack](#playback-acks-client--server) protocol. `stt_provider` is the config id actually in effect (`elevenlabs` / `deepgram-flux` / `deepgram-nova`), not the Python class name. `turn_detector` is the class name of the detector actually in effect (e.g. `LocalAudioTurnDetector`). `vad_endpointing` is a bool: whether the local [VAD endpointing](#config-overrides) fast path actually armed for this session (not merely what was requested). | | `transcript_partial` | `text` | Live STT (may change). | | `transcript_committed` | `text` | Final user transcript for the utterance. | | `agent_text_delta` | `text` | Streaming assistant text (captions / UI). | @@ -109,26 +109,28 @@ Right before the server sends `session_ended`, it sends a `session_transcript` m ```json { "type": "session_transcript", + "started_at": 1713099998.500, "entries": [ - { "role": "user", "text": "Hola, ¿qué tal?", "timestamp": 1713100000.123 }, - { "role": "assistant", "text": "¡Hola! Todo bien, ¿en qué puedo ayudarte?", "timestamp": 1713100002.456 }, - { "role": "user", "text": "Cuéntame una historia", "timestamp": 1713100010.789 }, - { "role": "assistant", "text": "Había una vez…", "timestamp": 1713100012.012 } + { "role": "user", "text": "Hola, ¿qué tal?", "timestamp": 1713100000.123, "offset_ms": 1623 }, + { "role": "assistant", "text": "¡Hola! Todo bien, ¿en qué puedo ayudarte?", "timestamp": 1713100002.456, "offset_ms": 3956 }, + { "role": "user", "text": "Cuéntame una historia", "timestamp": 1713100010.789, "offset_ms": 12289 }, + { "role": "assistant", "text": "Había una vez…", "timestamp": 1713100012.012, "offset_ms": 13512 } ] } ``` -Each entry has: +`started_at` is the session's wall-clock start (Unix seconds). Each entry has: | Field | Type | Description | |-------------|--------|-------------| | `role` | string | `"user"` or `"assistant"`. | | `text` | string | Final committed text for that turn. | | `timestamp` | float | Unix timestamp (seconds) when the text was committed. | +| `offset_ms` | int | Milliseconds since `started_at` — lines up with the call recording's playback position. | This lets the frontend persist the conversation without having to accumulate `transcript_committed` / `agent_text_done` messages itself. -**Audio recording** is available server-side via the `VoiceSession` Python API (`record_audio=True`) but is not sent over the WebSocket (PCM dumps are too large for a single JSON frame). Use the `session.input_audio` / `session.output_audio` properties to access raw PCM bytes after the session closes for server-side storage, conversion, or upload. +**Audio recording** is available server-side via the `VoiceSession` Python API (`record_audio=True`) but is not sent over the WebSocket (PCM dumps are too large for a single JSON frame). Use the `session.input_audio` / `session.output_audio` properties to access raw PCM bytes after the session closes for server-side storage, conversion, or upload. For a persistent, playable per-call file see [Call recording](#call-recording) below. ### Turn metrics @@ -168,5 +170,80 @@ Once per turn — after `agent_text_done`, and also when the turn is interrupted ### What this is not - Not the bundled **`GET /voice/`** HTML demo — only the **`/voice/ws`** contract. -- Not REST/SSE for the live voice loop; real-time voice uses this WebSocket. +- Not REST/SSE for the live voice loop; real-time voice uses this WebSocket (or WebRTC, below). - The LLM/agent runs on the server; the client only captures audio, plays TTS, and renders text events. + +## Voice over WebRTC: `POST /voice/rtc` + +The same voice session over WebRTC instead of a WebSocket. Requires the **`timbal[voice]`** extra (which ships aiortc) on the server — without it the route answers **501** with an install hint. The bundled playground has a transport dropdown that exercises this path. + +**Why choose it over `/voice/ws`:** Opus instead of raw PCM (~10x less bandwidth), jitter buffering and packet-loss concealment on lossy/mobile networks, and **server-paced playback** — the server sends TTS at real time from its own clock, so it always knows exactly what the caller has heard. Barge-in truncation (`interrupted.heard_text`, memory truncation) is exact with **no playback acks**, and the unspoken tail of an interrupted reply is dropped server-side instead of asking the client to clear buffers. + +### Signaling + +One HTTP round trip, WHIP-style — no trickle ICE (wait for ICE gathering to complete before posting the offer): + +``` +POST /voice/rtc +{ "sdp": "", "type": "offer", "config": { ...same keys as the WS config frame... } } + +200 → { "sdp": "", "type": "answer" } +400 → bad offer / no audio track / runnable is not an Agent +501 → timbal[voice] extra (aiortc) not installed +``` + +The offer **must** contain: + +- **One audio track** — the mic. Sent Opus-encoded; the server decodes and resamples to the session rate, so the client-side `sample_rate` config key is irrelevant on this transport. Keep `echoCancellation: true` in `getUserMedia` constraints. +- **A data channel** (any label) — created by the client so its SCTP m-line rides the offer. All non-audio session events arrive here as JSON: the exact payloads of the WS protocol, except there are **no `audio` messages** (TTS is a real audio track the browser plays natively) and **no `playback` acks** (`session_started.playback_acks` is `"native"`, and `transport` is `"webrtc"`). + +The server answers with the TTS audio track on the same m-line. Client teardown = close the peer connection; server teardown (session end/error) closes the connection and fires `session_ended` on the data channel first. + +### ICE configuration + +| Env var | Meaning | +|---|---| +| `TIMBAL_STUN_URL` | STUN server; defaults to `stun:stun.l.google.com:19302`. Set to empty to disable (loopback/LAN). | +| `TIMBAL_TURN_URL` | Optional TURN server for clients behind symmetric NATs. | +| `TIMBAL_TURN_USERNAME` / `TIMBAL_TURN_PASSWORD` | TURN credentials. | + +## Call recording + +Server-side, transport-agnostic (WS and WebRTC share the wiring): every session writes **one playable MP3** — the call as heard, on the call timeline — plus a **JSON manifest** with the timestamped transcript and per-turn latency metrics. This is the data an ElevenLabs-style conversation review UI needs: audio player, transcript entries at `offset_ms`, latency chips per turn. + +**Enable** (server-side only — client config frames cannot switch recording on or off): + +```python +# via the runnable, full control: +agent.voice_config = { + "recording": { + "dir": "recordings", # required — files land here + "layout": "mixed", # "mixed" (default): mono, both voices summed + # "split": stereo, caller left / agent right + "bitrate_kbps": 32, # MP3 bitrate (32 kbps mono ≈ 0.25 MB/min) + "on_saved": my_async_hook, # optional: async (RecordingResult) -> None + }, +} +``` + +or via env (the platform's config surface — all read **per session**, never cached at boot, so late-injected env e.g. after a CRIU restore still applies): + +| Env var | Meaning | +|---|---| +| `TIMBAL_VOICE_RECORDING_DIR` | Enables recording; files land here. | +| `TIMBAL_VOICE_RECORDING_LAYOUT` | `mixed` (default) or `split`. | +| `TIMBAL_VOICE_RECORDING_BITRATE_KBPS` | MP3 bitrate, default `32`. | +| `TIMBAL_VOICE_RECORDING_UPLOAD` | `platform` → push files to the platform API after each call (see below). | + +Keys in `voice_config["recording"]` win over env, per key. Requires the `timbal[voice]` extra (av + numpy); without it the server logs a warning and runs the call unrecorded. + +When `TIMBAL_ORG_ID` / `TIMBAL_PROJECT_ID` / `TIMBAL_PROJECT_ENV_ID` / `TIMBAL_APP_ID` / `TIMBAL_PROJECT_REV` are present in env, they are stamped into the manifest `meta` (as `org_id`, `project_id`, ...) so the files are self-describing for ingest. + +Per session, keyed by the `session_id` from `session_started`: + +- **`{session_id}.mp3`** — mic and TTS mixed on the real call timeline. TTS synthesizes faster than real time, so the agent side is paced by the mic clock; on barge-in the **unheard tail is dropped** exactly like `interrupted.heard_text` truncates the text (exact on WebRTC, ack/estimate-based on WS). Encoded progressively — a crashed process still leaves a playable file. +- **`{session_id}.json`** — manifest: `session_id`, `started_at`/`ended_at`, resolved config (`transport`, `model`, `stt_provider`, ...), `transcript` entries with `offset_ms`, `turns` (the full `TurnMetrics` per turn), and the audio descriptor (`layout`, `sample_rate`, `bitrate_kbps`, `duration_secs`). Written **atomically** (tmp + rename) and always *after* the MP3 is finalized — "manifest exists" reliably means "recording complete"; an MP3 without a manifest is a crashed call, playable up to the crash. + +`on_saved` fires after both files are written (e.g. upload to platform storage and delete the local copy); its failures are logged, never crash the session. + +**Platform push** (`TIMBAL_VOICE_RECORDING_UPLOAD=platform`): after each call, a background task PUTs both files as multipart to `{host}/orgs/{org}/projects/{project}/sessions/{session_id}` — host, Bearer credential and org/project resolved by the standard `resolve_platform_config` (env / ~/.timbal), fresh per session. 2xx → local files deleted; 4xx → keep files, log, no retry; 5xx/429/network → exponential backoff (1s base, ×5, cap 5 min, ~1h budget). Uploads never block session teardown, and files are only deleted after a confirmed 2xx — a crash mid-upload leaves them intact for re-ingest. A user-provided `on_saved` in `voice_config` takes precedence over the platform push. diff --git a/python/timbal/server/http.py b/python/timbal/server/http.py index b715d3c6..07cf3077 100644 --- a/python/timbal/server/http.py +++ b/python/timbal/server/http.py @@ -80,9 +80,11 @@ def create_app() -> FastAPI: allow_headers=["*"], ) + from .rtc import router as rtc_router from .voice import router as voice_router app.include_router(voice_router) + app.include_router(rtc_router) @app.get("/healthcheck") async def healthcheck() -> Response: diff --git a/python/timbal/server/recording_upload.py b/python/timbal/server/recording_upload.py new file mode 100644 index 00000000..8fec01e5 --- /dev/null +++ b/python/timbal/server/recording_upload.py @@ -0,0 +1,174 @@ +"""Push call recordings to the platform (option C of the recording handoff). + +Activated per session by ``TIMBAL_VOICE_RECORDING_UPLOAD=platform``. The +contract, agreed with the platform team (final, general-sessions model): + + PUT {host}/orgs/{org}/projects/{project}/sessions/{session_id} + Authorization: Bearer {token} + multipart: "manifest" (application/json) + "audio" (audio/mpeg) + +Platform-side constraints we satisfy by construction: the path session_id +equals ``manifest.session_id`` (both derive from the recording filename), +session ids are 32 hex chars (within their ``[A-Za-z0-9_-]{1,128}``), and +32 kbps MP3 keeps multi-hour calls far below the 100 MB cap. + +Identity and auth come from :func:`resolve_platform_config` — the same +resolution every other platform call uses (env ``TIMBAL_API_HOST`` / +``TIMBAL_API_TOKEN``/``TIMBAL_API_KEY`` / ``TIMBAL_ORG_ID`` / +``TIMBAL_PROJECT_ID``, with ~/.timbal file fallback) — and the request goes +through :func:`timbal.platform.utils._request` with a recording-specific +retry policy: + +* 2xx → the platform upserted (idempotent by session_id) → delete both files. +* 4xx (except 429) → permanent (auth/validation): keep files, log once, + no retry (``_request`` raises immediately). +* 5xx / 429 / network → exponential backoff (1s base, ×5, cap 5 min), + ~1h total budget. Files stay on disk the whole time — the platform + sweeper (or nothing, on ephemeral ECS disk) is the backstop. + +Uploads are fire-and-forget background tasks: they never block session +teardown, and a crash mid-upload leaves both files intact because deletion +only ever happens after a 2xx. +""" + +from __future__ import annotations + +import asyncio +from collections.abc import Awaitable, Callable +from pathlib import Path +from typing import Any + +import httpx +import structlog + +from ..errors import PlatformError +from ..platform.utils import _request +from ..state.config_loader import resolve_platform_config + +logger = structlog.get_logger("timbal.server.recording_upload") + +# Keep strong refs so fire-and-forget upload tasks aren't garbage-collected. +_upload_tasks: set[asyncio.Task[Any]] = set() + + +def _recording_backoff(attempt: int) -> float: + """1, 5, 25, 125, then 300s flat — the agreed recording retry curve.""" + return min(1.0 * 5.0**attempt, 300.0) + + +# 14 retries on that curve ≈ 53 min of waits (+ request time) ≈ the ~1h budget. +_MAX_RETRIES = 14 + +# Multipart bodies up to ~100 MB (platform cap) on slow links: no write/read +# deadline, keep only the connect/pool timeout. +_UPLOAD_TIMEOUT = httpx.Timeout(30.0, read=None, write=None) + + +async def upload_recording( + audio_path: Path, + manifest_path: Path, + *, + path: str, + max_retries: int = _MAX_RETRIES, + backoff: Callable[[int], float] = _recording_backoff, +) -> bool: + """PUT both files to the platform; delete them on success. Returns success. + + ``path`` is the platform API path (relative to the platform host, which + ``_request`` resolves together with auth headers). + """ + session_id = audio_path.stem + try: + files = { + "manifest": (manifest_path.name, manifest_path.read_bytes(), "application/json"), + "audio": (audio_path.name, audio_path.read_bytes(), "audio/mpeg"), + } + except OSError as e: + logger.error("recording_upload_files_missing", session_id=session_id, error=str(e)) + return False + + try: + response = await _request( + "PUT", + path, + files=files, + max_retries=max_retries, + backoff=backoff, + timeout=_UPLOAD_TIMEOUT, + ) + except PlatformError as e: + if e.status_code is not None and 400 <= e.status_code < 500 and e.status_code != 429: + # Permanent (auth/validation): keep the files, don't hammer. + logger.error("recording_upload_rejected", session_id=session_id, status=e.status_code, path=path) + else: + logger.error( + "recording_upload_failed", + session_id=session_id, + status=e.status_code, + error=str(e), + hint="files kept on disk for sweeper/manual ingest", + ) + return False + except Exception as e: # network/timeout retries exhausted + logger.error( + "recording_upload_failed", + session_id=session_id, + error=str(e), + hint="files kept on disk for sweeper/manual ingest", + ) + return False + + audio_path.unlink(missing_ok=True) + manifest_path.unlink(missing_ok=True) + try: + body = response.json() # {"session_id", "created", "item_count"} + except Exception: + body = {} + logger.info( + "recording_upload_ok", + session_id=session_id, + created=body.get("created"), + item_count=body.get("item_count"), + ) + return True + + +def platform_recording_upload_hook() -> Callable[[Any], Awaitable[None]] | None: + """Build the ``on_saved`` hook from platform config, or None (with a log). + + ``force_refresh``: serverless session boxes are CRIU-restored from warm + snapshots with env arriving at restore time — a boot-time default + resolution may have cached ``None``, and this hook runs per session. + """ + config = resolve_platform_config(force_refresh=True) + subject = config.subject if config is not None else None + if config is None or subject is None or not subject.org_id or not subject.project_id: + logger.warning( + "recording_upload_misconfigured", + has_platform_config=config is not None, + has_org=bool(subject and subject.org_id), + has_project=bool(subject and subject.project_id), + hint=( + "TIMBAL_VOICE_RECORDING_UPLOAD=platform needs TIMBAL_API_HOST, " + "TIMBAL_API_TOKEN|TIMBAL_API_KEY, TIMBAL_ORG_ID and TIMBAL_PROJECT_ID" + ), + ) + return None + path_prefix = f"orgs/{subject.org_id}/projects/{subject.project_id}/sessions" + + async def _hook(result: Any) -> None: + """on_saved: schedule the upload and return immediately (never blocks teardown).""" + if result.manifest_path is None: + logger.warning("recording_upload_skipped", reason="no_manifest", path=str(result.audio_path)) + return + task = asyncio.create_task( + upload_recording( + result.audio_path, + result.manifest_path, + path=f"{path_prefix}/{result.audio_path.stem}", + ) + ) + _upload_tasks.add(task) + task.add_done_callback(_upload_tasks.discard) + + return _hook diff --git a/python/timbal/server/rtc.py b/python/timbal/server/rtc.py new file mode 100644 index 00000000..cec21d30 --- /dev/null +++ b/python/timbal/server/rtc.py @@ -0,0 +1,213 @@ +"""Voice over WebRTC — offer/answer signaling for the same runnable as ``/voice/ws``. + +``POST /voice/rtc`` takes ``{"sdp": ..., "type": "offer", "config": {...}}`` +(``config`` uses the same keys as the WebSocket hello) and returns the SDP +answer — WHIP-style, one round trip, no trickle ICE (aiortc finishes +gathering before answering). + +Protocol expectations for clients: + +* The offer must contain one audio track (the mic) **and** a data channel — + the channel rides the offer's SCTP m-line, so the client creates it; the + server binds to whatever channel arrives, whatever its label. +* Session events arrive as JSON on the data channel — the same payloads as + the WebSocket transport, except ``audio``: TTS is a real audio track here, + paced by the server, so there are no base64 audio messages and **no + playback acks** — the server's own pacing clock is the played position, + which makes barge-in ``heard_text`` exact instead of estimated. + +Requires the ``timbal[voice]`` extra (aiortc); the route answers 501 without it. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import json +import os +from contextlib import aclosing +from typing import Any + +import structlog +from fastapi import APIRouter, Request +from fastapi.responses import JSONResponse + +from .voice import build_voice_session, event_to_payloads, merge_client_voice_overrides + +logger = structlog.get_logger("timbal.server.rtc") + +router = APIRouter(prefix="/voice", tags=["voice"]) + +# Peer connections (and their driver tasks) stay alive via these references; +# entries are discarded when the session driver tears the connection down. +_pcs: set[Any] = set() +_drivers: set[asyncio.Task] = set() + + +def _ice_servers() -> list[Any]: + from aiortc import RTCIceServer + + servers = [] + # Empty TIMBAL_STUN_URL disables STUN (loopback/tests); unset keeps the default. + stun_url = os.environ.get("TIMBAL_STUN_URL", "stun:stun.l.google.com:19302") + if stun_url: + servers.append(RTCIceServer(urls=stun_url)) + turn_url = os.environ.get("TIMBAL_TURN_URL") + if turn_url: + servers.append( + RTCIceServer( + urls=turn_url, + username=os.environ.get("TIMBAL_TURN_USERNAME"), + credential=os.environ.get("TIMBAL_TURN_PASSWORD"), + ) + ) + return servers + + +@router.post("/rtc") +async def voice_rtc(request: Request) -> JSONResponse: + try: + from aiortc import RTCConfiguration, RTCPeerConnection, RTCSessionDescription + + from ..voice.webrtc import PacedPlaybackTracker, PcmQueueTrack, track_to_pcm + except ImportError: + return JSONResponse( + status_code=501, + content={"error": "The WebRTC voice transport requires the timbal[voice] extra."}, + ) + + from ..core.agent import Agent + from ..voice import AudioOutput + + body = await request.json() + sdp = body.get("sdp") + if not isinstance(sdp, str) or body.get("type") != "offer": + return JSONResponse(status_code=400, content={"error": "Body must carry an SDP offer."}) + config = body.get("config") + if not isinstance(config, dict): + config = {} + + runnable = request.app.state.runnable + if not isinstance(runnable, Agent): + logger.error("voice_rtc_rejected", reason="runnable is not an Agent", type=type(runnable).__name__) + return JSONResponse(status_code=400, content={"error": "Voice requires an Agent runnable."}) + + defaults: dict = getattr(request.app.state, "voice_config", None) or {} + sample_rate = int(merge_client_voice_overrides(defaults, config).get("sample_rate", 16_000)) + + downlink = PcmQueueTrack(sample_rate=sample_rate) + session, meta = build_voice_session( + runnable, defaults, config, playback_tracker=PacedPlaybackTracker(downlink) + ) + meta = {"playback_acks": "native", "transport": "webrtc", **meta} + session.recording_meta = meta + + pc = RTCPeerConnection(configuration=RTCConfiguration(iceServers=_ice_servers())) + _pcs.add(pc) + + mic_track: Any = None + channel: Any = None + channel_ready = asyncio.Event() + # Session events can fire before SCTP is up; buffer until the client's + # data channel arrives. + pending_payloads: list[str] = [] + + def _dc_send(payload: dict) -> None: + msg = json.dumps(payload) + if channel is not None and channel.readyState == "open": + try: + channel.send(msg) + except Exception as e: + logger.debug("voice_rtc_dc_send_failed", error=str(e), msg_type=payload.get("type")) + else: + pending_payloads.append(msg) + + @pc.on("track") + def on_track(track: Any) -> None: + nonlocal mic_track + if track.kind == "audio" and mic_track is None: + mic_track = track + + @pc.on("datachannel") + def on_datachannel(ch: Any) -> None: + nonlocal channel + channel = ch + for msg in pending_payloads: + try: + ch.send(msg) + except Exception as e: + logger.debug("voice_rtc_dc_flush_failed", error=str(e)) + break + pending_payloads.clear() + channel_ready.set() + + @pc.on("connectionstatechange") + async def on_connectionstatechange() -> None: + logger.debug("voice_rtc_connection_state", state=pc.connectionState) + if pc.connectionState in ("failed", "closed"): + # Client is gone: end the session now rather than waiting for the + # STT provider to time out on silence. Idempotent when the driver + # already closed it. + await session.close() + + try: + await pc.setRemoteDescription(RTCSessionDescription(sdp=sdp, type="offer")) + except Exception as e: + _pcs.discard(pc) + logger.warning("voice_rtc_bad_offer", error=str(e)) + return JSONResponse(status_code=400, content={"error": f"Invalid SDP offer: {e}"}) + + if mic_track is None: + _pcs.discard(pc) + with contextlib.suppress(Exception): + await pc.close() + return JSONResponse(status_code=400, content={"error": "Offer must contain an audio track."}) + + # After setRemoteDescription, so the TTS track reuses the offer's audio + # transceiver instead of adding a second m-line the client never offered. + pc.addTrack(downlink) + + async def _drive() -> None: + # Do not start the session before the transport is usable: STT starts + # its clock on connect, and a session that dies immediately (bad API + # key) must still get its error payload to a client whose SCTP + # handshake hasn't finished. ICE failure lands in + # connectionstatechange, so this only times out on a client that + # connected but never offered a data channel. + try: + await asyncio.wait_for(channel_ready.wait(), timeout=15.0) + except TimeoutError: + logger.warning("voice_rtc_no_datachannel") + with contextlib.suppress(Exception): + await pc.close() + _pcs.discard(pc) + return + mic_pcm = track_to_pcm(mic_track, sample_rate=sample_rate) + try: + async with aclosing(session.run(mic_pcm)) as events: + async for event in events: + if isinstance(event, AudioOutput): + downlink.write(event.data) + continue + for payload in event_to_payloads(event, session, meta): + _dc_send(payload) + except Exception as e: + logger.error("voice_rtc_session_error", error=str(e), exc_info=True) + finally: + downlink.stop() + with contextlib.suppress(Exception): + await pc.close() + _pcs.discard(pc) + logger.info("voice_rtc_disconnected") + + driver = asyncio.create_task(_drive()) + _drivers.add(driver) + driver.add_done_callback(_drivers.discard) + + answer = await pc.createAnswer() + # Completes ICE gathering before returning — the answer is complete. + await pc.setLocalDescription(answer) + logger.info("voice_rtc_connected") + return JSONResponse( + content={"sdp": pc.localDescription.sdp, "type": pc.localDescription.type} + ) diff --git a/python/timbal/server/voice.html b/python/timbal/server/voice.html index 9f0195a3..e99518e3 100644 --- a/python/timbal/server/voice.html +++ b/python/timbal/server/voice.html @@ -102,6 +102,13 @@ border-color: var(--primary); box-shadow: 0 0 0 3px rgba(0,0,0,.07); } + .panel select:disabled { + color: var(--text-muted); + background: var(--bg); + border-color: var(--border); + cursor: not-allowed; + opacity: 0.65; + } .model-picker { display: flex; flex-direction: column; @@ -601,6 +608,13 @@

Voice session

+

Echo cancellation stays on in every mode. Settings apply on the next Start.

@@ -633,6 +647,28 @@

Voice session

localStorage.setItem(TD_STORAGE_KEY, tdSelect.value); }; +const transportSelect = document.getElementById('transport-select'); +const TRANSPORT_STORAGE_KEY = 'timbal-voice-transport'; +transportSelect.value = new URLSearchParams(location.search).get('transport') + || localStorage.getItem(TRANSPORT_STORAGE_KEY) || 'ws'; +if (!transportSelect.value) transportSelect.value = 'ws'; +function syncNsForTransport() { + // WebRTC sends the raw mic track, so the worklet's RNNoise output never + // reaches the wire — the transport forces browser noise suppression + // (startMic mirrors this). Gray the control out instead of lying with it. + const rtc = transportSelect.value === 'webrtc'; + nsSelect.disabled = rtc; + nsSelect.title = rtc + ? 'Not configurable over WebRTC — the browser applies its own noise suppression to the mic track.' + : "Mic noise suppression — applied on the next Start. Echo cancellation stays on in every mode."; +} +transportSelect.onchange = () => { + // Applied on the next Start — a live session keeps its transport. + localStorage.setItem(TRANSPORT_STORAGE_KEY, transportSelect.value); + syncNsForTransport(); +}; +syncNsForTransport(); + const sttSelect = document.getElementById('stt-select'); const STT_STORAGE_KEY = 'timbal-voice-stt-provider'; sttSelect.value = localStorage.getItem(STT_STORAGE_KEY) || ''; @@ -822,6 +858,8 @@

Voice session

} let ws, audioCtx, micStream, worklet, analyser, currentSource, meterRAF; +/** WebRTC transport state: the peer connection and the element playing the server's TTS track. */ +let pc = null, remoteAudioEl = null; /** False until WS is open, config frame sent, and AudioContext is running (worklet processes mic). */ let micAudioToWs = false; let nextPlayTime = 0; @@ -1098,7 +1136,9 @@

Voice session

if (analyser) { analyser.disconnect(); analyser = null; } if (micStream) { micStream.getTracks().forEach(t => t.stop()); micStream = null; } - const nsMode = nsSelect.value; + // WebRTC ships the raw getUserMedia track; RNNoise lives in the worklet + // and would be silently bypassed — force the browser suppressor instead. + const nsMode = transportSelect.value === 'webrtc' ? 'browser' : nsSelect.value; let useRnnoise = nsMode === 'rnnoise'; if (useRnnoise && rnnoiseAssets === null) { try { @@ -1199,7 +1239,12 @@

Voice session

micSelect.onchange = () => { if (ws && ws.readyState === WebSocket.OPEN) ws.send(JSON.stringify({type: 'mic_change'})); - startMic(micSelect.value); + void startMic(micSelect.value).then(() => { + // On WebRTC the pc holds the (now stopped) old track — swap in the new one. + const track = micStream?.getAudioTracks()[0]; + const sender = pc?.getSenders?.().find((s) => s.track?.kind === 'audio'); + if (sender && track) void sender.replaceTrack(track); + }); }; navigator.mediaDevices.addEventListener('devicechange', async () => { @@ -1252,6 +1297,17 @@

Voice session

sock.onmessage = null; try { sock.close(); } catch (_) {} } + if (pc) { + const conn = pc; + pc = null; + conn.onconnectionstatechange = null; + conn.ontrack = null; + try { conn.close(); } catch (_) {} + } + if (remoteAudioEl) { + try { remoteAudioEl.pause(); remoteAudioEl.srcObject = null; } catch (_) {} + remoteAudioEl = null; + } if (worklet) { try { worklet.disconnect(); } catch (_) {} worklet = null; } if (currentSource) { try { currentSource.disconnect(); } catch (_) {} currentSource = null; } if (micStream) { micStream.getTracks().forEach(t => t.stop()); micStream = null; } @@ -1300,6 +1356,30 @@

Voice session

void startSession(); }; +/** Config keys shared by the WS hello and the RTC offer body. */ +function sessionConfigPayload() { + const cfg = {}; + if (tdSelect.value) cfg.turn_detector = tdSelect.value; + if (sttSelect.value) cfg.stt_provider = sttSelect.value; + if (modelSelect.value) cfg.model = modelSelect.value; + return cfg; +} + +/** Fresh-session UI/playback state, shared by both transports. */ +function resetSessionUi() { + transcriptEl.innerHTML = ''; + window._assistantEl = null; + window._lastAssistantEl = null; + window._turnBubbles = []; + window._replyDone = false; + partialEl = null; + statusEl = null; + resetPlaybackTracking(); + suppressAssistantAudio = false; + stopAssistantAudio(); + sessionReady = false; +} + async function start() { const gen = startGen; setConn('connecting', 'Getting ready', 'Microphone and server connection…'); @@ -1317,6 +1397,11 @@

Voice session

await populateDevices(); } + if (transportSelect.value === 'webrtc') { + await startRtc(gen); + return; + } + setConn('connecting', 'Connecting', 'Opening WebSocket to this server…'); const proto = location.protocol === 'https:' ? 'wss' : 'ws'; micAudioToWs = false; @@ -1338,23 +1423,9 @@

Voice session

const ctxRate = audioCtx ? Math.round(audioCtx.sampleRate) : SAMPLE_RATE; const sr = rnnoiseCtxActive ? Math.round(ctxRate / 3) : ctxRate; sessionSampleRate = sr; - const hello = { sample_rate: sr }; - if (tdSelect.value) hello.turn_detector = tdSelect.value; - if (sttSelect.value) hello.stt_provider = sttSelect.value; - if (modelSelect.value) hello.model = modelSelect.value; - ws.send(JSON.stringify(hello)); - transcriptEl.innerHTML = ''; - window._assistantEl = null; - window._lastAssistantEl = null; - window._turnBubbles = []; - window._replyDone = false; - partialEl = null; - statusEl = null; + ws.send(JSON.stringify({ sample_rate: sr, ...sessionConfigPayload() })); + resetSessionUi(); setConn('live', 'Starting…', 'Loading STT / turn models (first session can take a few seconds)…'); - resetPlaybackTracking(); - suppressAssistantAudio = false; - stopAssistantAudio(); - sessionReady = false; if (playbackAckTimer) clearInterval(playbackAckTimer); playbackAckTimer = setInterval(() => sendPlaybackAck(false), PLAYBACK_ACK_INTERVAL_MS); const tryArmMic = async () => { @@ -1420,7 +1491,13 @@

Voice session

console.error('voice ws: bad json', err); return; } - switch (msg.type) { + handleServerMessage(msg); + }; +} + +/** One handler for both transports: WS frames and RTC data-channel messages. */ +function handleServerMessage(msg) { + switch (msg.type) { case 'session_started': sessionReady = true; { @@ -1531,8 +1608,95 @@

Voice session

case 'session_ended': setConn('ended', 'Session ended', 'Click Start for a new session.'); break; + } +} + +async function startRtc(gen) { + setConn('connecting', 'Connecting', 'Negotiating WebRTC with this server…'); + pc = new RTCPeerConnection(); + const conn = pc; + + // Raw mic track (browser AEC/NS applied via getUserMedia constraints). The + // worklet graph from startMic keeps driving the level meter; its PCM output + // goes nowhere because micAudioToWs stays false on this transport. + pc.addTrack(micStream.getAudioTracks()[0], micStream); + + // The server binds to whatever channel the offer carries — creating it + // here puts the SCTP m-line in the offer. + const dc = pc.createDataChannel('events'); + dc.onmessage = (e) => { + if (gen !== startGen) return; + let msg; + try { + msg = JSON.parse(e.data); + } catch (err) { + console.error('voice rtc: bad json', err); + return; } + handleServerMessage(msg); }; + + pc.ontrack = (e) => { + // TTS arrives as a real audio track: the browser handles decode, jitter + // and clock — no playPCM, no playback acks (the server paces playback + // itself, so barge-in truncation is exact without them). + remoteAudioEl = remoteAudioEl || new Audio(); + remoteAudioEl.autoplay = true; + remoteAudioEl.srcObject = e.streams[0] || new MediaStream([e.track]); + remoteAudioEl.play?.().catch(() => { /* autoplay policy — resumes on gesture */ }); + }; + + pc.onconnectionstatechange = () => { + if (gen !== startGen || pc !== conn) return; + if (['failed', 'closed', 'disconnected'].includes(conn.connectionState)) { + sessionReady = false; + micViz.classList.remove('active'); + if (connVisual === 'ended' || connVisual === 'error') { updateEmptyState(); return; } + if (userEndedSession) { + setConn('ended', 'Session ended', 'Click Start for a new session.'); + } else { + setConn('ended', 'Disconnected', 'Connection closed. Click Start for a new session.'); + } + updateEmptyState(); + } + }; + + const offer = await pc.createOffer(); + await pc.setLocalDescription(offer); + // One-round-trip signaling: wait for ICE gathering so the offer is complete + // (bounded — host candidates alone are enough against a local server). + if (pc.iceGatheringState !== 'complete') { + await new Promise((resolve) => { + const timer = setTimeout(resolve, 2000); + pc.addEventListener('icegatheringstatechange', () => { + if (conn.iceGatheringState === 'complete') { clearTimeout(timer); resolve(); } + }); + }); + } + if (gen !== startGen) return; + + const resp = await fetch('/voice/rtc', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + sdp: conn.localDescription.sdp, + type: 'offer', + config: sessionConfigPayload(), + }), + }); + if (!resp.ok) { + let detail = `Server answered ${resp.status}.`; + try { detail = (await resp.json()).error || detail; } catch (_) {} + if (resp.status === 501) detail += ' Install it: uv pip install "timbal[voice]".'; + throw new Error(detail); + } + const answer = await resp.json(); + if (gen !== startGen) return; + await conn.setRemoteDescription(answer); + + resetSessionUi(); + setConn('live', 'Starting…', 'Loading STT / turn models (first session can take a few seconds)…'); + updateEmptyState(); } initRunnableStrip(); diff --git a/python/timbal/server/voice.py b/python/timbal/server/voice.py index 9df818e4..9ebe706d 100644 --- a/python/timbal/server/voice.py +++ b/python/timbal/server/voice.py @@ -37,7 +37,7 @@ def default_voice_config_from_env() -> dict[str, Any]: """STT/TTS defaults for ``/voice/ws`` (ElevenLabs). Override with env or ``runnable.voice_config``.""" - return { + cfg: dict[str, Any] = { "stt_provider": os.environ.get("TIMBAL_STT_PROVIDER", "elevenlabs"), "stt_model": os.environ.get("TIMBAL_STT_MODEL", "scribe_v2_realtime"), "tts_model": os.environ.get("TIMBAL_TTS_MODEL", "eleven_flash_v2_5"), @@ -55,6 +55,37 @@ def default_voice_config_from_env() -> dict[str, Any]: }, "tts_extra": {"auto_mode": True}, } + return cfg + + +def _recording_config_from_env() -> dict[str, Any]: + """Recording knobs the platform injects as env vars. + + Deliberately *not* part of :func:`default_voice_config_from_env`: that + runs once at server boot, while serverless session boxes can be + CRIU-restored from a warm snapshot with env arriving at restore time. + Recording env must therefore be read per session (call sites are in + :func:`build_voice_session`). + """ + cfg: dict[str, Any] = {} + if recording_dir := os.environ.get("TIMBAL_VOICE_RECORDING_DIR"): + cfg["dir"] = recording_dir + if layout := os.environ.get("TIMBAL_VOICE_RECORDING_LAYOUT"): + cfg["layout"] = layout + if bitrate := os.environ.get("TIMBAL_VOICE_RECORDING_BITRATE_KBPS"): + cfg["bitrate_kbps"] = bitrate + return cfg + + +# Platform identity env → manifest ``meta`` keys. Makes recording files +# self-describing for sweeper ingest and crash recovery (platform ask). +_RECORDING_IDENTITY_ENV = ( + ("TIMBAL_ORG_ID", "org_id"), + ("TIMBAL_PROJECT_ID", "project_id"), + ("TIMBAL_PROJECT_ENV_ID", "project_env_id"), + ("TIMBAL_APP_ID", "app_id"), + ("TIMBAL_PROJECT_REV", "project_rev"), +) def merge_voice_config(runnable: Any) -> dict[str, Any]: @@ -162,11 +193,12 @@ def _import_stack() -> None: elif isinstance(td, str) and td.strip().lower() in ("local", "audio", "smart_turn"): detector = resolve_turn_detector(td) elif td is None: - # Playground often switches to Smart Turn on first Start — warm it. + # "local" is the default for non-Flux STT (see the session setup), and + # the playground also switches to Smart Turn on first Start. detector = resolve_turn_detector("local") if isinstance(detector, LocalAudioTurnDetector): - from ..voice.session import AudioInputConfig + from ..voice.providers import AudioInputConfig await detector.start(AudioInputConfig(sample_rate=16_000)) await SileroVad().start(sample_rate=16_000) @@ -194,26 +226,79 @@ async def voice_page(request: Request) -> HTMLResponse: return HTMLResponse(html) -@router.websocket("/ws") -async def voice_ws(ws: WebSocket) -> None: - from ..core.agent import Agent - from ..voice import ( - AgentStatus, - AgentTextDelta, - AgentTextDone, - AudioInputConfig, - AudioOutput, - AudioOutputConfig, - SessionEnded, - SessionError, - SessionInterrupted, - SessionStarted, - TranscriptCommitted, - TranscriptPartial, - TurnMetricsEvent, - VoiceSession, - VoiceSessionEvent, - ) +def select_turn_detector_spec(server_spec: Any, client_spec: Any, *, stt_is_flux: bool) -> Any: + """Choose the turn detector for a session: a mode name, instance, or factory. + + ``server_spec`` comes from ``runnable.voice_config`` and may be any of those + three. ``client_spec`` comes over the wire and is honoured only as a mode + name — a browser must not be able to inject a callable — but is otherwise + preferred, so the playground can A/B detectors. + + The choice is made here rather than in + :func:`~timbal.voice.turn_detection.resolve_turn_detector` because it turns + on the STT: whichever mode is best depends on how well the provider + endpoints, and only the session setup knows which provider is in play. + """ + spec = server_spec + if isinstance(client_spec, str) and client_spec.strip(): + spec = client_spec + elif client_spec is not None and not isinstance(client_spec, str): + logger.warning("voice_ws_bad_turn_detector", error="client turn_detector must be a mode name string") + + mode = spec.strip().lower() if isinstance(spec, str) else None + if stt_is_flux: + # Flux owns EOU (~260ms). Local/lexical run *after* EndOfTurn and add a + # second HOLD tax; they also disable the useful Provider path. Force + # provider unless the caller explicitly picked heuristic/raw/provider. + if mode is None or mode in ("local", "audio", "smart_turn", "lexical"): + if spec is not None: + # Includes an instance or factory from voice_config, which this + # overrides too — worth saying so rather than silently ignoring. + logger.info("voice_ws_flux_overrides_turn_detector", requested=str(spec), using="provider") + return "provider" + return spec + if spec is not None: + return spec + + # Nothing chosen. The holdless heuristic is the worst of the four on every + # backend that endpoints on silence — 65-69% against 96-100% for detectors + # that can hold — because Nova and ElevenLabs commit each fragment of a + # paused utterance separately, and only a hold puts them back together. + # + # The warmup path already resolved "local" for this case, so a server with no + # `turn_detector` configured was loading Smart Turn, Namo and Silero at + # startup and then handing the session a detector that used none of them. + # + # Without `timbal[voice]`, `local` returns the heuristic decision verbatim + # (its punctuation fallback sits behind the audio-EOU branch), so it would buy + # nothing; `lexical` holds an unfinished transcript with no extra deps. Asking + # the resolver beats probing imports: the degradation rule stays in one place. + from ..voice.turn_detection import resolve_turn_detector + + mode = "local" if getattr(resolve_turn_detector("local"), "audio_eou", None) is not None else "lexical" + logger.info("voice_ws_default_turn_detector", using=mode) + return mode + + +def build_voice_session( + runnable: Any, + defaults: dict[str, Any], + client_config: dict[str, Any], + *, + playback_tracker: Any = None, +) -> tuple[Any, dict[str, Any]]: + """Resolve voice config into a ``VoiceSession`` plus ``session_started`` metadata. + + Transport-agnostic: the WebSocket handler and the WebRTC route both call + this with their own client config dict. ``playback_tracker`` lets a paced + transport substitute its own position source for the default + client-acked estimate. + + Returns ``(session, meta)`` where ``meta`` holds the resolved identity + keys (``stt_provider``, ``stt_model``, ``model``, ``turn_detector``) that + transports include in their ``session_started`` payload. + """ + from ..voice import VoiceSession from ..voice.deepgram import ( DeepgramFluxSTT, effective_stt_model, @@ -221,57 +306,10 @@ async def voice_ws(ws: WebSocket) -> None: stt_provider_id, ) from ..voice.elevenlabs import ElevenLabsStreamTTS + from ..voice.providers import AudioInputConfig, AudioOutputConfig + from ..voice.turn_detection import resolve_turn_detector - await ws.accept() - logger.info("voice_ws_connected") - - runnable = ws.app.state.runnable - if not isinstance(runnable, Agent): - logger.error("voice_ws_rejected", reason="runnable is not an Agent", type=type(runnable).__name__) - await ws.close(code=1008, reason="Voice requires an Agent runnable") - return - - audio_queue: asyncio.Queue[bytes] = asyncio.Queue() - - # Read the config hello. Protocol frames ("playback" acks, "audio", - # "mic_change") can race ahead of it; the hello is the only JSON message - # *without* a "type" field, so skip typed frames — however many arrive — - # until the hello shows up or the 2s deadline passes (a swallowed hello - # silently drops sample_rate / turn_detector overrides). A *binary* first - # frame means a client that streams raw PCM without a hello: start with - # defaults immediately instead of burning the deadline. - config: dict = {} - deadline = time.monotonic() + 2.0 - try: - while (remaining := deadline - time.monotonic()) > 0: - first = await asyncio.wait_for(ws.receive(), timeout=remaining) - if "text" in first and first["text"]: - # Per-frame errors (invalid JSON, malformed audio payload) must - # not end the scan: a valid hello may still be in flight. - try: - data = json.loads(first["text"]) - except ValueError as e: - logger.warning("voice_ws_bad_handshake_frame", error=str(e)) - continue - if isinstance(data, dict) and data.get("type") is not None: - if data.get("type") == "audio": - try: - await audio_queue.put(base64.b64decode(data["data"])) - except (KeyError, TypeError, ValueError) as e: - logger.warning("voice_ws_bad_handshake_frame", error=str(e)) - # Playback acks before any TTS audio carry no information. - continue - config = data if isinstance(data, dict) else {} - elif "bytes" in first and first["bytes"]: - await audio_queue.put(first["bytes"]) - break - except TimeoutError: - pass - except Exception as e: - logger.warning("voice_ws_first_frame_error", error=str(e)) - - defaults: dict = getattr(ws.app.state, "voice_config", None) or {} - merged = merge_client_voice_overrides(defaults, config) + merged = merge_client_voice_overrides(defaults, client_config) stt_provider = merged.get("stt_provider") stt_model_requested = merged.get("stt_model") @@ -318,29 +356,15 @@ async def voice_ws(ws: WebSocket) -> None: # a mode name ("heuristic"|"provider"|"local"|"lexical"). The client JSON # may additionally select a *mode name* (string only — useful for A/B # testing detectors from the playground); instances and factories can never - # come over the wire. - from ..voice.turn_detection import resolve_turn_detector - + # come over the wire. When neither picks one, the server chooses by STT + # rather than taking ``resolve_turn_detector``'s zero-dep default, because + # only here is the STT's endpointing behaviour known. turn_detector = None - raw_td = defaults.get("turn_detector") - client_td = config.get("turn_detector") - if isinstance(client_td, str) and client_td.strip(): - raw_td = client_td - elif client_td is not None and not isinstance(client_td, str): - logger.warning("voice_ws_bad_turn_detector", error="client turn_detector must be a mode name string") - if stt_is_flux: - # Flux owns EOU (~260ms). Local/lexical run *after* EndOfTurn and add - # a second HOLD tax; they also disable the useful Provider path. Force - # provider unless the client explicitly picked heuristic/raw/provider. - td_mode = raw_td.strip().lower() if isinstance(raw_td, str) else None - if td_mode is None or td_mode in ("local", "audio", "smart_turn", "lexical"): - if td_mode is not None: - logger.info( - "voice_ws_flux_overrides_turn_detector", - requested=raw_td, - using="provider", - ) - raw_td = "provider" + raw_td = select_turn_detector_spec( + defaults.get("turn_detector"), + client_config.get("turn_detector"), + stt_is_flux=stt_is_flux, + ) if raw_td is not None: try: # voice_config is process-wide; VoiceSession clones the resolved @@ -372,7 +396,7 @@ async def voice_ws(ws: WebSocket) -> None: str(runnable.model) if isinstance(getattr(runnable, "model", None), str) else None ) logger.info( - "voice_ws_session_config", + "voice_session_config", stt=type(stt).__name__, stt_provider=stt_provider, stt_model=stt_model, @@ -385,6 +409,55 @@ async def voice_ws(ws: WebSocket) -> None: # TODO(tool-filler): read a server-side `filler_phrases` voice_config key # (list of phrases or `(tool_name) -> str | None` callable) and pass it to # `VoiceSession(filler=...)` once tool-call filler speech returns. + session_kwargs: dict[str, Any] = {} + if "turn_timeout_secs" in merged: + try: + session_kwargs["turn_timeout_secs"] = float(merged["turn_timeout_secs"]) + except (TypeError, ValueError): + logger.warning("voice_ws_bad_turn_timeout_secs", value=repr(merged.get("turn_timeout_secs"))) + if "turn_timeout_fallback" in merged: + fb = merged["turn_timeout_fallback"] + session_kwargs["turn_timeout_fallback"] = None if fb is None else str(fb) + if playback_tracker is not None: + session_kwargs["playback_tracker"] = playback_tracker + + # Call recording is read from *server* config only — env (per session, + # CRIU-safe) under ``runnable.voice_config["recording"]`` (user keys win) + # — never from the merged dict: merge_client_voice_overrides applies + # client keys freely, and a browser must not be able to switch recording + # on or off. + user_recording = defaults.get("recording") + recording_cfg = { + **_recording_config_from_env(), + **(user_recording if isinstance(user_recording, dict) else {}), + } + if recording_cfg.get("dir"): + try: + from uuid_extensions import uuid7 + + from ..voice.recording import CallRecorder + + on_saved = recording_cfg.get("on_saved") + if on_saved is None and os.environ.get("TIMBAL_VOICE_RECORDING_UPLOAD") == "platform": + from .recording_upload import platform_recording_upload_hook + + on_saved = platform_recording_upload_hook() + session_id = uuid7(as_type="str").replace("-", "") + session_kwargs["session_id"] = session_id + session_kwargs["recorder"] = CallRecorder( + Path(recording_cfg["dir"]) / f"{session_id}.mp3", + sample_rate=int(merged.get("sample_rate", 16_000)), + layout=recording_cfg.get("layout", "mixed"), + bitrate_kbps=int(recording_cfg.get("bitrate_kbps", 32)), + on_saved=on_saved, + meta={k: v for env_key, k in _RECORDING_IDENTITY_ENV if (v := os.environ.get(env_key))} or None, + ) + except ImportError: + logger.warning("voice_recording_unavailable", hint="call recording requires timbal[voice] (av + numpy)") + except Exception as e: + # A misconfigured recorder must not take voice down with it. + logger.error("voice_recording_setup_failed", error=str(e), exc_info=True) + session = VoiceSession( agent=runnable, stt=stt, @@ -394,7 +467,150 @@ async def voice_ws(ws: WebSocket) -> None: turn_detector=turn_detector, vad_endpointing=vad_endpointing, model=model_override, + **session_kwargs, ) + meta = { + "session_id": session.session_id, + "stt_provider": stt_provider, + "stt_model": stt_model, + "model": llm_model, + "turn_detector": turn_detector_label, + } + return session, meta + + +def event_to_payloads(event: Any, session: Any, meta: dict[str, Any]) -> list[dict[str, Any]]: + """Map one ``VoiceSessionEvent`` to the JSON payloads a transport sends. + + Shared by the WebSocket handler and the WebRTC data channel — the wire + format is identical. ``AudioOutput`` is serialized as base64 here; a paced + transport (WebRTC) intercepts it *before* calling this and feeds the PCM + to its audio track instead. + + ``meta`` is the dict from :func:`build_voice_session`, extended by the + transport with its own keys (``transport``, ``playback_acks``). + """ + from ..voice import ( + AgentStatus, + AgentTextDelta, + AgentTextDone, + AudioOutput, + SessionEnded, + SessionError, + SessionInterrupted, + SessionStarted, + TranscriptCommitted, + TranscriptPartial, + TurnMetricsEvent, + ) + + if isinstance(event, SessionStarted): + return [ + { + "type": "session_started", + **meta, + # The endpointer arms during session startup (before this event + # is emitted), so this reflects the real state — not the + # requested config. + "vad_endpointing": session._endpointer is not None, + } + ] + if isinstance(event, TranscriptPartial): + return [{"type": "transcript_partial", "text": event.text}] + if isinstance(event, TranscriptCommitted): + payload: dict[str, Any] = {"type": "transcript_committed", "text": event.text} + if event.replace: + payload["replace"] = True + return [payload] + if isinstance(event, AgentStatus): + return [{"type": "agent_status", "text": event.text}] + if isinstance(event, AgentTextDelta): + return [{"type": "agent_text_delta", "text": event.text}] + if isinstance(event, AgentTextDone): + return [{"type": "agent_text_done", "text": event.text}] + if isinstance(event, AudioOutput): + return [{"type": "audio", "data": base64.b64encode(event.data).decode("ascii")}] + if isinstance(event, TurnMetricsEvent): + return [{"type": "metrics", "metrics": event.metrics.model_dump()}] + if isinstance(event, SessionInterrupted): + return [{"type": "interrupted", "heard_text": event.heard_text}] + if isinstance(event, SessionError): + return [{"type": "error", "message": event.message}] + if isinstance(event, SessionEnded): + # Entries carry absolute wall-clock timestamps; offset_ms (relative to + # started_at) saves every client the subtraction — it's what a + # conversation-review UI actually renders. + started_at = getattr(session, "started_at", None) + entries = [] + for e in session.transcript: + d = e.model_dump() + if started_at is not None: + d["offset_ms"] = max(0, round((e.timestamp - started_at) * 1000)) + entries.append(d) + transcript_payload: dict[str, Any] = {"type": "session_transcript", "entries": entries} + if started_at is not None: + transcript_payload["started_at"] = started_at + return [transcript_payload, {"type": "session_ended"}] + return [] + + +@router.websocket("/ws") +async def voice_ws(ws: WebSocket) -> None: + from ..core.agent import Agent + from ..voice import SessionInterrupted, VoiceSessionEvent + + await ws.accept() + logger.info("voice_ws_connected") + + runnable = ws.app.state.runnable + if not isinstance(runnable, Agent): + logger.error("voice_ws_rejected", reason="runnable is not an Agent", type=type(runnable).__name__) + await ws.close(code=1008, reason="Voice requires an Agent runnable") + return + + audio_queue: asyncio.Queue[bytes] = asyncio.Queue() + + # Read the config hello. Protocol frames ("playback" acks, "audio", + # "mic_change") can race ahead of it; the hello is the only JSON message + # *without* a "type" field, so skip typed frames — however many arrive — + # until the hello shows up or the 2s deadline passes (a swallowed hello + # silently drops sample_rate / turn_detector overrides). A *binary* first + # frame means a client that streams raw PCM without a hello: start with + # defaults immediately instead of burning the deadline. + config: dict = {} + deadline = time.monotonic() + 2.0 + try: + while (remaining := deadline - time.monotonic()) > 0: + first = await asyncio.wait_for(ws.receive(), timeout=remaining) + if "text" in first and first["text"]: + # Per-frame errors (invalid JSON, malformed audio payload) must + # not end the scan: a valid hello may still be in flight. + try: + data = json.loads(first["text"]) + except ValueError as e: + logger.warning("voice_ws_bad_handshake_frame", error=str(e)) + continue + if isinstance(data, dict) and data.get("type") is not None: + if data.get("type") == "audio": + try: + await audio_queue.put(base64.b64decode(data["data"])) + except (KeyError, TypeError, ValueError) as e: + logger.warning("voice_ws_bad_handshake_frame", error=str(e)) + # Playback acks before any TTS audio carry no information. + continue + config = data if isinstance(data, dict) else {} + elif "bytes" in first and first["bytes"]: + await audio_queue.put(first["bytes"]) + break + except TimeoutError: + pass + except Exception as e: + logger.warning("voice_ws_first_frame_error", error=str(e)) + + defaults: dict = getattr(ws.app.state, "voice_config", None) or {} + session, meta = build_voice_session(runnable, defaults, config) + meta = {"playback_acks": "recommended", "transport": "websocket", **meta} + session.recording_meta = meta async def _recv_loop() -> None: """Read frames from the browser and feed PCM into the audio queue.""" @@ -459,68 +675,16 @@ async def _send_json(data: dict) -> None: async def _handle(event: VoiceSessionEvent) -> None: """Forward session events to the browser over WebSocket.""" nonlocal warned_ack_degraded - if isinstance(event, SessionStarted): - # ``playback_acks`` advertises that the server understands - # ``{"type": "playback", "played_ms": ...}`` and expects clients that - # play audio to send them (see server/README.md). - await _send_json( - { - "type": "session_started", - "playback_acks": "recommended", - # Config id (``elevenlabs`` / ``deepgram-flux`` / …), not - # the STT class name — matches playground select values. - "stt_provider": stt_provider, - "stt_model": stt_model, - "model": llm_model, - "turn_detector": turn_detector_label, - # The endpointer arms during session startup (before this - # event is emitted), so this reflects the real state — not - # the requested config. - "vad_endpointing": session._endpointer is not None, - } + if isinstance(event, SessionInterrupted) and not warned_ack_degraded and not session.playback.ack_received: + warned_ack_degraded = True + logger.warning( + "voice_ws_truncation_degraded", + hint="client sent no playback acks before a barge-in; heard-text " + "truncation used the wall-clock estimate. Send " + '{"type": "playback", "played_ms": ...} every ~250ms while audio plays.', ) - elif isinstance(event, TranscriptPartial): - await _send_json({"type": "transcript_partial", "text": event.text}) - elif isinstance(event, TranscriptCommitted): - payload: dict = {"type": "transcript_committed", "text": event.text} - if event.replace: - payload["replace"] = True + for payload in event_to_payloads(event, session, meta): await _send_json(payload) - elif isinstance(event, AgentStatus): - await _send_json({"type": "agent_status", "text": event.text}) - elif isinstance(event, AgentTextDelta): - await _send_json({"type": "agent_text_delta", "text": event.text}) - elif isinstance(event, AgentTextDone): - await _send_json({"type": "agent_text_done", "text": event.text}) - elif isinstance(event, AudioOutput): - await _send_json( - { - "type": "audio", - "data": base64.b64encode(event.data).decode("ascii"), - } - ) - elif isinstance(event, TurnMetricsEvent): - await _send_json({"type": "metrics", "metrics": event.metrics.model_dump()}) - elif isinstance(event, SessionInterrupted): - if not warned_ack_degraded and not session.playback.ack_received: - warned_ack_degraded = True - logger.warning( - "voice_ws_truncation_degraded", - hint="client sent no playback acks before a barge-in; heard-text " - "truncation used the wall-clock estimate. Send " - '{"type": "playback", "played_ms": ...} every ~250ms while audio plays.', - ) - await _send_json({"type": "interrupted", "heard_text": event.heard_text}) - elif isinstance(event, SessionError): - await _send_json({"type": "error", "message": event.message}) - elif isinstance(event, SessionEnded): - await _send_json( - { - "type": "session_transcript", - "entries": [e.model_dump() for e in session.transcript], - } - ) - await _send_json({"type": "session_ended"}) recv_task = asyncio.create_task(_recv_loop()) try: diff --git a/python/timbal/voice/__init__.py b/python/timbal/voice/__init__.py index 487c7c10..eb58a991 100644 --- a/python/timbal/voice/__init__.py +++ b/python/timbal/voice/__init__.py @@ -10,6 +10,20 @@ PunctuationEouPredictor, TextEouPredictor, ) +from .events import ( + AgentStatus, + AgentTextDelta, + AgentTextDone, + AudioOutput, + SessionEnded, + SessionError, + SessionInterrupted, + SessionStarted, + TranscriptCommitted, + TranscriptEntry, + TranscriptPartial, + VoiceSessionEvent, +) from .metrics import ( TurnMetrics, TurnMetricsEvent, @@ -18,32 +32,20 @@ BufferedPlaybackTracker, PlaybackTracker, ) -from .realtime import ( - RealtimeEvent, - RealtimeModel, - RealtimeSession, -) -from .session import ( - AgentStatus, - AgentTextDelta, - AgentTextDone, +from .providers import ( AudioInputConfig, - AudioOutput, AudioOutputConfig, - SessionEnded, - SessionError, - SessionInterrupted, - SessionStarted, SpeechToText, TextToSpeech, - TranscriptCommitted, - TranscriptEntry, TranscriptEvent, - TranscriptPartial, TTSStream, - VoiceSession, - VoiceSessionEvent, ) +from .realtime import ( + RealtimeEvent, + RealtimeModel, + RealtimeSession, +) +from .session import VoiceSession from .turn_detection import ( CommitAction, CommitDecision, diff --git a/python/timbal/voice/deepgram.py b/python/timbal/voice/deepgram.py index a852795f..7db199b0 100644 --- a/python/timbal/voice/deepgram.py +++ b/python/timbal/voice/deepgram.py @@ -31,7 +31,7 @@ from websockets.asyncio.client import connect as ws_connect from websockets.exceptions import ConnectionClosed -from .session import ( +from .providers import ( AudioInputConfig, SpeechToText, TranscriptEvent, @@ -183,13 +183,15 @@ async def _receive_loop(self) -> None: continue await self._handle_message(msg) except ConnectionClosed as e: - logger.debug("dg_stt_ws_closed", error=str(e)) - # Deepgram closes normally after CloseStream; only surface abnormal - # closures as errors so the session tears down loudly. - if not self._stop.is_set() and e.rcvd is not None and e.rcvd.code not in (1000, 1001): - await self._queue.put( - TranscriptEvent(type="error", text=f"STT connection closed: {e}") - ) + if self._stop.is_set(): + # Requested teardown (close() sent CloseStream); silence is correct. + logger.debug("dg_stt_ws_closed", error=str(e)) + else: + # Provider-initiated, any code — 1000/1001 included. The provider + # hanging up mid-call is not the user hanging up; ending the + # session silently made the two indistinguishable. + logger.warning("dg_stt_ws_closed_unrequested", error=str(e)) + await self._queue.put(TranscriptEvent(type="error", text=f"STT connection closed: {e}")) except Exception as e: logger.error("dg_stt_receive_error", error=str(e), exc_info=True) await self._queue.put(TranscriptEvent(type="error", text=f"STT receive error: {e}")) @@ -269,6 +271,15 @@ def _build_uri(self, config: AudioInputConfig) -> str: # EndOfTurn; keep this under the session stale window (2.5s). if "eot_timeout_ms" not in extra: params.append(("eot_timeout_ms", "2000")) + # Deepgram's 0.7 ends a turn on a mid-sentence pause: swept over the + # pause-merge family, it merges 68% against 82% at 0.8 and 91% at 0.9, + # monotonically, for 635ms / 859ms / 1944ms of dead air. 0.8 buys fourteen + # points for 214ms and takes the cell to 100% with no ghost turns; 0.9 buys + # nine more for a further second, which is worse dead air than ElevenLabs. + # `provider` has no hold of its own, so on Flux this value *is* the turn + # policy — it is the only setting that can merge a pause there. + if "eot_threshold" not in extra: + params.append(("eot_threshold", "0.8")) for k, v in extra.items(): if v is None or k.startswith("_") or k not in _FLUX_QUERY_KEYS: continue diff --git a/python/timbal/voice/elevenlabs.py b/python/timbal/voice/elevenlabs.py index 7a58db3d..06169842 100644 --- a/python/timbal/voice/elevenlabs.py +++ b/python/timbal/voice/elevenlabs.py @@ -24,7 +24,7 @@ from websockets.asyncio.client import connect as ws_connect from websockets.exceptions import ConnectionClosed -from .session import ( +from .providers import ( AudioInputConfig, AudioOutputConfig, SpeechToText, @@ -166,7 +166,14 @@ async def _flush_audio(self, commit: bool = False) -> None: "commit": commit, "sample_rate": self._input_config.sample_rate if self._input_config else 16_000, } - await self._ws.send(json.dumps(msg)) + try: + await self._ws.send(json.dumps(msg)) + except ConnectionClosed: + # Dead socket: drop the audio. Acceptable only because the receive + # loop's sentinel is already tearing the session down — and required, + # because this path is called from the periodic flusher and from + # commit() (VAD endpointing), neither of which may raise. + pass async def _periodic_flush(self) -> None: try: @@ -208,8 +215,13 @@ async def _receive_loop(self) -> None: await self._queue.put(TranscriptEvent(type="error", text=f"STT fatal ({mt}): {err}")) break except ConnectionClosed as e: - logger.debug("el_stt_ws_closed", error=str(e)) - await self._queue.put(TranscriptEvent(type="error", text=f"STT connection closed: {e}")) + if self._stop.is_set(): + # Requested teardown; an error event here races session close and + # can surface a spurious SessionError on a normal hangup. + logger.debug("el_stt_ws_closed", error=str(e)) + else: + logger.warning("el_stt_ws_closed_unrequested", error=str(e)) + await self._queue.put(TranscriptEvent(type="error", text=f"STT connection closed: {e}")) except Exception as e: logger.error("el_stt_receive_error", error=str(e), exc_info=True) await self._queue.put(TranscriptEvent(type="error", text=f"STT receive error: {e}")) diff --git a/python/timbal/voice/endpointing.py b/python/timbal/voice/endpointing.py index 3f80c78b..7623a38c 100644 --- a/python/timbal/voice/endpointing.py +++ b/python/timbal/voice/endpointing.py @@ -149,9 +149,7 @@ def __init__( min_commit_interval_secs if min_commit_interval_secs is not None else self.MIN_COMMIT_INTERVAL_SECS ) self.text_incomplete_delay_secs = ( - text_incomplete_delay_secs - if text_incomplete_delay_secs is not None - else self.TEXT_INCOMPLETE_DELAY_SECS + text_incomplete_delay_secs if text_incomplete_delay_secs is not None else self.TEXT_INCOMPLETE_DELAY_SECS ) self._score: Callable[[], Awaitable[float | None]] | None = None @@ -302,11 +300,24 @@ async def _run_endpoint(self) -> None: return t0 = time.monotonic() p = await self._score() + p_text: float | None = None + if self._text_score is not None: + try: + p_text = await self._text_score() + except Exception as e: + logger.warning("vad_text_score_failed", error=str(e)) + p_text = None + driver = "audio" if p is None: - # No audio EOU available / not enough buffered signal: never - # force-commit blind — the provider debounce handles it. - logger.debug("vad_endpoint_skipped", reason="no_eou_score") - return + # No audio EOU (text-only detector, or not enough buffered + # signal). Size the delay from the text score instead of + # skipping: the mapping only needs a P(complete), and refusing + # to commit here leaves the turn waiting on the provider's own + # debounce — 7.5s on a spoken account number under Nova. + if p_text is None: + logger.debug("vad_endpoint_skipped", reason="no_eou_score") + return + p, driver = p_text, "text" delay = endpointing_delay( p, min_delay=self.min_delay_secs, @@ -314,22 +325,19 @@ async def _run_endpoint(self) -> None: curve=self.delay_curve, ) text_bumped = False - p_text: float | None = None - if self._text_score is not None: - try: - p_text = await self._text_score() - except Exception as e: - logger.warning("vad_text_score_failed", error=str(e)) - p_text = None - if p_text is not None and p_text < self.TEXT_INCOMPLETE_THRESHOLD: - bumped = max(delay, self.text_incomplete_delay_secs) - text_bumped = bumped > delay - delay = bumped + # Only a bump when text is the *second* opinion. With text driving, + # the incomplete case is already priced into the delay above, and + # applying the floor again would double-count it. + if driver == "audio" and p_text is not None and p_text < self.TEXT_INCOMPLETE_THRESHOLD: + bumped = max(delay, self.text_incomplete_delay_secs) + text_bumped = bumped > delay + delay = bumped # INFO on purpose: fires once per speech-stop and is the whole # observable behaviour of the endpointing fast path. logger.info( "vad_eou_score", p=round(p, 3), + driver=driver, delay_secs=round(delay, 3), score_ms=round((time.monotonic() - t0) * 1000, 1), p_text=round(p_text, 3) if p_text is not None else None, diff --git a/python/timbal/voice/eou.py b/python/timbal/voice/eou.py index 360b0221..9c18e5e4 100644 --- a/python/timbal/voice/eou.py +++ b/python/timbal/voice/eou.py @@ -81,16 +81,91 @@ async def predict_eou(self, text: str) -> float: _DANGLING_TOKENS = frozenset( { # English - "and", "or", "but", "so", "because", "if", "when", "while", "that", - "which", "who", "to", "of", "for", "with", "from", "in", "on", "at", - "by", "the", "a", "an", "my", "your", "our", "their", "his", "her", - "its", "i", "we", "you", "they", "it", "is", "are", "was", "were", - "um", "uh", "like", "then", "as", "than", "into", "about", "over", + "and", + "or", + "but", + "so", + "because", + "if", + "when", + "while", + "that", + "which", + "who", + "to", + "of", + "for", + "with", + "from", + "in", + "on", + "at", + "by", + "the", + "a", + "an", + "my", + "your", + "our", + "their", + "his", + "her", + "its", + "i", + "we", + "you", + "they", + "it", + "is", + "are", + "was", + "were", + "um", + "uh", + "like", + "then", + "as", + "than", + "into", + "about", + "over", # Spanish - "y", "e", "o", "u", "pero", "porque", "que", "si", "cuando", - "mientras", "de", "del", "para", "por", "con", "sin", "en", - "la", "el", "los", "las", "un", "una", "unos", "unas", "mi", "tu", - "su", "es", "está", "estoy", "eh", "este", "esto", "como", "más", + "y", + "e", + "o", + "u", + "pero", + "porque", + "que", + "si", + "cuando", + "mientras", + "de", + "del", + "para", + "por", + "con", + "sin", + "en", + "la", + "el", + "los", + "las", + "un", + "una", + "unos", + "unas", + "mi", + "tu", + "su", + "es", + "está", + "estoy", + "eh", + "este", + "esto", + "como", + "más", "muy", } ) @@ -102,8 +177,22 @@ async def predict_eou(self, text: str) -> float: # Namo-class text EOU behind the same :class:`TextEouPredictor` interface. _SHORT_HEDGES = frozenset( { - "uh", "um", "uhm", "hmm", "mm", "mhm", "mmhmm", "well", "so", "like", - "maybe", "perhaps", "idk", "pues", "este", "eh", + "uh", + "um", + "uhm", + "hmm", + "mm", + "mhm", + "mmhmm", + "well", + "so", + "like", + "maybe", + "perhaps", + "idk", + "pues", + "este", + "eh", } ) _PHRASE_HEDGES = frozenset( diff --git a/python/timbal/voice/events.py b/python/timbal/voice/events.py new file mode 100644 index 00000000..ac98ba6d --- /dev/null +++ b/python/timbal/voice/events.py @@ -0,0 +1,85 @@ +"""Observable surface of a voice session: transcript records and events. + +Shared by :class:`~timbal.voice.session.VoiceSession` (STT → agent → TTS) and +:class:`~timbal.voice.realtime.RealtimeSession` (speech-to-speech). +""" + +from __future__ import annotations + +import time +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field + + +class TranscriptEntry(BaseModel): + """Single entry in the session transcript.""" + + model_config = ConfigDict(extra="forbid") + + role: Literal["user", "assistant"] + text: str + timestamp: float = Field(default_factory=time.time) + # TODO(tool-filler): re-add `filler: bool = False` when tool-call filler + # speech returns, so transcript entries for spoken latency-masking phrases + # ("let me check that") are distinguishable from real reply text. + + +class VoiceSessionEvent(BaseModel): + """Base for all events emitted by a :class:`VoiceSession`.""" + + type: str + + +class SessionStarted(VoiceSessionEvent): + type: Literal["session_started"] = "session_started" + + +class SessionEnded(VoiceSessionEvent): + type: Literal["session_ended"] = "session_ended" + + +class TranscriptPartial(VoiceSessionEvent): + type: Literal["transcript_partial"] = "transcript_partial" + text: str + + +class TranscriptCommitted(VoiceSessionEvent): + type: Literal["transcript_committed"] = "transcript_committed" + text: str + # True → rewrite last user bubble (CONTINUE_TURN merge), don't append another. + replace: bool = False + + +class AgentTextDelta(VoiceSessionEvent): + type: Literal["agent_text_delta"] = "agent_text_delta" + text: str + + +class AgentTextDone(VoiceSessionEvent): + type: Literal["agent_text_done"] = "agent_text_done" + text: str + + +class AgentStatus(VoiceSessionEvent): + """Non-transcript status for the UI (e.g. tool calls while the mic is idle).""" + + type: Literal["agent_status"] = "agent_status" + text: str + + +class AudioOutput(VoiceSessionEvent): + model_config = ConfigDict(arbitrary_types_allowed=True) + type: Literal["audio_output"] = "audio_output" + data: bytes + + +class SessionInterrupted(VoiceSessionEvent): + type: Literal["interrupted"] = "interrupted" + heard_text: str | None = None + """Assistant text the user actually heard before the interruption (None if unknown/none).""" + + +class SessionError(VoiceSessionEvent): + type: Literal["error"] = "error" + message: str diff --git a/python/timbal/voice/metrics.py b/python/timbal/voice/metrics.py index 118de48a..dd8351f4 100644 --- a/python/timbal/voice/metrics.py +++ b/python/timbal/voice/metrics.py @@ -15,7 +15,7 @@ from pydantic import BaseModel, ConfigDict -from .session import VoiceSessionEvent +from .events import VoiceSessionEvent class TurnMetrics(BaseModel): @@ -24,6 +24,8 @@ class TurnMetrics(BaseModel): model_config = ConfigDict(extra="forbid") turn_index: int + run_id: str | None = None + """Id of the agent run this turn produced, to join metrics against the trace.""" user_text_chars: int eou_to_llm_first_token_ms: float | None = None """Committed transcript -> first LLM text delta.""" diff --git a/python/timbal/voice/namo.py b/python/timbal/voice/namo.py index d70c4d36..1d2f3720 100644 --- a/python/timbal/voice/namo.py +++ b/python/timbal/voice/namo.py @@ -120,9 +120,7 @@ def _prepare_text_for_namo(text: str) -> tuple[str, float | None]: @lru_cache(maxsize=4) -def _load_bundle( - repo_id: str, filename: str, cpu_count: int, max_length: int -) -> tuple[ort.InferenceSession, object]: +def _load_bundle(repo_id: str, filename: str, cpu_count: int, max_length: int) -> tuple[ort.InferenceSession, object]: """One (ort session, tokenizer) per repo — shared process-wide.""" path = _hf_download_cached_first(repo_id, filename) logger.debug("namo_loading_model", path=path, repo_id=repo_id) @@ -179,11 +177,7 @@ def __init__( model_path: str | None = None, cpu_count: int = 1, ) -> None: - self.repo_id = ( - repo_id - or os.environ.get("TIMBAL_NAMO_REPO_ID") - or DEFAULT_REPO_ID - ) + self.repo_id = repo_id or os.environ.get("TIMBAL_NAMO_REPO_ID") or DEFAULT_REPO_ID self.model_path = model_path self.cpu_count = cpu_count self.max_length = _MAX_LENGTH_BY_REPO.get(self.repo_id, _DEFAULT_MAX_LENGTH) diff --git a/python/timbal/voice/providers.py b/python/timbal/voice/providers.py new file mode 100644 index 00000000..09e9f652 --- /dev/null +++ b/python/timbal/voice/providers.py @@ -0,0 +1,119 @@ +"""STT/TTS provider interfaces and their cross-provider configuration. + +Implement these to plug a new speech provider into +:class:`~timbal.voice.session.VoiceSession`. +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from collections.abc import AsyncIterator +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, Field + + +class AudioInputConfig(BaseModel): + """Cross-provider STT configuration; provider-specific knobs go in ``extra``.""" + + model_config = ConfigDict(extra="forbid") + + model: str | None = None + language: str | None = None + sample_rate: int = 16_000 + encoding: str = "pcm_s16le" + extra: dict[str, Any] = Field(default_factory=dict) + + +class AudioOutputConfig(BaseModel): + """Cross-provider TTS configuration; provider-specific knobs go in ``extra``.""" + + model_config = ConfigDict(extra="forbid") + + model: str | None = None + voice: str | None = None + sample_rate: int = 16_000 + encoding: str = "pcm_s16le" + extra: dict[str, Any] = Field(default_factory=dict) + + +class TranscriptEvent(BaseModel): + """Single event from an STT provider.""" + + type: Literal["partial", "committed", "error"] + text: str + + +class SpeechToText(ABC): + """Abstract STT provider. + + Lifecycle: ``connect`` → push audio / consume ``events`` → ``close``. + """ + + @abstractmethod + async def connect(self, config: AudioInputConfig) -> None: ... + + @abstractmethod + async def push_audio(self, chunk: bytes) -> None: ... + + @abstractmethod + async def commit(self) -> None: ... + + @abstractmethod + def events(self) -> AsyncIterator[TranscriptEvent]: ... + + @abstractmethod + async def close(self) -> None: ... + + +class TTSStream(ABC): + """One streaming synthesis unit (e.g. an ElevenLabs *context*): text is fed + incrementally while audio is read concurrently from :meth:`audio`. + + Contract: + + * ``feed`` may be called any number of times (in order). + * ``end`` signals no more text; ``audio()`` finishes once the provider + drains the remaining synthesis. + * ``abort`` (barge-in) stops generation ASAP and unblocks ``audio()``. + * All methods are idempotent-safe after ``end``/``abort``. + """ + + @abstractmethod + async def feed(self, text: str) -> None: ... + + @abstractmethod + async def end(self) -> None: ... + + @abstractmethod + async def abort(self) -> None: ... + + @abstractmethod + def audio(self) -> AsyncIterator[bytes]: ... + + +class TextToSpeech(ABC): + """Abstract TTS provider. + + Lifecycle: ``connect`` → ``synthesize`` (repeatable) → ``close``. + """ + + @abstractmethod + async def connect(self, config: AudioOutputConfig) -> None: ... + + @abstractmethod + def synthesize(self, text: str) -> AsyncIterator[bytes]: ... + + def open_stream(self) -> TTSStream | None: + """Optional capability: a per-reply streaming context. + + Providers supporting incremental text input with prosody continuity + (ElevenLabs multi-context WS) return a fresh :class:`TTSStream` per + call, so the session feeds every flush into ONE context. Independent + segments each get "final sentence" intonation and an audible seam + between them. Default ``None`` → per-segment ``synthesize``. + """ + return None + + @abstractmethod + async def close(self) -> None: ... diff --git a/python/timbal/voice/realtime.py b/python/timbal/voice/realtime.py index af9c6496..6328c5ec 100644 --- a/python/timbal/voice/realtime.py +++ b/python/timbal/voice/realtime.py @@ -52,21 +52,15 @@ import time from abc import ABC, abstractmethod from collections.abc import AsyncIterable, AsyncIterator -from typing import TYPE_CHECKING, Literal +from typing import Literal import structlog from pydantic import BaseModel, ConfigDict -if TYPE_CHECKING: - from .metrics import TurnMetrics - -from .playback import BufferedPlaybackTracker, PlaybackTracker, map_played_bytes_to_text -from .session import ( +from .events import ( AgentTextDelta, AgentTextDone, - AudioInputConfig, AudioOutput, - AudioOutputConfig, SessionEnded, SessionError, SessionInterrupted, @@ -76,6 +70,9 @@ TranscriptPartial, VoiceSessionEvent, ) +from .metrics import TurnMetrics, TurnMetricsEvent +from .playback import BufferedPlaybackTracker, PlaybackTracker, map_played_bytes_to_text +from .providers import AudioInputConfig, AudioOutputConfig logger = structlog.get_logger("timbal.voice.realtime") @@ -382,9 +379,7 @@ async def _handle_interruption(self, *, notify_model: bool) -> None: if self._turn_text or self._turn_audio_bytes: # S2S interleaves text and audio without segment boundaries: map the # heard position proportionally over the whole turn. - heard_text = map_played_bytes_to_text( - [(self._turn_text, self._turn_audio_bytes)], heard_bytes - ) + heard_text = map_played_bytes_to_text([(self._turn_text, self._turn_audio_bytes)], heard_bytes) if self._turn_active: if heard_text: self._transcript.append(TranscriptEntry(role="assistant", text=heard_text)) @@ -412,8 +407,6 @@ async def _handle_interruption(self, *, notify_model: bool) -> None: await self._emit(SessionInterrupted(heard_text=heard_text)) async def _emit_turn_metrics(self, *, interrupted: bool, heard_bytes: int | None = None) -> None: - from .metrics import TurnMetrics, TurnMetricsEvent - def _ms(t0: float | None, t1: float | None) -> float | None: if t0 is None or t1 is None: return None diff --git a/python/timbal/voice/recording.py b/python/timbal/voice/recording.py new file mode 100644 index 00000000..54ec2e91 --- /dev/null +++ b/python/timbal/voice/recording.py @@ -0,0 +1,322 @@ +"""Call recording: one playable audio file per voice session, plus a manifest. + +The recorder reconstructs the *call as heard*, not the TTS as synthesized. +Mic PCM is the master clock — it arrives continuously in real time on both +transports — and every mic chunk drains an equal span of the agent's queued +TTS onto the same timeline (silence when the agent wasn't talking). TTS +synthesizes faster than real time, so the agent side is a queue; a barge-in +drops its unheard tail (:meth:`CallRecorder.drop_agent_tail`) exactly like +interruption truncation drops unheard text. + +Layouts: + +* ``"mixed"`` (default) — mono, both voices summed with a clipping guard. + Sounds like the call did; what a human reviewing the conversation wants. +* ``"split"`` — stereo, caller left / agent right. The call-center + convention for re-transcription/diarization where voice bleed matters. + +Output is MP3 (universal ``