From ad87ad07686b8df535b43aea9788683b7d92a999 Mon Sep 17 00:00:00 2001 From: berges99 Date: Thu, 23 Jul 2026 17:10:55 +0200 Subject: [PATCH 01/15] feat(voice): RNNoise mic denoising option in the playground MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a "Noise: browser / RNNoise / off" selector to voice.html. RNNoise (@jitsi/rnnoise-wasm from jsDelivr) runs inside the capture worklet as a mic preprocessor: 48 kHz context, 480-sample frames denoised in place, then 3:1 decimation back to 16 kHz so the wire format is unchanged. Transport-agnostic by design — the same preprocessing carries over to a future WebRTC channel. Browser noiseSuppression is disabled while RNNoise is active (never two suppressors); echoCancellation stays on in every mode since the barge-in echo gates rely on it. Asset fetch failure, wasm init failure, and a browser refusing a 48 kHz context all degrade gracefully to the previous behavior. The hello reports the worklet's output rate (ctx/3 when denoising) so STT negotiation stays correct. Footer layout: mode selects (Turns, Noise) move to their own row so labels stay readable. --- python/timbal/server/voice.html | 236 +++++++++++++++++++++++++++----- 1 file changed, 199 insertions(+), 37 deletions(-) diff --git a/python/timbal/server/voice.html b/python/timbal/server/voice.html index 50e50052..cd60315b 100644 --- a/python/timbal/server/voice.html +++ b/python/timbal/server/voice.html @@ -164,7 +164,7 @@ 50% { opacity: 0.5; transform: scale(0.85); } } - #mic-select, #td-select { + #mic-select, #td-select, #ns-select { flex: 1; min-width: 0; font: inherit; @@ -177,14 +177,10 @@ outline: none; cursor: pointer; } - #mic-select:focus, #td-select:focus { + #mic-select:focus, #td-select:focus, #ns-select:focus { border-color: var(--primary); box-shadow: 0 0 0 3px rgba(0,0,0,.07); } - #td-select { - flex: 0 1 auto; - max-width: 46%; - } .mic-viz { display: flex; align-items: flex-end; @@ -367,6 +363,16 @@ flex: 1; min-width: 0; } + .foot-row--modes { + display: flex; + align-items: center; + gap: 0.5rem; + min-width: 0; + } + .foot-row--modes select { + flex: 1 1 0; + min-width: 0; + } .foot-actions { display: flex; align-items: center; @@ -486,6 +492,11 @@
+ +
+
- +

Audio is captured in your browser (microphone) and streamed to this server. Use a private session and only on sites you trust; anyone with access to this page may be able to use the same agent.

@@ -518,6 +531,7 @@ const btnReconnect = document.getElementById('btn-reconnect'); const micSelect = document.getElementById('mic-select'); const tdSelect = document.getElementById('td-select'); +const nsSelect = document.getElementById('ns-select'); const micViz = document.getElementById('mic-viz'); const TD_STORAGE_KEY = 'timbal-voice-turn-detector'; @@ -528,6 +542,15 @@ localStorage.setItem(TD_STORAGE_KEY, tdSelect.value); location.reload(); }; + +const NS_STORAGE_KEY = 'timbal-voice-noise-suppression'; +nsSelect.value = localStorage.getItem(NS_STORAGE_KEY) || 'browser'; +if (!nsSelect.value) nsSelect.value = 'browser'; // unknown stored value +nsSelect.onchange = () => { + // The AudioContext rate and worklet graph are fixed at session start. + localStorage.setItem(NS_STORAGE_KEY, nsSelect.value); + location.reload(); +}; const micBars = () => [...micViz.querySelectorAll('.mic-bar')]; function truncateMiddle(s, max) { @@ -716,52 +739,188 @@ return mics.length ? mics[0].deviceId : undefined; } +// ---- RNNoise (client-side neural noise suppression) ---------------------- +// Runs as a mic *preprocessor* inside the capture worklet, before PCM is sent +// to the server — deliberately transport-agnostic (a WebRTC channel later +// would reuse the same denoised track). Assets come from jsDelivr; any load +// failure falls back to the browser's built-in suppressor for the session. +const RNNOISE_CDN = 'https://cdn.jsdelivr.net/npm/@jitsi/rnnoise-wasm@0.2.1/dist'; +/** RNNoise operates on 480-sample frames of 48 kHz audio (10 ms). */ +const RNNOISE_RATE = 48000; +const RNNOISE_FRAME = 480; +/** null = not fetched yet; false = fetch failed (don't retry this page load). */ +let rnnoiseAssets = null; +/** True when the shared AudioContext was built for the RNNoise graph (48 kHz + glue in the worklet module). */ +let rnnoiseCtxActive = false; + +async function loadRnnoiseAssets() { + const withTimeout = (p) => Promise.race([ + p, new Promise((_, rej) => setTimeout(() => rej(new Error('fetch timeout')), 6000)), + ]); + const [glue, wasm] = await Promise.all([ + withTimeout(fetch(`${RNNOISE_CDN}/rnnoise.js`).then(r => { + if (!r.ok) throw new Error(`HTTP ${r.status}`); + return r.text(); + })), + withTimeout(fetch(`${RNNOISE_CDN}/rnnoise.wasm`).then(r => { + if (!r.ok) throw new Error(`HTTP ${r.status}`); + return r.arrayBuffer(); + })), + ]); + return { glue, wasm }; +} + +/** + * Capture worklet module source. With `glue` (the emscripten factory + * `createRNNWasmModule`) prepended, the processor can denoise: it buffers + * 480-sample frames at the context rate (48 kHz), runs rnnoise_process_frame + * in-place, then decimates 3:1 (mean of 3 — cheap anti-alias; speech lives + * well under the resulting 8 kHz Nyquist) so the server keeps receiving the + * same 16 kHz PCM as the plain path. Until the wasm module resolves, frames + * pass through un-denoised at the same output rate, so the negotiated + * sample_rate never changes mid-session. + */ +function captureWorkletSource(glue) { + return (glue || '') + ` + class Capture extends AudioWorkletProcessor { + constructor(options) { + super(); + this._buf = []; + this._frame = []; + this._denoise = !!(options && options.processorOptions && options.processorOptions.denoise); + this._outRate = this._denoise ? sampleRate / 3 : sampleRate; + this._rn = null; // { mod, state, ptr } once the wasm module is ready + this.port.onmessage = (e) => { + const msg = e.data; + if (msg && msg.type === 'rnnoise-init') this._initRnnoise(msg.wasm); + }; + } + _initRnnoise(wasmBytes) { + try { + createRNNWasmModule({ wasmBinary: new Uint8Array(wasmBytes) }).then((mod) => { + const state = mod._rnnoise_create(); + const ptr = mod._malloc(${RNNOISE_FRAME} * 4); + this._rn = { mod, state, ptr }; + this.port.postMessage({ type: 'rnnoise-ready' }); + }).catch((err) => this.port.postMessage({ type: 'rnnoise-error', error: String(err) })); + } catch (err) { + this.port.postMessage({ type: 'rnnoise-error', error: String(err) }); + } + } + _pushDenoised(ch) { + for (let i = 0; i < ch.length; i++) this._frame.push(ch[i]); + while (this._frame.length >= ${RNNOISE_FRAME}) { + const frame = this._frame.splice(0, ${RNNOISE_FRAME}); + if (this._rn) { + const { mod, state, ptr } = this._rn; + const base = ptr >> 2; + // Re-read HEAPF32 around the call: wasm memory growth detaches old views. + let heap = mod.HEAPF32; + // RNNoise expects float samples in int16 range. + for (let j = 0; j < ${RNNOISE_FRAME}; j++) heap[base + j] = frame[j] * 32768; + mod._rnnoise_process_frame(state, ptr, ptr); + heap = mod.HEAPF32; + for (let j = 0; j < ${RNNOISE_FRAME}; j++) frame[j] = heap[base + j] / 32768; + } + for (let j = 0; j < ${RNNOISE_FRAME}; j += 3) { + this._buf.push((frame[j] + frame[j + 1] + frame[j + 2]) / 3); + } + } + } + process(inputs) { + const ch = inputs[0][0]; + if (!ch) return true; + if (this._denoise) { + this._pushDenoised(ch); + } else { + for (let i = 0; i < ch.length; i++) this._buf.push(ch[i]); + } + const needed = Math.floor(this._outRate * ${CHUNK_MS / 1000}); + while (this._buf.length >= needed) { + const slice = this._buf.splice(0, needed); + const i16 = new Int16Array(slice.length); + for (let j = 0; j < slice.length; j++) { + let s = Math.max(-1, Math.min(1, slice[j])); + i16[j] = s < 0 ? s * 32768 : s * 32767; + } + this.port.postMessage(i16.buffer, [i16.buffer]); + } + return true; + } + } + registerProcessor('capture', Capture); + `; +} + async function startMic(deviceId) { if (currentSource) { currentSource.disconnect(); currentSource = null; } if (worklet) { worklet.disconnect(); worklet = null; } if (analyser) { analyser.disconnect(); analyser = null; } if (micStream) { micStream.getTracks().forEach(t => t.stop()); micStream = null; } + const nsMode = nsSelect.value; + let useRnnoise = nsMode === 'rnnoise'; + if (useRnnoise && rnnoiseAssets === null) { + try { + rnnoiseAssets = await loadRnnoiseAssets(); + } catch (err) { + console.warn('RNNoise assets failed to load — falling back to browser noise suppression.', err); + rnnoiseAssets = false; + } + } + if (useRnnoise && !rnnoiseAssets) useRnnoise = false; + // The context (rate + worklet module) is created once per page; a mic change + // mid-session must keep matching it, whatever the current fetch state says. + if (audioCtx) useRnnoise = rnnoiseCtxActive; + const constraints = { audio: { - echoCancellation: true, noiseSuppression: true, autoGainControl: true, - sampleRate: SAMPLE_RATE, + echoCancellation: true, // always: the barge-in echo gates assume the browser AEC is doing its job + // Never stack two suppressors — browser NS stays on only in 'browser' + // mode or as the fallback when RNNoise assets failed to load. + noiseSuppression: nsMode === 'browser' || (nsMode === 'rnnoise' && !useRnnoise), + autoGainControl: true, + sampleRate: useRnnoise ? RNNOISE_RATE : SAMPLE_RATE, ...(deviceId ? { deviceId: { exact: deviceId } } : {}), } }; micStream = await navigator.mediaDevices.getUserMedia(constraints); if (!audioCtx) { - audioCtx = new AudioContext({ sampleRate: SAMPLE_RATE }); - await audioCtx.audioWorklet.addModule('data:text/javascript,' + encodeURIComponent(` - class Capture extends AudioWorkletProcessor { - constructor() { super(); this._buf = []; } - process(inputs) { - const ch = inputs[0][0]; - if (!ch) return true; - for (let i = 0; i < ch.length; i++) this._buf.push(ch[i]); - const needed = Math.floor(sampleRate * ${CHUNK_MS / 1000}); - while (this._buf.length >= needed) { - const slice = this._buf.splice(0, needed); - const i16 = new Int16Array(slice.length); - for (let j = 0; j < slice.length; j++) { - let s = Math.max(-1, Math.min(1, slice[j])); - i16[j] = s < 0 ? s * 32768 : s * 32767; - } - this.port.postMessage(i16.buffer, [i16.buffer]); - } - return true; - } - } - registerProcessor('capture', Capture); - `)); + audioCtx = new AudioContext({ sampleRate: useRnnoise ? RNNOISE_RATE : SAMPLE_RATE }); + if (useRnnoise && Math.round(audioCtx.sampleRate) !== RNNOISE_RATE) { + // Decimating a non-48k context by 3 would yield a wire rate the STT + // provider rejects (e.g. 44100/3 = 14700). Rebuild the plain graph and + // restore the browser suppressor for this session. + console.warn(`RNNoise needs a 48 kHz AudioContext; got ${audioCtx.sampleRate} — falling back to browser noise suppression.`); + await audioCtx.close(); + audioCtx = new AudioContext({ sampleRate: SAMPLE_RATE }); + useRnnoise = false; + try { await micStream.getAudioTracks()[0].applyConstraints({ noiseSuppression: true }); } catch (_) {} + } + rnnoiseCtxActive = useRnnoise; + await audioCtx.audioWorklet.addModule('data:text/javascript,' + + encodeURIComponent(captureWorkletSource(useRnnoise ? rnnoiseAssets.glue : ''))); } currentSource = audioCtx.createMediaStreamSource(micStream); - worklet = new AudioWorkletNode(audioCtx, 'capture'); + worklet = new AudioWorkletNode(audioCtx, 'capture', { + processorOptions: { denoise: rnnoiseCtxActive }, + }); + if (rnnoiseCtxActive) { + // Copy: the transferable is consumed per node, and mic changes make new nodes. + const wasmBytes = rnnoiseAssets.wasm.slice(0); + worklet.port.postMessage({ type: 'rnnoise-init', wasm: wasmBytes }, [wasmBytes]); + } worklet.port.onmessage = (e) => { + const d = e.data; + if (d && d.type === 'rnnoise-ready') { console.info('RNNoise active on mic input.'); return; } + if (d && d.type === 'rnnoise-error') { + console.warn('RNNoise failed in the audio worklet — mic continues un-denoised.', d.error); + return; + } if (!micAudioToWs || !ws || ws.readyState !== WebSocket.OPEN) return; - ws.send(new Uint8Array(e.data)); + ws.send(new Uint8Array(d)); }; currentSource.connect(worklet); const silent = audioCtx.createGain(); @@ -857,7 +1016,10 @@ // The config hello MUST be the first frame on the socket — the server // reads one frame as session config. stopAssistantAudio() below sends a // playback ack, which used to race ahead and eat the config slot. - const sr = audioCtx ? Math.round(audioCtx.sampleRate) : SAMPLE_RATE; + // Report the worklet's OUTPUT rate: the RNNoise graph runs the context at + // 48 kHz but decimates 3:1 before sending, so the wire rate is ctx/3. + const ctxRate = audioCtx ? Math.round(audioCtx.sampleRate) : SAMPLE_RATE; + const sr = rnnoiseCtxActive ? Math.round(ctxRate / 3) : ctxRate; sessionSampleRate = sr; const hello = { sample_rate: sr }; if (tdSelect.value) hello.turn_detector = tdSelect.value; From 44f309b3a92b077e5a9f95a76d71f6e693e7e160 Mon Sep 17 00:00:00 2001 From: berges99 Date: Thu, 23 Jul 2026 17:11:05 +0200 Subject: [PATCH 02/15] fix(voice): cap the incomplete-EOU hold at 3s (was 4.8s) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A wrong "incomplete" from Smart Turn (observed on short closers like "Thank you." p=0.036 and interjections like "No, no." p=0.378) costs the full hold before the turn runs — and no mid-hold re-score can rescue it, since the EOU backend trims trailing silence and would reproduce the same window and score. 3.0s matches both references: LiveKit's current endpointing max_delay default (6.0s was the old default) and Pipecat's SmartTurnParams stop_secs fallback. With our ~1.2s STT VAD silence before the hold arms, worst-case total is ~4.2s. --- python/timbal/voice/turn_detection.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/python/timbal/voice/turn_detection.py b/python/timbal/voice/turn_detection.py index 5af95573..201b61c5 100644 --- a/python/timbal/voice/turn_detection.py +++ b/python/timbal/voice/turn_detection.py @@ -583,11 +583,16 @@ class LocalAudioTurnDetector(HeuristicTurnDetector): completion_threshold: float = 0.5 # Grace window after an incomplete-scored commit before the fragment runs - # anyway. LiveKit's equivalent (max_endpointing_delay) defaults to 6.0s of - # *total* silence after speech when their EOU model says "not done". The - # HOLD only arms after the STT VAD silence (~1.2s with the server default), - # so 4.8s here reproduces that 6s total thinking budget. - DEFAULT_HOLD_TIMEOUT_SECS = 4.8 + # anyway. Reference points: LiveKit's max_endpointing_delay is 6.0s of + # *total* silence when their EOU model says "not done"; Pipecat's + # smart-turn fallback (stop_secs) is 3.0s of continued silence. The HOLD + # only arms after the STT VAD silence (~1.2s with the server default), so + # 3.0s here gives ~4.2s total — between the two. This is also the full + # price of a *wrong* "incomplete" score (e.g. Smart Turn on a bare + # "Thank you."), and no re-score can rescue those mid-hold: the backend + # trims trailing silence before scoring, so waiting longer reproduces the + # same window and the same score. + DEFAULT_HOLD_TIMEOUT_SECS = 3.0 # The model consumes the last 8s of *speech*; the EOU backend trims the # trailing silence (STT commit debounce, hold pauses) before scoring, so # buffer extra raw PCM to keep a full 8s of signal after the trim. From 190cf3a7bca043193453ba097341d75436c63294 Mon Sep 17 00:00:00 2001 From: berges99 Date: Thu, 23 Jul 2026 18:02:05 +0200 Subject: [PATCH 03/15] fix(voice): text-confidence tiers + stop mid-thought false commits Mix transcript confidence into endpointing: audio-incomplete + finished text shortens the HOLD; audio-complete + hedge text (e.g. "I don't know.") delays via a short HOLD and a VAD delay bump. Floor VAD min_delay at 0.45s so confident scores never commit in ~0ms. Also fix hold expiry stretching on the commit's own partials, rescue stuck STT partials that never commit, and stop the playground from splitting one agent reply across two bubbles on stray mic partials. --- python/tests/core/test_eou.py | 27 ++- python/tests/core/test_turn_detection.py | 51 ++++++ python/tests/core/test_vad_endpointing.py | 56 +++++- python/tests/core/test_voice_session.py | 205 ++++++++++++++++++++++ python/timbal/server/voice.html | 10 +- python/timbal/voice/endpointing.py | 58 +++++- python/timbal/voice/eou.py | 75 +++++++- python/timbal/voice/session.py | 114 +++++++++++- python/timbal/voice/turn_detection.py | 82 ++++++++- 9 files changed, 641 insertions(+), 37 deletions(-) diff --git a/python/tests/core/test_eou.py b/python/tests/core/test_eou.py index 310c19e8..4d04aa2e 100644 --- a/python/tests/core/test_eou.py +++ b/python/tests/core/test_eou.py @@ -18,12 +18,14 @@ async def test_empty_is_complete(self) -> None: async def test_terminal_punctuation_complete(self) -> None: p = PunctuationEouPredictor() - for text in ("Tell me a story.", "How are you?", "Stop!", "Vale…", "¿Qué tal?"): + for text in ("Tell me a story.", "How are you?", "Stop!", "¿Qué tal?"): assert await p.predict_eou(text) == p.P_TERMINAL async def test_continuing_punctuation_incomplete(self) -> None: p = PunctuationEouPredictor() - for text in ("I want to say,", "First this;", "Listen —"): + # Ellipsis included: STT writes "…" / "..." when the speaker audibly + # trails off mid-thought — the opposite of terminal, despite the dots. + for text in ("I want to say,", "First this;", "Listen —", "Vale…", "I was thinking about..."): assert await p.predict_eou(text) == p.P_CONTINUING async def test_trailing_dangling_token_incomplete_english(self) -> None: @@ -66,3 +68,24 @@ async def test_lifecycle_noops(self) -> None: p = PunctuationEouPredictor() await p.start() await p.close() + + async def test_hedge_beats_terminal_punctuation(self) -> None: + """Live failure: STT writes "Uh, I don't know." with a period; audio + EOU agrees complete — hedges must score incomplete anyway.""" + p = PunctuationEouPredictor() + for text in ( + "Uh, I don't know.", + "I don't know.", + "I'm not sure.", + "uh", + "Well", + "No sé.", + "Let me think.", + ): + assert await p.predict_eou(text) == p.P_HEDGE, text + assert await p.predict_eou(text) < 0.4 + + async def test_hedge_does_not_match_real_completes(self) -> None: + p = PunctuationEouPredictor() + for text in ("Tell me a story.", "Quite incredible.", "Yeah, sure. Anything else?"): + assert await p.predict_eou(text) == p.P_TERMINAL, text diff --git a/python/tests/core/test_turn_detection.py b/python/tests/core/test_turn_detection.py index 5212a6ff..b65b332f 100644 --- a/python/tests/core/test_turn_detection.py +++ b/python/tests/core/test_turn_detection.py @@ -386,9 +386,60 @@ async def test_incomplete_audio_holds(self) -> None: decision = await det.on_committed("I was wondering", _state()) assert decision.action is CommitAction.HOLD assert decision.reason == "audio_hold" + # Neutral (unpunctuated) text must not shorten the hold. + assert decision.hold_timeout_secs == det.hold_timeout_secs assert model.calls == 1 await det.close() + async def test_incomplete_audio_complete_text_short_hold(self) -> None: + """Confidence tier: Smart Turn under-scores short closers ("Thank you." + p=0.036 live) — terminal punctuation disagrees, so the hold shrinks to + the short tier instead of eating the full budget as dead air.""" + det = LocalAudioTurnDetector(audio_eou=_FixedAudioEou(0.1)) + await det.start(type("C", (), {"sample_rate": 16000})()) + det.push_audio(b"\x00\x01" * 8000) + decision = await det.on_committed("Thank you.", _state()) + assert decision.action is CommitAction.HOLD + assert decision.reason == "audio_hold_text_complete" + assert decision.hold_timeout_secs == det.text_complete_hold_timeout_secs + assert det.text_complete_hold_timeout_secs < det.hold_timeout_secs + # Per-session clones keep the knob. + assert det.clone().text_complete_hold_timeout_secs == det.text_complete_hold_timeout_secs + await det.close() + + async def test_complete_audio_hedge_text_short_hold(self) -> None: + """Inverse tier: Smart Turn over-scores thinking pauses + ("Uh, I don't know." p=0.825 live) — hedge text disagrees, so HOLD + short instead of NEW_TURN immediately.""" + det = LocalAudioTurnDetector(audio_eou=_FixedAudioEou(0.9)) + await det.start(type("C", (), {"sample_rate": 16000})()) + det.push_audio(b"\x00\x01" * 8000) + decision = await det.on_committed("Uh, I don't know.", _state()) + assert decision.action is CommitAction.HOLD + assert decision.reason == "audio_complete_text_incomplete" + assert decision.hold_timeout_secs == det.text_incomplete_hold_timeout_secs + await det.close() + + async def test_complete_audio_real_complete_still_new_turn(self) -> None: + det = LocalAudioTurnDetector(audio_eou=_FixedAudioEou(0.9)) + await det.start(type("C", (), {"sample_rate": 16000})()) + det.push_audio(b"\x00\x01" * 8000) + decision = await det.on_committed("Tell me a story.", _state()) + assert decision.action is CommitAction.NEW_TURN + await det.close() + + async def test_incomplete_audio_ellipsis_full_hold(self) -> None: + """An STT ellipsis means the speaker trailed off — it must not count + as terminal punctuation for the short tier.""" + det = LocalAudioTurnDetector(audio_eou=_FixedAudioEou(0.1)) + await det.start(type("C", (), {"sample_rate": 16000})()) + det.push_audio(b"\x00\x01" * 8000) + decision = await det.on_committed("I was thinking about...", _state()) + assert decision.action is CommitAction.HOLD + assert decision.reason == "audio_hold" + assert decision.hold_timeout_secs == det.hold_timeout_secs + await det.close() + async def test_complete_audio_new_turn(self) -> None: det = LocalAudioTurnDetector(audio_eou=_FixedAudioEou(0.9)) await det.start(type("C", (), {"sample_rate": 16000})()) diff --git a/python/tests/core/test_vad_endpointing.py b/python/tests/core/test_vad_endpointing.py index a64bb246..21954d29 100644 --- a/python/tests/core/test_vad_endpointing.py +++ b/python/tests/core/test_vad_endpointing.py @@ -77,16 +77,28 @@ def _endpointer(vad: FakeVad | None = None, **knobs) -> VadEndpointer: class _Recorder: """Bind targets that record calls and return scripted values.""" - def __init__(self, *, p: float | None = 0.95, gate: bool = True) -> None: + def __init__( + self, + *, + p: float | None = 0.95, + gate: bool = True, + p_text: float | None = None, + ) -> None: self.p = p self.gate = gate + self.p_text = p_text self.score_calls = 0 self.commit_calls = 0 + self.text_score_calls = 0 async def score(self) -> float | None: self.score_calls += 1 return self.p + async def text_score(self) -> float | None: + self.text_score_calls += 1 + return self.p_text + async def commit(self) -> None: self.commit_calls += 1 @@ -95,7 +107,12 @@ def should_commit(self) -> bool: async def _started(ep: VadEndpointer, rec: _Recorder) -> VadEndpointer: - ep.bind(score=rec.score, commit=rec.commit, should_commit=rec.should_commit) + ep.bind( + score=rec.score, + commit=rec.commit, + should_commit=rec.should_commit, + text_score=rec.text_score if rec.p_text is not None else None, + ) await ep.start(sample_rate=16_000) return ep @@ -112,9 +129,11 @@ async def _settle(secs: float = 0.15) -> None: class TestEndpointingDelay: def test_confident_maps_to_min(self): assert endpointing_delay(1.0, min_delay=0.0, max_delay=3.0) == 0.0 + assert endpointing_delay(1.0) == VadEndpointer.MIN_DELAY_SECS def test_zero_maps_to_max(self): assert endpointing_delay(0.0, min_delay=0.0, max_delay=3.0) == 3.0 + assert endpointing_delay(0.0) == 3.0 def test_monotonic_decreasing(self): delays = [endpointing_delay(p / 10) for p in range(11)] @@ -128,10 +147,15 @@ def test_incomplete_exceeds_provider_debounce(self): # p below the completion threshold must compute a delay longer than # the ~1.2s ElevenLabs VAD debounce — the provider commit wins and the # existing HOLD machinery handles the incomplete utterance. - assert endpointing_delay(0.4) > 1.0 + assert endpointing_delay(0.4, min_delay=0.0) > 1.0 + + def test_confident_respects_livekit_floor(self): + # LiveKit min_delay shape: even p≈1 never collapses to ~0. + assert endpointing_delay(0.95) >= VadEndpointer.MIN_DELAY_SECS - 1e-9 + assert VadEndpointer.MIN_DELAY_SECS >= 0.4 - def test_confident_is_fast(self): - assert endpointing_delay(0.95) < 0.05 + def test_curve_still_fast_without_floor(self): + assert endpointing_delay(0.95, min_delay=0.0) < 0.05 # --------------------------------------------------------------------------- @@ -204,6 +228,28 @@ async def test_resumed_speech_cancels_pending(self): await _settle(0.05) assert rec.commit_calls == 0 + async def test_text_incomplete_bumps_delay(self): + """Audio-complete + hedge text → delay floored at TEXT_INCOMPLETE_DELAY, + so a continuation can cancel before commit (live: "I don't know" → story).""" + rec = _Recorder(p=0.98, p_text=0.2) + ep = await _started( + _endpointer( + min_delay_secs=0.0, + max_delay_secs=0.05, + text_incomplete_delay_secs=0.4, + ), + rec, + ) + ep.push(b"SSS") + ep.push(b"\x00\x00") + await _settle(0.1) # past audio-only delay (0.05), still in text bump + assert rec.score_calls == 1 + assert rec.text_score_calls == 1 + assert rec.commit_calls == 0 + ep.push(b"SS") # continuation cancels during the bumped wait + await _settle(0.05) + assert rec.commit_calls == 0 + async def test_single_noise_frame_does_not_cancel(self): rec = _Recorder(p=0.5) # delay(0.5) = 0.25 * max → 0.1s with max 0.4: long enough to inject a blip. diff --git a/python/tests/core/test_voice_session.py b/python/tests/core/test_voice_session.py index 2c40af07..d18b616d 100644 --- a/python/tests/core/test_voice_session.py +++ b/python/tests/core/test_voice_session.py @@ -30,6 +30,12 @@ VoiceSessionEvent, _strip_markdown, ) +from timbal.voice.turn_detection import ( + CommitAction, + CommitDecision, + PartialDecision, + TurnDetector, +) # --------------------------------------------------------------------------- # Mock STT / TTS @@ -1555,3 +1561,202 @@ async def _run() -> None: assert session._turn_tts_stream is None assert session._turn_tts_pump is None assert any(isinstance(e, SessionInterrupted) for e in events) + + +# --------------------------------------------------------------------------- +# Tests: hold expiry timing (partial-grace anchor) +# --------------------------------------------------------------------------- + + +class _AlwaysHoldOnceDetector(TurnDetector): + """HOLDs the first fresh commit with a fixed short timeout; anything while + holding supersedes into a NEW_TURN (minimal detector for expiry timing).""" + + def __init__(self, timeout_secs: float) -> None: + self._timeout_secs = timeout_secs + + async def on_partial(self, text: str, state) -> PartialDecision: + return PartialDecision.IGNORE + + async def on_committed(self, text: str, state) -> CommitDecision: + if state.holding: + return CommitDecision(action=CommitAction.NEW_TURN, text=text, reason="new_turn") + return CommitDecision( + action=CommitAction.HOLD, text=text, reason="hold", hold_timeout_secs=self._timeout_secs + ) + + +class TestHoldExpiryTiming: + async def _run_hold_scenario( + self, + *, + grace_secs: float, + timeout_secs: float, + partial_after_commit_delay: float | None, + run_timeout: float = 10.0, + ) -> tuple[float, VoiceSession]: + """Inject partial→commit (HOLD) and return seconds from commit to turn start.""" + agent = Agent(name="t", model=TestModel(responses=["ok"]), tools=[]) + stt = DelayedMockSTT() + tts = MockTTS() + session = VoiceSession( + agent=agent, stt=stt, tts=tts, turn_detector=_AlwaysHoldOnceDetector(timeout_secs) + ) + session._hold_partial_grace_secs = grace_secs + + events: list[VoiceSessionEvent] = [] + elapsed = 0.0 + + async def _empty_audio() -> AsyncIterator[bytes]: + return + yield # noqa: RET504 + + async def _drive() -> None: + nonlocal elapsed + while not any(isinstance(e, SessionStarted) for e in events): + await asyncio.sleep(0.01) + # The fragment's own partial precedes its commit (STT ordering). + await stt.inject(TranscriptEvent(type="partial", text="Thank you")) + await stt.inject(TranscriptEvent(type="committed", text="Thank you.")) + t0 = asyncio.get_event_loop().time() + if partial_after_commit_delay is not None: + await asyncio.sleep(partial_after_commit_delay) + # User resumed speaking mid-hold. + await stt.inject(TranscriptEvent(type="partial", text="and one more")) + while not any(isinstance(e, TranscriptCommitted) for e in events): + await asyncio.sleep(0.01) + elapsed = asyncio.get_event_loop().time() - t0 + while not any(isinstance(e, AgentTextDone) for e in events): + await asyncio.sleep(0.01) + await stt.finish() + + async def _run() -> None: + async with aclosing(session.run(_empty_audio())) as stream: + driver = asyncio.create_task(_drive()) + async for ev in stream: + events.append(ev) + await driver + + await asyncio.wait_for(_run(), timeout=run_timeout) + return elapsed, session + + async def test_pre_commit_partial_does_not_stretch_hold(self) -> None: + """Live bug: every commit is preceded by its own partials, so anchoring + the grace window on *any* recent partial floored each hold at the grace + window (~2s) instead of the armed timeout (1.2s tier).""" + elapsed, session = await self._run_hold_scenario( + grace_secs=5.0, # deliberately huge: failure mode is unmissable + timeout_secs=0.2, + partial_after_commit_delay=None, + ) + assert elapsed < 2.0, f"hold expiry took {elapsed:.2f}s — pre-commit partial stretched it" + assert session.transcript[0].text == "Thank you." + + async def test_post_arm_partial_extends_hold(self) -> None: + """A partial *after* the hold is armed means the user resumed speaking + — the expiry must wait out the grace window (no VAD in this harness → + extension is conservative).""" + elapsed, _ = await self._run_hold_scenario( + grace_secs=0.8, + timeout_secs=0.1, + partial_after_commit_delay=0.05, + ) + # Must have waited past the bare timeout into the grace window. + assert elapsed > 0.5, f"hold expired after only {elapsed:.2f}s despite fresh partial" + + +# --------------------------------------------------------------------------- +# Tests: stale partial sweeper +# --------------------------------------------------------------------------- + + +class _StuckPartialSTT(DelayedMockSTT): + """STT that transcribes a partial but never commits it on its own — the + live 'quiet speech during playback' failure. ``commit()`` releases it.""" + + def __init__(self) -> None: + super().__init__() + self.commit_calls = 0 + self.pending_text: str | None = None + + async def commit(self) -> None: + self.commit_calls += 1 + if self.pending_text is not None: + text, self.pending_text = self.pending_text, None + await self.inject(TranscriptEvent(type="committed", text=text)) + + +class TestStalePartialSweeper: + async def test_stuck_partial_is_force_committed(self) -> None: + agent = Agent(name="t", model=TestModel(responses=["ok"]), tools=[]) + stt = _StuckPartialSTT() + tts = MockTTS() + session = VoiceSession(agent=agent, stt=stt, tts=tts) + session._stale_partial_poll_secs = 0.05 + session._stale_partial_commit_secs = 0.2 + + events: list[VoiceSessionEvent] = [] + + async def _empty_audio() -> AsyncIterator[bytes]: + return + yield # noqa: RET504 + + async def _drive() -> None: + while not any(isinstance(e, SessionStarted) for e in events): + await asyncio.sleep(0.01) + stt.pending_text = "Cool." + await stt.inject(TranscriptEvent(type="partial", text="Cool.")) + while not any(isinstance(e, AgentTextDone) for e in events): + await asyncio.sleep(0.01) + await stt.finish() + + async def _run() -> None: + async with aclosing(session.run(_empty_audio())) as stream: + driver = asyncio.create_task(_drive()) + async for ev in stream: + events.append(ev) + await driver + + await asyncio.wait_for(_run(), timeout=10) + + assert stt.commit_calls >= 1 + assert session.transcript[0].role == "user" + assert session.transcript[0].text == "Cool." + + async def test_sweeper_does_not_refire_on_ignored_commit(self) -> None: + """An IGNOREd commit (noise) leaves _last_commit_at stale; the sweeper + must force at most one commit per stranded partial, not one per poll.""" + agent = Agent(name="t", model=TestModel(responses=["ok"]), tools=[]) + stt = _StuckPartialSTT() + tts = MockTTS() + session = VoiceSession(agent=agent, stt=stt, tts=tts) + session._stale_partial_poll_secs = 0.05 + session._stale_partial_commit_secs = 0.15 + + events: list[VoiceSessionEvent] = [] + + async def _empty_audio() -> AsyncIterator[bytes]: + return + yield # noqa: RET504 + + async def _drive() -> None: + while not any(isinstance(e, SessionStarted) for e in events): + await asyncio.sleep(0.01) + # Hesitation-only: the resulting forced commit is IGNOREd by the + # heuristic detector, so no turn starts and no _last_commit_at bump. + stt.pending_text = "Mm-hmm." + await stt.inject(TranscriptEvent(type="partial", text="Mm-hmm.")) + await asyncio.sleep(1.0) # several poll+stale windows + await stt.finish() + + async def _run() -> None: + async with aclosing(session.run(_empty_audio())) as stream: + driver = asyncio.create_task(_drive()) + async for ev in stream: + events.append(ev) + await driver + + await asyncio.wait_for(_run(), timeout=10) + + assert stt.commit_calls == 1 + assert all(entry.role != "user" for entry in session.transcript) diff --git a/python/timbal/server/voice.html b/python/timbal/server/voice.html index cd60315b..7baa9402 100644 --- a/python/timbal/server/voice.html +++ b/python/timbal/server/voice.html @@ -697,12 +697,10 @@ updateEmptyState(); return; } - // Seal any in-flight assistant bubble so live STT can't read as a - // continuation of the previous reply. - if (window._assistantEl) { - window._lastAssistantEl = window._assistantEl; - window._assistantEl = null; - } + // Do NOT seal the in-flight assistant bubble here: a stray mic partial + // (breath, echo blip) mid-reply would split one reply across two bubbles. + // The live caption is parked *after* the assistant bubble (appendChild + // below), so ordering stays readable while the reply keeps growing. if (!partialEl) { partialEl = log('partial', ''); } else { diff --git a/python/timbal/voice/endpointing.py b/python/timbal/voice/endpointing.py index 279bcc2e..3f80c78b 100644 --- a/python/timbal/voice/endpointing.py +++ b/python/timbal/voice/endpointing.py @@ -51,23 +51,26 @@ def endpointing_delay( p: float, *, - min_delay: float = 0.0, + min_delay: float = 0.45, max_delay: float = 3.0, curve: float = 2.0, ) -> float: """Map EOU probability to extra silence to wait before force-committing. ``delay = min + (max - min) * (1 - p) ** curve`` — smooth, monotonic - decreasing in ``p``. With the defaults (fired ~0.3s after speech stop: - 0.2s VAD silence + ~0.1s Smart Turn inference): + decreasing in ``p``. ``min_delay`` is a LiveKit-shaped floor: even a + confident-complete score still waits before force-committing, so a brief + thinking pause ("Uh, I don't know." → "tell me a story") can cancel the + pending endpoint via resumed speech. Defaults (fired ~0.3s after speech + stop: 0.2s VAD silence + ~0.1s Smart Turn inference): ======== ========= ========================================== p delay total silence before commit ======== ========= ========================================== - 0.95 ~0.01s ~0.3s (confident: LiveKit-class response) - 0.7 ~0.27s ~0.6s - 0.5 0.75s ~1.05s (unsure: still beats the provider) - <0.4 >1.1s provider debounce (~1.2s) commits first — + 0.95 0.45s ~0.75s (confident: LiveKit min_delay floor) + 0.7 ~0.68s ~1.0s + 0.5 ~1.1s ~1.4s (unsure) + <0.4 >1.4s provider debounce (~1.2s) often commits first — incomplete utterances fall through to the existing HOLD machinery untouched ======== ========= ========================================== @@ -108,12 +111,19 @@ class VadEndpointer: SPEECH_RESUME_SECS = 0.064 """Consecutive speech (2 frames) that cancels a pending endpoint — a single noise-spike frame must not kill a valid pending commit.""" - MIN_DELAY_SECS = 0.0 + MIN_DELAY_SECS = 0.45 + """LiveKit-shaped floor: never force-commit instantly on high-p scores.""" MAX_DELAY_SECS = 3.0 DELAY_CURVE = 2.0 """See :func:`endpointing_delay`.""" MIN_COMMIT_INTERVAL_SECS = 2.0 """Floor between force-commits (ElevenLabs throttles rapid commits).""" + TEXT_INCOMPLETE_DELAY_SECS = 1.2 + """When the latest STT partial looks incomplete (hedge / trailing-off), + bump the delay to at least this — same short tier as the commit-path + inverse HOLD. Gives the continuation time to cancel the pending endpoint.""" + TEXT_INCOMPLETE_THRESHOLD = 0.4 + """``P(complete)`` below this from the text scorer → treat as incomplete.""" def __init__( self, @@ -126,6 +136,7 @@ def __init__( max_delay_secs: float | None = None, delay_curve: float | None = None, min_commit_interval_secs: float | None = None, + text_incomplete_delay_secs: float | None = None, ) -> None: self._vad = vad self.speech_threshold = speech_threshold if speech_threshold is not None else self.SPEECH_THRESHOLD @@ -137,10 +148,16 @@ def __init__( self.min_commit_interval_secs = ( min_commit_interval_secs if min_commit_interval_secs is not None else self.MIN_COMMIT_INTERVAL_SECS ) + self.text_incomplete_delay_secs = ( + text_incomplete_delay_secs + if text_incomplete_delay_secs is not None + else self.TEXT_INCOMPLETE_DELAY_SECS + ) self._score: Callable[[], Awaitable[float | None]] | None = None self._commit: Callable[[], Awaitable[None]] | None = None self._should_commit: Callable[[], bool] | None = None + self._text_score: Callable[[], Awaitable[float | None]] | None = None self._started = False self._closed = False @@ -168,11 +185,20 @@ def bind( score: Callable[[], Awaitable[float | None]], commit: Callable[[], Awaitable[None]], should_commit: Callable[[], bool], + text_score: Callable[[], Awaitable[float | None]] | None = None, ) -> None: - """Attach the session callbacks. Must run before :meth:`start`.""" + """Attach the session callbacks. Must run before :meth:`start`. + + ``text_score`` is optional: when provided, returns ``P(complete)`` for + the latest STT partial (or ``None``). Used to bump the delay when the + transcript looks mid-thought even if the audio model is confident — + same LiveKit min/max shape, text as the confidence signal. Swappable + later for a DistilBERT-class text EOU without changing this wiring. + """ self._score = score self._commit = commit self._should_commit = should_commit + self._text_score = text_score async def start(self, *, sample_rate: int) -> None: """Load Silero (downloading/instantiating off the event loop) and arm. @@ -287,6 +313,18 @@ async def _run_endpoint(self) -> None: max_delay=self.max_delay_secs, curve=self.delay_curve, ) + text_bumped = False + p_text: float | None = None + if self._text_score is not None: + try: + p_text = await self._text_score() + except Exception as e: + logger.warning("vad_text_score_failed", error=str(e)) + p_text = None + if p_text is not None and p_text < self.TEXT_INCOMPLETE_THRESHOLD: + bumped = max(delay, self.text_incomplete_delay_secs) + text_bumped = bumped > delay + delay = bumped # INFO on purpose: fires once per speech-stop and is the whole # observable behaviour of the endpointing fast path. logger.info( @@ -294,6 +332,8 @@ async def _run_endpoint(self) -> None: p=round(p, 3), delay_secs=round(delay, 3), score_ms=round((time.monotonic() - t0) * 1000, 1), + p_text=round(p_text, 3) if p_text is not None else None, + text_incomplete_bump=text_bumped, ) remaining = delay - (time.monotonic() - t0) if remaining > 0: diff --git a/python/timbal/voice/eou.py b/python/timbal/voice/eou.py index ea6a5ec6..45221cb9 100644 --- a/python/timbal/voice/eou.py +++ b/python/timbal/voice/eou.py @@ -95,30 +95,99 @@ async def predict_eou(self, text: str) -> float: } ) +# Thinking-pause / hedge phrases: often punctuated as complete by STT +# ("Uh, I don't know.") but the speaker continues. Short single-token hedges +# match only as the *entire* utterance; multi-word phrases also match as a +# trailing clause ("well i don't know"). Swappable later for a DistilBERT / +# Namo-class text EOU behind the same :class:`TextEouPredictor` interface. +_SHORT_HEDGES = frozenset( + { + "uh", "um", "uhm", "hmm", "mm", "mhm", "mmhmm", "well", "so", "like", + "maybe", "perhaps", "idk", "pues", "este", "eh", + } +) +_PHRASE_HEDGES = frozenset( + { + "i don't know", + "i dont know", + "i do not know", + "not sure", + "i'm not sure", + "im not sure", + "i am not sure", + "let me think", + "i mean", + "you know", + "i guess", + "no se", + "no sé", + "a ver", + "no lo se", + "no lo sé", + } +) +_PUNCT_STRIP_RE = re.compile(r"[.?!…。？！,;:—–\-]+") + + +def _normalize_utterance(text: str) -> str: + """Lowercase, strip punctuation to spaces, collapse whitespace, unify apostrophes.""" + t = text.lower().strip() + for a in ("'", "'", "`"): + t = t.replace(a, "'") + t = _PUNCT_STRIP_RE.sub(" ", t) + return re.sub(r"\s+", " ", t).strip() + + +def _looks_like_hedge(text: str) -> bool: + """True when the utterance is (or ends in) a thinking-pause hedge.""" + norm = _normalize_utterance(text) + if not norm: + return False + if norm in _SHORT_HEDGES or norm in _PHRASE_HEDGES: + return True + for phrase in _PHRASE_HEDGES: + if norm.endswith(" " + phrase): + return True + return False + class PunctuationEouPredictor(TextEouPredictor): - """Zero-dep lexical EOU: terminal punctuation + trailing dangling tokens. + """Zero-dep lexical EOU: punctuation, dangling tokens, and hedges. Scores (tunable via subclass attributes): + * thinking-pause hedge ("i don't know", bare "uh"/"well") → :attr:`P_HEDGE` + — wins over terminal punctuation (STT writes "Uh, I don't know.") * ends with terminal punctuation → :attr:`P_TERMINAL` - * ends with continuing punctuation → :attr:`P_CONTINUING` + * ends with continuing punctuation / ellipsis → :attr:`P_CONTINUING` * last word is a dangling conjunction/preposition/filler → :attr:`P_DANGLING` * otherwise → :attr:`P_NEUTRAL` (slightly complete-leaning; STT often drops the final period and over-merging is worse than under-merging) - English + Spanish dangling lists match the server's ``es`` default. + English + Spanish dangling lists match the server's ``es`` default. This + is the dumb baseline behind :class:`TextEouPredictor` — swap for a small + ONNX text classifier later without changing turn-detection wiring. """ P_TERMINAL: float = 0.95 P_CONTINUING: float = 0.15 P_DANGLING: float = 0.15 + P_HEDGE: float = 0.2 P_NEUTRAL: float = 0.60 async def predict_eou(self, text: str) -> float: stripped = text.strip() if not stripped: return 1.0 + # Hedges before terminal punct: "Uh, I don't know." must not score + # complete just because STT stuck a period on a mid-thought pause. + if _looks_like_hedge(stripped): + return self.P_HEDGE + # Ellipsis before the terminal check: STT writes "..." / "…" exactly + # when the speaker trails off mid-thought — the opposite of terminal, + # despite ending in ".". + if stripped.endswith("...") or stripped.endswith("…"): + return self.P_CONTINUING last_char = stripped[-1] if last_char in _TERMINAL_PUNCT: return self.P_TERMINAL diff --git a/python/timbal/voice/session.py b/python/timbal/voice/session.py index 5e0f2ac4..e77468d0 100644 --- a/python/timbal/voice/session.py +++ b/python/timbal/voice/session.py @@ -431,11 +431,29 @@ def __init__( self._last_commit_at: float = 0.0 self._partials_since_last_commit: int = 0 self._last_partial_at: float = 0.0 + # Latest non-empty STT partial text — fed to the VAD endpointer's + # optional text_score so a mid-thought hedge can bump the delay even + # before the provider commits. + self._latest_partial_text: str = "" # A pending HOLD must not expire while the user is audibly mid-utterance # (recent STT partial): the upcoming commit merges with / supersedes the # hold. Must exceed the STT VAD silence threshold (~1.2s default) so the # commit always lands before the extended expiry re-fires. self._hold_partial_grace_secs = 2.0 + # When the last STT commit event arrived. Anchors the hold-expiry + # extension: only partials *newer* than the commit mean the user + # resumed speaking — the committed fragment's own trailing partial + # refinements must not stretch the hold. + self._commit_event_at: float = 0.0 + # Watchdog for transcripts the provider never commits: STT can emit a + # partial (e.g. quiet speech ducked by AEC during assistant playback) + # whose VAD never registers an utterance — no commit ever fires and the + # words hang as a "…" caption forever. After this much silence past the + # last partial (comfortably beyond the provider's ~1.2s debounce, so it + # only fires when the provider clearly won't), force ``stt.commit()``. + self._stale_partial_commit_secs = 2.5 + self._stale_partial_poll_secs = 0.5 + self._stale_commit_sent_at = 0.0 # Serial TTS runs off the agent ``async for`` critical path so we keep pulling # LLM/Agent events (and emit trace OUTPUT) while audio still synthesizes. @@ -527,6 +545,7 @@ async def run(self, audio_in: AsyncIterable[bytes]) -> AsyncIterator[VoiceSessio audio_task = asyncio.create_task(self._forward_audio(audio_in)) stt_task = asyncio.create_task(self._process_stt_events()) + sweep_task = asyncio.create_task(self._sweep_stale_partials()) try: while True: @@ -535,10 +554,10 @@ async def run(self, audio_in: AsyncIterable[bytes]) -> AsyncIterator[VoiceSessio break yield event finally: - for task in (audio_task, stt_task): + for task in (audio_task, stt_task, sweep_task): if not task.done(): task.cancel() - await asyncio.gather(audio_task, stt_task, return_exceptions=True) + await asyncio.gather(audio_task, stt_task, sweep_task, return_exceptions=True) except Exception as e: logger.error("voice_session_error", error=str(e), exc_info=True) @@ -716,6 +735,7 @@ async def _maybe_start_endpointer(self) -> None: score=score_fn, commit=self._endpoint_commit, should_commit=self._endpoint_should_commit, + text_score=self._endpoint_text_score, ) try: await endpointer.start(sample_rate=self.audio_input.sample_rate) @@ -748,6 +768,24 @@ def _endpoint_should_commit(self) -> bool: return False return self._last_partial_at > self._last_commit_at + async def _endpoint_text_score(self) -> float | None: + """``P(complete)`` for the latest STT partial, for VAD delay bumping. + + Uses the turn detector's text EOU when present (``fallback_text_eou`` + on :class:`~timbal.voice.LocalAudioTurnDetector`); otherwise the + shared :class:`~timbal.voice.PunctuationEouPredictor` baseline. Returns + ``None`` when there is no fresh partial — endpointer keeps the + audio-only delay. + """ + if self._last_partial_at <= self._last_commit_at or not self._latest_partial_text: + return None + text_eou = getattr(self.turn_detector, "fallback_text_eou", None) + if text_eou is None: + from .eou import PunctuationEouPredictor + + text_eou = PunctuationEouPredictor() + return await text_eou.predict_eou(self._latest_partial_text) + async def _endpoint_commit(self) -> None: self._endpoint_commit_sent_at = time.monotonic() await self.stt.commit() @@ -787,6 +825,17 @@ def _vad_vetoes_barge_in(self, text: str) -> bool: ) return True + def _vad_contradicts_recent_partial(self) -> bool: + """True when the local VAD is healthy and saw no real speech energy + recently — a fresh STT partial is then a hallucination, not the user + mid-utterance. Conservative on missing evidence (no endpointer / + starved VAD → ``False``), mirroring :meth:`_vad_vetoes_barge_in`. + """ + if self._endpointer is None: + return False + speech_secs = self._endpointer.speech_secs_in_window(self.BARGE_IN_VAD_WINDOW_SECS) + return speech_secs is not None and speech_secs < self.MIN_BARGE_IN_VAD_SPEECH_SECS + # -- Internal: audio → STT --------------------------------------------- async def _forward_audio(self, audio_in: AsyncIterable[bytes]) -> None: @@ -804,6 +853,44 @@ async def _forward_audio(self, audio_in: AsyncIterable[bytes]) -> None: logger.error("audio_forward_error", error=str(e), exc_info=True) await self._emit(SessionError(message=f"Audio input error: {e}")) + async def _sweep_stale_partials(self) -> None: + """Watchdog: force-commit transcripts the provider never commits. + + Failure mode (seen live): the user speaks quietly while the assistant + is playing — AEC ducks the near-end audio, the STT still transcribes a + partial, but neither the provider VAD nor Silero registers an + utterance. No commit ever fires and the words hang as a "…" caption + until the session dies. When a partial has gone + ``_stale_partial_commit_secs`` with no commit and no newer partial, + force ``stt.commit()`` so the transcript flows through the normal + ``_handle_committed`` path (providers without manual commit no-op). + """ + try: + while not self._closed: + await asyncio.sleep(self._stale_partial_poll_secs) + if self._closed: + return + if self._last_partial_at <= self._last_commit_at: + continue + # One forced commit per stranded partial: an IGNOREd commit + # does not bump _last_commit_at, so without this guard a noise + # partial would retrigger a commit every poll. + if self._last_partial_at <= self._stale_commit_sent_at: + continue + stale_secs = time.monotonic() - self._last_partial_at + if stale_secs < self._stale_partial_commit_secs: + continue + self._stale_commit_sent_at = time.monotonic() + # INFO on purpose: this is the only trace that a transcript was + # rescued from a provider that silently refused to commit. + logger.info("stt_stale_partial_commit", stale_secs=round(stale_secs, 1)) + try: + await self.stt.commit() + except Exception as e: + logger.warning("stt_stale_partial_commit_failed", error=str(e)) + except asyncio.CancelledError: + return + # -- Internal: STT → turns --------------------------------------------- async def _process_stt_events(self) -> None: @@ -814,6 +901,7 @@ async def _process_stt_events(self) -> None: self._partials_since_last_commit += 1 if text: self._last_partial_at = time.monotonic() + self._latest_partial_text = text await self._emit(TranscriptPartial(text=text)) decision = await self.turn_detector.on_partial(text, self._turn_state()) if decision is PartialDecision.BARGE_IN and self._vad_vetoes_barge_in(text): @@ -876,17 +964,30 @@ async def _arm_hold(self, text: str, timeout_secs: float) -> None: self._hold_armed_timeout_secs = timeout_secs self._last_commit_at = time.monotonic() + # Anchor for "user resumed speaking": partials older than the commit + # event are the held fragment's own trailing refinements and must not + # stretch the hold (they otherwise floor every expiry at the grace + # window instead of ``timeout_secs``). + anchor = self._commit_event_at if self._commit_event_at > 0 else time.monotonic() + async def _expire() -> None: me = asyncio.current_task() try: remaining = timeout_secs while True: await asyncio.sleep(remaining) - # Never fire mid-utterance: a recent STT partial means the - # user resumed speaking, and their commit is about to merge - # with / supersede this hold. Fire only after real silence. + # Never fire mid-utterance: a *new* STT partial since the + # commit means the user resumed speaking, and their commit + # is about to merge with / supersede this hold. But require + # mic energy to corroborate: STT partials can hallucinate + # from silence (same failure as barge-ins) and would extend + # the hold into dead air. since_partial = time.monotonic() - self._last_partial_at - if since_partial < self._hold_partial_grace_secs: + if ( + self._last_partial_at > anchor + and since_partial < self._hold_partial_grace_secs + and not self._vad_contradicts_recent_partial() + ): remaining = self._hold_partial_grace_secs - since_partial continue break @@ -954,6 +1055,7 @@ async def _begin_user_turn( async def _handle_committed(self, text: str) -> None: if self._closed: return + self._commit_event_at = time.monotonic() # Any commit (endpointer-forced or provider debounce) makes a pending # VAD endpoint stale — the STT segment it targeted is already closed. if self._endpointer is not None: diff --git a/python/timbal/voice/turn_detection.py b/python/timbal/voice/turn_detection.py index 201b61c5..1b86a758 100644 --- a/python/timbal/voice/turn_detection.py +++ b/python/timbal/voice/turn_detection.py @@ -593,6 +593,24 @@ class LocalAudioTurnDetector(HeuristicTurnDetector): # trims trailing silence before scoring, so waiting longer reproduces the # same window and the same score. DEFAULT_HOLD_TIMEOUT_SECS = 3.0 + # Confidence tier for the HOLD (LiveKit's min/max endpointing delay shape, + # with the transcript as the confidence signal): when the audio model says + # "incomplete" but the text looks finished (terminal punctuation — Smart + # Turn systematically under-scores short closers like "Thank you." / + # "No, no."), the hold shrinks to this. The audio model is never overruled + # outright — the disagreeing text just shortens how long it gets to be + # proven right. Both-signals-incomplete keeps the full timeout. + TEXT_COMPLETE_HOLD_TIMEOUT_SECS = 1.2 + # The tier needs *confidently* finished text (terminal punctuation scores + # P_TERMINAL=0.95), not the predictor's complete-leaning neutral (0.60) — + # unpunctuated text must not shorten the hold. + TEXT_COMPLETE_TIER_THRESHOLD = 0.9 + # Inverse tier: audio says complete but text looks mid-thought (hedges + # score P_HEDGE=0.2, dangling/continuing ~0.15). Don't NEW_TURN — short + # HOLD so a continuation ("…tell me a story") can merge. Neutral (0.60) + # must NOT trigger this, or every unpunctuated complete fires a hold. + TEXT_INCOMPLETE_TIER_THRESHOLD = 0.4 + TEXT_INCOMPLETE_HOLD_TIMEOUT_SECS = 1.2 # The model consumes the last 8s of *speech*; the EOU backend trims the # trailing silence (STT commit debounce, hold pauses) before scoring, so # buffer extra raw PCM to keep a full 8s of signal after the trim. @@ -605,6 +623,8 @@ def __init__( *, completion_threshold: float | None = None, hold_timeout_secs: float | None = None, + text_complete_hold_timeout_secs: float | None = None, + text_incomplete_hold_timeout_secs: float | None = None, fallback_text_eou: TextEouPredictor | None = None, ) -> None: self.audio_eou = audio_eou @@ -613,10 +633,20 @@ def __init__( self.hold_timeout_secs = ( hold_timeout_secs if hold_timeout_secs is not None else self.DEFAULT_HOLD_TIMEOUT_SECS ) - # Used only when the buffered PCM is too short to score (fast commit at - # session start / mic audio not routed): a zero-dep lexical check so an - # obviously unfinished utterance still HOLDs instead of inheriting the - # heuristic NEW_TURN. + self.text_complete_hold_timeout_secs = ( + text_complete_hold_timeout_secs + if text_complete_hold_timeout_secs is not None + else self.TEXT_COMPLETE_HOLD_TIMEOUT_SECS + ) + self.text_incomplete_hold_timeout_secs = ( + text_incomplete_hold_timeout_secs + if text_incomplete_hold_timeout_secs is not None + else self.TEXT_INCOMPLETE_HOLD_TIMEOUT_SECS + ) + # Used when the buffered PCM is too short to score, and as the text + # confidence signal for both HOLD tiers (complete-text shortens; + # incomplete-text delays an audio-complete commit). Zero-dep lexical + # baseline — swap for a DistilBERT-class TextEouPredictor later. self.fallback_text_eou = fallback_text_eou or PunctuationEouPredictor() self._sample_rate = 16_000 self._pcm: deque[bytes] = deque() @@ -644,6 +674,8 @@ def clone(self) -> LocalAudioTurnDetector: audio_eou=self.audio_eou, completion_threshold=self.completion_threshold, hold_timeout_secs=self.hold_timeout_secs, + text_complete_hold_timeout_secs=self.text_complete_hold_timeout_secs, + text_incomplete_hold_timeout_secs=self.text_incomplete_hold_timeout_secs, fallback_text_eou=self.fallback_text_eou, ) @@ -756,6 +788,32 @@ async def on_committed(self, text: str, state: TurnState) -> CommitDecision: text_preview=candidate[:80], ) if p >= self.completion_threshold: + # Inverse tier (see TEXT_INCOMPLETE_HOLD_TIMEOUT_SECS): audio says + # done but the transcript looks mid-thought — hold short so a + # continuation can merge. Never overrule into IGNORE; just delay. + try: + p_text = await self.fallback_text_eou.predict_eou(candidate) + except Exception as e: + logger.warning("text_eou_predict_failed", error=str(e)) + p_text = None + if p_text is not None and p_text < self.TEXT_INCOMPLETE_TIER_THRESHOLD: + # Mid-agent barge-in that looks incomplete still merges via + # CONTINUE rather than parking a HOLD over the reply. + if state.active_user_text and state.assistant_active: + combined = state.active_user_text.rstrip(", ") + " " + text + return CommitDecision( + action=CommitAction.CONTINUE_TURN, + text=combined, + reason="audio_complete_text_incomplete_continue", + ) + return CommitDecision( + action=CommitAction.HOLD, + text=candidate, + reason="audio_complete_text_incomplete", + hold_timeout_secs=min( + self.text_incomplete_hold_timeout_secs, self.hold_timeout_secs + ), + ) return CommitDecision( action=CommitAction.NEW_TURN, text=candidate, @@ -769,11 +827,23 @@ async def on_committed(self, text: str, state: TurnState) -> CommitDecision: text=combined, reason="audio_continuation", ) + # Confidence tier (see TEXT_COMPLETE_HOLD_TIMEOUT_SECS): a transcript + # that reads finished disagrees with the audio score — hold, but short. + timeout = self.hold_timeout_secs + reason = "audio_hold" + try: + p_text = await self.fallback_text_eou.predict_eou(candidate) + except Exception as e: + logger.warning("text_eou_predict_failed", error=str(e)) + p_text = None + if p_text is not None and p_text >= self.TEXT_COMPLETE_TIER_THRESHOLD: + timeout = min(self.text_complete_hold_timeout_secs, self.hold_timeout_secs) + reason = "audio_hold_text_complete" return CommitDecision( action=CommitAction.HOLD, text=candidate, - reason="audio_hold", - hold_timeout_secs=self.hold_timeout_secs, + reason=reason, + hold_timeout_secs=timeout, ) From ba59ebb08f6df267890892cde412f22feabc6276 Mon Sep 17 00:00:00 2001 From: berges99 Date: Fri, 24 Jul 2026 11:52:59 +0200 Subject: [PATCH 04/15] fix(voice): Namo text EOU, shorter complete-text holds, fix hold grace stretch Wire VideoSDK Namo as the local text EOU with a terminal-punct lexical rescue so finished questions stop eating the incomplete HOLD. Cut the audio-incomplete/text-complete hold to 0.35s, and only extend HOLD on post-commit Silero speech so late STT refinements cannot floor short holds at the 2s grace window. Playground auto-starts; demo defaults to Groq Llama 3.1 8B Instant. --- examples/voice_turn_modes.py | 2 +- pyproject.toml | 1 + python/tests/core/test_namo.py | 182 +++++++++++++++++ python/tests/core/test_smart_turn.py | 1 + python/tests/core/test_turn_detection.py | 71 ++++++- python/tests/core/test_voice_session.py | 111 ++++++++++ python/timbal/server/voice.html | 198 ++++++++++++++---- python/timbal/server/voice.py | 5 +- python/timbal/voice/__init__.py | 10 +- python/timbal/voice/namo.py | 247 +++++++++++++++++++++++ python/timbal/voice/session.py | 71 +++++-- python/timbal/voice/turn_detection.py | 85 ++++++-- uv.lock | 202 ++++++++++++++++++ 13 files changed, 1104 insertions(+), 82 deletions(-) create mode 100644 python/tests/core/test_namo.py create mode 100644 python/timbal/voice/namo.py diff --git a/examples/voice_turn_modes.py b/examples/voice_turn_modes.py index 6b32b0d8..d17c10cd 100644 --- a/examples/voice_turn_modes.py +++ b/examples/voice_turn_modes.py @@ -27,7 +27,7 @@ agent = Agent( name="voice_turn_modes", - model=os.environ.get("TIMBAL_VOICE_DEMO_MODEL", "openai/gpt-4o-mini"), + model=os.environ.get("TIMBAL_VOICE_DEMO_MODEL", "groq/llama-3.1-8b-instant"), system_prompt=( "You are a concise voice assistant. Keep replies to 1–2 short sentences. " f"(turn_detector mode: {_MODE})" diff --git a/pyproject.toml b/pyproject.toml index 606c2dec..aacc1e22 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,6 +63,7 @@ server = [ voice = [ "huggingface-hub>=0.28.0", "onnxruntime>=1.20.0", + "transformers>=4.40.0", ] all = [ "timbal[codegen]", diff --git a/python/tests/core/test_namo.py b/python/tests/core/test_namo.py new file mode 100644 index 00000000..cfa95094 --- /dev/null +++ b/python/tests/core/test_namo.py @@ -0,0 +1,182 @@ +"""Tests for the Namo text EOU backend (``timbal[voice]`` extra).""" + +from __future__ import annotations + +import os +import sys +import time + +import pytest + +np = pytest.importorskip("numpy", reason="timbal[voice] extra not installed") +pytest.importorskip("onnxruntime", reason="timbal[voice] extra not installed") +pytest.importorskip("transformers", reason="timbal[voice] extra not installed") + +from timbal.voice import LocalAudioTurnDetector, resolve_turn_detector # noqa: E402 +from timbal.voice import turn_detection as turn_detection_module # noqa: E402 +from timbal.voice.eou import PunctuationEouPredictor # noqa: E402 +from timbal.voice.namo import ( # noqa: E402 + DEFAULT_REPO_ID, + MULTILINGUAL_REPO_ID, + NamoTextEouPredictor, + _ACK_FORCE_P, + _prepare_text_for_namo, +) + + +class _FakeTokenizer: + def __call__(self, text, truncation=True, max_length=512, return_tensors="np"): + n = min(len(text.split()) + 2, max_length) + return { + "input_ids": np.ones((1, n), dtype=np.int64), + "attention_mask": np.ones((1, n), dtype=np.int64), + } + + +class _FakeOrtSession: + """Returns fixed 2-class logits: [incomplete, complete].""" + + def __init__(self, p_complete: float = 0.9) -> None: + self.p_complete = p_complete + self.calls: list[dict] = [] + + def run(self, output_names, feeds): # noqa: ARG002 + self.calls.append(feeds) + # Inverse-softmax-ish logits so softmax → [1-p, p]. + p = min(1.0 - 1e-6, max(1e-6, self.p_complete)) + logit_complete = float(np.log(p / (1.0 - p))) + logits = np.array([[0.0, logit_complete]], dtype=np.float32) + return [logits] + + +def _reset_caches() -> None: + turn_detection_module._DEFAULT_AUDIO_EOU = turn_detection_module._AUDIO_EOU_UNSET + turn_detection_module._DEFAULT_TEXT_EOU = turn_detection_module._TEXT_EOU_UNSET + + +class TestNamoPredict: + async def test_empty_is_complete(self) -> None: + model = NamoTextEouPredictor() + model._session = _FakeOrtSession(0.1) + model._tokenizer = _FakeTokenizer() + assert await model.predict_eou("") == 1.0 + assert await model.predict_eou(" ") == 1.0 + assert model._session.calls == [] + + async def test_softmax_complete_label(self) -> None: + model = NamoTextEouPredictor() + model._session = _FakeOrtSession(0.87) + model._tokenizer = _FakeTokenizer() + p = await model.predict_eou("Tell me a story.") + assert p == pytest.approx(0.87, abs=1e-3) + assert len(model._session.calls) == 1 + + async def test_softmax_incomplete(self) -> None: + model = NamoTextEouPredictor() + model._session = _FakeOrtSession(0.15) + model._tokenizer = _FakeTokenizer() + p = await model.predict_eou("Uh, I don't know.") + assert p == pytest.approx(0.15, abs=1e-3) + + async def test_short_ack_with_period_forced_complete(self) -> None: + """STT loves ``Yeah.`` / ``Okay.`` — Namo scores those incomplete.""" + model = NamoTextEouPredictor() + model._session = _FakeOrtSession(0.05) # would be incomplete if consulted + model._tokenizer = _FakeTokenizer() + for text in ("Yeah.", "Okay.", "ok!", "Bye.", "Thank you."): + assert await model.predict_eou(text) == _ACK_FORCE_P, text + assert model._session.calls == [] + + async def test_ack_prep_strips_terminal_punct(self) -> None: + text, override = _prepare_text_for_namo("Yeah.") + assert text == "Yeah" + assert override == _ACK_FORCE_P + # Longer hedges must still hit the model (no force). + text, override = _prepare_text_for_namo("Uh, I don't know.") + assert override is None + assert "know" in text.lower() + + def test_default_repo_is_english_specialist(self) -> None: + model = NamoTextEouPredictor() + assert model.repo_id == DEFAULT_REPO_ID + assert "English" in model.repo_id + assert model.max_length == 512 + + def test_multilingual_repo_longer_context(self) -> None: + model = NamoTextEouPredictor(repo_id=MULTILINGUAL_REPO_ID) + assert model.max_length == 8192 + + +class TestResolveLocalInjectsNamo: + def test_local_gets_namo_text_eou(self) -> None: + _reset_caches() + try: + detector = resolve_turn_detector("local") + assert isinstance(detector, LocalAudioTurnDetector) + assert isinstance(detector.fallback_text_eou, NamoTextEouPredictor) + assert "English" in detector.fallback_text_eou.repo_id + finally: + _reset_caches() + + def test_local_shares_one_text_eou_instance(self) -> None: + _reset_caches() + try: + a = resolve_turn_detector("local") + b = resolve_turn_detector("local") + assert a.fallback_text_eou is b.fallback_text_eou + assert a.clone().fallback_text_eou is a.fallback_text_eou + finally: + _reset_caches() + + def test_local_falls_back_without_namo(self, monkeypatch) -> None: + _reset_caches() + monkeypatch.setitem(sys.modules, "timbal.voice.namo", None) + try: + detector = resolve_turn_detector("local") + assert isinstance(detector, LocalAudioTurnDetector) + assert isinstance(detector.fallback_text_eou, PunctuationEouPredictor) + finally: + _reset_caches() + + +@pytest.mark.skipif( + not os.environ.get("TIMBAL_NAMO_INTEGRATION"), + reason="set TIMBAL_NAMO_INTEGRATION=1 to download and run the real checkpoint", +) +class TestRealCheckpoint: + async def test_real_model_scores_text(self) -> None: + model = NamoTextEouPredictor() + await model.start() + p_complete = await model.predict_eou("Tell me a story.") + p_hedge = await model.predict_eou("Uh, I don't know.") + p_dangling = await model.predict_eou("I was thinking about") + assert 0.0 <= p_complete <= 1.0 + assert 0.0 <= p_hedge <= 1.0 + # English specialist is decisive on the live failure mode. + assert p_complete > 0.9 + assert p_hedge < 0.1 + assert p_dangling < 0.1 + + async def test_inference_latency_under_50ms(self) -> None: + """Warm inference should stay well under a STT commit debounce (~1.2s). + + Model card claims <11ms; we allow 50ms headroom for CI noise / cold CPU. + """ + model = NamoTextEouPredictor() + await model.start() + # Discard one more warm call after start()'s own warmup. + await model.predict_eou("warmup again") + samples = ( + "Tell me a story.", + "Uh, I don't know.", + "Quite incredible.", + "Yeah, sure. Anything else?", + "I was thinking about", + ) + times: list[float] = [] + for text in samples: + t0 = time.perf_counter() + await model.predict_eou(text) + times.append(time.perf_counter() - t0) + p50 = sorted(times)[len(times) // 2] + assert p50 < 0.050, f"p50 inference {p50 * 1000:.1f}ms exceeded 50ms budget" diff --git a/python/tests/core/test_smart_turn.py b/python/tests/core/test_smart_turn.py index 038d8109..31fe9db4 100644 --- a/python/tests/core/test_smart_turn.py +++ b/python/tests/core/test_smart_turn.py @@ -197,6 +197,7 @@ async def test_works_inside_local_detector(self): class TestResolveLocalMode: def _reset_default_cache(self): turn_detection_module._DEFAULT_AUDIO_EOU = turn_detection_module._AUDIO_EOU_UNSET + turn_detection_module._DEFAULT_TEXT_EOU = turn_detection_module._TEXT_EOU_UNSET def test_local_gets_smart_turn_model(self): self._reset_default_cache() diff --git a/python/tests/core/test_turn_detection.py b/python/tests/core/test_turn_detection.py index b65b332f..61e44cfa 100644 --- a/python/tests/core/test_turn_detection.py +++ b/python/tests/core/test_turn_detection.py @@ -394,7 +394,11 @@ async def test_incomplete_audio_holds(self) -> None: async def test_incomplete_audio_complete_text_short_hold(self) -> None: """Confidence tier: Smart Turn under-scores short closers ("Thank you." p=0.036 live) — terminal punctuation disagrees, so the hold shrinks to - the short tier instead of eating the full budget as dead air.""" + the short tier instead of eating the full budget as dead air. + + Keep this well under the incomplete-text HOLD (1.2s): VAD has usually + already waited, and a second 1.2s tax is the cold-start "slow" feel. + """ det = LocalAudioTurnDetector(audio_eou=_FixedAudioEou(0.1)) await det.start(type("C", (), {"sample_rate": 16000})()) det.push_audio(b"\x00\x01" * 8000) @@ -402,6 +406,8 @@ async def test_incomplete_audio_complete_text_short_hold(self) -> None: assert decision.action is CommitAction.HOLD assert decision.reason == "audio_hold_text_complete" assert decision.hold_timeout_secs == det.text_complete_hold_timeout_secs + assert det.text_complete_hold_timeout_secs == 0.35 + assert det.text_complete_hold_timeout_secs < det.text_incomplete_hold_timeout_secs assert det.text_complete_hold_timeout_secs < det.hold_timeout_secs # Per-session clones keep the knob. assert det.clone().text_complete_hold_timeout_secs == det.text_complete_hold_timeout_secs @@ -428,6 +434,49 @@ async def test_complete_audio_real_complete_still_new_turn(self) -> None: assert decision.action is CommitAction.NEW_TURN await det.close() + async def test_complete_audio_question_despite_namo_zero(self) -> None: + """Namo often scores finished questions ~0 — lexical gate must skip the + 1.2s incomplete HOLD (live cold-start: "How are you?" / "What's 2+2?").""" + det = LocalAudioTurnDetector( + audio_eou=_FixedAudioEou(0.98), + fallback_text_eou=_FixedTextEou(0.0), + ) + await det.start(type("C", (), {"sample_rate": 16000})()) + det.push_audio(b"\x00\x01" * 8000) + decision = await det.on_committed("Hello, hello. How are you?", _state()) + assert decision.action is CommitAction.NEW_TURN + decision = await det.on_committed("What's two plus two?", _state()) + assert decision.action is CommitAction.NEW_TURN + await det.close() + + async def test_complete_audio_hedge_still_holds_with_namo_zero(self) -> None: + """Lexical hedge (~0.2) + Namo 0 → still incomplete tier HOLD.""" + det = LocalAudioTurnDetector( + audio_eou=_FixedAudioEou(0.9), + fallback_text_eou=_FixedTextEou(0.0), + ) + await det.start(type("C", (), {"sample_rate": 16000})()) + det.push_audio(b"\x00\x01" * 8000) + decision = await det.on_committed("Uh, I don't know.", _state()) + assert decision.action is CommitAction.HOLD + assert decision.reason == "audio_complete_text_incomplete" + await det.close() + + async def test_complete_audio_neutral_midthought_keeps_namo_incomplete(self) -> None: + """Unpunctuated mid-thought: lexical ~0.60 must NOT rescue Namo 0.""" + det = LocalAudioTurnDetector( + audio_eou=_FixedAudioEou(0.9), + fallback_text_eou=_FixedTextEou(0.0), + ) + await det.start(type("C", (), {"sample_rate": 16000})()) + det.push_audio(b"\x00\x01" * 8000) + decision = await det.on_committed( + "I was thinking about something my dad told me", _state() + ) + assert decision.action is CommitAction.HOLD + assert decision.reason == "audio_complete_text_incomplete" + await det.close() + async def test_incomplete_audio_ellipsis_full_hold(self) -> None: """An STT ellipsis means the speaker trailed off — it must not count as terminal punctuation for the short tier.""" @@ -970,8 +1019,10 @@ async def _run() -> None: assert any(e.type == "agent_text_done" for e in events) assert session.transcript[0].text == "I was wondering about" - async def test_hold_interrupts_playing_audio(self) -> None: - """HOLD while TTS is still buffered must interrupt like NEW_TURN.""" + async def test_hold_defers_while_audio_playing(self) -> None: + """HOLD while TTS is still buffered arms the fragment but does not + interrupt — chopping the reply for a deferred commit was wiping + greetings on echo-ish ``Hello, hello.`` commits.""" import asyncio from .test_voice_session import DelayedMockSTT, FakePlaybackTracker @@ -1001,7 +1052,6 @@ async def on_committed(self, text, state): # noqa: ARG002 hold_timeout_secs=5.0, ) events: list[VoiceSessionEvent] = [] - held_during: list[str | None] = [] async def _empty(): return @@ -1012,10 +1062,16 @@ async def _drive() -> None: await asyncio.sleep(0.01) await stt.inject(TranscriptEvent(type="committed", text="I was wondering about")) for _ in range(50): - if tracker.interrupt_calls > 0 and session._held_user_text: - held_during.append(session._held_user_text) + if session._held_user_text == "I was wondering about": break await asyncio.sleep(0.01) + else: + raise AssertionError("HOLD never armed while audio playing") + # Assert *before* finish: session.close() interrupts when the + # tracker still reports playing (unrelated to HOLD deferral). + assert tracker.interrupt_calls == 0 + assert not any(e.type == "interrupted" for e in events) + tracker.playing = False await stt.finish() async def _run() -> None: @@ -1026,9 +1082,6 @@ async def _run() -> None: await driver await asyncio.wait_for(_run(), timeout=5) - assert tracker.interrupt_calls >= 1 - assert held_during == ["I was wondering about"] - assert any(e.type == "interrupted" for e in events) async def test_hold_refinement_updates_session_fragment(self) -> None: """Longer STT re-commit while HOLDing must replace the held fragment.""" diff --git a/python/tests/core/test_voice_session.py b/python/tests/core/test_voice_session.py index d18b616d..54c9477f 100644 --- a/python/tests/core/test_voice_session.py +++ b/python/tests/core/test_voice_session.py @@ -1209,6 +1209,12 @@ async def _run() -> None: assert [e.role for e in session.transcript] == ["user", "assistant"] assert session.transcript[0].text == combined assert session.transcript[1].text == "Second reply" + committed = [e for e in events if isinstance(e, TranscriptCommitted)] + assert len(committed) == 2 + assert committed[0].replace is False + assert committed[0].text == fragment + assert committed[1].replace is True + assert committed[1].text == combined # The merged turn's LLM input must contain the combined utterance # exactly once — the old memory *rewrite* (instead of pop) left it in # both the parent memory and the new prompt. @@ -1244,6 +1250,80 @@ async def test_align_continue_memory_drops_heard_assistant(self) -> None: await session._align_continue_memory(fragment_user_text="hello can") assert root.memory == [] + async def test_hold_during_tts_does_not_truncate_reply(self) -> None: + """HOLD while the client is still playing must defer — not amputate the + greeting for an echo-ish fragment (session log: Hello! How can I…).""" + + class _HoldThenExpire(TurnDetector): + async def on_partial(self, text, state): # noqa: ARG002 + return PartialDecision.IGNORE + + async def on_committed(self, text, state): # noqa: ARG002 + if state.holding: + return CommitDecision(action=CommitAction.NEW_TURN, text=text, reason="hold_supersede") + return CommitDecision( + action=CommitAction.HOLD, + text=text, + reason="audio_hold", + hold_timeout_secs=0.15, + ) + + tracker = FakePlaybackTracker() + agent = Agent(name="t", model=TestModel(responses=[self.RESPONSE, "Second"]), tools=[]) + stt = DelayedMockSTT() + tts = SlowMockTTS(delay=0.02, chunk=b"\x00\x01" * 50, num_chunks=4) + session = VoiceSession( + agent=agent, + stt=stt, + tts=tts, + turn_detector=_HoldThenExpire(), + playback_tracker=tracker, + ) + events: list[VoiceSessionEvent] = [] + + async def _empty_audio() -> AsyncIterator[bytes]: + return + yield # noqa: RET504 + + async def _drive() -> None: + while not any(isinstance(e, SessionStarted) for e in events): + await asyncio.sleep(0.01) + await stt.inject(TranscriptEvent(type="committed", text="Julia.")) + while not any(isinstance(e, AgentTextDone) for e in events): + await asyncio.sleep(0.01) + # Client still has queued TTS — HOLD must not emit interrupt/truncate. + tracker.playing = True + tracker.played = 80 + await stt.inject(TranscriptEvent(type="committed", text="Hello, hello.")) + await asyncio.sleep(0.05) + interrupts = [e for e in events if isinstance(e, SessionInterrupted)] + assert interrupts == [], "HOLD during TTS must not interrupt" + # Full reply still in transcript (not truncated to a heard prefix). + assistants = [e for e in session.transcript if e.role == "assistant"] + assert assistants and assistants[0].text == self.RESPONSE + tracker.playing = False + # Let hold expire → turn for the held fragment. + for _ in range(200): + if sum(1 for e in events if isinstance(e, AgentTextDone)) >= 2: + break + await asyncio.sleep(0.01) + else: + raise AssertionError("second turn never finished after hold expiry") + await stt.finish() + + async def _run() -> None: + async with aclosing(session.run(_empty_audio())) as stream: + driver = asyncio.create_task(_drive()) + async for ev in stream: + events.append(ev) + await driver + + await asyncio.wait_for(_run(), timeout=10) + assert sum(1 for e in events if isinstance(e, AgentTextDone)) >= 2 + # First assistant entry survived the HOLD (may later be joined by turn 2). + assert session.transcript[1].role == "assistant" + assert session.transcript[1].text == self.RESPONSE + async def test_interrupt_cancels_orphan_turn_before_speaking(self) -> None: """``interrupt()`` must cancel a scheduled turn even if ``_is_speaking`` is still False — otherwise HOLD expiry / commit races double-run agents.""" @@ -1594,6 +1674,7 @@ async def _run_hold_scenario( timeout_secs: float, partial_after_commit_delay: float | None, run_timeout: float = 10.0, + endpointer: object | None = None, ) -> tuple[float, VoiceSession]: """Inject partial→commit (HOLD) and return seconds from commit to turn start.""" agent = Agent(name="t", model=TestModel(responses=["ok"]), tools=[]) @@ -1603,6 +1684,8 @@ async def _run_hold_scenario( agent=agent, stt=stt, tts=tts, turn_detector=_AlwaysHoldOnceDetector(timeout_secs) ) session._hold_partial_grace_secs = grace_secs + if endpointer is not None: + session._endpointer = endpointer # type: ignore[assignment] events: list[VoiceSessionEvent] = [] elapsed = 0.0 @@ -1664,6 +1747,34 @@ async def test_post_arm_partial_extends_hold(self) -> None: # Must have waited past the bare timeout into the grace window. assert elapsed > 0.5, f"hold expired after only {elapsed:.2f}s despite fresh partial" + async def test_pre_commit_speech_does_not_stretch_via_vad_lookback(self) -> None: + """Live bug: Silero's 2s lookback still contains the held utterance, so + ``not vad_contradicts`` was always true after commit — any late STT + refinement floored a 0.35s text-complete HOLD at the 2s grace window. + """ + + class _StaleSpeechEP: + """Reports speech only in wide windows (pre-commit residue).""" + + def speech_secs_in_window(self, window_secs: float) -> float: + return 0.5 if window_secs >= 1.0 else 0.0 + + def notify_committed(self) -> None: + return None + + async def close(self) -> None: + return None + + elapsed, _ = await self._run_hold_scenario( + grace_secs=2.0, + timeout_secs=0.2, + partial_after_commit_delay=0.05, + endpointer=_StaleSpeechEP(), + ) + assert elapsed < 0.8, ( + f"hold expiry took {elapsed:.2f}s — pre-commit VAD speech stretched the grace" + ) + # --------------------------------------------------------------------------- # Tests: stale partial sweeper diff --git a/python/timbal/server/voice.html b/python/timbal/server/voice.html index 7baa9402..2558613f 100644 --- a/python/timbal/server/voice.html +++ b/python/timbal/server/voice.html @@ -405,32 +405,48 @@ opacity: 0.4; cursor: not-allowed; } - .btn-stop { + .btn-session { flex-shrink: 0; width: 2.5rem; height: 2.5rem; padding: 0; border: none; border-radius: 10px; - background: var(--danger); display: inline-flex; align-items: center; justify-content: center; box-shadow: 0 1px 2px rgba(0, 0, 0, 0.1); } - .btn-stop:hover:not(:disabled) { - background: #b91c1c; - } - .btn-stop:focus-visible { + .btn-session:focus-visible { outline: 2px solid var(--primary); outline-offset: 2px; } - .btn-stop-icon { + .btn-session[data-mode="stop"] { + background: var(--danger); + } + .btn-session[data-mode="stop"]:hover:not(:disabled) { + background: #b91c1c; + } + .btn-session[data-mode="start"] { + background: var(--primary); + } + .btn-session[data-mode="start"]:hover:not(:disabled) { + opacity: 0.9; + } + .btn-session-icon--stop { width: 11px; height: 11px; border-radius: 2px; background: var(--primary-fg); } + .btn-session-icon--start { + width: 0; + height: 0; + margin-left: 2px; + border-style: solid; + border-width: 7px 0 7px 12px; + border-color: transparent transparent transparent var(--primary-fg); + } button.primary { background: var(--primary); color: var(--primary-fg); @@ -476,17 +492,17 @@
-
+
- Getting ready - Microphone and server connection… + Ready + Connecting…
-
@@ -497,15 +513,15 @@
- - - + + - @@ -527,7 +543,7 @@ const footStatusDetail = document.getElementById('foot-status-detail'); const transcriptEl = document.getElementById('transcript'); const voiceEmpty = document.getElementById('voice-empty'); -const btnStop = document.getElementById('btn-stop'); +const btnSession = document.getElementById('btn-session'); const btnReconnect = document.getElementById('btn-reconnect'); const micSelect = document.getElementById('mic-select'); const tdSelect = document.getElementById('td-select'); @@ -538,19 +554,28 @@ // Unknown stored values leave the select on '' (server default) automatically. tdSelect.value = localStorage.getItem(TD_STORAGE_KEY) || ''; tdSelect.onchange = () => { - // A detector is fixed at session start, so apply by starting a new session. + // Applied on the next Start (no reload) — modes are fixed per WebSocket hello. localStorage.setItem(TD_STORAGE_KEY, tdSelect.value); - location.reload(); }; const NS_STORAGE_KEY = 'timbal-voice-noise-suppression'; nsSelect.value = localStorage.getItem(NS_STORAGE_KEY) || 'browser'; if (!nsSelect.value) nsSelect.value = 'browser'; // unknown stored value nsSelect.onchange = () => { - // The AudioContext rate and worklet graph are fixed at session start. + // Applied on the next Start — AudioContext rate / worklet are rebuilt then. localStorage.setItem(NS_STORAGE_KEY, nsSelect.value); - location.reload(); }; + +function setSessionButton(mode) { + const start = mode === 'start'; + btnSession.dataset.mode = start ? 'start' : 'stop'; + btnSession.disabled = false; + btnSession.setAttribute('aria-label', start ? 'Start session' : 'End session'); + btnSession.title = start ? 'Start a new voice session' : 'End session'; + btnSession.innerHTML = start + ? '' + : ''; +} const micBars = () => [...micViz.querySelectorAll('.mic-bar')]; function truncateMiddle(s, max) { @@ -657,7 +682,10 @@ footStatusDetail.textContent = detail; cardEl.classList.toggle('card--inactive', state === 'ended' || state === 'error'); cardEl.classList.toggle('card--live', state === 'live'); - btnReconnect.classList.toggle('visible', state === 'ended' || state === 'error'); + // Hard failures only — normal stop/disconnect uses Start for a new session. + btnReconnect.classList.toggle('visible', state === 'error'); + if (state === 'ended' || state === 'error') setSessionButton('start'); + else if (state === 'live' || state === 'connecting') setSessionButton('stop'); } function updateEmptyState() { @@ -990,17 +1018,88 @@ btnReconnect.onclick = () => location.reload(); +let sessionBusy = false; +/** Bumped on every teardown so an in-flight `start()` can't resurrect a dead session. */ +let startGen = 0; + +async function endSession({ userInitiated = false, silent = false } = {}) { + startGen += 1; + userEndedSession = userInitiated; + stopAssistantAudio(); + if (playbackAckTimer) { clearInterval(playbackAckTimer); playbackAckTimer = null; } + micAudioToWs = false; + sessionReady = false; + micViz.classList.remove('active'); + if (ws) { + const sock = ws; + ws = null; + sock.onclose = null; + sock.onerror = null; + sock.onmessage = null; + try { sock.close(); } catch (_) {} + } + if (worklet) { try { worklet.disconnect(); } catch (_) {} worklet = null; } + if (currentSource) { try { currentSource.disconnect(); } catch (_) {} currentSource = null; } + if (micStream) { micStream.getTracks().forEach(t => t.stop()); micStream = null; } + if (audioCtx) { + try { await audioCtx.close(); } catch (_) {} + audioCtx = null; + rnnoiseCtxActive = false; + } + analyser = null; + if (meterRAF) { cancelAnimationFrame(meterRAF); meterRAF = null; } + setSessionButton('start'); + if (!silent) { + if (userInitiated) { + setConn('ended', 'Session ended', 'Click Start for a new session.'); + } else { + setConn('ended', 'Disconnected', 'Click Start for a new session.'); + } + } + updateEmptyState(); +} + +async function startSession() { + if (sessionBusy) return; + sessionBusy = true; + btnSession.disabled = true; + try { + // Always a fresh session: tear down mic/WS/graph, then open a new one. + await endSession({ userInitiated: false, silent: true }); + userEndedSession = false; + await start(); + } catch (err) { + console.error(err); + setSessionButton('start'); + setConn('error', 'Could not start', err?.message || 'Microphone or connection failed.'); + } finally { + sessionBusy = false; + btnSession.disabled = false; + } +} + +btnSession.onclick = () => { + if (btnSession.dataset.mode === 'stop') { + void endSession({ userInitiated: true }); + return; + } + void startSession(); +}; + async function start() { + const gen = startGen; setConn('connecting', 'Getting ready', 'Microphone and server connection…'); userEndedSession = false; sessionReady = false; await startMic(); + if (gen !== startGen) return; const mics = await populateDevices(); const preferred = pickPreferredMic(mics); const activeId = micStream?.getAudioTracks()[0]?.getSettings().deviceId; if (preferred && preferred !== activeId) { await startMic(preferred); + if (gen !== startGen) return; await populateDevices(); } @@ -1009,8 +1108,14 @@ micAudioToWs = false; ws = new WebSocket(`${proto}://${location.host}/voice/ws`); ws.binaryType = 'arraybuffer'; + if (gen !== startGen) { + try { ws.close(); } catch (_) {} + ws = null; + return; + } ws.onopen = () => { + if (gen !== startGen) return; // The config hello MUST be the first frame on the socket — the server // reads one frame as session config. stopAssistantAudio() below sends a // playback ack, which used to race ahead and eat the config slot. @@ -1061,26 +1166,25 @@ window.addEventListener('keydown', kick); })(); setConn('live', 'Connected', 'Waiting for voice session…'); - btnStop.disabled = false; updateEmptyState(); }; ws.onclose = (ev) => { micAudioToWs = false; - btnStop.disabled = true; if (playbackAckTimer) { clearInterval(playbackAckTimer); playbackAckTimer = null; } sessionReady = false; micViz.classList.remove('active'); + // Teardown from endSession/startSession already nulled `ws` — ignore. if (connVisual === 'ended' || connVisual === 'error') { updateEmptyState(); return; } if (userEndedSession) { - setConn('ended', 'Session ended', 'You stopped the session. Reload to start again.'); + setConn('ended', 'Session ended', 'Click Start for a new session.'); } else { setConn('ended', 'Disconnected', ev.wasClean - ? 'The connection closed. Reload if you want another session.' - : 'Connection dropped — check the server and reload.'); + ? 'Connection closed. Click Start for a new session.' + : 'Connection dropped — check the server, then click Start.'); } updateEmptyState(); }; @@ -1110,6 +1214,23 @@ break; case 'transcript_committed': setPartial(null); + // Seal the prior assistant turn. interrupt() for a not-yet-spoken + // CONTINUE rewrite arrives after this commit's predecessor already + // sealed — without sealing here, an empty heard_text interrupt can + // wipe the *previous* completed reply (still in _turnBubbles). + window._turnBubbles = []; + window._replyDone = true; + window._assistantEl = null; + if (msg.replace) { + const users = transcriptEl.querySelectorAll('.msg.user'); + const last = users[users.length - 1]; + if (last) { + last.textContent = msg.text; + transcriptEl.scrollTop = transcriptEl.scrollHeight; + updateEmptyState(); + break; + } + } log('user', msg.text); break; case 'agent_text_delta': @@ -1141,7 +1262,13 @@ // A mid-stream STT partial can split one reply across several bubbles // (setPartial seals the in-flight bubble); rewrite the *whole* set, // not just the last fragment. - if (msg.heard_text !== undefined && msg.heard_text !== null) { + // + // Skip when nothing is in flight for *this* turn: CONTINUE can interrupt + // a just-started LLM call (heard="") and must not delete the prior + // completed assistant bubble still referenced by _lastAssistantEl. + const inFlight = Boolean(window._assistantEl) + || (window._turnBubbles && window._turnBubbles.length && !window._replyDone); + if (msg.heard_text !== undefined && msg.heard_text !== null && (inFlight || msg.heard_text)) { const bubbles = (window._turnBubbles && window._turnBubbles.length) ? window._turnBubbles : [window._assistantEl || window._lastAssistantEl].filter(Boolean); @@ -1167,21 +1294,18 @@ window._sessionTranscript = msg.entries; break; case 'session_ended': - setConn('ended', 'Session ended', 'The agent closed this voice session.'); + setConn('ended', 'Session ended', 'Click Start for a new session.'); break; } }; } -btnStop.onclick = () => { - userEndedSession = true; - stopAssistantAudio(); - if (ws) ws.close(); - micStream?.getTracks().forEach(t => t.stop()); -}; - initRunnableStrip(); -start(); +setSessionButton('start'); +setConn('connecting', 'Getting ready', 'Microphone and server connection…'); +cardEl.classList.add('card--inactive'); +// Auto-start on load — no idle play click. Start remains after Stop for a new session. +void startSession(); diff --git a/python/timbal/server/voice.py b/python/timbal/server/voice.py index e32b783a..b0034e50 100644 --- a/python/timbal/server/voice.py +++ b/python/timbal/server/voice.py @@ -384,7 +384,10 @@ async def _handle(event: VoiceSessionEvent) -> None: elif isinstance(event, TranscriptPartial): await _send_json({"type": "transcript_partial", "text": event.text}) elif isinstance(event, TranscriptCommitted): - await _send_json({"type": "transcript_committed", "text": event.text}) + payload: dict = {"type": "transcript_committed", "text": event.text} + if event.replace: + payload["replace"] = True + await _send_json(payload) elif isinstance(event, AgentTextDelta): await _send_json({"type": "agent_text_delta", "text": event.text}) elif isinstance(event, AgentTextDone): diff --git a/python/timbal/voice/__init__.py b/python/timbal/voice/__init__.py index e035d59c..552a6a99 100644 --- a/python/timbal/voice/__init__.py +++ b/python/timbal/voice/__init__.py @@ -60,12 +60,17 @@ def __getattr__(name: str): - # Lazy: importing smart_turn / vad pulls numpy/onnxruntime (timbal[voice] - # extra), which must not be required just to import timbal.voice. + # Lazy: importing smart_turn / namo / vad pulls numpy/onnxruntime / + # transformers (timbal[voice] extra), which must not be required just to + # import timbal.voice. if name == "SmartTurnEouModel": from .smart_turn import SmartTurnEouModel return SmartTurnEouModel + if name == "NamoTextEouPredictor": + from .namo import NamoTextEouPredictor + + return NamoTextEouPredictor if name == "SileroVad": from .vad import SileroVad @@ -87,6 +92,7 @@ def __getattr__(name: str): "HeuristicTurnDetector", "LexicalTurnDetector", "LocalAudioTurnDetector", + "NamoTextEouPredictor", "PartialDecision", "PlaybackTracker", "ProviderTurnDetector", diff --git a/python/timbal/voice/namo.py b/python/timbal/voice/namo.py new file mode 100644 index 00000000..d70c4d36 --- /dev/null +++ b/python/timbal/voice/namo.py @@ -0,0 +1,247 @@ +"""Namo Turn Detector v1 — DistilBERT text EOU behind :class:`TextEouPredictor`. + +Runs VideoSDK's open +`Namo-Turn-Detector-v1-English `_ +(Apache-2.0, DistilBERT, quantized ONNX, ~135MB, <11ms CPU) on the STT +transcript and returns ``P(complete)``. Complements Smart Turn (audio) with a +semantic text signal — the missing half for mid-thought hedges like +"Uh, I don't know." that audio alone over-scores as complete. + +The multilingual mmBERT checkpoint is available via ``repo_id`` / +``TIMBAL_NAMO_REPO_ID``, but measured English polarity is mushy on that +graph (card examples like "so that's all I have for today" score incomplete); +prefer the English specialist until VideoSDK ships a sharper multilingual. + +Requires the ``timbal[voice]`` extra (``onnxruntime`` + ``huggingface_hub`` + +``transformers``). Import fails without it; +:func:`~timbal.voice.resolve_turn_detector` falls back to +:class:`~timbal.voice.PunctuationEouPredictor`. + +Usage:: + + from timbal.voice import LocalAudioTurnDetector + from timbal.voice.namo import NamoTextEouPredictor + + detector = LocalAudioTurnDetector( + audio_eou=..., + fallback_text_eou=NamoTextEouPredictor(), + ) + +Or ``turn_detector="local"`` — the resolver injects Namo automatically when +the extra is installed. + +Label contract (per the model card): class ``1`` = end of turn, ``0`` = not +end of turn. We expose ``softmax(logits)[1]`` as ``P(complete)``. +""" + +from __future__ import annotations + +import asyncio +import os +import re +from functools import lru_cache, partial + +import numpy as np +import onnxruntime as ort +import structlog +from transformers import AutoTokenizer + +from .eou import TextEouPredictor +from .smart_turn import _hf_download_cached_first + +logger = structlog.get_logger("timbal.voice.namo") + +# Default: English DistilBERT specialist — decisive on measured hedges +# ("Uh, I don't know." → ~0.0 complete). Multilingual mmBERT is opt-in. +DEFAULT_REPO_ID = "videosdk-live/Namo-Turn-Detector-v1-English" +ENGLISH_REPO_ID = DEFAULT_REPO_ID +MULTILINGUAL_REPO_ID = "videosdk-live/Namo-Turn-Detector-v1-Multilingual" +DEFAULT_FILENAME = "model_quant.onnx" +# English DistilBERT uses 512; multilingual card uses 8192. Cap per-repo. +_MAX_LENGTH_BY_REPO = { + DEFAULT_REPO_ID: 512, + MULTILINGUAL_REPO_ID: 8192, +} +_DEFAULT_MAX_LENGTH = 512 +# Class index for "end of turn" / complete (model card). +_COMPLETE_LABEL = 1 + +_WORD_RE = re.compile(r"[^\W\d_]+", re.UNICODE) +_TRAILING_TERMINAL_RE = re.compile(r"[.!？。…]+$") +# Short backchannels Namo (and STT punctuation) mis-score as incomplete — +# measured: "Yeah."→0.04, "Yeah"→0.92, "Okay."/"Okay"→~0.00. Force complete. +_FORCE_COMPLETE_ACKS = frozenset( + { + "yeah", + "yep", + "yup", + "yes", + "ok", + "okay", + "sure", + "bye", + "goodbye", + "thanks", + "thank you", + "hi", + "hello", + "hey", + "no", + "nah", + "nope", + "alright", + "right", + "cool", + "fine", + "got it", + "mhmm", + "mmhmm", + "mhm", + } +) +_ACK_FORCE_P = 0.95 + + +def _prepare_text_for_namo(text: str) -> tuple[str, float | None]: + """Normalize short STT acks; optionally force ``P(complete)``. + + Returns ``(text_for_model, override_or_None)``. Empty → complete. + """ + stripped = text.strip() + if not stripped: + return "", 1.0 + words = _WORD_RE.findall(stripped) + if len(words) <= 3: + stripped = _TRAILING_TERMINAL_RE.sub("", stripped).strip() + key = " ".join(w.lower() for w in _WORD_RE.findall(stripped)) + if key in _FORCE_COMPLETE_ACKS: + return stripped, _ACK_FORCE_P + return stripped, None + + +@lru_cache(maxsize=4) +def _load_bundle( + repo_id: str, filename: str, cpu_count: int, max_length: int +) -> tuple[ort.InferenceSession, object]: + """One (ort session, tokenizer) per repo — shared process-wide.""" + path = _hf_download_cached_first(repo_id, filename) + logger.debug("namo_loading_model", path=path, repo_id=repo_id) + so = ort.SessionOptions() + so.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL + so.inter_op_num_threads = 1 + so.intra_op_num_threads = cpu_count + so.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL + session = ort.InferenceSession(path, sess_options=so) + tokenizer = AutoTokenizer.from_pretrained(repo_id) + # Warm the graph + tokenizer so the first live score isn't the cold path. + inputs = tokenizer( + "warmup", + truncation=True, + max_length=max_length, + return_tensors="np", + ) + session.run( + None, + { + "input_ids": inputs["input_ids"], + "attention_mask": inputs["attention_mask"], + }, + ) + logger.info("namo_text_eou_loaded", path=path, repo_id=repo_id) + return session, tokenizer + + +class NamoTextEouPredictor(TextEouPredictor): + """Local text end-of-turn scoring with the Namo DistilBERT ONNX checkpoint. + + Parameters + ---------- + repo_id: + Hugging Face repo. Defaults to the English DistilBERT specialist + (:data:`DEFAULT_REPO_ID`); pass :data:`MULTILINGUAL_REPO_ID` (or set + ``TIMBAL_NAMO_REPO_ID``) for mmBERT multilingual weights. Constructor + wins over the env var. + model_path: + Optional local ``.onnx`` path (skips the Hub download for the graph; + the tokenizer still loads from ``repo_id``). + cpu_count: + ``intra_op_num_threads`` for the ONNX session (1 is plenty). + + ``start()`` loads off the event loop; ``predict_eou()`` runs tokenize + + inference on the default executor. Instances are safe to share across + sessions. + """ + + def __init__( + self, + repo_id: str | None = None, + *, + model_path: str | None = None, + cpu_count: int = 1, + ) -> None: + self.repo_id = ( + repo_id + or os.environ.get("TIMBAL_NAMO_REPO_ID") + or DEFAULT_REPO_ID + ) + self.model_path = model_path + self.cpu_count = cpu_count + self.max_length = _MAX_LENGTH_BY_REPO.get(self.repo_id, _DEFAULT_MAX_LENGTH) + self._session: ort.InferenceSession | None = None + self._tokenizer: object | None = None + self._load_lock = asyncio.Lock() + + async def start(self) -> None: + async with self._load_lock: + if self._session is not None: + return + loop = asyncio.get_running_loop() + self._session, self._tokenizer = await loop.run_in_executor(None, self._load) + + async def close(self) -> None: + # Shared process-wide; keep alive. + pass + + def _load(self) -> tuple[ort.InferenceSession, object]: + if self.model_path is not None: + # Custom graph path: still need the matching tokenizer from the repo. + so = ort.SessionOptions() + so.intra_op_num_threads = self.cpu_count + session = ort.InferenceSession(self.model_path, sess_options=so) + tokenizer = AutoTokenizer.from_pretrained(self.repo_id) + return session, tokenizer + return _load_bundle(self.repo_id, DEFAULT_FILENAME, self.cpu_count, self.max_length) + + async def predict_eou(self, text: str) -> float: + if self._session is None: + await self.start() + loop = asyncio.get_running_loop() + return await loop.run_in_executor(None, partial(self._predict_sync, text)) + + def _predict_sync(self, text: str) -> float: + stripped, override = _prepare_text_for_namo(text) + if override is not None: + return override + if not stripped: + return 1.0 + inputs = self._tokenizer( + stripped, + truncation=True, + max_length=self.max_length, + return_tensors="np", + ) + outputs = self._session.run( + None, + { + "input_ids": inputs["input_ids"].astype(np.int64), + "attention_mask": inputs["attention_mask"].astype(np.int64), + }, + ) + logits = np.asarray(outputs[0][0], dtype=np.float64) + # Stable softmax → P(complete) = P(label=1). + shifted = logits - np.max(logits) + exp = np.exp(shifted) + probs = exp / np.sum(exp) + if probs.shape[0] <= _COMPLETE_LABEL: + return float(probs[-1]) + return float(probs[_COMPLETE_LABEL]) diff --git a/python/timbal/voice/session.py b/python/timbal/voice/session.py index e77468d0..861f42cb 100644 --- a/python/timbal/voice/session.py +++ b/python/timbal/voice/session.py @@ -167,6 +167,8 @@ class TranscriptPartial(VoiceSessionEvent): class TranscriptCommitted(VoiceSessionEvent): type: Literal["transcript_committed"] = "transcript_committed" text: str + # True → rewrite last user bubble (CONTINUE_TURN merge), don't append another. + replace: bool = False class AgentTextDelta(VoiceSessionEvent): @@ -771,14 +773,17 @@ def _endpoint_should_commit(self) -> bool: async def _endpoint_text_score(self) -> float | None: """``P(complete)`` for the latest STT partial, for VAD delay bumping. - Uses the turn detector's text EOU when present (``fallback_text_eou`` - on :class:`~timbal.voice.LocalAudioTurnDetector`); otherwise the - shared :class:`~timbal.voice.PunctuationEouPredictor` baseline. Returns - ``None`` when there is no fresh partial — endpointer keeps the - audio-only delay. + Prefers :meth:`~timbal.voice.LocalAudioTurnDetector.effective_text_eou` + (Namo blended with lexical) so finished questions don't inflate the + incomplete-text delay when the model under-scores. Falls back to raw + ``fallback_text_eou`` / punctuation baseline. Returns ``None`` when + there is no fresh partial — endpointer keeps the audio-only delay. """ if self._last_partial_at <= self._last_commit_at or not self._latest_partial_text: return None + effective = getattr(self.turn_detector, "effective_text_eou", None) + if callable(effective): + return await effective(self._latest_partial_text) text_eou = getattr(self.turn_detector, "fallback_text_eou", None) if text_eou is None: from .eou import PunctuationEouPredictor @@ -836,6 +841,26 @@ def _vad_contradicts_recent_partial(self) -> bool: speech_secs = self._endpointer.speech_secs_in_window(self.BARGE_IN_VAD_WINDOW_SECS) return speech_secs is not None and speech_secs < self.MIN_BARGE_IN_VAD_SPEECH_SECS + def _vad_confirms_speech_since(self, since_monotonic: float) -> bool: + """True when Silero saw real speech *after* ``since_monotonic``. + + HOLD expiry used to call :meth:`_vad_contradicts_recent_partial`, whose + 2s lookback still contains the utterance that armed the hold — so any + late STT refinement after commit "confirmed" speech and floored every + short text-complete HOLD at the 2s grace window (live: armed 0.35s, + expired ~2.0s). Missing VAD → ``False`` (don't stretch; a real resume + still supersedes via COMMIT). No endpointer → ``True`` so unit tests + without Silero keep the partial-extends-hold behavior. + """ + if self._endpointer is None: + return True + window = time.monotonic() - since_monotonic + if window <= 0: + return False + window = min(window, self.BARGE_IN_VAD_WINDOW_SECS) + speech_secs = self._endpointer.speech_secs_in_window(window) + return speech_secs is not None and speech_secs >= self.MIN_BARGE_IN_VAD_SPEECH_SECS + # -- Internal: audio → STT --------------------------------------------- async def _forward_audio(self, audio_in: AsyncIterable[bytes]) -> None: @@ -978,17 +1003,22 @@ async def _expire() -> None: await asyncio.sleep(remaining) # Never fire mid-utterance: a *new* STT partial since the # commit means the user resumed speaking, and their commit - # is about to merge with / supersede this hold. But require - # mic energy to corroborate: STT partials can hallucinate - # from silence (same failure as barge-ins) and would extend - # the hold into dead air. + # is about to merge with / supersede this hold. Require + # post-commit mic energy — not "any speech in the last 2s", + # which still includes the held utterance itself and used + # to stretch every short HOLD out to the grace window. since_partial = time.monotonic() - self._last_partial_at if ( self._last_partial_at > anchor and since_partial < self._hold_partial_grace_secs - and not self._vad_contradicts_recent_partial() + and self._vad_confirms_speech_since(anchor) ): remaining = self._hold_partial_grace_secs - since_partial + logger.debug( + "stt_hold_extended", + remaining=round(remaining, 3), + timeout_secs=timeout_secs, + ) continue break except asyncio.CancelledError: @@ -1042,7 +1072,8 @@ async def _begin_user_turn( self._transcript[-1] = TranscriptEntry(role="user", text=final_text) else: self._transcript.append(TranscriptEntry(role="user", text=final_text)) - await self._emit(TranscriptCommitted(text=final_text)) + replace_user_entry = False + await self._emit(TranscriptCommitted(text=final_text, replace=replace_user_entry)) self._active_turn_user_text = final_text self._turn_eou_at = time.monotonic() self._current_turn_task = asyncio.create_task(self._run_turn(final_text)) @@ -1134,11 +1165,19 @@ async def _handle_committed(self, text: str) -> None: ) if decision.action is CommitAction.HOLD: - # Stop any still-playing TTS (common right after agent_text_done). - # NEW_TURN / CONTINUE_TURN always interrupt first; HOLD must too or - # assistant audio keeps talking over the deferred user fragment. - await self.interrupt() - self._cancel_turn.clear() + # HOLD = "not sure yet" — do NOT chop an audible reply for a + # deferred fragment (echo-ish "Hello, hello." mid-TTS was wiping + # greetings). Partials that meant barge-in already interrupted. + # NEW_TURN / hold expiry / CONTINUE interrupt when the turn starts. + if self._assistant_audio_playing: + logger.info( + "stt_hold_defer_during_tts", + text_preview=final_text[:80], + reason=decision.reason, + ) + else: + await self.interrupt() + self._cancel_turn.clear() # Detector returns the full utterance to hold (refine/merge already applied). timeout = ( decision.hold_timeout_secs if decision.hold_timeout_secs is not None else self.hold_timeout_secs diff --git a/python/timbal/voice/turn_detection.py b/python/timbal/voice/turn_detection.py index 1b86a758..dad3f4de 100644 --- a/python/timbal/voice/turn_detection.py +++ b/python/timbal/voice/turn_detection.py @@ -597,10 +597,12 @@ class LocalAudioTurnDetector(HeuristicTurnDetector): # with the transcript as the confidence signal): when the audio model says # "incomplete" but the text looks finished (terminal punctuation — Smart # Turn systematically under-scores short closers like "Thank you." / - # "No, no."), the hold shrinks to this. The audio model is never overruled - # outright — the disagreeing text just shortens how long it gets to be - # proven right. Both-signals-incomplete keeps the full timeout. - TEXT_COMPLETE_HOLD_TIMEOUT_SECS = 1.2 + # "I am David." / "Quite good."), the hold shrinks to this. Keep it short: + # the VAD endpointer has usually already paid ~0.5–3s of silence delay + # before the commit, so a second 1.2s tax felt like dead air on every + # finished utterance Smart Turn under-scored. Both-signals-incomplete + # keeps the full timeout. + TEXT_COMPLETE_HOLD_TIMEOUT_SECS = 0.35 # The tier needs *confidently* finished text (terminal punctuation scores # P_TERMINAL=0.95), not the predictor's complete-leaning neutral (0.60) — # unpunctuated text must not shorten the hold. @@ -645,14 +647,36 @@ def __init__( ) # Used when the buffered PCM is too short to score, and as the text # confidence signal for both HOLD tiers (complete-text shortens; - # incomplete-text delays an audio-complete commit). Zero-dep lexical - # baseline — swap for a DistilBERT-class TextEouPredictor later. + # incomplete-text delays an audio-complete commit). Default is the + # zero-dep lexical baseline; ``resolve_turn_detector("local")`` injects + # Namo when ``timbal[voice]`` is installed. self.fallback_text_eou = fallback_text_eou or PunctuationEouPredictor() + # Namo under-scores many finished questions (~0.0 on "How are you?" / + # "What's two plus two?"). Lexical gate corroborates so we only pay the + # incomplete-text tax when both agree the user is mid-thought. + self._lexical_gate = PunctuationEouPredictor() self._sample_rate = 16_000 self._pcm: deque[bytes] = deque() self._pcm_bytes = 0 self._max_pcm_bytes = 0 + async def effective_text_eou(self, text: str) -> float: + """``P(complete)`` for HOLD tiers / VAD delay — Namo with lexical rescue. + + Namo under-scores many finished questions (~0.0 on "How are you?"). + Only lift the score when the lexical baseline is *confidently* complete + (terminal punct ≥ :attr:`TEXT_COMPLETE_TIER_THRESHOLD`) — a blunt + ``max(namo, lexical)`` would also promote neutral unpunctuated + mid-thoughts (lexical ~0.60) over the incomplete tier and neuter Namo. + """ + p = await self.fallback_text_eou.predict_eou(text) + if isinstance(self.fallback_text_eou, PunctuationEouPredictor): + return p + p_lex = await self._lexical_gate.predict_eou(text) + if p_lex >= self.TEXT_COMPLETE_TIER_THRESHOLD: + return max(p, p_lex) + return p + async def start(self, config: AudioInputConfig) -> None: self._sample_rate = int(getattr(config, "sample_rate", None) or 16_000) # PCM16 mono: 2 bytes/sample. @@ -661,10 +685,12 @@ async def start(self, config: AudioInputConfig) -> None: self._pcm_bytes = 0 if self.audio_eou is not None: await self.audio_eou.start(sample_rate=self._sample_rate) + await self.fallback_text_eou.start() async def close(self) -> None: if self.audio_eou is not None: await self.audio_eou.close() + await self.fallback_text_eou.close() self._pcm.clear() self._pcm_bytes = 0 @@ -763,7 +789,7 @@ async def on_committed(self, text: str, state: TurnState) -> CommitDecision: # score the text lexically so an incomplete fast commit still HOLDs. if decision.action is not CommitAction.NEW_TURN: return decision - p_text = await self.fallback_text_eou.predict_eou(candidate) + p_text = await self.effective_text_eou(candidate) if p_text >= self.completion_threshold: return decision return CommitDecision( @@ -792,7 +818,7 @@ async def on_committed(self, text: str, state: TurnState) -> CommitDecision: # done but the transcript looks mid-thought — hold short so a # continuation can merge. Never overrule into IGNORE; just delay. try: - p_text = await self.fallback_text_eou.predict_eou(candidate) + p_text = await self.effective_text_eou(candidate) except Exception as e: logger.warning("text_eou_predict_failed", error=str(e)) p_text = None @@ -832,7 +858,7 @@ async def on_committed(self, text: str, state: TurnState) -> CommitDecision: timeout = self.hold_timeout_secs reason = "audio_hold" try: - p_text = await self.fallback_text_eou.predict_eou(candidate) + p_text = await self.effective_text_eou(candidate) except Exception as e: logger.warning("text_eou_predict_failed", error=str(e)) p_text = None @@ -871,8 +897,31 @@ def _default_audio_eou() -> AudioEouModel | None: return _DEFAULT_AUDIO_EOU +def _default_text_eou() -> TextEouPredictor: + """Namo English DistilBERT when ``timbal[voice]`` is installed, else punctuation. + + Shared process-wide (same rationale as :func:`_default_audio_eou`). + """ + global _DEFAULT_TEXT_EOU + if _DEFAULT_TEXT_EOU is not _TEXT_EOU_UNSET: + return _DEFAULT_TEXT_EOU + try: + from .namo import NamoTextEouPredictor + except ImportError: + logger.debug( + "namo_text_eou_unavailable", + hint="falling back to PunctuationEouPredictor; install timbal[voice] for Namo", + ) + _DEFAULT_TEXT_EOU = PunctuationEouPredictor() + return _DEFAULT_TEXT_EOU + _DEFAULT_TEXT_EOU = NamoTextEouPredictor() + return _DEFAULT_TEXT_EOU + + _AUDIO_EOU_UNSET: Any = object() _DEFAULT_AUDIO_EOU: AudioEouModel | None = _AUDIO_EOU_UNSET +_TEXT_EOU_UNSET: Any = object() +_DEFAULT_TEXT_EOU: TextEouPredictor | Any = _TEXT_EOU_UNSET def resolve_turn_detector(spec: Any = None) -> TurnDetector: @@ -880,11 +929,12 @@ def resolve_turn_detector(spec: Any = None) -> TurnDetector: Accepted names: ``heuristic`` (default), ``provider``, ``local``, ``lexical``, ``raw`` (debug: no silence/noise/echo filtering at all). - ``local`` auto-loads the Smart Turn v3 :class:`~timbal.voice.eou.AudioEouModel` - when the ``timbal[voice]`` extra is installed (heuristic degradation otherwise). - Instances are returned unchanged — callers that reuse one spec across - concurrent sessions (server ``voice_config``) must :meth:`~TurnDetector.clone` - per session, or pass a zero-arg factory callable instead. + ``local`` auto-loads Smart Turn (audio) + Namo (text) when the + ``timbal[voice]`` extra is installed (heuristic / punctuation degradation + otherwise). Instances are returned unchanged — callers that reuse one spec + across concurrent sessions (server ``voice_config``) must + :meth:`~TurnDetector.clone` per session, or pass a zero-arg factory + callable instead. """ if spec is None: return HeuristicTurnDetector() @@ -897,9 +947,12 @@ def resolve_turn_detector(spec: Any = None) -> TurnDetector: if key in ("provider", "stt"): return ProviderTurnDetector() if key in ("local", "audio", "smart_turn"): - return LocalAudioTurnDetector(audio_eou=_default_audio_eou()) + return LocalAudioTurnDetector( + audio_eou=_default_audio_eou(), + fallback_text_eou=_default_text_eou(), + ) if key in ("lexical", "semantic", "punctuation"): - return LexicalTurnDetector() + return LexicalTurnDetector(text_eou=_default_text_eou()) if key in ("raw", "none", "off"): return RawTurnDetector() raise ValueError( diff --git a/uv.lock b/uv.lock index 6bc225a2..a88ac520 100644 --- a/uv.lock +++ b/uv.lock @@ -1836,6 +1836,110 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, ] +[[package]] +name = "regex" +version = "2026.7.19" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/20/98/04b13f1ddfb63158025291c02e03eb42fbb7acb51d091d541050eb4e35e8/regex-2026.7.19.tar.gz", hash = "sha256:7e77b324909c1617cbb4c668677e2c6ae13f44d7c1de0d4f15f2e3c10f3315b5", size = 416440, upload-time = "2026-07-19T00:19:48.923Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/e5/cef4de2bac939280b68d32adc659478845238a8274f2f79c465063f590ad/regex-2026.7.19-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ac777001cdfc28b72477d93c8564bb7583081ea8fb45cdca3d568e0a4f87183c", size = 494012, upload-time = "2026-07-19T00:16:39.927Z" }, + { url = "https://files.pythonhosted.org/packages/ff/87/e86f51eb117457bb7803132ffe5cb6e2841e2b5bea4cc85d397f3c6e257d/regex-2026.7.19-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:59787bd5f8c70aa339084e961d2996b53fbdeab4d5393bba5c1fe1fc32e02bae", size = 295281, upload-time = "2026-07-19T00:16:41.433Z" }, + { url = "https://files.pythonhosted.org/packages/41/2e/2360c41d8080a3d9ec7e5c90fad6eab3b50192869d10e9a5609e48c8177b/regex-2026.7.19-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:90c633e7e8d6bf4e992b8b36ce69e018f834b641dd6de8cea6d78c06ffa119c5", size = 290615, upload-time = "2026-07-19T00:16:43.058Z" }, + { url = "https://files.pythonhosted.org/packages/cf/69/b65ba4344efbc771b28fe5dde84cbbb6c8f9551165952fe78def5b9dde6a/regex-2026.7.19-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:87ccab0db8d5f4fbb0272642113c1adb2ffc698c16d3a0944580222331fa7a20", size = 791804, upload-time = "2026-07-19T00:16:44.662Z" }, + { url = "https://files.pythonhosted.org/packages/81/b6/a40dfa0dc6224b36f620c00296eacc830489cbf8c2837b6750dfe6170375/regex-2026.7.19-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9e50d748a32da622f256e8d505867f5d3c43a837c6a9f0efb149655fadd1042a", size = 861723, upload-time = "2026-07-19T00:16:46.412Z" }, + { url = "https://files.pythonhosted.org/packages/e3/02/735991dee71abd83196a7962f7ed8bf5aa05720ff06e2d3ff896a85e2bbb/regex-2026.7.19-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bf1516fe58fc104f39b2d1dbe2d5e27d0cd45c4be2e42ba6ee0cc763701ec3c7", size = 905932, upload-time = "2026-07-19T00:16:47.956Z" }, + { url = "https://files.pythonhosted.org/packages/45/6c/e7098d8b846ccdbf431d8c081b61e496526a27a28094ed09e0dce21b3f54/regex-2026.7.19-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:09f3e5287f94f17b709dc9a9e70865855feee835c861613be144218ce4ca82cc", size = 801407, upload-time = "2026-07-19T00:16:49.43Z" }, + { url = "https://files.pythonhosted.org/packages/8a/18/34b69274e2649bcc7d9b089c2b2983fb2632d8ecf667e359593be9072e79/regex-2026.7.19-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6383cd2ed53a646c659ba1fe65727db76437fdaa069e697a0b44a51d5843d864", size = 774448, upload-time = "2026-07-19T00:16:51.352Z" }, + { url = "https://files.pythonhosted.org/packages/bb/e6/0a72247d025585fd3800b98e040b84d562a88af6303347100484849f4f01/regex-2026.7.19-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:09d3007fc76249a83cdd33de160d50e6cb77f54e09d8fa9e7148e10607ce24af", size = 783297, upload-time = "2026-07-19T00:16:53.071Z" }, + { url = "https://files.pythonhosted.org/packages/b1/aa/c4f65ae7dd02a36b323a70c4cff326e1f3442361aaebc9311100a130d54f/regex-2026.7.19-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6f8c6e7a1cfa3dc9d0ee2de0e65e834537fa29992cc3976ffec914afc35c5dd5", size = 854736, upload-time = "2026-07-19T00:16:54.607Z" }, + { url = "https://files.pythonhosted.org/packages/62/c3/668082bcc817b9e694189b84997aeba7385b7779faa6711788679c482e35/regex-2026.7.19-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:b2ea4a3e8357be8849e833beeae757ac3c7a6b3fc055c03c808a53c91ad30d82", size = 763298, upload-time = "2026-07-19T00:16:56.289Z" }, + { url = "https://files.pythonhosted.org/packages/4b/fb/2d07ad555e7af88aa5f867fdafa47a8d945ee237c20af3ebceb46a820835/regex-2026.7.19-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:80115dd39481fd3a4b4080220799dbcacb921a844de4b827264ececacbe17c78", size = 844430, upload-time = "2026-07-19T00:16:57.933Z" }, + { url = "https://files.pythonhosted.org/packages/51/15/c82a471fe3dce56f03745635b43aa456c40dc0db089e07ef148b331507d1/regex-2026.7.19-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d6ce43a0269d68cee79a7d1ade7def53c20f8f2a047b92d7b5d5bcc73ae88327", size = 789683, upload-time = "2026-07-19T00:16:59.583Z" }, + { url = "https://files.pythonhosted.org/packages/b5/f4/7532a2c59d56f5398902c20de60f0c9a5d1cd364e42a051b48e1b210be7b/regex-2026.7.19-cp311-cp311-win32.whl", hash = "sha256:9be2a6647740dd3cca6acb24e87f03d7632cd280dbce9bbe40c26353a215a45d", size = 266778, upload-time = "2026-07-19T00:17:01.032Z" }, + { url = "https://files.pythonhosted.org/packages/83/2b/cf1bc631db154eb95520d9d5dbc2371ff77a0f014bbf7d748fed8496aa63/regex-2026.7.19-cp311-cp311-win_amd64.whl", hash = "sha256:8d3469c91dd92ee41b7c95280edbd975ef1ba9195086686623a1c6e8935ce965", size = 277983, upload-time = "2026-07-19T00:17:02.571Z" }, + { url = "https://files.pythonhosted.org/packages/8d/bd/56ceaf170e875d5a6761bf2bfd0d040f1cacc896850d5e40cb29b11bbd06/regex-2026.7.19-cp311-cp311-win_arm64.whl", hash = "sha256:36aacfb15faaff3ced55afbf35ec72f50d4aee22082c4f7fe0573a33e2fca92e", size = 276961, upload-time = "2026-07-19T00:17:04.135Z" }, + { url = "https://files.pythonhosted.org/packages/3b/b9/d11d7e501ac8fd7d617684423ebb9561e0b998481c1e4cbc0cb212c5d74a/regex-2026.7.19-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2cc3460cedf7579948486eab03bc9ad7089df4d7281c0f47f4afe03e8d13f02d", size = 496778, upload-time = "2026-07-19T00:17:05.677Z" }, + { url = "https://files.pythonhosted.org/packages/3f/a9/a5ab6f312f24318019170dc485d5421fe4f89e43a98640da50d95a8a7041/regex-2026.7.19-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0e9554c8785eac5cffe6300f69a91f58ba72bc88a5f8d661235ad7c6aa5b8ccd", size = 297122, upload-time = "2026-07-19T00:17:07.59Z" }, + { url = "https://files.pythonhosted.org/packages/b3/63/4cab4d7f2d384a144d420b763d97674cb70619c878ea6fcd7640d0e62143/regex-2026.7.19-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d7da47a0f248977f08e2cb659ff3c17ddc13a4d39b3a7baa0a81bf5b415430f6", size = 292009, upload-time = "2026-07-19T00:17:09.648Z" }, + { url = "https://files.pythonhosted.org/packages/22/85/102a81b218298957d4ea7d2f084fae537a71add9d6ff93c8e67284c5f45e/regex-2026.7.19-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93db40c8de0815baab96a06e08a984bac71f989d13bab789e382158c5d426797", size = 796708, upload-time = "2026-07-19T00:17:11.542Z" }, + { url = "https://files.pythonhosted.org/packages/78/b5/dc136af5629938a037cd2b304c12240e132ec92f38be8ff9cc89af2a1f2d/regex-2026.7.19-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:66bd62c59a5427746e8c44becae1d9b99d22fb13f30f492083dfb9ad7c45cc18", size = 865651, upload-time = "2026-07-19T00:17:13.312Z" }, + { url = "https://files.pythonhosted.org/packages/e0/75/67402ae3cd9c8c988a4c805d15ee3eef015e7ca4cb112cf3e640fc1f4153/regex-2026.7.19-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1649eb39fcc9ea80c4d2f110fde2b8ab2aef3877b98f02ab9b14e961f418c511", size = 911756, upload-time = "2026-07-19T00:17:15.015Z" }, + { url = "https://files.pythonhosted.org/packages/2a/8e/096d00c7c480ef2ff4265349b14e2261d4ab787ba1f74e2e80d1c58079c3/regex-2026.7.19-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9dce8ec9695f531a1b8a6f314fd4b393adcccf2ea861db480cdf97a301d01a68", size = 801798, upload-time = "2026-07-19T00:17:17.208Z" }, + { url = "https://files.pythonhosted.org/packages/f0/41/e7ecac6edb5722417f85cc67eaf386322fbe8acf6918ec2fdc37c20dd9d0/regex-2026.7.19-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3080a7fd38ef049bd489e01c970c97dd84ff446a885b0f1f6b26d9b1ad13ce11", size = 776933, upload-time = "2026-07-19T00:17:19.347Z" }, + { url = "https://files.pythonhosted.org/packages/6f/69/03c9b3f058d66403e0ca2c938696e81d51cd4c6d47ec5265f02f96948d9a/regex-2026.7.19-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1d793a7988e04fcb1e2e135567443d82173225d657419ec09414a9b5a145b986", size = 784338, upload-time = "2026-07-19T00:17:21.057Z" }, + { url = "https://files.pythonhosted.org/packages/f6/f7/b38ab3d43f284afbb618fcd15d0e77eb786ae461ce1f6bc7494619ddc0f2/regex-2026.7.19-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e8b0abe7d870f53ca5143895fef7d1041a0c831a140d3dc2c760dd7ba25d4a8b", size = 860452, upload-time = "2026-07-19T00:17:23.119Z" }, + { url = "https://files.pythonhosted.org/packages/15/5c/ff60ef0571121714f3cf9920bc183071e384a10b556d042e0fdb06cc07a5/regex-2026.7.19-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:4e5413bd5f13d3a4e3539ca98f70f75e7fca92518dd7f117f030ebedd10b60cb", size = 765958, upload-time = "2026-07-19T00:17:24.81Z" }, + { url = "https://files.pythonhosted.org/packages/aa/0f/bd34021162c0ab47f9a315bd56cd5642e920c8e5668a75ef6c6a6fca590d/regex-2026.7.19-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:73b133a9e6fb512858e7f065e96f1180aa46646bc74a83aea62f1d314f3dd035", size = 851765, upload-time = "2026-07-19T00:17:26.993Z" }, + { url = "https://files.pythonhosted.org/packages/2a/20/a2ca43edade0595cccfdc98636739f536d9e26898e7dbddc2b9e98898953/regex-2026.7.19-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dbe6493fbd27321b1d1f2dd4f5c7e5bd4d8b1d7cab7f32fd67db3d0b2ed8248a", size = 789714, upload-time = "2026-07-19T00:17:28.699Z" }, + { url = "https://files.pythonhosted.org/packages/5d/47/e02db4015d424fc83c00ea0ac8c5e5ec14397943de9abf909d5ce3a25931/regex-2026.7.19-cp312-cp312-win32.whl", hash = "sha256:ddd67571c10869f65a5d7dde536d1e066e306cc90de57d7de4d5f34802428bb5", size = 267157, upload-time = "2026-07-19T00:17:31.051Z" }, + { url = "https://files.pythonhosted.org/packages/08/8e/c780c131f79b42ed22d1bd7da4096c2c35f813e835acd02ef0f018bd892c/regex-2026.7.19-cp312-cp312-win_amd64.whl", hash = "sha256:e30d40268a28d54ce0437031750497004c22602b8e3ab891f759b795a003b312", size = 277777, upload-time = "2026-07-19T00:17:32.848Z" }, + { url = "https://files.pythonhosted.org/packages/3e/4c/e4d7e086449bdf379d89774bf1f89dc4a41943f3c5a6125a03905b34b5fb/regex-2026.7.19-cp312-cp312-win_arm64.whl", hash = "sha256:de9208bb427130c82a5dbfd104f92c8876fc9559278c880b3002755bbbe9c83d", size = 277136, upload-time = "2026-07-19T00:17:34.803Z" }, + { url = "https://files.pythonhosted.org/packages/5d/3d/84165e4299ff76f3a40fe1f2abf939e976f693383a08d2beea6af62bd2c1/regex-2026.7.19-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f035d9dc1d25eff9d361456572231c7d27b5ccd473ca7dc0adfce732bd006d40", size = 496552, upload-time = "2026-07-19T00:17:36.808Z" }, + { url = "https://files.pythonhosted.org/packages/02/a2/a65293e6e4cf28eb7ee1be5335a5386c40d6742e9f47fafc8fec785e16c7/regex-2026.7.19-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c42572142ed0b9d5d261ba727157c426510da78e20828b66bbb855098b8a4e38", size = 296983, upload-time = "2026-07-19T00:17:38.816Z" }, + { url = "https://files.pythonhosted.org/packages/95/47/2d0564e93d87bc48618360ddca232a2ca612bbdf53ce8465d45ca5ce14ee/regex-2026.7.19-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:40b34dd88658e4fedd2fddbf0275ac970d00614b731357f425722a3ed1983d11", size = 291832, upload-time = "2026-07-19T00:17:40.726Z" }, + { url = "https://files.pythonhosted.org/packages/07/cd/42dfbabff3dfc9603c501c0e2e2c5adbb09d127b267bf5348de0af338c15/regex-2026.7.19-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c41c63992bf1874cebb6e7f56fd7d3c007924659a604ae3d90e427d40d4fd13", size = 796775, upload-time = "2026-07-19T00:17:42.382Z" }, + { url = "https://files.pythonhosted.org/packages/df/5d/f6a4839f2b934e3eed5973fd07f5929ee97d4c98939fb275ea23c274ee16/regex-2026.7.19-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d3372064506b94dd2c67c845f2db8062e9e9ba84d04e33cb96d7d33c11fe1ae", size = 865687, upload-time = "2026-07-19T00:17:44.185Z" }, + { url = "https://files.pythonhosted.org/packages/14/b0/b47d6c36049bc59806a50bd4c86ced70bbe058d787f80281b1d7a9b0e024/regex-2026.7.19-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fce7760bf283405b2c7999cab3da4e72f7deca6396013115e3f7a955db9760da", size = 911962, upload-time = "2026-07-19T00:17:46.442Z" }, + { url = "https://files.pythonhosted.org/packages/2a/be/ff61f28f9273658cfe23acbbac5217221f6519960ed401e61dfdab12bc35/regex-2026.7.19-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0d702548d89d572b2929879bc883bb7a4c4709efafe4512cadee56c55c9bd15", size = 801817, upload-time = "2026-07-19T00:17:48.25Z" }, + { url = "https://files.pythonhosted.org/packages/c3/bb/8b4f7f26b333f9f79e1b453613c39bb4776f51d38ae66dd0ba31d6b354ca/regex-2026.7.19-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d446c6ac40bb6e05025ccee55b84d80fe9bf8e93010ffc4bb9484f13d498835f", size = 776908, upload-time = "2026-07-19T00:17:50.183Z" }, + { url = "https://files.pythonhosted.org/packages/09/13/610110fc5921d380516d03c26b652555f08aa0d23ea78a771231873c3638/regex-2026.7.19-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4c3501bfa814ab07b5580741f9bf78dfdfe146a04057f82df9e2402d2a975939", size = 784426, upload-time = "2026-07-19T00:17:52.454Z" }, + { url = "https://files.pythonhosted.org/packages/ca/f5/1ef9e2a83a5947c57ebff0b377cb5727c3d5ec1992317a320d035cd0dbb6/regex-2026.7.19-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c4585c3e64b4f9e583b4d2683f18f5d5d872b3d71dcf24594b74ecc23602fa96", size = 860600, upload-time = "2026-07-19T00:17:54.229Z" }, + { url = "https://files.pythonhosted.org/packages/a0/02/073af33a3ec149241d11c80acea91e722aa0adbf05addd50f251c4fe89c3/regex-2026.7.19-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:571fde9741eb0ccde23dd4e0c1d50fbae910e901fa7e629faf39b2dda740d220", size = 765950, upload-time = "2026-07-19T00:17:56.041Z" }, + { url = "https://files.pythonhosted.org/packages/81/a9/d1e9f819dc394a568ef370cd56cf25394e957a2235f8370f23b576e5a475/regex-2026.7.19-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:15b364b9b98d6d2fe1a85034c23a3180ff913f46caddc3895f6fd65186255ccc", size = 851794, upload-time = "2026-07-19T00:17:57.897Z" }, + { url = "https://files.pythonhosted.org/packages/03/3a/8ae83eda7579feacdf984e71fb9e70635fb6f832eeddca58427ec4fca926/regex-2026.7.19-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffd8893ccc1c2fce6e0d6ca402d716fe1b29db70c7132609a05955e31b2aa8f2", size = 789845, upload-time = "2026-07-19T00:17:59.97Z" }, + { url = "https://files.pythonhosted.org/packages/4b/23/c195cbfe5a75fdec64d8f6554fd15237b837919d2c61bdc141d7c807b08b/regex-2026.7.19-cp313-cp313-win32.whl", hash = "sha256:f0fa4fa9c3632d708742baf2282f2055c11d888a790362670a403cbf48a2c404", size = 267135, upload-time = "2026-07-19T00:18:01.958Z" }, + { url = "https://files.pythonhosted.org/packages/b2/80/a11de8404b7272b70acb45c1c05987cce60b45d5693da2e176f0e390d564/regex-2026.7.19-cp313-cp313-win_amd64.whl", hash = "sha256:d51ffd3427640fa2da6ade574ceba932f210ad095f65fcc450a2b0a0d454868e", size = 277747, upload-time = "2026-07-19T00:18:04.121Z" }, + { url = "https://files.pythonhosted.org/packages/d1/29/0f5c8eff1b4f1f3d83276d365fccecf666afcc7d947420943bf394d07adb/regex-2026.7.19-cp313-cp313-win_arm64.whl", hash = "sha256:c670fe7be5b6020b76bc6e8d2196074657e1327595bca93a389e1a76ab130ad8", size = 277129, upload-time = "2026-07-19T00:18:05.821Z" }, + { url = "https://files.pythonhosted.org/packages/dc/4c/44b74742052cedda40f9ae469532a037112f7311a36669a891fba8984bb0/regex-2026.7.19-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:db47b561c9afd884baa1f96f797c9ca369872c4b65912bc691cfa99e68340af2", size = 501134, upload-time = "2026-07-19T00:18:07.567Z" }, + { url = "https://files.pythonhosted.org/packages/f0/45/bbd038b5e39ee5613a5a689290145b40058cc152c41de9cc23639d2b9734/regex-2026.7.19-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:65dcd28d3eba2ab7c2fd906485cc301392b47cc2234790d27d4e4814e02cdfda", size = 299418, upload-time = "2026-07-19T00:18:09.38Z" }, + { url = "https://files.pythonhosted.org/packages/65/38/c5bde94b4cedfd5850d64c3f08222d8e1600e84f6ee71d9b44b4b8163f74/regex-2026.7.19-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f2e7f8e2ab6c2922be02c7ec45185aa5bd771e2e57b95455ee343a44d8130dff", size = 294486, upload-time = "2026-07-19T00:18:11.188Z" }, + { url = "https://files.pythonhosted.org/packages/d7/6a/2f5e107cb26c960b781967178899daf2787a7ab151844ed3c01d6fc95474/regex-2026.7.19-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe31f28c94402043161876a258a9c6f757cb485905c7614ce8d6cd40e6b7bdc1", size = 811643, upload-time = "2026-07-19T00:18:12.975Z" }, + { url = "https://files.pythonhosted.org/packages/37/d4/a2f963406d7d73a62eed84ba05a258afb6cad1b21aa4517443ce40506b78/regex-2026.7.19-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8f6fa298bb4f7f58a33334406218ba74716e68feddf5e4e54cd5d8082705abf", size = 871081, upload-time = "2026-07-19T00:18:14.733Z" }, + { url = "https://files.pythonhosted.org/packages/45/a3/44be546340bedb15f13063f5e7fe16793ea4d9ea2e805d09bd174ac27724/regex-2026.7.19-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cc1b2440423a851fad781309dd87843868f4f66a6bcd1ddb9225cf4ec2c84732", size = 917372, upload-time = "2026-07-19T00:18:16.724Z" }, + { url = "https://files.pythonhosted.org/packages/f8/f6/e0870b0fd2a40dba0074e4b76e514b21313d37946c9248453e34ec43923e/regex-2026.7.19-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ac59a0900474a52b7c04af8196affc22bd9842acb0950df12f7b813e983609a", size = 816089, upload-time = "2026-07-19T00:18:18.617Z" }, + { url = "https://files.pythonhosted.org/packages/ae/27/957e8e22690ad6634572b39b71f130a6105f4d0718bb16849eac00fff147/regex-2026.7.19-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4896db1f4ce0576765b8272aa922df324e0f5b9bb2c3d03044ff32a7234a9aba", size = 785206, upload-time = "2026-07-19T00:18:20.464Z" }, + { url = "https://files.pythonhosted.org/packages/76/a4/186e410941e731037c01166069ab86da9f65e8f8110c18009ccf4bd623ee/regex-2026.7.19-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4e6883a021db30511d9fb8cfb0f222ce1f2c369f7d4d8b0448f449a93ba0bdfc", size = 800431, upload-time = "2026-07-19T00:18:22.716Z" }, + { url = "https://files.pythonhosted.org/packages/73/9f/e4e10e023d291d64a33e246610b724493bf1ce98e0e59c9b7c837e5acfb7/regex-2026.7.19-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:09523a592938aa9f587fb74467c63ff0cf88fc3df14c82ab0f0517dcf76aaa62", size = 864906, upload-time = "2026-07-19T00:18:24.772Z" }, + { url = "https://files.pythonhosted.org/packages/24/57/ccb20b6be5f1f52a053d1ba2a8f7a077edb9d918248b8490d7506c6832b3/regex-2026.7.19-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:1ebac3474b8589fce2f9b225b650afd61448f7c73a5d0255a10cc6366471aed1", size = 773559, upload-time = "2026-07-19T00:18:27.008Z" }, + { url = "https://files.pythonhosted.org/packages/a3/82/f3b263cf8fad927dc102891da8502e718b7ff9d19af7a2a07c03865d7188/regex-2026.7.19-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4a0530bb1b8c1c985e7e2122e2b4d3aedd8a3c21c6bfddae6767c4405668b56e", size = 857739, upload-time = "2026-07-19T00:18:29.107Z" }, + { url = "https://files.pythonhosted.org/packages/47/2e/1687bd1b6c2aed5e672ccf845fc11557821fe7366d921b50889ea5ce57bf/regex-2026.7.19-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2ef7eeb108c47ce7bcc9513e51bcb1bf57e8f483d52fce68a8642e3527141ae0", size = 804522, upload-time = "2026-07-19T00:18:31.362Z" }, + { url = "https://files.pythonhosted.org/packages/76/7c/cc4e7655181b2d9235b704f2c5e19d8eff002bbc437bae59baee0e381aca/regex-2026.7.19-cp313-cp313t-win32.whl", hash = "sha256:64b6ca7391a1395c2638dd5c7456d67bea44fc6c5e8e92c5dc8aa6a8f23292b4", size = 269141, upload-time = "2026-07-19T00:18:33.479Z" }, + { url = "https://files.pythonhosted.org/packages/bb/14/961b4c7b05a2391c32dbc85e27773076671ef8f97f36cec70fe414734c02/regex-2026.7.19-cp313-cp313t-win_amd64.whl", hash = "sha256:f04b9f56b0e0614c0126be12c2c2d9f8850c1e57af302bd0a63bed379d4af974", size = 280036, upload-time = "2026-07-19T00:18:35.419Z" }, + { url = "https://files.pythonhosted.org/packages/ce/67/795644550d788ddbb6dc458c95895f8009978ea6d6ea76b005eb3f45e8c9/regex-2026.7.19-cp313-cp313t-win_arm64.whl", hash = "sha256:fcee38cd8e5089d6d4f048ba1233b3ad76e5954f545382180889112ff5cb712d", size = 279394, upload-time = "2026-07-19T00:18:37.454Z" }, + { url = "https://files.pythonhosted.org/packages/d2/25/0c4c452f8ef3efe456745b2f33195f5904b573fb4c2ff3f0cb9ec188461e/regex-2026.7.19-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:a81758ed242b861b72e778ba34d41366441a2e10b16b472784c88da2dea7e2dd", size = 496750, upload-time = "2026-07-19T00:18:39.633Z" }, + { url = "https://files.pythonhosted.org/packages/24/9e/b70ca6c1704f6c7cd32a9e143c86cc5968d10981eca284bad670c245ea7d/regex-2026.7.19-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4aa5435cdb3eb6f55fe98a171b05e3fbcd95fadaa4aa32acf62afd9b0cfdbcac", size = 297093, upload-time = "2026-07-19T00:18:41.583Z" }, + { url = "https://files.pythonhosted.org/packages/87/74/0b692da2520d51fbff19c88b83d97e4c702909dd02386c585998b7e2dbed/regex-2026.7.19-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:60be8693a1dadc210bbcbc0db3e26da5f7d01d1d5a3da594e99b4fa42df404f5", size = 292043, upload-time = "2026-07-19T00:18:43.347Z" }, + { url = "https://files.pythonhosted.org/packages/e3/a7/1d478e614016045a33feae57446215f9fd65b665a5ceb2f891fb3183bc52/regex-2026.7.19-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19662dbedbe783d323196312d38f5ba53cf56296378252171985da6899887d3", size = 797214, upload-time = "2026-07-19T00:18:45.362Z" }, + { url = "https://files.pythonhosted.org/packages/aa/ae/11b9c9411d92c30e3d2db32df5a31133e4a99a8fc397a604fd08f6c4bffb/regex-2026.7.19-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d15df07081d91b76ff20d43f94592ee110330152d617b730fdbe5ef9fb680053", size = 866433, upload-time = "2026-07-19T00:18:47.315Z" }, + { url = "https://files.pythonhosted.org/packages/b1/62/2b2efc4992f91d6d204b24c647c9f9412e85379d92b7c0ab9fdae622327e/regex-2026.7.19-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:56ad4d9f77df871a99e25c37091052a02528ec0eb059de928ee33956b854b45b", size = 911360, upload-time = "2026-07-19T00:18:49.588Z" }, + { url = "https://files.pythonhosted.org/packages/14/71/986ceea9aa3da548bf1357cad89b63915ec6d21ec957c8113b29ece567df/regex-2026.7.19-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7322ec6cc9fba9d49ab888bb82d67ac5625627aa168f0165139b17018df3fb8a", size = 801275, upload-time = "2026-07-19T00:18:51.767Z" }, + { url = "https://files.pythonhosted.org/packages/15/be/ce9d9534b2cda96eab32c548261224b9b4e220a4126f098f60f42ae7b4cd/regex-2026.7.19-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9c7472192ebfad53a6be7c4a8bfb2d64b81c0e93a1fc8c57e1dd0b638297b5d1", size = 777131, upload-time = "2026-07-19T00:18:54.053Z" }, + { url = "https://files.pythonhosted.org/packages/61/2b/58b5c710f2c3929515a25f3a1ca0dad0dcd4518d4fff3cf23bc7adb8dcd2/regex-2026.7.19-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c10b82c2634df08dfb13b1f04e38fe310d086ee092f4f69c0c8da234251e556e", size = 785020, upload-time = "2026-07-19T00:18:56.579Z" }, + { url = "https://files.pythonhosted.org/packages/84/03/5fe091935b74f15fe0f97998c215cae418d1c0413f6258c7d4d2e83aa37f/regex-2026.7.19-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:17ed5692f6acc4183e98331101a5f9e4f64d72fe58b753da4d444a2c77d05b12", size = 861263, upload-time = "2026-07-19T00:18:58.64Z" }, + { url = "https://files.pythonhosted.org/packages/d8/fa/d60bf82e10841eef62a9e32aac401468f05fddfbcb2942e342b1ba3d2433/regex-2026.7.19-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:22a992de9a0d91bda927bf02b94351d737a0302905432c88a53de7c4b9ce62e2", size = 766199, upload-time = "2026-07-19T00:19:00.705Z" }, + { url = "https://files.pythonhosted.org/packages/bf/5d/11e64d151b0662b81d6bf644c74dc118d461df85bdf2577fadbbf751788a/regex-2026.7.19-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:618a0aed532be87294c4477b0481f3aa0f1520f4014a4374dd4cf789b4cd2c97", size = 851317, upload-time = "2026-07-19T00:19:03.015Z" }, + { url = "https://files.pythonhosted.org/packages/7c/34/532efb87488d90807bae6a443d357ee5e2728a478c597619c8aaa17cc0bd/regex-2026.7.19-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ce9e679f776649746729b6c86382da519ef649c8e34cc41df0d2e5e0f6c36d4", size = 789557, upload-time = "2026-07-19T00:19:05.338Z" }, + { url = "https://files.pythonhosted.org/packages/d6/90/3a8d5ca977171ec3ae21a71207d2228b2663bde14d7f7ef0e6363ecf9290/regex-2026.7.19-cp314-cp314-win32.whl", hash = "sha256:73f272fba87b8ccfe70a137d02a54af386f6d27aa509fbffdd978f5947aae1aa", size = 272531, upload-time = "2026-07-19T00:19:07.487Z" }, + { url = "https://files.pythonhosted.org/packages/96/e1/8862885e70409de70e8c005f57fb2e7be8d9ef0317250d60f4c9660a300d/regex-2026.7.19-cp314-cp314-win_amd64.whl", hash = "sha256:d721e53758b2cca74990185eb0671dd466d7a388a1a45d0c6f4c13cef41a68ac", size = 280831, upload-time = "2026-07-19T00:19:09.46Z" }, + { url = "https://files.pythonhosted.org/packages/08/82/2693e53e29f9104d9de95d37ce4dd826bd32d5f9c0085d3aa6ac042675c4/regex-2026.7.19-cp314-cp314-win_arm64.whl", hash = "sha256:65fa6cb38ed5e9c3637e68e544f598b39c3b86b808ed0627a67b68320384b459", size = 281099, upload-time = "2026-07-19T00:19:11.398Z" }, + { url = "https://files.pythonhosted.org/packages/92/b7/9a01aa16461a18cde9d7b9c3ab21e501db2ce33725f53014342b91df2b0a/regex-2026.7.19-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:5a2721c8720e2cb3c209925dfb9200199b4b07361c9e01d321719404b21458b3", size = 501121, upload-time = "2026-07-19T00:19:13.425Z" }, + { url = "https://files.pythonhosted.org/packages/f3/5e/bbaeca815dc9191c424c94a4fdc5c87c75748a64a6271821212ebdd4e1a3/regex-2026.7.19-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:199535629f25caf89698039af3d1ad5fcae7f933e2112c73f1cdf49165c99518", size = 299415, upload-time = "2026-07-19T00:19:15.43Z" }, + { url = "https://files.pythonhosted.org/packages/cd/d6/0dd1a321afaab95eb7ff44aa0f637301786f1dc71c6b797b9ed236ed8890/regex-2026.7.19-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9b60d7814174f059e5de4ab98271cc5ba9259cfea55273a81544dceea32dc8d9", size = 294483, upload-time = "2026-07-19T00:19:17.879Z" }, + { url = "https://files.pythonhosted.org/packages/92/5f/40bacf91d0904f812e13bbbab3864604c463eced8afdc54aeaa50492ea95/regex-2026.7.19-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dbece16025afda5e3031af0c4059207e61dcf73ef13af844964f57f387d1c435", size = 811833, upload-time = "2026-07-19T00:19:20.102Z" }, + { url = "https://files.pythonhosted.org/packages/94/7c/4902744261f775aeede8b5627314b38482da29cf49a57b66a6fb753246c5/regex-2026.7.19-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d24ecb4f5e009ea0bd275ee37ad9953b32005e2e5e60f8bbae16da0dbbf0d3a0", size = 871270, upload-time = "2026-07-19T00:19:22.365Z" }, + { url = "https://files.pythonhosted.org/packages/16/70/6980c9be6bf21c0a60ed3e0aea39cf419ecf3b08d1d9947bc56e196ef186/regex-2026.7.19-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8cae6fd77a5b72dae505084b1a2ee0360139faf72fedbab667cd7cc65aae7a6a", size = 917534, upload-time = "2026-07-19T00:19:24.529Z" }, + { url = "https://files.pythonhosted.org/packages/52/92/8b2bd872782ce8c42691e39acb38eb8efe014e5ddb78ad7d943d6f197ce9/regex-2026.7.19-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9724e6cb5e478cd7d8cabf027826178739cb18cf0e117d0e32814d479fa02276", size = 816135, upload-time = "2026-07-19T00:19:26.919Z" }, + { url = "https://files.pythonhosted.org/packages/de/2d/33a602f657bdc4041f17d79f92ab18261d255d91a06117a6e29df023e5e2/regex-2026.7.19-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:572fc57b0009c735ee56c175ea021b637a15551a312f56734277f923d6fd0f6c", size = 785492, upload-time = "2026-07-19T00:19:29.192Z" }, + { url = "https://files.pythonhosted.org/packages/9e/36/0987cf4cb271680064a70d24a475873775a151d0b7058698a006cb0cae4a/regex-2026.7.19-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:20568e182eb82d39a6bf7cff3fd58566f14c75c6f74b2c8c96537eecf9010e3a", size = 800658, upload-time = "2026-07-19T00:19:31.392Z" }, + { url = "https://files.pythonhosted.org/packages/a8/24/c14f31c135e1ba55fa4f9a58ca98d0842512bf6188230763c31c8f449e3b/regex-2026.7.19-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1d58561843f0ff7dc78b4c28b5e2dc388f3eff94ebc8a232a3adba961fc00009", size = 865073, upload-time = "2026-07-19T00:19:33.485Z" }, + { url = "https://files.pythonhosted.org/packages/14/85/181a12211f22469f24d2de1ebddfe397d2396e2c29013b9a58134a91069a/regex-2026.7.19-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:61bb1bd45520aacd56dd80943bd34991fb5350afdd1f36f2282230fd5154a218", size = 773684, upload-time = "2026-07-19T00:19:35.599Z" }, + { url = "https://files.pythonhosted.org/packages/23/58/bd1a0c1a62251366f8d21f41b1ea3c76994962071b8b6ea42f72d505c0f0/regex-2026.7.19-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:cd3584591ea4429026cdb931b054342c2bcf189b44ff367f8d5c15bc092a2966", size = 857769, upload-time = "2026-07-19T00:19:37.738Z" }, + { url = "https://files.pythonhosted.org/packages/e4/4f/f7e2dad6756b2fe1fe75dd90a628c3b45f249d39f948dd90cd2476325417/regex-2026.7.19-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cc26a66e212fa5d6c6170c3a40d99d888db3020c6fdab1523250d4341382e44", size = 804546, upload-time = "2026-07-19T00:19:40.229Z" }, + { url = "https://files.pythonhosted.org/packages/2b/d7/01d31d5bdb09bc026fab77f59a371fdf8f9b292e4810546c56182ca70498/regex-2026.7.19-cp314-cp314t-win32.whl", hash = "sha256:2c4e61e2e1be56f63ec3cc618aa9e0de81ef6f43d177205451840022e24f5b78", size = 274526, upload-time = "2026-07-19T00:19:42.398Z" }, + { url = "https://files.pythonhosted.org/packages/52/0e/cea4ce73bc0a8247a0748228ae6669984c7e1f8134b6fa66e59c0572e0ea/regex-2026.7.19-cp314-cp314t-win_amd64.whl", hash = "sha256:c639ea314df70a7b2811e8020448c75af8c9445f5a60f8a4ced81c306a9380c2", size = 283763, upload-time = "2026-07-19T00:19:44.644Z" }, + { url = "https://files.pythonhosted.org/packages/6f/b6/26e41975febae63b7a6e3e02f32cff6cff2e4f10d19c929082f56aebf7c6/regex-2026.7.19-cp314-cp314t-win_arm64.whl", hash = "sha256:9a15e785f244f3e07847b984ce8773fc3da10a9f3c131cc49a4c5b4d672b4547", size = 283451, upload-time = "2026-07-19T00:19:46.639Z" }, +] + [[package]] name = "requests" version = "2.34.2" @@ -1997,6 +2101,39 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3e/0a/9e1be9035b37448ce2e68c978f0591da94389ade5a5abafa4cf99985d1b2/ruff-0.15.4-py3-none-win_arm64.whl", hash = "sha256:60d5177e8cfc70e51b9c5fad936c634872a74209f934c1e79107d11787ad5453", size = 10966776, upload-time = "2026-02-26T20:03:56.908Z" }, ] +[[package]] +name = "safetensors" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/45/06/f955dbbb1859e3bd23c8ac6141af5106e7ad5fedec4a3a6e3d60f94b7001/safetensors-0.8.0.tar.gz", hash = "sha256:fabaf3e0f18a6618d9b36560682562157f77c2b71fcffc7b432be2baed9d753d", size = 325846, upload-time = "2026-06-09T07:52:25.563Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/a0/f718cda65b05407d228f97602cf60dca269c979867aa5beb25410de26cd3/safetensors-0.8.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c554f85858e05226d3c2828e32395e677434685d6d94594a41643361c5e837f0", size = 473568, upload-time = "2026-06-09T07:52:18.829Z" }, + { url = "https://files.pythonhosted.org/packages/f5/b1/fa7c600e7dceae12e9606c7578cbc9ff1e1ed55844883ee5c92205e86226/safetensors-0.8.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:c80201d22cbf405b80647a60ada77bba06c8fba2da2743ba1e89cdcc39a81f25", size = 484562, upload-time = "2026-06-09T07:52:17.518Z" }, + { url = "https://files.pythonhosted.org/packages/09/7d/65a7de0af421317bb36a067241e4235fff194eed60b961ed6d3f59a3fc60/safetensors-0.8.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a46e5ff292c356d6991e60942ba7f79817682d3a2cef0702136448cb9c4d235", size = 502844, upload-time = "2026-06-09T07:52:07.624Z" }, + { url = "https://files.pythonhosted.org/packages/91/4f/3175c9d75634e0e0dda0082794193521035edd7c70a6f212bf33ca06ddf4/safetensors-0.8.0-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4124502b78f03534117c848f87a39b8f31e577b15eff423bf8bfb95f2a8c30d0", size = 511823, upload-time = "2026-06-09T07:52:09.565Z" }, + { url = "https://files.pythonhosted.org/packages/20/87/846c289e7aa2299eff406335717cf43ce8777194ece8aad75772e0411615/safetensors-0.8.0-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7bc0a787ba8a35be368ee3574edfa2b1ad389eebd0a72e482ae275490e3f6c98", size = 633461, upload-time = "2026-06-09T07:52:11.128Z" }, + { url = "https://files.pythonhosted.org/packages/76/22/8d64d9df2c45d5ded401df889d0ad90882804ca172d79ec4f0df8f727fe0/safetensors-0.8.0-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:040070828e36dc8e122178bbbd5830ff9e97920affb84cbe0f46442497bed358", size = 545148, upload-time = "2026-06-09T07:52:13.603Z" }, + { url = "https://files.pythonhosted.org/packages/28/50/f203ff3a3ddfe19308efc83c5a3a29ed02bf786732ec35e68bf9162f3365/safetensors-0.8.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd6f3f93c9a0a7cc2788ee63fb763353d4bd2e89b0751bc78fcf7dda00bea774", size = 516040, upload-time = "2026-06-09T07:52:16.29Z" }, + { url = "https://files.pythonhosted.org/packages/46/fb/cdaed17ceb2948784fd9c36b6fd3e951b608547cea81a48e8ee6f8cfdfcb/safetensors-0.8.0-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:fcdd41ec4628fee5799f807c73c353629130fbd942aa23d83c623dd6c9d52d78", size = 513832, upload-time = "2026-06-09T07:52:12.37Z" }, + { url = "https://files.pythonhosted.org/packages/0d/49/1e15de264dcc3b77943d2d0c56a95809956883b1c2d6d585c792523f180b/safetensors-0.8.0-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8e9f537aa183a38ace122d27303dcd986b26bd2a7591f9181d7f0c396f4677ca", size = 559930, upload-time = "2026-06-09T07:52:14.743Z" }, + { url = "https://files.pythonhosted.org/packages/2a/43/bf38443278eab4b1be1fce2931e2b012ad9cb7df52ada751d0aab8f7659a/safetensors-0.8.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:87eec7ffed2b809f05a398a8becb7d013f19f7837cd15d9748580d6cf30dbaf4", size = 678670, upload-time = "2026-06-09T07:52:20.032Z" }, + { url = "https://files.pythonhosted.org/packages/72/e3/68cd3fa5b48488e84add63e04cb12f3bc28ae4638c06d4508c6e88823d0e/safetensors-0.8.0-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:4a95ae2b05d7726d751da4ebf626a2ca782b706e101bd894c95bc2450b1cffcc", size = 786679, upload-time = "2026-06-09T07:52:21.322Z" }, + { url = "https://files.pythonhosted.org/packages/29/4b/1c19c509d56e01f4fbb3d0a2e597450f6cc04d1d56cf52defb0a62dfd715/safetensors-0.8.0-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:3ae091f16662658bdc019a4ff6cb4c085bb7d725eb5978b183ffd265863b6d2d", size = 765683, upload-time = "2026-06-09T07:52:22.594Z" }, + { url = "https://files.pythonhosted.org/packages/27/43/41c1621732edd934d868a00d1b891584c892a7b62a9aab82ea5a0a5623ee/safetensors-0.8.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8e080062fcde23be189565e1c3305d16751a218ecf9412c8601e64204eb6f846", size = 722361, upload-time = "2026-06-09T07:52:23.924Z" }, + { url = "https://files.pythonhosted.org/packages/8e/3f/73ccf82579412b4a71c4ca673f10b5f1f888d7cf5af7fe24f27d30307be4/safetensors-0.8.0-cp310-abi3-win32.whl", hash = "sha256:2ddf52eac562eda224f99acfa7889d02968c1fd59a5b011ae7d8137c37e9c02d", size = 342401, upload-time = "2026-06-09T07:52:28.895Z" }, + { url = "https://files.pythonhosted.org/packages/1b/6d/3fba214c1e5e0f69991677ec3bc17023f0421776975e1de0c682dca475e2/safetensors-0.8.0-cp310-abi3-win_amd64.whl", hash = "sha256:096ec1a98435df7beb08853bb5aa9081a84f23d0adc67ed1a0a10550f608373f", size = 355540, upload-time = "2026-06-09T07:52:27.832Z" }, + { url = "https://files.pythonhosted.org/packages/8d/fc/7eedc3510d97878876e32774eebbeb61c43f148a96e915c84229a3e967aa/safetensors-0.8.0-cp310-abi3-win_arm64.whl", hash = "sha256:f7838e5135a406ad3e02efdcb8cf2e5397d368b0154537c4fec682dbc544d452", size = 340500, upload-time = "2026-06-09T07:52:26.745Z" }, +] + +[[package]] +name = "shellingham" +version = "1.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310, upload-time = "2023-10-24T04:13:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, +] + [[package]] name = "six" version = "1.17.0" @@ -2086,6 +2223,7 @@ all = [ { name = "python-docx", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "rich", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "ruff", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "transformers", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "uvicorn", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "websockets", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] @@ -2109,6 +2247,7 @@ server = [ voice = [ { name = "huggingface-hub", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "onnxruntime", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "transformers", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] [package.dev-dependencies] @@ -2149,6 +2288,8 @@ requires-dist = [ { name = "ruff", marker = "extra == 'all'", specifier = ">=0.9.3" }, { name = "ruff", marker = "extra == 'codegen'", specifier = ">=0.9.3" }, { name = "structlog", specifier = ">=25.1.0" }, + { name = "transformers", marker = "extra == 'all'", specifier = ">=4.40.0" }, + { name = "transformers", marker = "extra == 'voice'", specifier = ">=4.40.0" }, { name = "typing-extensions", specifier = ">=4.14.1" }, { name = "uuid7", specifier = ">=0.1.0" }, { name = "uvicorn", marker = "extra == 'all'", specifier = ">=0.34.0" }, @@ -2168,6 +2309,32 @@ dev = [ { name = "yappi", specifier = ">=1.7.6" }, ] +[[package]] +name = "tokenizers" +version = "0.22.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/73/6f/f80cfef4a312e1fb34baf7d85c72d4411afde10978d4657f8cdd811d3ccc/tokenizers-0.22.2.tar.gz", hash = "sha256:473b83b915e547aa366d1eee11806deaf419e17be16310ac0a14077f1e28f917", size = 372115, upload-time = "2026-01-05T10:45:15.988Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/92/97/5dbfabf04c7e348e655e907ed27913e03db0923abb5dfdd120d7b25630e1/tokenizers-0.22.2-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:544dd704ae7238755d790de45ba8da072e9af3eea688f698b137915ae959281c", size = 3100275, upload-time = "2026-01-05T10:41:02.158Z" }, + { url = "https://files.pythonhosted.org/packages/2e/47/174dca0502ef88b28f1c9e06b73ce33500eedfac7a7692108aec220464e7/tokenizers-0.22.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:1e418a55456beedca4621dbab65a318981467a2b188e982a23e117f115ce5001", size = 2981472, upload-time = "2026-01-05T10:41:00.276Z" }, + { url = "https://files.pythonhosted.org/packages/d6/84/7990e799f1309a8b87af6b948f31edaa12a3ed22d11b352eaf4f4b2e5753/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2249487018adec45d6e3554c71d46eb39fa8ea67156c640f7513eb26f318cec7", size = 3290736, upload-time = "2026-01-05T10:40:32.165Z" }, + { url = "https://files.pythonhosted.org/packages/78/59/09d0d9ba94dcd5f4f1368d4858d24546b4bdc0231c2354aa31d6199f0399/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:25b85325d0815e86e0bac263506dd114578953b7b53d7de09a6485e4a160a7dd", size = 3168835, upload-time = "2026-01-05T10:40:38.847Z" }, + { url = "https://files.pythonhosted.org/packages/47/50/b3ebb4243e7160bda8d34b731e54dd8ab8b133e50775872e7a434e524c28/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bfb88f22a209ff7b40a576d5324bf8286b519d7358663db21d6246fb17eea2d5", size = 3521673, upload-time = "2026-01-05T10:40:56.614Z" }, + { url = "https://files.pythonhosted.org/packages/e0/fa/89f4cb9e08df770b57adb96f8cbb7e22695a4cb6c2bd5f0c4f0ebcf33b66/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1c774b1276f71e1ef716e5486f21e76333464f47bece56bbd554485982a9e03e", size = 3724818, upload-time = "2026-01-05T10:40:44.507Z" }, + { url = "https://files.pythonhosted.org/packages/64/04/ca2363f0bfbe3b3d36e95bf67e56a4c88c8e3362b658e616d1ac185d47f2/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:df6c4265b289083bf710dff49bc51ef252f9d5be33a45ee2bed151114a56207b", size = 3379195, upload-time = "2026-01-05T10:40:51.139Z" }, + { url = "https://files.pythonhosted.org/packages/2e/76/932be4b50ef6ccedf9d3c6639b056a967a86258c6d9200643f01269211ca/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:369cc9fc8cc10cb24143873a0d95438bb8ee257bb80c71989e3ee290e8d72c67", size = 3274982, upload-time = "2026-01-05T10:40:58.331Z" }, + { url = "https://files.pythonhosted.org/packages/1d/28/5f9f5a4cc211b69e89420980e483831bcc29dade307955cc9dc858a40f01/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:29c30b83d8dcd061078b05ae0cb94d3c710555fbb44861139f9f83dcca3dc3e4", size = 9478245, upload-time = "2026-01-05T10:41:04.053Z" }, + { url = "https://files.pythonhosted.org/packages/6c/fb/66e2da4704d6aadebf8cb39f1d6d1957df667ab24cff2326b77cda0dcb85/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:37ae80a28c1d3265bb1f22464c856bd23c02a05bb211e56d0c5301a435be6c1a", size = 9560069, upload-time = "2026-01-05T10:45:10.673Z" }, + { url = "https://files.pythonhosted.org/packages/16/04/fed398b05caa87ce9b1a1bb5166645e38196081b225059a6edaff6440fac/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:791135ee325f2336f498590eb2f11dc5c295232f288e75c99a36c5dbce63088a", size = 9899263, upload-time = "2026-01-05T10:45:12.559Z" }, + { url = "https://files.pythonhosted.org/packages/05/a1/d62dfe7376beaaf1394917e0f8e93ee5f67fea8fcf4107501db35996586b/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:38337540fbbddff8e999d59970f3c6f35a82de10053206a7562f1ea02d046fa5", size = 10033429, upload-time = "2026-01-05T10:45:14.333Z" }, + { url = "https://files.pythonhosted.org/packages/fd/18/a545c4ea42af3df6effd7d13d250ba77a0a86fb20393143bbb9a92e434d4/tokenizers-0.22.2-cp39-abi3-win32.whl", hash = "sha256:a6bf3f88c554a2b653af81f3204491c818ae2ac6fbc09e76ef4773351292bc92", size = 2502363, upload-time = "2026-01-05T10:45:20.593Z" }, + { url = "https://files.pythonhosted.org/packages/65/71/0670843133a43d43070abeb1949abfdef12a86d490bea9cd9e18e37c5ff7/tokenizers-0.22.2-cp39-abi3-win_amd64.whl", hash = "sha256:c9ea31edff2968b44a88f97d784c2f16dc0729b8b143ed004699ebca91f05c48", size = 2747786, upload-time = "2026-01-05T10:45:18.411Z" }, + { url = "https://files.pythonhosted.org/packages/72/f4/0de46cfa12cdcbcd464cc59fde36912af405696f687e53a091fb432f694c/tokenizers-0.22.2-cp39-abi3-win_arm64.whl", hash = "sha256:9ce725d22864a1e965217204946f830c37876eee3b2ba6fc6255e8e903d5fcbc", size = 2612133, upload-time = "2026-01-05T10:45:17.232Z" }, +] + [[package]] name = "tomli" version = "2.4.0" @@ -2234,6 +2401,41 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl", hash = "sha256:ee1e4c0e59148062281c49d80b25b67771a127c85fc9676d3be5f243206826bf", size = 78374, upload-time = "2026-02-03T17:35:50.982Z" }, ] +[[package]] +name = "transformers" +version = "5.14.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "numpy", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "packaging", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "pyyaml", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "regex", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "safetensors", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "tokenizers", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "tqdm", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "typer", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5a/fb/2a2ba88f325e68a921d8b69ff63b477830b2e73ade9a3c8c8cab2f06d741/transformers-5.14.1.tar.gz", hash = "sha256:60d196c27781eacf8637e2b533f517582907ad6f9ae142046d6b69431a5b2173", size = 9295927, upload-time = "2026-07-16T09:41:57.773Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6f/67/8d85ca2323233ae3c0365a659c4e52ee1f587b440e4bc577e7d8e4416d0f/transformers-5.14.1-py3-none-any.whl", hash = "sha256:9db974c4079ede2d1a3ea7ca5a240df33f2cc26fc2b36ba64c5f2a4f43b6e725", size = 11625234, upload-time = "2026-07-16T09:41:54.143Z" }, +] + +[[package]] +name = "typer" +version = "0.27.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "rich", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "shellingham", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/37/78/fda3361b56efc27944f24225f6ecd13d96d6fcfe37bd0eb34e2f4c63f9fc/typer-0.27.0.tar.gz", hash = "sha256:629bd12ea5d13a17148125d9a264f949eb171fb3f120f9b04d85873cab054fa5", size = 203430, upload-time = "2026-07-15T19:21:07.007Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/03/26a383c9e58c213199d1aad1c3d353cfc22d4444ec6d2c0bf8ad02523843/typer-0.27.0-py3-none-any.whl", hash = "sha256:6f4b27631e47f077871b7dc30e933ec0131c1390fbe0e387ea5574b5bac9ccf1", size = 122716, upload-time = "2026-07-15T19:21:05.553Z" }, +] + [[package]] name = "typing-extensions" version = "4.15.0" From 2336bd820b889d988207c9a3fe5ed5d1bbc45ed7 Mon Sep 17 00:00:00 2001 From: berges99 Date: Fri, 24 Jul 2026 11:59:00 +0200 Subject: [PATCH 05/15] test(voice): move voice/turn tests under python/tests/voice Mirror timbal.voice module layout (session, turn_detection, namo, smart_turn, endpointing, eou, vad, playback, realtime) out of tests/core. --- python/tests/voice/__init__.py | 0 .../test_endpointing.py} | 0 python/tests/{core => voice}/test_eou.py | 0 python/tests/{core => voice}/test_namo.py | 0 python/tests/{core => voice}/test_playback.py | 0 .../test_realtime.py} | 0 .../test_voice_session.py => voice/test_session.py} | 0 .../test_session_stt_refinement.py} | 0 python/tests/{core => voice}/test_smart_turn.py | 0 python/tests/{core => voice}/test_turn_detection.py | 12 ++++++------ .../test_turn_detection_adversarial.py | 2 +- .../{core/test_silero_vad.py => voice/test_vad.py} | 0 12 files changed, 7 insertions(+), 7 deletions(-) create mode 100644 python/tests/voice/__init__.py rename python/tests/{core/test_vad_endpointing.py => voice/test_endpointing.py} (100%) rename python/tests/{core => voice}/test_eou.py (100%) rename python/tests/{core => voice}/test_namo.py (100%) rename python/tests/{core => voice}/test_playback.py (100%) rename python/tests/{core/test_realtime_session.py => voice/test_realtime.py} (100%) rename python/tests/{core/test_voice_session.py => voice/test_session.py} (100%) rename python/tests/{core/test_voice_session_stt_refinement.py => voice/test_session_stt_refinement.py} (100%) rename python/tests/{core => voice}/test_smart_turn.py (100%) rename python/tests/{core => voice}/test_turn_detection.py (99%) rename python/tests/{core => voice}/test_turn_detection_adversarial.py (99%) rename python/tests/{core/test_silero_vad.py => voice/test_vad.py} (100%) diff --git a/python/tests/voice/__init__.py b/python/tests/voice/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/tests/core/test_vad_endpointing.py b/python/tests/voice/test_endpointing.py similarity index 100% rename from python/tests/core/test_vad_endpointing.py rename to python/tests/voice/test_endpointing.py diff --git a/python/tests/core/test_eou.py b/python/tests/voice/test_eou.py similarity index 100% rename from python/tests/core/test_eou.py rename to python/tests/voice/test_eou.py diff --git a/python/tests/core/test_namo.py b/python/tests/voice/test_namo.py similarity index 100% rename from python/tests/core/test_namo.py rename to python/tests/voice/test_namo.py diff --git a/python/tests/core/test_playback.py b/python/tests/voice/test_playback.py similarity index 100% rename from python/tests/core/test_playback.py rename to python/tests/voice/test_playback.py diff --git a/python/tests/core/test_realtime_session.py b/python/tests/voice/test_realtime.py similarity index 100% rename from python/tests/core/test_realtime_session.py rename to python/tests/voice/test_realtime.py diff --git a/python/tests/core/test_voice_session.py b/python/tests/voice/test_session.py similarity index 100% rename from python/tests/core/test_voice_session.py rename to python/tests/voice/test_session.py diff --git a/python/tests/core/test_voice_session_stt_refinement.py b/python/tests/voice/test_session_stt_refinement.py similarity index 100% rename from python/tests/core/test_voice_session_stt_refinement.py rename to python/tests/voice/test_session_stt_refinement.py diff --git a/python/tests/core/test_smart_turn.py b/python/tests/voice/test_smart_turn.py similarity index 100% rename from python/tests/core/test_smart_turn.py rename to python/tests/voice/test_smart_turn.py diff --git a/python/tests/core/test_turn_detection.py b/python/tests/voice/test_turn_detection.py similarity index 99% rename from python/tests/core/test_turn_detection.py rename to python/tests/voice/test_turn_detection.py index 61e44cfa..400fd89c 100644 --- a/python/tests/core/test_turn_detection.py +++ b/python/tests/voice/test_turn_detection.py @@ -31,7 +31,7 @@ resolve_turn_detector, ) -from .test_voice_session import MockSTT, MockTTS +from .test_session import MockSTT, MockTTS # --------------------------------------------------------------------------- # Moved heuristic functions (ported from test_voice_session_stt_refinement.py) @@ -900,7 +900,7 @@ async def test_hold_expiry_deferred_while_user_speaking(self) -> None: import asyncio import time - from .test_voice_session import DelayedMockSTT + from .test_session import DelayedMockSTT class _HoldOnce(TurnDetector): def __init__(self) -> None: @@ -963,7 +963,7 @@ async def test_hold_timeout_starts_turn(self) -> None: """HOLD must not start the agent until the hold timer expires.""" import asyncio - from .test_voice_session import DelayedMockSTT + from .test_session import DelayedMockSTT class _HoldOnce(TurnDetector): def __init__(self) -> None: @@ -1025,7 +1025,7 @@ async def test_hold_defers_while_audio_playing(self) -> None: greetings on echo-ish ``Hello, hello.`` commits.""" import asyncio - from .test_voice_session import DelayedMockSTT, FakePlaybackTracker + from .test_session import DelayedMockSTT, FakePlaybackTracker class _AlwaysHold(TurnDetector): async def on_partial(self, text, state): # noqa: ARG002 @@ -1087,7 +1087,7 @@ async def test_hold_refinement_updates_session_fragment(self) -> None: """Longer STT re-commit while HOLDing must replace the held fragment.""" import asyncio - from .test_voice_session import DelayedMockSTT + from .test_session import DelayedMockSTT class _HoldRefine(TurnDetector): async def on_partial(self, text, state): # noqa: ARG002 @@ -1177,7 +1177,7 @@ async def test_hold_expiry_does_not_wipe_rearmed_hold(self) -> None: """ import asyncio - from .test_voice_session import DelayedMockSTT + from .test_session import DelayedMockSTT class _HoldThenRefine(TurnDetector): def __init__(self) -> None: diff --git a/python/tests/core/test_turn_detection_adversarial.py b/python/tests/voice/test_turn_detection_adversarial.py similarity index 99% rename from python/tests/core/test_turn_detection_adversarial.py rename to python/tests/voice/test_turn_detection_adversarial.py index 793cf718..6dc9f4a3 100644 --- a/python/tests/core/test_turn_detection_adversarial.py +++ b/python/tests/voice/test_turn_detection_adversarial.py @@ -25,8 +25,8 @@ _looks_like_fresh_hold_utterance, ) +from .test_session import DelayedMockSTT, MockTTS from .test_turn_detection import _FixedAudioEou, _state -from .test_voice_session import DelayedMockSTT, MockTTS # --------------------------------------------------------------------------- # Pure helpers — edge cases that have bitten us / look sharp diff --git a/python/tests/core/test_silero_vad.py b/python/tests/voice/test_vad.py similarity index 100% rename from python/tests/core/test_silero_vad.py rename to python/tests/voice/test_vad.py From f8be453e2fe7299f3dbbf39ef6f94fae7a26ffa5 Mon Sep 17 00:00:00 2001 From: berges99 Date: Fri, 24 Jul 2026 12:14:36 +0200 Subject: [PATCH 06/15] fix(voice): ignore STT crumbs after finished commits, hide vetoed partials Refuse CONTINUE-merge of ultra-early short trailers onto punctuated turns so the LLM isn't cancelled/restarted; keep question VAD-splits and late barge-ins. Preserve LocalAudio NEW_TURN for those barge-ins (don't HOLD during TTS). Skip TranscriptPartial for VAD-vetoed barge-in hallucinations. --- python/tests/voice/test_session.py | 10 +++ python/tests/voice/test_turn_detection.py | 85 ++++++++++++++++++++ python/timbal/voice/session.py | 6 +- python/timbal/voice/turn_detection.py | 94 +++++++++++++++++++++-- 4 files changed, 189 insertions(+), 6 deletions(-) diff --git a/python/tests/voice/test_session.py b/python/tests/voice/test_session.py index 54c9477f..e11fe254 100644 --- a/python/tests/voice/test_session.py +++ b/python/tests/voice/test_session.py @@ -1561,10 +1561,20 @@ async def test_hallucinated_partial_does_not_interrupt(self) -> None: # Reply survived intact. assert session.transcript[-1].role == "assistant" assert session.transcript[-1].text == "This is a fairly long reply that keeps on playing for a while." + # Vetoed barge-in partials must not flash in the playground caption. + vetoed = [ + e + for e in events + if isinstance(e, TranscriptPartial) and "not not too bad" in e.text.lower() + ] + assert vetoed == [] async def test_real_speech_partial_still_interrupts(self) -> None: _, events = await self._run_barge_in_scenario(speech_secs=1.0) assert any(isinstance(e, SessionInterrupted) for e in events) + assert any( + isinstance(e, TranscriptPartial) and "not not too bad" in e.text.lower() for e in events + ) class TestStreamingTTS: diff --git a/python/tests/voice/test_turn_detection.py b/python/tests/voice/test_turn_detection.py index 400fd89c..443c4885 100644 --- a/python/tests/voice/test_turn_detection.py +++ b/python/tests/voice/test_turn_detection.py @@ -243,6 +243,70 @@ async def test_fast_short_fragment_becomes_continuation(self) -> None: assert decision.text == "Hola, ¿qué tal estás?" assert decision.reason == "continuation" + async def test_trailing_crumb_on_finished_turn_is_ignored(self) -> None: + """Scribe ghost after a finished commit must not CONTINUE-cancel the LLM. + + Live: commit "…killer." → LLM START → "No." at +55ms → CONTINUE merge + restarted the turn on a frankenstein prompt. + """ + det = HeuristicTurnDetector() + decision = await det.on_committed( + "No.", + _state( + assistant_active=True, + active_user_text="Yeah, that guy was a real killer.", + seconds_since_turn_start=0.1, + seconds_since_last_commit=0.055, + ), + ) + assert decision.action is CommitAction.IGNORE + assert decision.reason == "trailing_crumb" + + async def test_late_short_barge_in_on_finished_turn_is_new_turn(self) -> None: + """Past the crumb window, a short fresh interrupt is a real barge-in.""" + det = HeuristicTurnDetector() + decision = await det.on_committed( + "Stop.", + _state( + assistant_active=True, + active_user_text="Yeah, that guy was a real killer.", + seconds_since_turn_start=1.5, + seconds_since_last_commit=1.0, + ), + ) + assert decision.action is CommitAction.NEW_TURN + assert decision.text == "Stop." + + async def test_question_split_after_finished_question_still_continues(self) -> None: + """VAD can punctuate mid-question; don't drop the second half as a crumb.""" + det = HeuristicTurnDetector() + decision = await det.on_committed( + "estás?", + _state( + assistant_active=True, + active_user_text="Hola, ¿qué tal?", + seconds_since_turn_start=1.0, + seconds_since_last_commit=0.2, + ), + ) + assert decision.action is CommitAction.CONTINUE_TURN + assert decision.text == "Hola, ¿qué tal? estás?" + + async def test_lowercase_glue_still_continues_unfinished_active(self) -> None: + """Single-word lowercase trailers stay CONTINUE when active is unfinished.""" + det = HeuristicTurnDetector() + decision = await det.on_committed( + "weather?", + _state( + assistant_active=True, + active_user_text="Tell me about the", + seconds_since_turn_start=1.0, + seconds_since_last_commit=0.2, + ), + ) + assert decision.action is CommitAction.CONTINUE_TURN + assert decision.reason == "continuation" + async def test_long_new_query_during_turn_is_new_turn(self) -> None: det = HeuristicTurnDetector() decision = await det.on_committed( @@ -391,6 +455,27 @@ async def test_incomplete_audio_holds(self) -> None: assert model.calls == 1 await det.close() + async def test_late_barge_in_on_finished_turn_not_held(self) -> None: + """Parent NEW_TURN for finished+fresh must survive incomplete Smart Turn. + + Otherwise session HOLDs during TTS and the barge-in never interrupts. + """ + det = LocalAudioTurnDetector(audio_eou=_FixedAudioEou(0.1)) + await det.start(type("C", (), {"sample_rate": 16000})()) + det.push_audio(b"\x00\x01" * 8000) + decision = await det.on_committed( + "Stop.", + _state( + assistant_active=True, + active_user_text="Yeah, that guy was a real killer.", + seconds_since_turn_start=1.5, + seconds_since_last_commit=1.0, + ), + ) + assert decision.action is CommitAction.NEW_TURN + assert decision.text == "Stop." + await det.close() + async def test_incomplete_audio_complete_text_short_hold(self) -> None: """Confidence tier: Smart Turn under-scores short closers ("Thank you." p=0.036 live) — terminal punctuation disagrees, so the hold shrinks to diff --git a/python/timbal/voice/session.py b/python/timbal/voice/session.py index 861f42cb..bbc4a191 100644 --- a/python/timbal/voice/session.py +++ b/python/timbal/voice/session.py @@ -927,10 +927,14 @@ async def _process_stt_events(self) -> None: if text: self._last_partial_at = time.monotonic() self._latest_partial_text = text - await self._emit(TranscriptPartial(text=text)) decision = await self.turn_detector.on_partial(text, self._turn_state()) if decision is PartialDecision.BARGE_IN and self._vad_vetoes_barge_in(text): + # Hallucinated multi-word partials during TTS (no mic + # energy) — do not flash them in the playground caption. + # (_vad_vetoes_barge_in already logs stt_partial_barge_in_vetoed.) decision = PartialDecision.IGNORE + else: + await self._emit(TranscriptPartial(text=text)) if decision is PartialDecision.BARGE_IN: # INFO on purpose: a barge-in cancels TTS and truncates the # committed reply — when debugging "the agent went silent / diff --git a/python/timbal/voice/turn_detection.py b/python/timbal/voice/turn_detection.py index dad3f4de..263fa3ab 100644 --- a/python/timbal/voice/turn_detection.py +++ b/python/timbal/voice/turn_detection.py @@ -282,6 +282,24 @@ def _looks_like_fresh_hold_utterance(text: str) -> bool: return True +_TERMINAL_END_CHARS = frozenset(".?!。？！") + + +def _looks_like_finished_utterance(text: str) -> bool: + """True when ``text`` ends with terminal punctuation (not an ellipsis trail-off). + + Used to refuse CONTINUE-merge of short STT crumbs onto an already-complete + turn ("…killer." + "No." → cancel+restart). Ellipsis / bare mid-thought + fragments stay "unfinished" so VAD splits can still CONTINUE. + """ + stripped = text.strip() + if not stripped: + return False + if stripped.endswith("...") or stripped.endswith("…"): + return False + return stripped[-1] in _TERMINAL_END_CHARS + + class HeuristicTurnDetector(TurnDetector): """Default detector: the regex/similarity heuristics tuned for ElevenLabs Scribe. @@ -304,6 +322,12 @@ class HeuristicTurnDetector(TurnDetector): EARLY_DUPLICATE_RATIO = 0.58 CONTINUATION_WINDOW_SECS = 3.0 CONTINUATION_MAX_CHARS = 30 + # After a finished commit has already started a turn, Scribe often emits a + # trailing crumb ("No.", "He", "Okay.") within tens of ms. CONTINUE-merging + # those cancels the LLM for a frankenstein — IGNORE ultra-early crumbs; + # later short barge-ins ("Stop.") still fall through to NEW_TURN. + TRAILING_CRUMB_WINDOW_SECS = 0.4 + TRAILING_CRUMB_MAX_WORDS = 2 async def on_partial(self, text: str, state: TurnState) -> PartialDecision: if not state.audio_playing or not text: @@ -398,8 +422,40 @@ async def on_committed(self, text: str, state: TurnState) -> CommitDecision: state.seconds_since_last_commit < self.CONTINUATION_WINDOW_SECS and len(text) < self.CONTINUATION_MAX_CHARS ): - combined = state.active_user_text.rstrip(", ") + " " + text - return CommitDecision(action=CommitAction.CONTINUE_TURN, text=combined, reason="continuation") + # Active already looks finished + trailer looks like its own + # utterance → not a VAD mid-phrase split. Ultra-early crumbs + # are STT ghosts (live: "…killer." then "No." at +55ms); + # anything later is a real short barge-in → NEW_TURN below. + if _looks_like_finished_utterance(state.active_user_text) and _looks_like_fresh_hold_utterance( + text + ): + words = _WORD_RE.findall(text) + if ( + state.seconds_since_last_commit < self.TRAILING_CRUMB_WINDOW_SECS + and len(words) <= self.TRAILING_CRUMB_MAX_WORDS + ): + # Question-final trailers after a finished active are usually + # VAD splits ("¿qué tal?" + "estás?"), not Scribe ghosts + # ("…killer." + "No."). Ghost crumbs are statement/ack-shaped. + if text.rstrip().endswith(("?", "？")): + combined = state.active_user_text.rstrip(", ") + " " + text + return CommitDecision( + action=CommitAction.CONTINUE_TURN, + text=combined, + reason="continuation", + ) + return CommitDecision( + action=CommitAction.IGNORE, + text=text, + reason="trailing_crumb", + ) + else: + combined = state.active_user_text.rstrip(", ") + " " + text + return CommitDecision( + action=CommitAction.CONTINUE_TURN, + text=combined, + reason="continuation", + ) return CommitDecision(action=CommitAction.NEW_TURN, text=text, reason="new_turn") @@ -745,6 +801,17 @@ async def on_committed(self, text: str, state: TurnState) -> CommitDecision: return decision if decision.action is CommitAction.CONTINUE_TURN: return decision + # Mid-reply barge-in the parent already classified as NEW_TURN (finished + # active + fresh trailer past the crumb window). Do not rescore into HOLD + # — session defers HOLD during TTS, which would mute the interrupt. + if ( + decision.action is CommitAction.NEW_TURN + and state.assistant_active + and state.active_user_text + and _looks_like_finished_utterance(state.active_user_text) + and _looks_like_fresh_hold_utterance(text) + ): + return decision # Parent hold_merge of a self-contained commit → start the turn now # instead of re-holding, but keep the held fragment in the turn text # (never drop transcribed user speech; see parent hold_supersede). @@ -824,8 +891,16 @@ async def on_committed(self, text: str, state: TurnState) -> CommitDecision: p_text = None if p_text is not None and p_text < self.TEXT_INCOMPLETE_TIER_THRESHOLD: # Mid-agent barge-in that looks incomplete still merges via - # CONTINUE rather than parking a HOLD over the reply. - if state.active_user_text and state.assistant_active: + # CONTINUE rather than parking a HOLD over the reply — but not + # onto a finished active + fresh trailer (STT crumb / barge-in). + if ( + state.active_user_text + and state.assistant_active + and not ( + _looks_like_finished_utterance(state.active_user_text) + and _looks_like_fresh_hold_utterance(text) + ) + ): combined = state.active_user_text.rstrip(", ") + " " + text return CommitDecision( action=CommitAction.CONTINUE_TURN, @@ -846,7 +921,16 @@ async def on_committed(self, text: str, state: TurnState) -> CommitDecision: reason="audio_complete" if state.holding else (decision.reason or "new_turn"), ) # Incomplete: mid-agent-turn → merge+restart; else HOLD (session debounce). - if state.active_user_text and state.assistant_active: + # Skip merge when the active turn already looks finished and the trailer + # is a fresh utterance — Heuristic's trailing_crumb / NEW_TURN path. + if ( + state.active_user_text + and state.assistant_active + and not ( + _looks_like_finished_utterance(state.active_user_text) + and _looks_like_fresh_hold_utterance(text) + ) + ): combined = state.active_user_text.rstrip(", ") + " " + text return CommitDecision( action=CommitAction.CONTINUE_TURN, From 342370bffe8f5cd016677253fb7c7e5c5db7be55 Mon Sep 17 00:00:00 2001 From: berges99 Date: Fri, 24 Jul 2026 17:09:53 +0200 Subject: [PATCH 07/15] feat(voice): Deepgram STT, playground model/STT controls, fix long-reply barge-in MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add Deepgram Flux/Nova realtime STT alongside ElevenLabs Scribe, wire provider resolution + Flux→provider turn-detector pairing, and expand the /voice playground (STT picker, LLM catalog, Foundry-style layout, tool status, cold-start warmup). Drop queued AudioOutput on interrupt so long TTS replies cut immediately instead of draining the WS backlog. Also coerce nullish tool args in serialization and add a slow get_datetime demo tool for tool-call dead-air testing. --- examples/voice_turn_modes.py | 22 +- python/tests/server/test_voice_config.py | 3 + python/tests/server/voice_env.py | 1 + python/tests/utils/test_serialization.py | 14 + python/tests/voice/test_deepgram.py | 278 +++++++++++++ python/tests/voice/test_session.py | 89 +++++ python/timbal/server/README.md | 3 +- python/timbal/server/voice.html | 473 +++++++++++++++++------ python/timbal/server/voice.py | 128 ++++-- python/timbal/utils/serialization.py | 25 +- python/timbal/voice/__init__.py | 14 + python/timbal/voice/deepgram.py | 465 ++++++++++++++++++++++ python/timbal/voice/session.py | 116 +++++- 13 files changed, 1475 insertions(+), 156 deletions(-) create mode 100644 python/tests/voice/test_deepgram.py create mode 100644 python/timbal/voice/deepgram.py diff --git a/examples/voice_turn_modes.py b/examples/voice_turn_modes.py index d17c10cd..7f1bcec8 100644 --- a/examples/voice_turn_modes.py +++ b/examples/voice_turn_modes.py @@ -18,21 +18,41 @@ from __future__ import annotations +import asyncio import os +import random +from datetime import datetime, timezone from timbal import Agent +from timbal.core.tool import Tool from timbal.voice import resolve_turn_detector _MODE = os.environ.get("TIMBAL_VOICE_TURN_DETECTOR", "heuristic").strip().lower() + +async def get_datetime() -> str: + """Get the current UTC date and time (slow on purpose for voice tool-call testing).""" + await asyncio.sleep(random.uniform(3.0, 5.0)) + return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S %Z") + + agent = Agent( name="voice_turn_modes", model=os.environ.get("TIMBAL_VOICE_DEMO_MODEL", "groq/llama-3.1-8b-instant"), + max_tokens=1024, # required when switching to Anthropic from the playground system_prompt=( "You are a concise voice assistant. Keep replies to 1–2 short sentences. " + "When the user asks for the date and/or time, you MUST call get_datetime " + "and then speak the returned value clearly (include both date and time). " + "Never claim you cannot tell the time. " f"(turn_detector mode: {_MODE})" ), - tools=[], + tools=[ + Tool( + handler=get_datetime, + description="Return the current UTC date and time. Always use this for date/time questions.", + ) + ], ) # Server reads this on startup (instance or mode name). Client WS JSON cannot override it. diff --git a/python/tests/server/test_voice_config.py b/python/tests/server/test_voice_config.py index b9668ee4..db71e3ed 100644 --- a/python/tests/server/test_voice_config.py +++ b/python/tests/server/test_voice_config.py @@ -24,6 +24,7 @@ class TestDefaultVoiceConfigFromEnv: def test_defaults_when_unset(self) -> None: cfg = voice_routes.default_voice_config_from_env() + assert cfg["stt_provider"] == "elevenlabs" assert cfg["stt_model"] == "scribe_v2_realtime" assert cfg["tts_model"] == "eleven_flash_v2_5" assert cfg["voice"] == voice_routes._DEFAULT_VOICE_ID @@ -33,10 +34,12 @@ def test_defaults_when_unset(self) -> None: assert cfg["tts_extra"]["auto_mode"] is True def test_env_overrides(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("TIMBAL_STT_PROVIDER", "deepgram") monkeypatch.setenv("TIMBAL_STT_MODEL", "custom_stt") monkeypatch.setenv("TIMBAL_TTS_MODEL", "custom_tts") monkeypatch.setenv("TIMBAL_VOICE_LANGUAGE", "en") cfg = voice_routes.default_voice_config_from_env() + assert cfg["stt_provider"] == "deepgram" assert cfg["stt_model"] == "custom_stt" assert cfg["tts_model"] == "custom_tts" assert cfg["language"] == "en" diff --git a/python/tests/server/voice_env.py b/python/tests/server/voice_env.py index 22d030ac..fce58c97 100644 --- a/python/tests/server/voice_env.py +++ b/python/tests/server/voice_env.py @@ -4,6 +4,7 @@ # Env vars that affect ``default_voice_config_from_env`` / ``merge_voice_config``. VOICE_ENV_KEYS = ( + "TIMBAL_STT_PROVIDER", "TIMBAL_STT_MODEL", "TIMBAL_TTS_MODEL", "ELEVENLABS_VOICE_ID", diff --git a/python/tests/utils/test_serialization.py b/python/tests/utils/test_serialization.py index c5df9e2f..454952aa 100644 --- a/python/tests/utils/test_serialization.py +++ b/python/tests/utils/test_serialization.py @@ -169,6 +169,16 @@ def test_empty_string_returns_empty_dict(self): assert coerce_to_dict("") == {} assert coerce_to_dict(" ") == {} + def test_nullish_no_arg_tool_inputs(self): + # Groq / OpenAI-compat often emit null for tools with zero parameters. + assert coerce_to_dict(None) == {} + assert coerce_to_dict("null") == {} + assert coerce_to_dict("None") == {} + assert coerce_to_dict("NULL") == {} + + def test_json_null_literal_returns_empty_dict(self): + assert coerce_to_dict("null") == {} + def test_literal_eval_fallback(self): # Python dict literal that isn't valid JSON (single quotes) assert coerce_to_dict("{'a': 1}") == {"a": 1} @@ -177,6 +187,10 @@ def test_unparseable_string_raises(self): with pytest.raises(ValueError, match="Cannot coerce value to dict"): coerce_to_dict("not a dict at all!") + def test_non_dict_json_raises(self): + with pytest.raises(ValueError, match="Cannot coerce value to dict"): + coerce_to_dict("[1, 2, 3]") + def test_non_string_non_dict_raises(self): with pytest.raises(ValueError, match="Cannot coerce value to dict"): coerce_to_dict(42) diff --git a/python/tests/voice/test_deepgram.py b/python/tests/voice/test_deepgram.py new file mode 100644 index 00000000..77d61ca9 --- /dev/null +++ b/python/tests/voice/test_deepgram.py @@ -0,0 +1,278 @@ +"""Deepgram STT providers: message → TranscriptEvent mapping, URI building, resolve_stt. + +No live API calls — messages are fed straight into ``_handle_message`` and the +event queue is drained, mirroring how the receive loop feeds ``events()``. +""" + +from __future__ import annotations + +import json +from urllib.parse import parse_qs, urlparse + +import pytest +from timbal.voice.deepgram import ( + DEFAULT_FLUX_MODEL, + DEFAULT_NOVA_MODEL, + DeepgramFluxSTT, + DeepgramNovaSTT, + effective_stt_model, + is_flux_model, + resolve_stt, +) +from timbal.voice.elevenlabs import ElevenLabsRealtimeSTT +from timbal.voice.session import AudioInputConfig + + +def _drain(stt) -> list: + events = [] + while not stt._queue.empty(): + events.append(stt._queue.get_nowait()) + return events + + +def _flux_turn_info(event: str, transcript: str = "hello there", **kw) -> dict: + return { + "type": "TurnInfo", + "event": event, + "transcript": transcript, + "turn_index": 0, + "end_of_turn_confidence": 0.9, + **kw, + } + + +class _FakeWs: + def __init__(self) -> None: + self.sent: list = [] + + async def send(self, payload) -> None: + self.sent.append(payload) + + +class TestFluxEventMapping: + async def test_update_and_start_of_turn_are_partials(self) -> None: + stt = DeepgramFluxSTT() + await stt._handle_message(_flux_turn_info("StartOfTurn", "hi")) + await stt._handle_message(_flux_turn_info("Update", "hi there")) + events = _drain(stt) + assert [(e.type, e.text) for e in events] == [("partial", "hi"), ("partial", "hi there")] + + async def test_end_of_turn_is_committed(self) -> None: + stt = DeepgramFluxSTT() + await stt._handle_message(_flux_turn_info("EndOfTurn", "tell me a story.")) + events = _drain(stt) + assert [(e.type, e.text) for e in events] == [("committed", "tell me a story.")] + + async def test_eager_events_emit_nothing(self) -> None: + stt = DeepgramFluxSTT() + await stt._handle_message(_flux_turn_info("EagerEndOfTurn")) + await stt._handle_message(_flux_turn_info("TurnResumed")) + assert _drain(stt) == [] + + async def test_empty_transcripts_skipped(self) -> None: + stt = DeepgramFluxSTT() + await stt._handle_message(_flux_turn_info("Update", "")) + await stt._handle_message(_flux_turn_info("EndOfTurn", " ")) + assert _drain(stt) == [] + + async def test_connected_ignored_fatal_error_surfaces(self) -> None: + stt = DeepgramFluxSTT() + await stt._handle_message({"type": "Connected", "request_id": "r1", "sequence_id": 0}) + await stt._handle_message({"type": "FatalError", "description": "bad auth", "code": "AUTH"}) + events = _drain(stt) + assert len(events) == 1 + assert events[0].type == "error" + assert "bad auth" in events[0].text + + async def test_commit_is_noop(self) -> None: + stt = DeepgramFluxSTT() + stt._ws = _FakeWs() + await stt.commit() + assert stt._ws.sent == [] + + +class TestFluxUri: + def _parse(self, stt: DeepgramFluxSTT, config: AudioInputConfig) -> dict: + uri = stt._build_uri(config) + parsed = urlparse(uri) + assert parsed.scheme == "wss" + assert parsed.path == "/v2/listen" + return parse_qs(parsed.query) + + def test_defaults(self) -> None: + q = self._parse(DeepgramFluxSTT(), AudioInputConfig()) + assert q["model"] == [DEFAULT_FLUX_MODEL] + assert q["encoding"] == ["linear16"] + assert q["sample_rate"] == ["16000"] + assert q["eot_timeout_ms"] == ["2000"] + assert "language" not in q + + def test_language_hint_only_for_multi(self) -> None: + q = self._parse(DeepgramFluxSTT(), AudioInputConfig(language="es")) + assert q["language_hint"] == ["es"] + q = self._parse(DeepgramFluxSTT(), AudioInputConfig(model="flux-general-en", language="es")) + assert "language_hint" not in q + + def test_foreign_model_replaced(self) -> None: + # Only TIMBAL_STT_PROVIDER switched; env stt_model still Scribe's. + q = self._parse(DeepgramFluxSTT(), AudioInputConfig(model="scribe_v2_realtime")) + assert q["model"] == [DEFAULT_FLUX_MODEL] + + def test_extra_passthrough_filtered(self) -> None: + config = AudioInputConfig( + extra={ + "eot_threshold": 0.8, + "eot_timeout_ms": 3000, + "keyterm": ["Timbal", "Scribe"], + # Scribe-only knob must not leak into the Flux query. + "commit_strategy": "vad", + } + ) + q = self._parse(DeepgramFluxSTT(), config) + assert q["eot_threshold"] == ["0.8"] + assert q["eot_timeout_ms"] == ["3000"] + assert q["keyterm"] == ["Timbal", "Scribe"] + assert "commit_strategy" not in q + + +class TestNovaEventMapping: + def _results(self, text: str, *, is_final: bool, speech_final: bool = False, **kw) -> dict: + return { + "type": "Results", + "is_final": is_final, + "speech_final": speech_final, + "channel": {"alternatives": [{"transcript": text}]}, + **kw, + } + + async def test_interim_is_partial(self) -> None: + stt = DeepgramNovaSTT() + await stt._handle_message(self._results("hello", is_final=False)) + events = _drain(stt) + assert [(e.type, e.text) for e in events] == [("partial", "hello")] + + async def test_is_final_buffers_until_speech_final(self) -> None: + """Deepgram's documented recipe: concat is_final segments, flush on speech_final.""" + stt = DeepgramNovaSTT() + await stt._handle_message(self._results("yeah so my credit card number is two two", is_final=True)) + assert _drain(stt) == [] + await stt._handle_message(self._results("two two three three three three", is_final=True, speech_final=True)) + events = _drain(stt) + assert [(e.type, e.text) for e in events] == [ + ("committed", "yeah so my credit card number is two two two two three three three three") + ] + assert stt._segments == [] + + async def test_partial_includes_buffered_segments(self) -> None: + stt = DeepgramNovaSTT() + await stt._handle_message(self._results("hello there", is_final=True)) + await stt._handle_message(self._results("how are", is_final=False)) + events = _drain(stt) + assert [(e.type, e.text) for e in events] == [("partial", "hello there how are")] + + async def test_from_finalize_flushes(self) -> None: + """commit() → Finalize → is_final response with from_finalize must commit.""" + stt = DeepgramNovaSTT() + await stt._handle_message(self._results("stop the music", is_final=True, from_finalize=True)) + events = _drain(stt) + assert [(e.type, e.text) for e in events] == [("committed", "stop the music")] + + async def test_utterance_end_flushes_buffer(self) -> None: + stt = DeepgramNovaSTT() + await stt._handle_message(self._results("hello there", is_final=True)) + await stt._handle_message({"type": "UtteranceEnd", "last_word_end": 1.2}) + events = _drain(stt) + assert [(e.type, e.text) for e in events] == [("committed", "hello there")] + + async def test_empty_speech_final_no_commit(self) -> None: + stt = DeepgramNovaSTT() + await stt._handle_message(self._results("", is_final=True, speech_final=True)) + assert _drain(stt) == [] + + async def test_error_surfaces(self) -> None: + stt = DeepgramNovaSTT() + await stt._handle_message({"type": "Error", "description": "NET-0001"}) + events = _drain(stt) + assert events[0].type == "error" + assert "NET-0001" in events[0].text + + async def test_commit_sends_finalize(self) -> None: + stt = DeepgramNovaSTT() + ws = _FakeWs() + stt._ws = ws + await stt.commit() + assert [json.loads(m) for m in ws.sent if isinstance(m, str)] == [{"type": "Finalize"}] + + +class TestNovaUri: + def _parse(self, config: AudioInputConfig) -> dict: + uri = DeepgramNovaSTT()._build_uri(config) + parsed = urlparse(uri) + assert parsed.path == "/v1/listen" + return parse_qs(parsed.query) + + def test_defaults(self) -> None: + q = self._parse(AudioInputConfig()) + assert q["model"] == [DEFAULT_NOVA_MODEL] + assert q["encoding"] == ["linear16"] + assert q["interim_results"] == ["true"] + assert q["smart_format"] == ["true"] + assert q["punctuate"] == ["true"] + assert q["endpointing"] == ["300"] + assert q["channels"] == ["1"] + + def test_language_and_extra_override(self) -> None: + q = self._parse(AudioInputConfig(language="es", extra={"endpointing": 500, "utterance_end_ms": 1000})) + assert q["language"] == ["es"] + assert q["endpointing"] == ["500"] + assert q["utterance_end_ms"] == ["1000"] + + def test_foreign_model_replaced(self) -> None: + q = self._parse(AudioInputConfig(model="scribe_v2_realtime")) + assert q["model"] == [DEFAULT_NOVA_MODEL] + q = self._parse(AudioInputConfig(model="flux-general-en")) + assert q["model"] == [DEFAULT_NOVA_MODEL] + + def test_nova_variant_kept(self) -> None: + q = self._parse(AudioInputConfig(model="nova-3-medical")) + assert q["model"] == ["nova-3-medical"] + + +class TestResolveStt: + def test_explicit_providers(self) -> None: + assert isinstance(resolve_stt("elevenlabs"), ElevenLabsRealtimeSTT) + assert isinstance(resolve_stt("deepgram"), DeepgramFluxSTT) + assert isinstance(resolve_stt("deepgram", model="nova-3"), DeepgramNovaSTT) + assert isinstance(resolve_stt("deepgram", model="flux-general-en"), DeepgramFluxSTT) + # Leftover Scribe model + bare deepgram must NOT silently become Nova. + assert isinstance(resolve_stt("deepgram", model="scribe_v2_realtime"), DeepgramFluxSTT) + + def test_ui_labels(self) -> None: + assert isinstance(resolve_stt("deepgram-flux"), DeepgramFluxSTT) + assert isinstance(resolve_stt("deepgram-nova"), DeepgramNovaSTT) + # Label wins over a stale model id from env defaults. + assert isinstance(resolve_stt("deepgram-flux", model="scribe_v2_realtime"), DeepgramFluxSTT) + assert isinstance(resolve_stt("deepgram-nova", model="flux-general-multi"), DeepgramNovaSTT) + + def test_inference_from_model(self) -> None: + assert isinstance(resolve_stt(None, model="flux-general-multi"), DeepgramFluxSTT) + assert isinstance(resolve_stt(None, model="nova-3"), DeepgramNovaSTT) + assert isinstance(resolve_stt(None, model="scribe_v2_realtime"), ElevenLabsRealtimeSTT) + assert isinstance(resolve_stt(None, model=None), ElevenLabsRealtimeSTT) + + def test_effective_stt_model(self) -> None: + assert effective_stt_model(DeepgramFluxSTT(), "scribe_v2_realtime") == DEFAULT_FLUX_MODEL + assert effective_stt_model(DeepgramFluxSTT(), "flux-general-en") == "flux-general-en" + assert effective_stt_model(DeepgramNovaSTT(), "scribe_v2_realtime") == DEFAULT_NOVA_MODEL + assert effective_stt_model(DeepgramNovaSTT(), "nova-3-general") == "nova-3-general" + + def test_unknown_provider_raises(self) -> None: + with pytest.raises(ValueError, match="Unknown STT provider"): + resolve_stt("whisper-cpp") + + def test_is_flux_model(self) -> None: + assert is_flux_model("flux-general-multi") + assert is_flux_model("FLUX-GENERAL-EN") + assert not is_flux_model("nova-3") + assert not is_flux_model(None) + assert not is_flux_model("") diff --git a/python/tests/voice/test_session.py b/python/tests/voice/test_session.py index e11fe254..acd5ffa1 100644 --- a/python/tests/voice/test_session.py +++ b/python/tests/voice/test_session.py @@ -899,6 +899,26 @@ async def test_completed_turn_barge_in_rewrites_committed_entry(self) -> None: # Second turn's reply is intact. assert assistant_entries[1].text == "Second reply" + async def test_interrupt_drops_queued_audio_before_interrupted_event(self) -> None: + """TTS can enqueue far more AudioOutput than the WS drains. Barge-in must + not wait behind that backlog or long replies keep playing after interrupt.""" + agent = Agent(name="t", model=TestModel(responses=["ok"]), tools=[]) + session = VoiceSession(agent=agent, stt=MockSTT(), tts=MockTTS()) + await session._emit(AudioOutput(data=b"\x00" * 64)) + await session._emit(AudioOutput(data=b"\x01" * 64)) + await session._emit(AgentTextDone(text="already spoken")) + await session._emit(AudioOutput(data=b"\x02" * 64)) + dropped = session._drop_queued_audio_output() + assert dropped == 3 + remaining: list[VoiceSessionEvent] = [] + while True: + try: + remaining.append(session._event_queue.get_nowait()) + except asyncio.QueueEmpty: + break + assert len(remaining) == 1 + assert isinstance(remaining[0], AgentTextDone) + async def test_completed_turn_barge_in_resaves_serialized_trace(self) -> None: """Regression: interrupt() after AgentTextDone must re-persist the truncated memory. Serializing providers already saved the full reply in @@ -1785,6 +1805,39 @@ async def close(self) -> None: f"hold expiry took {elapsed:.2f}s — pre-commit VAD speech stretched the grace" ) + async def test_dangling_token_hold_is_dropped_on_expiry(self) -> None: + """Live bug: stale-force-committed \"I\" HOLDs for 3s then becomes a + user turn and the LLM invents a reply. Dangling tokens must die here.""" + agent = Agent(name="t", model=TestModel(responses=["should not run"]), tools=[]) + stt = DelayedMockSTT() + tts = MockTTS() + session = VoiceSession( + agent=agent, stt=stt, tts=tts, turn_detector=_AlwaysHoldOnceDetector(0.15) + ) + events: list[VoiceSessionEvent] = [] + + async def _empty_audio() -> AsyncIterator[bytes]: + return + yield # noqa: RET504 + + async def _drive() -> None: + while not any(isinstance(e, SessionStarted) for e in events): + await asyncio.sleep(0.01) + await stt.inject(TranscriptEvent(type="committed", text="I")) + await asyncio.sleep(0.5) + await stt.finish() + + async def _run() -> None: + async with aclosing(session.run(_empty_audio())) as stream: + driver = asyncio.create_task(_drive()) + async for ev in stream: + events.append(ev) + await driver + + await asyncio.wait_for(_run(), timeout=10) + assert all(entry.role != "user" for entry in session.transcript) + assert not any(isinstance(e, AgentTextDone) for e in events) + # --------------------------------------------------------------------------- # Tests: stale partial sweeper @@ -1881,3 +1934,39 @@ async def _run() -> None: assert stt.commit_calls == 1 assert all(entry.role != "user" for entry in session.transcript) + + async def test_noop_commit_provider_synthesizes_stranded_partial(self) -> None: + """Flux-style STT: commit() is a no-op, so the sweeper must promote the + stranded partial itself or the UI caption hangs forever.""" + agent = Agent(name="t", model=TestModel(responses=["ok"]), tools=[]) + stt = DelayedMockSTT() + tts = MockTTS() + session = VoiceSession(agent=agent, stt=stt, tts=tts) + session._stale_partial_poll_secs = 0.05 + session._stale_partial_commit_secs = 0.15 + + events: list[VoiceSessionEvent] = [] + + async def _empty_audio() -> AsyncIterator[bytes]: + return + yield # noqa: RET504 + + async def _drive() -> None: + while not any(isinstance(e, SessionStarted) for e in events): + await asyncio.sleep(0.01) + await stt.inject(TranscriptEvent(type="partial", text="What are you doing?")) + while not any(isinstance(e, AgentTextDone) for e in events): + await asyncio.sleep(0.01) + await stt.finish() + + async def _run() -> None: + async with aclosing(session.run(_empty_audio())) as stream: + driver = asyncio.create_task(_drive()) + async for ev in stream: + events.append(ev) + await driver + + await asyncio.wait_for(_run(), timeout=10) + + assert session.transcript[0].role == "user" + assert session.transcript[0].text == "What are you doing?" diff --git a/python/timbal/server/README.md b/python/timbal/server/README.md index e2b7920c..8c8bd83e 100644 --- a/python/timbal/server/README.md +++ b/python/timbal/server/README.md @@ -60,7 +60,8 @@ Only send keys you need; omitted keys keep server defaults. | Key | Description | |---------------|-------------| -| `stt_model` | Speech-to-text model id (ElevenLabs realtime). | +| `stt_provider` | `"elevenlabs"` (default), `"deepgram-flux"`, or `"deepgram-nova"` (bare `"deepgram"` routes by `stt_model`, defaulting to Flux). Deepgram needs `DEEPGRAM_API_KEY` on the server. Flux (`/v2/listen`) does model-native end-of-turn detection: the session auto-selects the `provider` turn detector (explicit `turn_detector` still wins) and disables local VAD endpointing. Nova-3 (`/v1/listen`) is plain ASR — Timbal turn detection and VAD endpointing work exactly as with ElevenLabs. Env default: `TIMBAL_STT_PROVIDER`. | +| `stt_model` | Speech-to-text model id (ElevenLabs realtime `scribe_*`, Deepgram `flux-general-en`/`flux-general-multi`/`nova-3*`). Model ids that don't belong to the selected provider are ignored (provider default used). | | `tts_model` | Text-to-speech model id. | | `voice` | ElevenLabs voice id string. | | `language` | e.g. `"es"`. | diff --git a/python/timbal/server/voice.html b/python/timbal/server/voice.html index 2558613f..9f0195a3 100644 --- a/python/timbal/server/voice.html +++ b/python/timbal/server/voice.html @@ -38,17 +38,96 @@ -webkit-font-smoothing: antialiased; } .shell { - max-width: 480px; + max-width: 1160px; margin: 0 auto; height: 100%; max-height: 100dvh; min-height: 0; - padding: 1.5rem 1rem 2rem; + padding: 1.5rem 1rem 1.5rem; display: flex; flex-direction: column; gap: 1rem; overflow: hidden; } + .layout { + flex: 1; + display: flex; + gap: 1rem; + min-height: 0; + } + .panel { + flex: 0 0 250px; + background: var(--surface); + border: 1px solid var(--border); + border-radius: var(--radius); + box-shadow: var(--shadow); + padding: 1rem; + display: flex; + flex-direction: column; + gap: 0.85rem; + min-height: 0; + overflow-y: auto; + } + .panel-title { + font-size: 0.6875rem; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.06em; + color: var(--text-muted); + } + .field { + display: flex; + flex-direction: column; + gap: 0.3rem; + } + .field-label { + font-size: 0.6875rem; + font-weight: 600; + color: var(--text-muted); + } + .panel select { + width: 100%; + min-width: 0; + font: inherit; + font-size: 0.8125rem; + color: var(--text); + background: var(--surface); + border: 1px solid var(--border-strong); + border-radius: var(--radius-sm); + padding: 0.5rem 0.65rem; + outline: none; + cursor: pointer; + } + .panel select:focus { + border-color: var(--primary); + box-shadow: 0 0 0 3px rgba(0,0,0,.07); + } + .model-picker { + display: flex; + flex-direction: column; + gap: 0.55rem; + } + .model-count { + font-size: 0.65625rem; + color: var(--text-muted); + margin-top: -0.15rem; + } + .panel-note { + font-size: 0.6875rem; + color: var(--text-muted); + line-height: 1.45; + margin-top: auto; + padding-top: 0.65rem; + border-top: 1px solid var(--border); + } + @media (max-width: 900px) { + body { overflow: auto; } + .shell { max-height: none; height: auto; } + .layout { flex-direction: column; overflow: visible; } + .panel { flex: none; width: auto; overflow: visible; } + .card { order: -1; min-height: 60dvh; } + .panel-note { margin-top: 0; } + } .top { display: flex; align-items: center; @@ -100,9 +179,8 @@ flex-direction: column; align-items: flex-start; gap: 0.35rem; - padding: 0.75rem 1rem; + padding-bottom: 0.75rem; border-bottom: 1px solid var(--border); - background: linear-gradient(180deg, #fafafa 0%, var(--surface) 100%); } .conn-version { font-size: 0.75rem; @@ -164,31 +242,20 @@ 50% { opacity: 0.5; transform: scale(0.85); } } - #mic-select, #td-select, #ns-select { - flex: 1; - min-width: 0; - font: inherit; - font-size: 0.8125rem; - color: var(--text); - background: var(--surface); - border: 1px solid var(--border-strong); - border-radius: var(--radius-sm); - padding: 0.5rem 0.65rem; - outline: none; - cursor: pointer; - } - #mic-select:focus, #td-select:focus, #ns-select:focus { - border-color: var(--primary); - box-shadow: 0 0 0 3px rgba(0,0,0,.07); + .mic-row { + display: flex; + align-items: flex-end; + gap: 0.5rem; } + .mic-row .field { flex: 1; min-width: 0; } .mic-viz { display: flex; align-items: flex-end; justify-content: center; gap: 3px; height: 28px; - flex: 0 0 56px; - padding-bottom: 2px; + flex: 0 0 40px; + padding-bottom: 5px; } .mic-bar { width: 4px; @@ -234,30 +301,6 @@ pointer-events: none; } #voice-empty[hidden] { display: none !important; } - .orb-wrap { - width: 72px; - height: 72px; - margin-bottom: 1rem; - position: relative; - } - .orb { - position: absolute; - inset: 12px; - border-radius: 50%; - background: radial-gradient(circle at 30% 30%, #fafafa, #e4e4e7); - border: 1px solid var(--border); - } - .card.card--live #voice-empty .orb-wrap .ring { - position: absolute; - inset: 0; - border-radius: 50%; - border: 2px solid var(--success-soft); - animation: ring-pulse 2s ease-out infinite; - } - @keyframes ring-pulse { - 0% { transform: scale(0.85); opacity: 0.8; } - 100% { transform: scale(1.35); opacity: 0; } - } .empty-title { font-size: 0.9375rem; font-weight: 600; @@ -268,9 +311,21 @@ font-size: 0.8125rem; color: var(--text-muted); margin-top: 0.35rem; - max-width: 240px; + max-width: 280px; line-height: 1.4; } + .msg.status { + margin: 0.35rem auto 0.65rem; + max-width: 90%; + text-align: center; + color: var(--text-muted); + font-size: 0.75rem; + font-style: italic; + padding: 0.35rem 0.65rem; + border-radius: 999px; + background: rgba(0, 0, 0, 0.03); + border: 1px solid var(--border); + } .msg { margin-bottom: 0.65rem; max-width: 90%; } .msg.user { @@ -353,26 +408,6 @@ color: var(--text-muted); line-height: 1.35; } - .foot-row--mic { - display: flex; - align-items: center; - gap: 0.5rem; - min-width: 0; - } - .foot-row--mic #mic-select { - flex: 1; - min-width: 0; - } - .foot-row--modes { - display: flex; - align-items: center; - gap: 0.5rem; - min-width: 0; - } - .foot-row--modes select { - flex: 1 1 0; - min-width: 0; - } .foot-actions { display: flex; align-items: center; @@ -471,71 +506,111 @@
-
-
- -
+ + +