diff --git a/examples/voice_turn_modes.py b/examples/voice_turn_modes.py index 6b32b0d8..7f1bcec8 100644 --- a/examples/voice_turn_modes.py +++ b/examples/voice_turn_modes.py @@ -18,21 +18,41 @@ from __future__ import annotations +import asyncio import os +import random +from datetime import datetime, timezone from timbal import Agent +from timbal.core.tool import Tool from timbal.voice import resolve_turn_detector _MODE = os.environ.get("TIMBAL_VOICE_TURN_DETECTOR", "heuristic").strip().lower() + +async def get_datetime() -> str: + """Get the current UTC date and time (slow on purpose for voice tool-call testing).""" + await asyncio.sleep(random.uniform(3.0, 5.0)) + return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S %Z") + + agent = Agent( name="voice_turn_modes", - model=os.environ.get("TIMBAL_VOICE_DEMO_MODEL", "openai/gpt-4o-mini"), + model=os.environ.get("TIMBAL_VOICE_DEMO_MODEL", "groq/llama-3.1-8b-instant"), + max_tokens=1024, # required when switching to Anthropic from the playground system_prompt=( "You are a concise voice assistant. Keep replies to 1–2 short sentences. " + "When the user asks for the date and/or time, you MUST call get_datetime " + "and then speak the returned value clearly (include both date and time). " + "Never claim you cannot tell the time. " f"(turn_detector mode: {_MODE})" ), - tools=[], + tools=[ + Tool( + handler=get_datetime, + description="Return the current UTC date and time. Always use this for date/time questions.", + ) + ], ) # Server reads this on startup (instance or mode name). Client WS JSON cannot override it. diff --git a/pyproject.toml b/pyproject.toml index 606c2dec..aacc1e22 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,6 +63,7 @@ server = [ voice = [ "huggingface-hub>=0.28.0", "onnxruntime>=1.20.0", + "transformers>=4.40.0", ] all = [ "timbal[codegen]", diff --git a/python/tests/server/test_voice_config.py b/python/tests/server/test_voice_config.py index b9668ee4..db71e3ed 100644 --- a/python/tests/server/test_voice_config.py +++ b/python/tests/server/test_voice_config.py @@ -24,6 +24,7 @@ class TestDefaultVoiceConfigFromEnv: def test_defaults_when_unset(self) -> None: cfg = voice_routes.default_voice_config_from_env() + assert cfg["stt_provider"] == "elevenlabs" assert cfg["stt_model"] == "scribe_v2_realtime" assert cfg["tts_model"] == "eleven_flash_v2_5" assert cfg["voice"] == voice_routes._DEFAULT_VOICE_ID @@ -33,10 +34,12 @@ def test_defaults_when_unset(self) -> None: assert cfg["tts_extra"]["auto_mode"] is True def test_env_overrides(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("TIMBAL_STT_PROVIDER", "deepgram") monkeypatch.setenv("TIMBAL_STT_MODEL", "custom_stt") monkeypatch.setenv("TIMBAL_TTS_MODEL", "custom_tts") monkeypatch.setenv("TIMBAL_VOICE_LANGUAGE", "en") cfg = voice_routes.default_voice_config_from_env() + assert cfg["stt_provider"] == "deepgram" assert cfg["stt_model"] == "custom_stt" assert cfg["tts_model"] == "custom_tts" assert cfg["language"] == "en" diff --git a/python/tests/server/voice_env.py b/python/tests/server/voice_env.py index 22d030ac..fce58c97 100644 --- a/python/tests/server/voice_env.py +++ b/python/tests/server/voice_env.py @@ -4,6 +4,7 @@ # Env vars that affect ``default_voice_config_from_env`` / ``merge_voice_config``. VOICE_ENV_KEYS = ( + "TIMBAL_STT_PROVIDER", "TIMBAL_STT_MODEL", "TIMBAL_TTS_MODEL", "ELEVENLABS_VOICE_ID", diff --git a/python/tests/utils/test_serialization.py b/python/tests/utils/test_serialization.py index c5df9e2f..454952aa 100644 --- a/python/tests/utils/test_serialization.py +++ b/python/tests/utils/test_serialization.py @@ -169,6 +169,16 @@ def test_empty_string_returns_empty_dict(self): assert coerce_to_dict("") == {} assert coerce_to_dict(" ") == {} + def test_nullish_no_arg_tool_inputs(self): + # Groq / OpenAI-compat often emit null for tools with zero parameters. + assert coerce_to_dict(None) == {} + assert coerce_to_dict("null") == {} + assert coerce_to_dict("None") == {} + assert coerce_to_dict("NULL") == {} + + def test_json_null_literal_returns_empty_dict(self): + assert coerce_to_dict("null") == {} + def test_literal_eval_fallback(self): # Python dict literal that isn't valid JSON (single quotes) assert coerce_to_dict("{'a': 1}") == {"a": 1} @@ -177,6 +187,10 @@ def test_unparseable_string_raises(self): with pytest.raises(ValueError, match="Cannot coerce value to dict"): coerce_to_dict("not a dict at all!") + def test_non_dict_json_raises(self): + with pytest.raises(ValueError, match="Cannot coerce value to dict"): + coerce_to_dict("[1, 2, 3]") + def test_non_string_non_dict_raises(self): with pytest.raises(ValueError, match="Cannot coerce value to dict"): coerce_to_dict(42) diff --git a/python/tests/voice/__init__.py b/python/tests/voice/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/tests/voice/test_deepgram.py b/python/tests/voice/test_deepgram.py new file mode 100644 index 00000000..2409709e --- /dev/null +++ b/python/tests/voice/test_deepgram.py @@ -0,0 +1,347 @@ +"""Deepgram STT providers: message → TranscriptEvent mapping, URI building, resolve_stt. + +No live API calls — messages are fed straight into ``_handle_message`` and the +event queue is drained, mirroring how the receive loop feeds ``events()``. +""" + +from __future__ import annotations + +import asyncio +import json +from urllib.parse import parse_qs, urlparse + +import pytest +from timbal.voice.deepgram import ( + DEFAULT_FLUX_MODEL, + DEFAULT_NOVA_MODEL, + DeepgramFluxSTT, + DeepgramNovaSTT, + _AUDIO_FLUSH_BYTES, + effective_stt_model, + is_flux_model, + resolve_stt, + stt_provider_id, +) +from timbal.voice.elevenlabs import ElevenLabsRealtimeSTT +from timbal.voice.session import AudioInputConfig + + +def _drain(stt) -> list: + events = [] + while not stt._queue.empty(): + events.append(stt._queue.get_nowait()) + return events + + +def _flux_turn_info(event: str, transcript: str = "hello there", **kw) -> dict: + return { + "type": "TurnInfo", + "event": event, + "transcript": transcript, + "turn_index": 0, + "end_of_turn_confidence": 0.9, + **kw, + } + + +class _FakeWs: + def __init__(self) -> None: + self.sent: list = [] + + async def send(self, payload) -> None: + self.sent.append(payload) + + +class _SlowFakeWs: + """WS that yields during send so concurrent callers can race without a lock.""" + + def __init__(self) -> None: + self.sent: list = [] + self.in_flight = 0 + self.max_in_flight = 0 + + async def send(self, payload) -> None: + self.in_flight += 1 + self.max_in_flight = max(self.max_in_flight, self.in_flight) + await asyncio.sleep(0.02) + self.sent.append(payload) + self.in_flight -= 1 + + +class TestWireSendSerialization: + async def test_threshold_push_and_flush_serialize_sends(self) -> None: + """push_audio and _flush_audio must not await ``_ws.send`` concurrently. + + Regression: buffer lock used to release before send, so the flush loop + and a threshold push could interleave PCM frames on one socket. + """ + stt = DeepgramNovaSTT() + ws = _SlowFakeWs() + stt._ws = ws + frame = b"\xab" * _AUDIO_FLUSH_BYTES + + async def _pushes() -> None: + await stt.push_audio(frame) + await stt.push_audio(frame) + + async def _flushes() -> None: + for _ in range(6): + await stt._flush_audio() + await asyncio.sleep(0.005) + + await asyncio.gather(_pushes(), _flushes()) + assert ws.max_in_flight == 1 + assert ws.sent == [frame, frame] + + async def test_flush_drains_remainder_after_threshold_push(self) -> None: + stt = DeepgramFluxSTT() + ws = _FakeWs() + stt._ws = ws + frame = b"\x11" * _AUDIO_FLUSH_BYTES + remainder = b"\x22" * 100 + await stt.push_audio(frame + remainder) + assert ws.sent == [frame + remainder] # one threshold send of all buffered + # Small chunk under threshold sits until flush. + await stt.push_audio(b"\x33" * 50) + assert len(ws.sent) == 1 + await stt._flush_audio() + assert ws.sent[-1] == b"\x33" * 50 + + +class TestFluxEventMapping: + async def test_update_and_start_of_turn_are_partials(self) -> None: + stt = DeepgramFluxSTT() + await stt._handle_message(_flux_turn_info("StartOfTurn", "hi")) + await stt._handle_message(_flux_turn_info("Update", "hi there")) + events = _drain(stt) + assert [(e.type, e.text) for e in events] == [("partial", "hi"), ("partial", "hi there")] + + async def test_end_of_turn_is_committed(self) -> None: + stt = DeepgramFluxSTT() + await stt._handle_message(_flux_turn_info("EndOfTurn", "tell me a story.")) + events = _drain(stt) + assert [(e.type, e.text) for e in events] == [("committed", "tell me a story.")] + + async def test_eager_events_emit_nothing(self) -> None: + stt = DeepgramFluxSTT() + await stt._handle_message(_flux_turn_info("EagerEndOfTurn")) + await stt._handle_message(_flux_turn_info("TurnResumed")) + assert _drain(stt) == [] + + async def test_empty_transcripts_skipped(self) -> None: + stt = DeepgramFluxSTT() + await stt._handle_message(_flux_turn_info("Update", "")) + await stt._handle_message(_flux_turn_info("EndOfTurn", " ")) + assert _drain(stt) == [] + + async def test_connected_ignored_fatal_error_surfaces(self) -> None: + stt = DeepgramFluxSTT() + await stt._handle_message({"type": "Connected", "request_id": "r1", "sequence_id": 0}) + await stt._handle_message({"type": "FatalError", "description": "bad auth", "code": "AUTH"}) + events = _drain(stt) + assert len(events) == 1 + assert events[0].type == "error" + assert "bad auth" in events[0].text + + async def test_commit_is_noop(self) -> None: + stt = DeepgramFluxSTT() + stt._ws = _FakeWs() + await stt.commit() + assert stt._ws.sent == [] + + +class TestFluxUri: + def _parse(self, stt: DeepgramFluxSTT, config: AudioInputConfig) -> dict: + uri = stt._build_uri(config) + parsed = urlparse(uri) + assert parsed.scheme == "wss" + assert parsed.path == "/v2/listen" + return parse_qs(parsed.query) + + def test_defaults(self) -> None: + q = self._parse(DeepgramFluxSTT(), AudioInputConfig()) + assert q["model"] == [DEFAULT_FLUX_MODEL] + assert q["encoding"] == ["linear16"] + assert q["sample_rate"] == ["16000"] + assert q["eot_timeout_ms"] == ["2000"] + assert "language" not in q + + def test_language_hint_only_for_multi(self) -> None: + q = self._parse(DeepgramFluxSTT(), AudioInputConfig(language="es")) + assert q["language_hint"] == ["es"] + q = self._parse(DeepgramFluxSTT(), AudioInputConfig(model="flux-general-en", language="es")) + assert "language_hint" not in q + + def test_foreign_model_replaced(self) -> None: + # Only TIMBAL_STT_PROVIDER switched; env stt_model still Scribe's. + q = self._parse(DeepgramFluxSTT(), AudioInputConfig(model="scribe_v2_realtime")) + assert q["model"] == [DEFAULT_FLUX_MODEL] + + def test_extra_passthrough_filtered(self) -> None: + config = AudioInputConfig( + extra={ + "eot_threshold": 0.8, + "eot_timeout_ms": 3000, + "keyterm": ["Timbal", "Scribe"], + # Scribe-only knob must not leak into the Flux query. + "commit_strategy": "vad", + } + ) + q = self._parse(DeepgramFluxSTT(), config) + assert q["eot_threshold"] == ["0.8"] + assert q["eot_timeout_ms"] == ["3000"] + assert q["keyterm"] == ["Timbal", "Scribe"] + assert "commit_strategy" not in q + + +class TestNovaEventMapping: + def _results(self, text: str, *, is_final: bool, speech_final: bool = False, **kw) -> dict: + return { + "type": "Results", + "is_final": is_final, + "speech_final": speech_final, + "channel": {"alternatives": [{"transcript": text}]}, + **kw, + } + + async def test_interim_is_partial(self) -> None: + stt = DeepgramNovaSTT() + await stt._handle_message(self._results("hello", is_final=False)) + events = _drain(stt) + assert [(e.type, e.text) for e in events] == [("partial", "hello")] + + async def test_is_final_buffers_until_speech_final(self) -> None: + """Deepgram's documented recipe: concat is_final segments, flush on speech_final.""" + stt = DeepgramNovaSTT() + await stt._handle_message(self._results("yeah so my credit card number is two two", is_final=True)) + assert _drain(stt) == [] + await stt._handle_message(self._results("two two three three three three", is_final=True, speech_final=True)) + events = _drain(stt) + assert [(e.type, e.text) for e in events] == [ + ("committed", "yeah so my credit card number is two two two two three three three three") + ] + assert stt._segments == [] + + async def test_partial_includes_buffered_segments(self) -> None: + stt = DeepgramNovaSTT() + await stt._handle_message(self._results("hello there", is_final=True)) + await stt._handle_message(self._results("how are", is_final=False)) + events = _drain(stt) + assert [(e.type, e.text) for e in events] == [("partial", "hello there how are")] + + async def test_from_finalize_flushes(self) -> None: + """commit() → Finalize → is_final response with from_finalize must commit.""" + stt = DeepgramNovaSTT() + await stt._handle_message(self._results("stop the music", is_final=True, from_finalize=True)) + events = _drain(stt) + assert [(e.type, e.text) for e in events] == [("committed", "stop the music")] + + async def test_utterance_end_flushes_buffer(self) -> None: + stt = DeepgramNovaSTT() + await stt._handle_message(self._results("hello there", is_final=True)) + await stt._handle_message({"type": "UtteranceEnd", "last_word_end": 1.2}) + events = _drain(stt) + assert [(e.type, e.text) for e in events] == [("committed", "hello there")] + + async def test_empty_speech_final_no_commit(self) -> None: + stt = DeepgramNovaSTT() + await stt._handle_message(self._results("", is_final=True, speech_final=True)) + assert _drain(stt) == [] + + async def test_error_surfaces(self) -> None: + stt = DeepgramNovaSTT() + await stt._handle_message({"type": "Error", "description": "NET-0001"}) + events = _drain(stt) + assert events[0].type == "error" + assert "NET-0001" in events[0].text + + async def test_commit_sends_finalize(self) -> None: + stt = DeepgramNovaSTT() + ws = _FakeWs() + stt._ws = ws + await stt.commit() + assert [json.loads(m) for m in ws.sent if isinstance(m, str)] == [{"type": "Finalize"}] + + +class TestNovaUri: + def _parse(self, config: AudioInputConfig) -> dict: + uri = DeepgramNovaSTT()._build_uri(config) + parsed = urlparse(uri) + assert parsed.path == "/v1/listen" + return parse_qs(parsed.query) + + def test_defaults(self) -> None: + q = self._parse(AudioInputConfig()) + assert q["model"] == [DEFAULT_NOVA_MODEL] + assert q["encoding"] == ["linear16"] + assert q["interim_results"] == ["true"] + assert q["smart_format"] == ["true"] + assert q["punctuate"] == ["true"] + assert q["endpointing"] == ["300"] + assert q["channels"] == ["1"] + + def test_language_and_extra_override(self) -> None: + q = self._parse(AudioInputConfig(language="es", extra={"endpointing": 500, "utterance_end_ms": 1000})) + assert q["language"] == ["es"] + assert q["endpointing"] == ["500"] + assert q["utterance_end_ms"] == ["1000"] + + def test_foreign_model_replaced(self) -> None: + q = self._parse(AudioInputConfig(model="scribe_v2_realtime")) + assert q["model"] == [DEFAULT_NOVA_MODEL] + q = self._parse(AudioInputConfig(model="flux-general-en")) + assert q["model"] == [DEFAULT_NOVA_MODEL] + + def test_nova_variant_kept(self) -> None: + q = self._parse(AudioInputConfig(model="nova-3-medical")) + assert q["model"] == ["nova-3-medical"] + + +class TestResolveStt: + def test_explicit_providers(self) -> None: + assert isinstance(resolve_stt("elevenlabs"), ElevenLabsRealtimeSTT) + assert isinstance(resolve_stt("deepgram"), DeepgramFluxSTT) + assert isinstance(resolve_stt("deepgram", model="nova-3"), DeepgramNovaSTT) + assert isinstance(resolve_stt("deepgram", model="flux-general-en"), DeepgramFluxSTT) + # Leftover Scribe model + bare deepgram must NOT silently become Nova. + assert isinstance(resolve_stt("deepgram", model="scribe_v2_realtime"), DeepgramFluxSTT) + + def test_ui_labels(self) -> None: + assert isinstance(resolve_stt("deepgram-flux"), DeepgramFluxSTT) + assert isinstance(resolve_stt("deepgram-nova"), DeepgramNovaSTT) + # Label wins over a stale model id from env defaults. + assert isinstance(resolve_stt("deepgram-flux", model="scribe_v2_realtime"), DeepgramFluxSTT) + assert isinstance(resolve_stt("deepgram-nova", model="flux-general-multi"), DeepgramNovaSTT) + + def test_inference_from_model(self) -> None: + assert isinstance(resolve_stt(None, model="flux-general-multi"), DeepgramFluxSTT) + assert isinstance(resolve_stt(None, model="nova-3"), DeepgramNovaSTT) + assert isinstance(resolve_stt(None, model="scribe_v2_realtime"), ElevenLabsRealtimeSTT) + assert isinstance(resolve_stt(None, model=None), ElevenLabsRealtimeSTT) + + def test_effective_stt_model(self) -> None: + assert effective_stt_model(DeepgramFluxSTT(), "scribe_v2_realtime") == DEFAULT_FLUX_MODEL + assert effective_stt_model(DeepgramFluxSTT(), "flux-general-en") == "flux-general-en" + assert effective_stt_model(DeepgramNovaSTT(), "scribe_v2_realtime") == DEFAULT_NOVA_MODEL + assert effective_stt_model(DeepgramNovaSTT(), "nova-3-general") == "nova-3-general" + # Unknown-provider fallback → ElevenLabs must not keep a Flux/Nova id. + assert effective_stt_model(ElevenLabsRealtimeSTT(), "flux-general-multi") is None + assert effective_stt_model(ElevenLabsRealtimeSTT(), "nova-3") is None + assert effective_stt_model(ElevenLabsRealtimeSTT(), "scribe_v2_realtime") == "scribe_v2_realtime" + assert effective_stt_model(ElevenLabsRealtimeSTT(), None) is None + + def test_unknown_provider_raises(self) -> None: + with pytest.raises(ValueError, match="Unknown STT provider"): + resolve_stt("whisper-cpp") + + def test_stt_provider_id(self) -> None: + assert stt_provider_id(DeepgramFluxSTT()) == "deepgram-flux" + assert stt_provider_id(DeepgramNovaSTT()) == "deepgram-nova" + assert stt_provider_id(ElevenLabsRealtimeSTT()) == "elevenlabs" + + def test_is_flux_model(self) -> None: + assert is_flux_model("flux-general-multi") + assert is_flux_model("FLUX-GENERAL-EN") + assert not is_flux_model("nova-3") + assert not is_flux_model(None) + assert not is_flux_model("") diff --git a/python/tests/core/test_vad_endpointing.py b/python/tests/voice/test_endpointing.py similarity index 90% rename from python/tests/core/test_vad_endpointing.py rename to python/tests/voice/test_endpointing.py index a64bb246..21954d29 100644 --- a/python/tests/core/test_vad_endpointing.py +++ b/python/tests/voice/test_endpointing.py @@ -77,16 +77,28 @@ def _endpointer(vad: FakeVad | None = None, **knobs) -> VadEndpointer: class _Recorder: """Bind targets that record calls and return scripted values.""" - def __init__(self, *, p: float | None = 0.95, gate: bool = True) -> None: + def __init__( + self, + *, + p: float | None = 0.95, + gate: bool = True, + p_text: float | None = None, + ) -> None: self.p = p self.gate = gate + self.p_text = p_text self.score_calls = 0 self.commit_calls = 0 + self.text_score_calls = 0 async def score(self) -> float | None: self.score_calls += 1 return self.p + async def text_score(self) -> float | None: + self.text_score_calls += 1 + return self.p_text + async def commit(self) -> None: self.commit_calls += 1 @@ -95,7 +107,12 @@ def should_commit(self) -> bool: async def _started(ep: VadEndpointer, rec: _Recorder) -> VadEndpointer: - ep.bind(score=rec.score, commit=rec.commit, should_commit=rec.should_commit) + ep.bind( + score=rec.score, + commit=rec.commit, + should_commit=rec.should_commit, + text_score=rec.text_score if rec.p_text is not None else None, + ) await ep.start(sample_rate=16_000) return ep @@ -112,9 +129,11 @@ async def _settle(secs: float = 0.15) -> None: class TestEndpointingDelay: def test_confident_maps_to_min(self): assert endpointing_delay(1.0, min_delay=0.0, max_delay=3.0) == 0.0 + assert endpointing_delay(1.0) == VadEndpointer.MIN_DELAY_SECS def test_zero_maps_to_max(self): assert endpointing_delay(0.0, min_delay=0.0, max_delay=3.0) == 3.0 + assert endpointing_delay(0.0) == 3.0 def test_monotonic_decreasing(self): delays = [endpointing_delay(p / 10) for p in range(11)] @@ -128,10 +147,15 @@ def test_incomplete_exceeds_provider_debounce(self): # p below the completion threshold must compute a delay longer than # the ~1.2s ElevenLabs VAD debounce — the provider commit wins and the # existing HOLD machinery handles the incomplete utterance. - assert endpointing_delay(0.4) > 1.0 + assert endpointing_delay(0.4, min_delay=0.0) > 1.0 + + def test_confident_respects_livekit_floor(self): + # LiveKit min_delay shape: even p≈1 never collapses to ~0. + assert endpointing_delay(0.95) >= VadEndpointer.MIN_DELAY_SECS - 1e-9 + assert VadEndpointer.MIN_DELAY_SECS >= 0.4 - def test_confident_is_fast(self): - assert endpointing_delay(0.95) < 0.05 + def test_curve_still_fast_without_floor(self): + assert endpointing_delay(0.95, min_delay=0.0) < 0.05 # --------------------------------------------------------------------------- @@ -204,6 +228,28 @@ async def test_resumed_speech_cancels_pending(self): await _settle(0.05) assert rec.commit_calls == 0 + async def test_text_incomplete_bumps_delay(self): + """Audio-complete + hedge text → delay floored at TEXT_INCOMPLETE_DELAY, + so a continuation can cancel before commit (live: "I don't know" → story).""" + rec = _Recorder(p=0.98, p_text=0.2) + ep = await _started( + _endpointer( + min_delay_secs=0.0, + max_delay_secs=0.05, + text_incomplete_delay_secs=0.4, + ), + rec, + ) + ep.push(b"SSS") + ep.push(b"\x00\x00") + await _settle(0.1) # past audio-only delay (0.05), still in text bump + assert rec.score_calls == 1 + assert rec.text_score_calls == 1 + assert rec.commit_calls == 0 + ep.push(b"SS") # continuation cancels during the bumped wait + await _settle(0.05) + assert rec.commit_calls == 0 + async def test_single_noise_frame_does_not_cancel(self): rec = _Recorder(p=0.5) # delay(0.5) = 0.25 * max → 0.1s with max 0.4: long enough to inject a blip. diff --git a/python/tests/core/test_eou.py b/python/tests/voice/test_eou.py similarity index 60% rename from python/tests/core/test_eou.py rename to python/tests/voice/test_eou.py index 310c19e8..ed2433a8 100644 --- a/python/tests/core/test_eou.py +++ b/python/tests/voice/test_eou.py @@ -18,12 +18,14 @@ async def test_empty_is_complete(self) -> None: async def test_terminal_punctuation_complete(self) -> None: p = PunctuationEouPredictor() - for text in ("Tell me a story.", "How are you?", "Stop!", "Vale…", "¿Qué tal?"): + for text in ("Tell me a story.", "How are you?", "Stop!", "¿Qué tal?"): assert await p.predict_eou(text) == p.P_TERMINAL async def test_continuing_punctuation_incomplete(self) -> None: p = PunctuationEouPredictor() - for text in ("I want to say,", "First this;", "Listen —"): + # Ellipsis included: STT writes "…" / "..." when the speaker audibly + # trails off mid-thought — the opposite of terminal, despite the dots. + for text in ("I want to say,", "First this;", "Listen —", "Vale…", "I was thinking about..."): assert await p.predict_eou(text) == p.P_CONTINUING async def test_trailing_dangling_token_incomplete_english(self) -> None: @@ -66,3 +68,40 @@ async def test_lifecycle_noops(self) -> None: p = PunctuationEouPredictor() await p.start() await p.close() + + async def test_hedge_beats_terminal_punctuation(self) -> None: + """Live failure: STT writes "Uh, I don't know." with a period; audio + EOU agrees complete — hedges must score incomplete anyway.""" + p = PunctuationEouPredictor() + for text in ( + "Uh, I don't know.", + "I don't know.", + "I'm not sure.", + "uh", + "Well", + "No sé.", + "Let me think.", + ): + assert await p.predict_eou(text) == p.P_HEDGE, text + assert await p.predict_eou(text) < 0.4 + + async def test_hedge_does_not_match_real_completes(self) -> None: + p = PunctuationEouPredictor() + for text in ( + "Tell me a story.", + "Quite incredible.", + "Yeah, sure. Anything else?", + # Contentful prefixes ending in hedge phrases must keep terminal + # scores — punctuation is stripped before the hedge match. + "I want you to know.", + "Tell me what you mean.", + "Know what I mean?", + "Do you know?", + ): + assert await p.predict_eou(text) == p.P_TERMINAL, text + + async def test_trailing_filler_hedge_still_incomplete(self) -> None: + """Filler + trailing phrase hedge stays incomplete (with/without punct).""" + p = PunctuationEouPredictor() + for text in ("so I mean", "uh you know", "Well, I mean.", "and you know"): + assert await p.predict_eou(text) == p.P_HEDGE, text diff --git a/python/tests/voice/test_namo.py b/python/tests/voice/test_namo.py new file mode 100644 index 00000000..cfa95094 --- /dev/null +++ b/python/tests/voice/test_namo.py @@ -0,0 +1,182 @@ +"""Tests for the Namo text EOU backend (``timbal[voice]`` extra).""" + +from __future__ import annotations + +import os +import sys +import time + +import pytest + +np = pytest.importorskip("numpy", reason="timbal[voice] extra not installed") +pytest.importorskip("onnxruntime", reason="timbal[voice] extra not installed") +pytest.importorskip("transformers", reason="timbal[voice] extra not installed") + +from timbal.voice import LocalAudioTurnDetector, resolve_turn_detector # noqa: E402 +from timbal.voice import turn_detection as turn_detection_module # noqa: E402 +from timbal.voice.eou import PunctuationEouPredictor # noqa: E402 +from timbal.voice.namo import ( # noqa: E402 + DEFAULT_REPO_ID, + MULTILINGUAL_REPO_ID, + NamoTextEouPredictor, + _ACK_FORCE_P, + _prepare_text_for_namo, +) + + +class _FakeTokenizer: + def __call__(self, text, truncation=True, max_length=512, return_tensors="np"): + n = min(len(text.split()) + 2, max_length) + return { + "input_ids": np.ones((1, n), dtype=np.int64), + "attention_mask": np.ones((1, n), dtype=np.int64), + } + + +class _FakeOrtSession: + """Returns fixed 2-class logits: [incomplete, complete].""" + + def __init__(self, p_complete: float = 0.9) -> None: + self.p_complete = p_complete + self.calls: list[dict] = [] + + def run(self, output_names, feeds): # noqa: ARG002 + self.calls.append(feeds) + # Inverse-softmax-ish logits so softmax → [1-p, p]. + p = min(1.0 - 1e-6, max(1e-6, self.p_complete)) + logit_complete = float(np.log(p / (1.0 - p))) + logits = np.array([[0.0, logit_complete]], dtype=np.float32) + return [logits] + + +def _reset_caches() -> None: + turn_detection_module._DEFAULT_AUDIO_EOU = turn_detection_module._AUDIO_EOU_UNSET + turn_detection_module._DEFAULT_TEXT_EOU = turn_detection_module._TEXT_EOU_UNSET + + +class TestNamoPredict: + async def test_empty_is_complete(self) -> None: + model = NamoTextEouPredictor() + model._session = _FakeOrtSession(0.1) + model._tokenizer = _FakeTokenizer() + assert await model.predict_eou("") == 1.0 + assert await model.predict_eou(" ") == 1.0 + assert model._session.calls == [] + + async def test_softmax_complete_label(self) -> None: + model = NamoTextEouPredictor() + model._session = _FakeOrtSession(0.87) + model._tokenizer = _FakeTokenizer() + p = await model.predict_eou("Tell me a story.") + assert p == pytest.approx(0.87, abs=1e-3) + assert len(model._session.calls) == 1 + + async def test_softmax_incomplete(self) -> None: + model = NamoTextEouPredictor() + model._session = _FakeOrtSession(0.15) + model._tokenizer = _FakeTokenizer() + p = await model.predict_eou("Uh, I don't know.") + assert p == pytest.approx(0.15, abs=1e-3) + + async def test_short_ack_with_period_forced_complete(self) -> None: + """STT loves ``Yeah.`` / ``Okay.`` — Namo scores those incomplete.""" + model = NamoTextEouPredictor() + model._session = _FakeOrtSession(0.05) # would be incomplete if consulted + model._tokenizer = _FakeTokenizer() + for text in ("Yeah.", "Okay.", "ok!", "Bye.", "Thank you."): + assert await model.predict_eou(text) == _ACK_FORCE_P, text + assert model._session.calls == [] + + async def test_ack_prep_strips_terminal_punct(self) -> None: + text, override = _prepare_text_for_namo("Yeah.") + assert text == "Yeah" + assert override == _ACK_FORCE_P + # Longer hedges must still hit the model (no force). + text, override = _prepare_text_for_namo("Uh, I don't know.") + assert override is None + assert "know" in text.lower() + + def test_default_repo_is_english_specialist(self) -> None: + model = NamoTextEouPredictor() + assert model.repo_id == DEFAULT_REPO_ID + assert "English" in model.repo_id + assert model.max_length == 512 + + def test_multilingual_repo_longer_context(self) -> None: + model = NamoTextEouPredictor(repo_id=MULTILINGUAL_REPO_ID) + assert model.max_length == 8192 + + +class TestResolveLocalInjectsNamo: + def test_local_gets_namo_text_eou(self) -> None: + _reset_caches() + try: + detector = resolve_turn_detector("local") + assert isinstance(detector, LocalAudioTurnDetector) + assert isinstance(detector.fallback_text_eou, NamoTextEouPredictor) + assert "English" in detector.fallback_text_eou.repo_id + finally: + _reset_caches() + + def test_local_shares_one_text_eou_instance(self) -> None: + _reset_caches() + try: + a = resolve_turn_detector("local") + b = resolve_turn_detector("local") + assert a.fallback_text_eou is b.fallback_text_eou + assert a.clone().fallback_text_eou is a.fallback_text_eou + finally: + _reset_caches() + + def test_local_falls_back_without_namo(self, monkeypatch) -> None: + _reset_caches() + monkeypatch.setitem(sys.modules, "timbal.voice.namo", None) + try: + detector = resolve_turn_detector("local") + assert isinstance(detector, LocalAudioTurnDetector) + assert isinstance(detector.fallback_text_eou, PunctuationEouPredictor) + finally: + _reset_caches() + + +@pytest.mark.skipif( + not os.environ.get("TIMBAL_NAMO_INTEGRATION"), + reason="set TIMBAL_NAMO_INTEGRATION=1 to download and run the real checkpoint", +) +class TestRealCheckpoint: + async def test_real_model_scores_text(self) -> None: + model = NamoTextEouPredictor() + await model.start() + p_complete = await model.predict_eou("Tell me a story.") + p_hedge = await model.predict_eou("Uh, I don't know.") + p_dangling = await model.predict_eou("I was thinking about") + assert 0.0 <= p_complete <= 1.0 + assert 0.0 <= p_hedge <= 1.0 + # English specialist is decisive on the live failure mode. + assert p_complete > 0.9 + assert p_hedge < 0.1 + assert p_dangling < 0.1 + + async def test_inference_latency_under_50ms(self) -> None: + """Warm inference should stay well under a STT commit debounce (~1.2s). + + Model card claims <11ms; we allow 50ms headroom for CI noise / cold CPU. + """ + model = NamoTextEouPredictor() + await model.start() + # Discard one more warm call after start()'s own warmup. + await model.predict_eou("warmup again") + samples = ( + "Tell me a story.", + "Uh, I don't know.", + "Quite incredible.", + "Yeah, sure. Anything else?", + "I was thinking about", + ) + times: list[float] = [] + for text in samples: + t0 = time.perf_counter() + await model.predict_eou(text) + times.append(time.perf_counter() - t0) + p50 = sorted(times)[len(times) // 2] + assert p50 < 0.050, f"p50 inference {p50 * 1000:.1f}ms exceeded 50ms budget" diff --git a/python/tests/core/test_playback.py b/python/tests/voice/test_playback.py similarity index 100% rename from python/tests/core/test_playback.py rename to python/tests/voice/test_playback.py diff --git a/python/tests/core/test_realtime_session.py b/python/tests/voice/test_realtime.py similarity index 100% rename from python/tests/core/test_realtime_session.py rename to python/tests/voice/test_realtime.py diff --git a/python/tests/core/test_voice_session.py b/python/tests/voice/test_session.py similarity index 76% rename from python/tests/core/test_voice_session.py rename to python/tests/voice/test_session.py index 2c40af07..741f1b83 100644 --- a/python/tests/core/test_voice_session.py +++ b/python/tests/voice/test_session.py @@ -7,6 +7,7 @@ from collections.abc import AsyncIterator from contextlib import aclosing +import pytest from timbal import Agent from timbal.core.test_model import TestModel from timbal.voice.metrics import TurnMetricsEvent @@ -30,6 +31,12 @@ VoiceSessionEvent, _strip_markdown, ) +from timbal.voice.turn_detection import ( + CommitAction, + CommitDecision, + PartialDecision, + TurnDetector, +) # --------------------------------------------------------------------------- # Mock STT / TTS @@ -414,6 +421,53 @@ async def _audio_source() -> AsyncIterator[bytes]: assert session.input_audio == b"" +# --------------------------------------------------------------------------- +# Tests: LLM connection warmup +# --------------------------------------------------------------------------- + + +class TestLlmWarmup: + async def test_warmup_uses_session_model_override(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Playground per-session model must warm that provider, not agent.model.""" + warmed: list[str] = [] + + async def _fake_warmup(model: str) -> None: + warmed.append(model) + + monkeypatch.setattr("timbal.core.llm_router.warmup_llm_connection", _fake_warmup) + agent = Agent(name="t", model="groq/llama-3.1-8b-instant", tools=[]) + session = VoiceSession( + agent=agent, + stt=MockSTT(), + tts=MockTTS(), + model="openai/gpt-4o-mini", + ) + session._start_llm_warmup() + assert session._llm_warmup_task is not None + await session._llm_warmup_task + assert warmed == ["openai/gpt-4o-mini"] + + async def test_warmup_falls_back_to_agent_model(self, monkeypatch: pytest.MonkeyPatch) -> None: + warmed: list[str] = [] + + async def _fake_warmup(model: str) -> None: + warmed.append(model) + + monkeypatch.setattr("timbal.core.llm_router.warmup_llm_connection", _fake_warmup) + agent = Agent(name="t", model="groq/llama-3.1-8b-instant", tools=[]) + session = VoiceSession(agent=agent, stt=MockSTT(), tts=MockTTS()) + session._start_llm_warmup() + assert session._llm_warmup_task is not None + await session._llm_warmup_task + assert warmed == ["groq/llama-3.1-8b-instant"] + + def test_warmup_skips_test_model(self) -> None: + agent = Agent(name="t", model=TestModel(responses=["ok"]), tools=[]) + session = VoiceSession(agent=agent, stt=MockSTT(), tts=MockTTS()) + session._start_llm_warmup() + assert session._llm_warmup_task is None + + # --------------------------------------------------------------------------- # Tests: event types emitted # --------------------------------------------------------------------------- @@ -893,6 +947,26 @@ async def test_completed_turn_barge_in_rewrites_committed_entry(self) -> None: # Second turn's reply is intact. assert assistant_entries[1].text == "Second reply" + async def test_interrupt_drops_queued_audio_before_interrupted_event(self) -> None: + """TTS can enqueue far more AudioOutput than the WS drains. Barge-in must + not wait behind that backlog or long replies keep playing after interrupt.""" + agent = Agent(name="t", model=TestModel(responses=["ok"]), tools=[]) + session = VoiceSession(agent=agent, stt=MockSTT(), tts=MockTTS()) + await session._emit(AudioOutput(data=b"\x00" * 64)) + await session._emit(AudioOutput(data=b"\x01" * 64)) + await session._emit(AgentTextDone(text="already spoken")) + await session._emit(AudioOutput(data=b"\x02" * 64)) + dropped = session._drop_queued_audio_output() + assert dropped == 3 + remaining: list[VoiceSessionEvent] = [] + while True: + try: + remaining.append(session._event_queue.get_nowait()) + except asyncio.QueueEmpty: + break + assert len(remaining) == 1 + assert isinstance(remaining[0], AgentTextDone) + async def test_completed_turn_barge_in_resaves_serialized_trace(self) -> None: """Regression: interrupt() after AgentTextDone must re-persist the truncated memory. Serializing providers already saved the full reply in @@ -1203,6 +1277,12 @@ async def _run() -> None: assert [e.role for e in session.transcript] == ["user", "assistant"] assert session.transcript[0].text == combined assert session.transcript[1].text == "Second reply" + committed = [e for e in events if isinstance(e, TranscriptCommitted)] + assert len(committed) == 2 + assert committed[0].replace is False + assert committed[0].text == fragment + assert committed[1].replace is True + assert committed[1].text == combined # The merged turn's LLM input must contain the combined utterance # exactly once — the old memory *rewrite* (instead of pop) left it in # both the parent memory and the new prompt. @@ -1238,6 +1318,80 @@ async def test_align_continue_memory_drops_heard_assistant(self) -> None: await session._align_continue_memory(fragment_user_text="hello can") assert root.memory == [] + async def test_hold_during_tts_does_not_truncate_reply(self) -> None: + """HOLD while the client is still playing must defer — not amputate the + greeting for an echo-ish fragment (session log: Hello! How can I…).""" + + class _HoldThenExpire(TurnDetector): + async def on_partial(self, text, state): # noqa: ARG002 + return PartialDecision.IGNORE + + async def on_committed(self, text, state): # noqa: ARG002 + if state.holding: + return CommitDecision(action=CommitAction.NEW_TURN, text=text, reason="hold_supersede") + return CommitDecision( + action=CommitAction.HOLD, + text=text, + reason="audio_hold", + hold_timeout_secs=0.15, + ) + + tracker = FakePlaybackTracker() + agent = Agent(name="t", model=TestModel(responses=[self.RESPONSE, "Second"]), tools=[]) + stt = DelayedMockSTT() + tts = SlowMockTTS(delay=0.02, chunk=b"\x00\x01" * 50, num_chunks=4) + session = VoiceSession( + agent=agent, + stt=stt, + tts=tts, + turn_detector=_HoldThenExpire(), + playback_tracker=tracker, + ) + events: list[VoiceSessionEvent] = [] + + async def _empty_audio() -> AsyncIterator[bytes]: + return + yield # noqa: RET504 + + async def _drive() -> None: + while not any(isinstance(e, SessionStarted) for e in events): + await asyncio.sleep(0.01) + await stt.inject(TranscriptEvent(type="committed", text="Julia.")) + while not any(isinstance(e, AgentTextDone) for e in events): + await asyncio.sleep(0.01) + # Client still has queued TTS — HOLD must not emit interrupt/truncate. + tracker.playing = True + tracker.played = 80 + await stt.inject(TranscriptEvent(type="committed", text="Hello, hello.")) + await asyncio.sleep(0.05) + interrupts = [e for e in events if isinstance(e, SessionInterrupted)] + assert interrupts == [], "HOLD during TTS must not interrupt" + # Full reply still in transcript (not truncated to a heard prefix). + assistants = [e for e in session.transcript if e.role == "assistant"] + assert assistants and assistants[0].text == self.RESPONSE + tracker.playing = False + # Let hold expire → turn for the held fragment. + for _ in range(200): + if sum(1 for e in events if isinstance(e, AgentTextDone)) >= 2: + break + await asyncio.sleep(0.01) + else: + raise AssertionError("second turn never finished after hold expiry") + await stt.finish() + + async def _run() -> None: + async with aclosing(session.run(_empty_audio())) as stream: + driver = asyncio.create_task(_drive()) + async for ev in stream: + events.append(ev) + await driver + + await asyncio.wait_for(_run(), timeout=10) + assert sum(1 for e in events if isinstance(e, AgentTextDone)) >= 2 + # First assistant entry survived the HOLD (may later be joined by turn 2). + assert session.transcript[1].role == "assistant" + assert session.transcript[1].text == self.RESPONSE + async def test_interrupt_cancels_orphan_turn_before_speaking(self) -> None: """``interrupt()`` must cancel a scheduled turn even if ``_is_speaking`` is still False — otherwise HOLD expiry / commit races double-run agents.""" @@ -1475,10 +1629,20 @@ async def test_hallucinated_partial_does_not_interrupt(self) -> None: # Reply survived intact. assert session.transcript[-1].role == "assistant" assert session.transcript[-1].text == "This is a fairly long reply that keeps on playing for a while." + # Vetoed barge-in partials must not flash in the playground caption. + vetoed = [ + e + for e in events + if isinstance(e, TranscriptPartial) and "not not too bad" in e.text.lower() + ] + assert vetoed == [] async def test_real_speech_partial_still_interrupts(self) -> None: _, events = await self._run_barge_in_scenario(speech_secs=1.0) assert any(isinstance(e, SessionInterrupted) for e in events) + assert any( + isinstance(e, TranscriptPartial) and "not not too bad" in e.text.lower() for e in events + ) class TestStreamingTTS: @@ -1555,3 +1719,302 @@ async def _run() -> None: assert session._turn_tts_stream is None assert session._turn_tts_pump is None assert any(isinstance(e, SessionInterrupted) for e in events) + + +# --------------------------------------------------------------------------- +# Tests: hold expiry timing (partial-grace anchor) +# --------------------------------------------------------------------------- + + +class _AlwaysHoldOnceDetector(TurnDetector): + """HOLDs the first fresh commit with a fixed short timeout; anything while + holding supersedes into a NEW_TURN (minimal detector for expiry timing).""" + + def __init__(self, timeout_secs: float) -> None: + self._timeout_secs = timeout_secs + + async def on_partial(self, text: str, state) -> PartialDecision: + return PartialDecision.IGNORE + + async def on_committed(self, text: str, state) -> CommitDecision: + if state.holding: + return CommitDecision(action=CommitAction.NEW_TURN, text=text, reason="new_turn") + return CommitDecision( + action=CommitAction.HOLD, text=text, reason="hold", hold_timeout_secs=self._timeout_secs + ) + + +class TestHoldExpiryTiming: + async def _run_hold_scenario( + self, + *, + grace_secs: float, + timeout_secs: float, + partial_after_commit_delay: float | None, + run_timeout: float = 10.0, + endpointer: object | None = None, + ) -> tuple[float, VoiceSession]: + """Inject partial→commit (HOLD) and return seconds from commit to turn start.""" + agent = Agent(name="t", model=TestModel(responses=["ok"]), tools=[]) + stt = DelayedMockSTT() + tts = MockTTS() + session = VoiceSession( + agent=agent, stt=stt, tts=tts, turn_detector=_AlwaysHoldOnceDetector(timeout_secs) + ) + session._hold_partial_grace_secs = grace_secs + if endpointer is not None: + session._endpointer = endpointer # type: ignore[assignment] + + events: list[VoiceSessionEvent] = [] + elapsed = 0.0 + + async def _empty_audio() -> AsyncIterator[bytes]: + return + yield # noqa: RET504 + + async def _drive() -> None: + nonlocal elapsed + while not any(isinstance(e, SessionStarted) for e in events): + await asyncio.sleep(0.01) + # The fragment's own partial precedes its commit (STT ordering). + await stt.inject(TranscriptEvent(type="partial", text="Thank you")) + await stt.inject(TranscriptEvent(type="committed", text="Thank you.")) + t0 = asyncio.get_event_loop().time() + if partial_after_commit_delay is not None: + await asyncio.sleep(partial_after_commit_delay) + # User resumed speaking mid-hold. + await stt.inject(TranscriptEvent(type="partial", text="and one more")) + while not any(isinstance(e, TranscriptCommitted) for e in events): + await asyncio.sleep(0.01) + elapsed = asyncio.get_event_loop().time() - t0 + while not any(isinstance(e, AgentTextDone) for e in events): + await asyncio.sleep(0.01) + await stt.finish() + + async def _run() -> None: + async with aclosing(session.run(_empty_audio())) as stream: + driver = asyncio.create_task(_drive()) + async for ev in stream: + events.append(ev) + await driver + + await asyncio.wait_for(_run(), timeout=run_timeout) + return elapsed, session + + async def test_pre_commit_partial_does_not_stretch_hold(self) -> None: + """Live bug: every commit is preceded by its own partials, so anchoring + the grace window on *any* recent partial floored each hold at the grace + window (~2s) instead of the armed timeout (1.2s tier).""" + elapsed, session = await self._run_hold_scenario( + grace_secs=5.0, # deliberately huge: failure mode is unmissable + timeout_secs=0.2, + partial_after_commit_delay=None, + ) + assert elapsed < 2.0, f"hold expiry took {elapsed:.2f}s — pre-commit partial stretched it" + assert session.transcript[0].text == "Thank you." + + async def test_post_arm_partial_extends_hold(self) -> None: + """A partial *after* the hold is armed means the user resumed speaking + — the expiry must wait out the grace window (no VAD in this harness → + extension is conservative).""" + elapsed, _ = await self._run_hold_scenario( + grace_secs=0.8, + timeout_secs=0.1, + partial_after_commit_delay=0.05, + ) + # Must have waited past the bare timeout into the grace window. + assert elapsed > 0.5, f"hold expired after only {elapsed:.2f}s despite fresh partial" + + async def test_pre_commit_speech_does_not_stretch_via_vad_lookback(self) -> None: + """Live bug: Silero's 2s lookback still contains the held utterance, so + ``not vad_contradicts`` was always true after commit — any late STT + refinement floored a 0.35s text-complete HOLD at the 2s grace window. + """ + + class _StaleSpeechEP: + """Reports speech only in wide windows (pre-commit residue).""" + + def speech_secs_in_window(self, window_secs: float) -> float: + return 0.5 if window_secs >= 1.0 else 0.0 + + def notify_committed(self) -> None: + return None + + async def close(self) -> None: + return None + + elapsed, _ = await self._run_hold_scenario( + grace_secs=2.0, + timeout_secs=0.2, + partial_after_commit_delay=0.05, + endpointer=_StaleSpeechEP(), + ) + assert elapsed < 0.8, ( + f"hold expiry took {elapsed:.2f}s — pre-commit VAD speech stretched the grace" + ) + + async def test_dangling_token_hold_is_dropped_on_expiry(self) -> None: + """Live bug: stale-force-committed \"I\" HOLDs for 3s then becomes a + user turn and the LLM invents a reply. Dangling tokens must die here.""" + agent = Agent(name="t", model=TestModel(responses=["should not run"]), tools=[]) + stt = DelayedMockSTT() + tts = MockTTS() + session = VoiceSession( + agent=agent, stt=stt, tts=tts, turn_detector=_AlwaysHoldOnceDetector(0.15) + ) + events: list[VoiceSessionEvent] = [] + + async def _empty_audio() -> AsyncIterator[bytes]: + return + yield # noqa: RET504 + + async def _drive() -> None: + while not any(isinstance(e, SessionStarted) for e in events): + await asyncio.sleep(0.01) + await stt.inject(TranscriptEvent(type="committed", text="I")) + await asyncio.sleep(0.5) + await stt.finish() + + async def _run() -> None: + async with aclosing(session.run(_empty_audio())) as stream: + driver = asyncio.create_task(_drive()) + async for ev in stream: + events.append(ev) + await driver + + await asyncio.wait_for(_run(), timeout=10) + assert all(entry.role != "user" for entry in session.transcript) + assert not any(isinstance(e, AgentTextDone) for e in events) + + +# --------------------------------------------------------------------------- +# Tests: stale partial sweeper +# --------------------------------------------------------------------------- + + +class _StuckPartialSTT(DelayedMockSTT): + """STT that transcribes a partial but never commits it on its own — the + live 'quiet speech during playback' failure. ``commit()`` releases it.""" + + def __init__(self) -> None: + super().__init__() + self.commit_calls = 0 + self.pending_text: str | None = None + + async def commit(self) -> None: + self.commit_calls += 1 + if self.pending_text is not None: + text, self.pending_text = self.pending_text, None + await self.inject(TranscriptEvent(type="committed", text=text)) + + +class TestStalePartialSweeper: + async def test_stuck_partial_is_force_committed(self) -> None: + agent = Agent(name="t", model=TestModel(responses=["ok"]), tools=[]) + stt = _StuckPartialSTT() + tts = MockTTS() + session = VoiceSession(agent=agent, stt=stt, tts=tts) + session._stale_partial_poll_secs = 0.05 + session._stale_partial_commit_secs = 0.2 + + events: list[VoiceSessionEvent] = [] + + async def _empty_audio() -> AsyncIterator[bytes]: + return + yield # noqa: RET504 + + async def _drive() -> None: + while not any(isinstance(e, SessionStarted) for e in events): + await asyncio.sleep(0.01) + stt.pending_text = "Cool." + await stt.inject(TranscriptEvent(type="partial", text="Cool.")) + while not any(isinstance(e, AgentTextDone) for e in events): + await asyncio.sleep(0.01) + await stt.finish() + + async def _run() -> None: + async with aclosing(session.run(_empty_audio())) as stream: + driver = asyncio.create_task(_drive()) + async for ev in stream: + events.append(ev) + await driver + + await asyncio.wait_for(_run(), timeout=10) + + assert stt.commit_calls >= 1 + assert session.transcript[0].role == "user" + assert session.transcript[0].text == "Cool." + + async def test_sweeper_does_not_refire_on_ignored_commit(self) -> None: + """An IGNOREd commit (noise) leaves _last_commit_at stale; the sweeper + must force at most one commit per stranded partial, not one per poll.""" + agent = Agent(name="t", model=TestModel(responses=["ok"]), tools=[]) + stt = _StuckPartialSTT() + tts = MockTTS() + session = VoiceSession(agent=agent, stt=stt, tts=tts) + session._stale_partial_poll_secs = 0.05 + session._stale_partial_commit_secs = 0.15 + + events: list[VoiceSessionEvent] = [] + + async def _empty_audio() -> AsyncIterator[bytes]: + return + yield # noqa: RET504 + + async def _drive() -> None: + while not any(isinstance(e, SessionStarted) for e in events): + await asyncio.sleep(0.01) + # Hesitation-only: the resulting forced commit is IGNOREd by the + # heuristic detector, so no turn starts and no _last_commit_at bump. + stt.pending_text = "Mm-hmm." + await stt.inject(TranscriptEvent(type="partial", text="Mm-hmm.")) + await asyncio.sleep(1.0) # several poll+stale windows + await stt.finish() + + async def _run() -> None: + async with aclosing(session.run(_empty_audio())) as stream: + driver = asyncio.create_task(_drive()) + async for ev in stream: + events.append(ev) + await driver + + await asyncio.wait_for(_run(), timeout=10) + + assert stt.commit_calls == 1 + assert all(entry.role != "user" for entry in session.transcript) + + async def test_noop_commit_provider_synthesizes_stranded_partial(self) -> None: + """Flux-style STT: commit() is a no-op, so the sweeper must promote the + stranded partial itself or the UI caption hangs forever.""" + agent = Agent(name="t", model=TestModel(responses=["ok"]), tools=[]) + stt = DelayedMockSTT() + tts = MockTTS() + session = VoiceSession(agent=agent, stt=stt, tts=tts) + session._stale_partial_poll_secs = 0.05 + session._stale_partial_commit_secs = 0.15 + + events: list[VoiceSessionEvent] = [] + + async def _empty_audio() -> AsyncIterator[bytes]: + return + yield # noqa: RET504 + + async def _drive() -> None: + while not any(isinstance(e, SessionStarted) for e in events): + await asyncio.sleep(0.01) + await stt.inject(TranscriptEvent(type="partial", text="What are you doing?")) + while not any(isinstance(e, AgentTextDone) for e in events): + await asyncio.sleep(0.01) + await stt.finish() + + async def _run() -> None: + async with aclosing(session.run(_empty_audio())) as stream: + driver = asyncio.create_task(_drive()) + async for ev in stream: + events.append(ev) + await driver + + await asyncio.wait_for(_run(), timeout=10) + + assert session.transcript[0].role == "user" + assert session.transcript[0].text == "What are you doing?" diff --git a/python/tests/core/test_voice_session_stt_refinement.py b/python/tests/voice/test_session_stt_refinement.py similarity index 100% rename from python/tests/core/test_voice_session_stt_refinement.py rename to python/tests/voice/test_session_stt_refinement.py diff --git a/python/tests/core/test_smart_turn.py b/python/tests/voice/test_smart_turn.py similarity index 99% rename from python/tests/core/test_smart_turn.py rename to python/tests/voice/test_smart_turn.py index 038d8109..31fe9db4 100644 --- a/python/tests/core/test_smart_turn.py +++ b/python/tests/voice/test_smart_turn.py @@ -197,6 +197,7 @@ async def test_works_inside_local_detector(self): class TestResolveLocalMode: def _reset_default_cache(self): turn_detection_module._DEFAULT_AUDIO_EOU = turn_detection_module._AUDIO_EOU_UNSET + turn_detection_module._DEFAULT_TEXT_EOU = turn_detection_module._TEXT_EOU_UNSET def test_local_gets_smart_turn_model(self): self._reset_default_cache() diff --git a/python/tests/core/test_turn_detection.py b/python/tests/voice/test_turn_detection.py similarity index 79% rename from python/tests/core/test_turn_detection.py rename to python/tests/voice/test_turn_detection.py index 5212a6ff..daa643b8 100644 --- a/python/tests/core/test_turn_detection.py +++ b/python/tests/voice/test_turn_detection.py @@ -31,7 +31,7 @@ resolve_turn_detector, ) -from .test_voice_session import MockSTT, MockTTS +from .test_session import MockSTT, MockTTS # --------------------------------------------------------------------------- # Moved heuristic functions (ported from test_voice_session_stt_refinement.py) @@ -243,6 +243,70 @@ async def test_fast_short_fragment_becomes_continuation(self) -> None: assert decision.text == "Hola, ¿qué tal estás?" assert decision.reason == "continuation" + async def test_trailing_crumb_on_finished_turn_is_ignored(self) -> None: + """Scribe ghost after a finished commit must not CONTINUE-cancel the LLM. + + Live: commit "…killer." → LLM START → "No." at +55ms → CONTINUE merge + restarted the turn on a frankenstein prompt. + """ + det = HeuristicTurnDetector() + decision = await det.on_committed( + "No.", + _state( + assistant_active=True, + active_user_text="Yeah, that guy was a real killer.", + seconds_since_turn_start=0.1, + seconds_since_last_commit=0.055, + ), + ) + assert decision.action is CommitAction.IGNORE + assert decision.reason == "trailing_crumb" + + async def test_late_short_barge_in_on_finished_turn_is_new_turn(self) -> None: + """Past the crumb window, a short fresh interrupt is a real barge-in.""" + det = HeuristicTurnDetector() + decision = await det.on_committed( + "Stop.", + _state( + assistant_active=True, + active_user_text="Yeah, that guy was a real killer.", + seconds_since_turn_start=1.5, + seconds_since_last_commit=1.0, + ), + ) + assert decision.action is CommitAction.NEW_TURN + assert decision.text == "Stop." + + async def test_question_split_after_finished_question_still_continues(self) -> None: + """VAD can punctuate mid-question; don't drop the second half as a crumb.""" + det = HeuristicTurnDetector() + decision = await det.on_committed( + "estás?", + _state( + assistant_active=True, + active_user_text="Hola, ¿qué tal?", + seconds_since_turn_start=1.0, + seconds_since_last_commit=0.2, + ), + ) + assert decision.action is CommitAction.CONTINUE_TURN + assert decision.text == "Hola, ¿qué tal? estás?" + + async def test_lowercase_glue_still_continues_unfinished_active(self) -> None: + """Single-word lowercase trailers stay CONTINUE when active is unfinished.""" + det = HeuristicTurnDetector() + decision = await det.on_committed( + "weather?", + _state( + assistant_active=True, + active_user_text="Tell me about the", + seconds_since_turn_start=1.0, + seconds_since_last_commit=0.2, + ), + ) + assert decision.action is CommitAction.CONTINUE_TURN + assert decision.reason == "continuation" + async def test_long_new_query_during_turn_is_new_turn(self) -> None: det = HeuristicTurnDetector() decision = await det.on_committed( @@ -386,9 +450,130 @@ async def test_incomplete_audio_holds(self) -> None: decision = await det.on_committed("I was wondering", _state()) assert decision.action is CommitAction.HOLD assert decision.reason == "audio_hold" + # Neutral (unpunctuated) text must not shorten the hold. + assert decision.hold_timeout_secs == det.hold_timeout_secs assert model.calls == 1 await det.close() + async def test_late_barge_in_on_finished_turn_not_held(self) -> None: + """Parent NEW_TURN for finished+fresh must survive incomplete Smart Turn. + + Otherwise session HOLDs during TTS and the barge-in never interrupts. + """ + det = LocalAudioTurnDetector(audio_eou=_FixedAudioEou(0.1)) + await det.start(type("C", (), {"sample_rate": 16000})()) + det.push_audio(b"\x00\x01" * 8000) + decision = await det.on_committed( + "Stop.", + _state( + assistant_active=True, + active_user_text="Yeah, that guy was a real killer.", + seconds_since_turn_start=1.5, + seconds_since_last_commit=1.0, + ), + ) + assert decision.action is CommitAction.NEW_TURN + assert decision.text == "Stop." + await det.close() + + async def test_incomplete_audio_complete_text_short_hold(self) -> None: + """Confidence tier: Smart Turn under-scores short closers ("Thank you." + p=0.036 live) — terminal punctuation disagrees, so the hold shrinks to + the short tier instead of eating the full budget as dead air. + + Keep this well under the incomplete-text HOLD (1.2s): VAD has usually + already waited, and a second 1.2s tax is the cold-start "slow" feel. + """ + det = LocalAudioTurnDetector(audio_eou=_FixedAudioEou(0.1)) + await det.start(type("C", (), {"sample_rate": 16000})()) + det.push_audio(b"\x00\x01" * 8000) + decision = await det.on_committed("Thank you.", _state()) + assert decision.action is CommitAction.HOLD + assert decision.reason == "audio_hold_text_complete" + assert decision.hold_timeout_secs == det.text_complete_hold_timeout_secs + assert det.text_complete_hold_timeout_secs == 0.35 + assert det.text_complete_hold_timeout_secs < det.text_incomplete_hold_timeout_secs + assert det.text_complete_hold_timeout_secs < det.hold_timeout_secs + # Per-session clones keep the knob. + assert det.clone().text_complete_hold_timeout_secs == det.text_complete_hold_timeout_secs + await det.close() + + async def test_complete_audio_hedge_text_short_hold(self) -> None: + """Inverse tier: Smart Turn over-scores thinking pauses + ("Uh, I don't know." p=0.825 live) — hedge text disagrees, so HOLD + short instead of NEW_TURN immediately.""" + det = LocalAudioTurnDetector(audio_eou=_FixedAudioEou(0.9)) + await det.start(type("C", (), {"sample_rate": 16000})()) + det.push_audio(b"\x00\x01" * 8000) + decision = await det.on_committed("Uh, I don't know.", _state()) + assert decision.action is CommitAction.HOLD + assert decision.reason == "audio_complete_text_incomplete" + assert decision.hold_timeout_secs == det.text_incomplete_hold_timeout_secs + await det.close() + + async def test_complete_audio_real_complete_still_new_turn(self) -> None: + det = LocalAudioTurnDetector(audio_eou=_FixedAudioEou(0.9)) + await det.start(type("C", (), {"sample_rate": 16000})()) + det.push_audio(b"\x00\x01" * 8000) + decision = await det.on_committed("Tell me a story.", _state()) + assert decision.action is CommitAction.NEW_TURN + await det.close() + + async def test_complete_audio_question_despite_namo_zero(self) -> None: + """Namo often scores finished questions ~0 — lexical gate must skip the + 1.2s incomplete HOLD (live cold-start: "How are you?" / "What's 2+2?").""" + det = LocalAudioTurnDetector( + audio_eou=_FixedAudioEou(0.98), + fallback_text_eou=_FixedTextEou(0.0), + ) + await det.start(type("C", (), {"sample_rate": 16000})()) + det.push_audio(b"\x00\x01" * 8000) + decision = await det.on_committed("Hello, hello. How are you?", _state()) + assert decision.action is CommitAction.NEW_TURN + decision = await det.on_committed("What's two plus two?", _state()) + assert decision.action is CommitAction.NEW_TURN + await det.close() + + async def test_complete_audio_hedge_still_holds_with_namo_zero(self) -> None: + """Lexical hedge (~0.2) + Namo 0 → still incomplete tier HOLD.""" + det = LocalAudioTurnDetector( + audio_eou=_FixedAudioEou(0.9), + fallback_text_eou=_FixedTextEou(0.0), + ) + await det.start(type("C", (), {"sample_rate": 16000})()) + det.push_audio(b"\x00\x01" * 8000) + decision = await det.on_committed("Uh, I don't know.", _state()) + assert decision.action is CommitAction.HOLD + assert decision.reason == "audio_complete_text_incomplete" + await det.close() + + async def test_complete_audio_neutral_midthought_keeps_namo_incomplete(self) -> None: + """Unpunctuated mid-thought: lexical ~0.60 must NOT rescue Namo 0.""" + det = LocalAudioTurnDetector( + audio_eou=_FixedAudioEou(0.9), + fallback_text_eou=_FixedTextEou(0.0), + ) + await det.start(type("C", (), {"sample_rate": 16000})()) + det.push_audio(b"\x00\x01" * 8000) + decision = await det.on_committed( + "I was thinking about something my dad told me", _state() + ) + assert decision.action is CommitAction.HOLD + assert decision.reason == "audio_complete_text_incomplete" + await det.close() + + async def test_incomplete_audio_ellipsis_full_hold(self) -> None: + """An STT ellipsis means the speaker trailed off — it must not count + as terminal punctuation for the short tier.""" + det = LocalAudioTurnDetector(audio_eou=_FixedAudioEou(0.1)) + await det.start(type("C", (), {"sample_rate": 16000})()) + det.push_audio(b"\x00\x01" * 8000) + decision = await det.on_committed("I was thinking about...", _state()) + assert decision.action is CommitAction.HOLD + assert decision.reason == "audio_hold" + assert decision.hold_timeout_secs == det.hold_timeout_secs + await det.close() + async def test_complete_audio_new_turn(self) -> None: det = LocalAudioTurnDetector(audio_eou=_FixedAudioEou(0.9)) await det.start(type("C", (), {"sample_rate": 16000})()) @@ -796,11 +981,17 @@ async def test_fresh_utterance_still_supersedes(self) -> None: class TestHoldInSession: async def test_hold_expiry_deferred_while_user_speaking(self) -> None: - """A pending HOLD must not fire mid-utterance (recent STT partials).""" + """HOLD must stay armed while STT partials keep being *processed*. + + Grace keys off ``_last_partial_at`` after the session drains the event + (``inject`` only enqueues). Wait for ``_latest_partial_text`` — not + ``_last_partial_at > before``: Windows monotonic can return the same + tick for consecutive partials, so the timestamp predicate hangs. + """ import asyncio import time - from .test_voice_session import DelayedMockSTT + from .test_session import DelayedMockSTT class _HoldOnce(TurnDetector): def __init__(self) -> None: @@ -813,34 +1004,77 @@ async def on_committed(self, text, state): # noqa: ARG002 self.n += 1 if self.n == 1: return CommitDecision( - action=CommitAction.HOLD, text=text, reason="test_hold", hold_timeout_secs=0.1 + action=CommitAction.HOLD, text=text, reason="test_hold", hold_timeout_secs=0.5 ) return CommitDecision(action=CommitAction.NEW_TURN, text=text, reason="test") + hold_timeout = 0.5 + grace = 0.5 + refresh_every = 0.05 # well under hold_timeout / grace agent = Agent(name="t", model=TestModel(responses=["ok"]), tools=[]) stt = DelayedMockSTT() session = VoiceSession(agent=agent, stt=stt, tts=MockTTS(), turn_detector=_HoldOnce()) - session._hold_partial_grace_secs = 0.3 + session._hold_partial_grace_secs = grace events: list[VoiceSessionEvent] = [] - turn_started_at: list[float] = [] - partials_end: list[float] = [] + refreshes_while_held = 0 async def _empty(): return yield # noqa: RET504 - async def _drive() -> None: - while not any(getattr(e, "type", None) == "session_started" for e in events): + async def _wait_until(pred, *, timeout: float = 2.0, msg: str = "condition") -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if pred(): + return await asyncio.sleep(0.01) + raise AssertionError(f"timed out waiting for {msg}") + + async def _inject_processed_partial(text: str) -> None: + await stt.inject(TranscriptEvent(type="partial", text=text)) + await _wait_until( + lambda t=text: session._latest_partial_text == t, + msg=f"partial processed ({text!r})", + ) + + async def _drive() -> None: + nonlocal refreshes_while_held + await _wait_until( + lambda: any(getattr(e, "type", None) == "session_started" for e in events), + msg="session_started", + ) await stt.inject(TranscriptEvent(type="committed", text="I was wondering about")) - # Keep "speaking" well past the 0.1s hold timeout. - for _ in range(6): - await stt.inject(TranscriptEvent(type="partial", text="the weather")) - await asyncio.sleep(0.1) - partials_end.append(time.monotonic()) - while not any(getattr(e, "type", None) == "agent_text_done" for e in events): - await asyncio.sleep(0.01) + await _wait_until(lambda: session._held_user_text is not None, msg="HOLD armed") + # Extend immediately — do not wait on _last_commit_at (expiry bumps it). + await _inject_processed_partial("the weather") + assert session._held_user_text is not None + + # Keep refreshing past the nominal hold timeout. + deadline = time.monotonic() + hold_timeout + 0.25 + i = 0 + while time.monotonic() < deadline: + await _inject_processed_partial(f"the weather {i}") + assert session._held_user_text is not None, ( + f"HOLD expired while partials were still being processed (iter {i})" + ) + refreshes_while_held += 1 + await asyncio.sleep(refresh_every) + i += 1 + + assert refreshes_while_held >= 3 + assert session._held_user_text is not None + + # Stop refreshing — hold should expire into a turn. + await _wait_until( + lambda: any(getattr(e, "type", None) == "transcript_committed" for e in events), + timeout=grace + 1.5, + msg="hold expiry → transcript_committed", + ) + await _wait_until( + lambda: any(getattr(e, "type", None) == "agent_text_done" for e in events), + msg="agent_text_done", + ) await stt.finish() async def _run() -> None: @@ -848,22 +1082,18 @@ async def _run() -> None: driver = asyncio.create_task(_drive()) async for ev in stream: events.append(ev) - if getattr(ev, "type", None) == "transcript_committed": - turn_started_at.append(time.monotonic()) await driver - await asyncio.wait_for(_run(), timeout=5) - assert turn_started_at, "hold never expired into a turn" - # The turn must have started only after the partial stream went quiet, - # not at the nominal 0.1s timeout (partials spanned ~0.6s). - assert partials_end, "driver never finished injecting partials" - assert turn_started_at[0] >= partials_end[0] - 0.15 + await asyncio.wait_for(_run(), timeout=12) + assert refreshes_while_held >= 3 + assert any(getattr(e, "type", None) == "transcript_committed" for e in events) + assert session._held_user_text is None async def test_hold_timeout_starts_turn(self) -> None: """HOLD must not start the agent until the hold timer expires.""" import asyncio - from .test_voice_session import DelayedMockSTT + from .test_session import DelayedMockSTT class _HoldOnce(TurnDetector): def __init__(self) -> None: @@ -919,11 +1149,13 @@ async def _run() -> None: assert any(e.type == "agent_text_done" for e in events) assert session.transcript[0].text == "I was wondering about" - async def test_hold_interrupts_playing_audio(self) -> None: - """HOLD while TTS is still buffered must interrupt like NEW_TURN.""" + async def test_hold_defers_while_audio_playing(self) -> None: + """HOLD while TTS is still buffered arms the fragment but does not + interrupt — chopping the reply for a deferred commit was wiping + greetings on echo-ish ``Hello, hello.`` commits.""" import asyncio - from .test_voice_session import DelayedMockSTT, FakePlaybackTracker + from .test_session import DelayedMockSTT, FakePlaybackTracker class _AlwaysHold(TurnDetector): async def on_partial(self, text, state): # noqa: ARG002 @@ -950,7 +1182,6 @@ async def on_committed(self, text, state): # noqa: ARG002 hold_timeout_secs=5.0, ) events: list[VoiceSessionEvent] = [] - held_during: list[str | None] = [] async def _empty(): return @@ -961,10 +1192,16 @@ async def _drive() -> None: await asyncio.sleep(0.01) await stt.inject(TranscriptEvent(type="committed", text="I was wondering about")) for _ in range(50): - if tracker.interrupt_calls > 0 and session._held_user_text: - held_during.append(session._held_user_text) + if session._held_user_text == "I was wondering about": break await asyncio.sleep(0.01) + else: + raise AssertionError("HOLD never armed while audio playing") + # Assert *before* finish: session.close() interrupts when the + # tracker still reports playing (unrelated to HOLD deferral). + assert tracker.interrupt_calls == 0 + assert not any(e.type == "interrupted" for e in events) + tracker.playing = False await stt.finish() async def _run() -> None: @@ -975,15 +1212,12 @@ async def _run() -> None: await driver await asyncio.wait_for(_run(), timeout=5) - assert tracker.interrupt_calls >= 1 - assert held_during == ["I was wondering about"] - assert any(e.type == "interrupted" for e in events) async def test_hold_refinement_updates_session_fragment(self) -> None: """Longer STT re-commit while HOLDing must replace the held fragment.""" import asyncio - from .test_voice_session import DelayedMockSTT + from .test_session import DelayedMockSTT class _HoldRefine(TurnDetector): async def on_partial(self, text, state): # noqa: ARG002 @@ -1073,7 +1307,7 @@ async def test_hold_expiry_does_not_wipe_rearmed_hold(self) -> None: """ import asyncio - from .test_voice_session import DelayedMockSTT + from .test_session import DelayedMockSTT class _HoldThenRefine(TurnDetector): def __init__(self) -> None: diff --git a/python/tests/core/test_turn_detection_adversarial.py b/python/tests/voice/test_turn_detection_adversarial.py similarity index 99% rename from python/tests/core/test_turn_detection_adversarial.py rename to python/tests/voice/test_turn_detection_adversarial.py index 793cf718..6dc9f4a3 100644 --- a/python/tests/core/test_turn_detection_adversarial.py +++ b/python/tests/voice/test_turn_detection_adversarial.py @@ -25,8 +25,8 @@ _looks_like_fresh_hold_utterance, ) +from .test_session import DelayedMockSTT, MockTTS from .test_turn_detection import _FixedAudioEou, _state -from .test_voice_session import DelayedMockSTT, MockTTS # --------------------------------------------------------------------------- # Pure helpers — edge cases that have bitten us / look sharp diff --git a/python/tests/core/test_silero_vad.py b/python/tests/voice/test_vad.py similarity index 100% rename from python/tests/core/test_silero_vad.py rename to python/tests/voice/test_vad.py diff --git a/python/timbal/server/README.md b/python/timbal/server/README.md index e2b7920c..b07cd40b 100644 --- a/python/timbal/server/README.md +++ b/python/timbal/server/README.md @@ -60,7 +60,8 @@ Only send keys you need; omitted keys keep server defaults. | Key | Description | |---------------|-------------| -| `stt_model` | Speech-to-text model id (ElevenLabs realtime). | +| `stt_provider` | `"elevenlabs"` (default), `"deepgram-flux"`, or `"deepgram-nova"` (bare `"deepgram"` routes by `stt_model`, defaulting to Flux). Deepgram needs `DEEPGRAM_API_KEY` on the server. Flux (`/v2/listen`) does model-native end-of-turn detection: the session auto-selects the `provider` turn detector (explicit `turn_detector` still wins) and disables local VAD endpointing. Nova-3 (`/v1/listen`) is plain ASR — Timbal turn detection and VAD endpointing work exactly as with ElevenLabs. Env default: `TIMBAL_STT_PROVIDER`. | +| `stt_model` | Speech-to-text model id (ElevenLabs realtime `scribe_*`, Deepgram `flux-general-en`/`flux-general-multi`/`nova-3*`). Model ids that don't belong to the selected provider are ignored (provider default used). | | `tts_model` | Text-to-speech model id. | | `voice` | ElevenLabs voice id string. | | `language` | e.g. `"es"`. | @@ -85,7 +86,7 @@ All downlink messages are **text JSON** with a **`type`** field. | `type` | Fields | Meaning | |-------------------------|---------------|--------| -| `session_started` | `playback_acks`, `turn_detector`, `vad_endpointing` | Voice session is live; safe to show “listening”. `playback_acks: "recommended"` advertises the [playback ack](#playback-acks-client--server) protocol. `turn_detector` is the class name of the detector actually in effect (e.g. `LocalAudioTurnDetector`), so clients can verify their requested mode. `vad_endpointing` is a bool: whether the local [VAD endpointing](#config-overrides) fast path actually armed for this session (not merely what was requested). | +| `session_started` | `playback_acks`, `stt_provider`, `stt_model`, `model`, `turn_detector`, `vad_endpointing` | Voice session is live; safe to show “listening”. `playback_acks: "recommended"` advertises the [playback ack](#playback-acks-client--server) protocol. `stt_provider` is the config id actually in effect (`elevenlabs` / `deepgram-flux` / `deepgram-nova`), not the Python class name. `turn_detector` is the class name of the detector actually in effect (e.g. `LocalAudioTurnDetector`). `vad_endpointing` is a bool: whether the local [VAD endpointing](#config-overrides) fast path actually armed for this session (not merely what was requested). | | `transcript_partial` | `text` | Live STT (may change). | | `transcript_committed` | `text` | Final user transcript for the utterance. | | `agent_text_delta` | `text` | Streaming assistant text (captions / UI). | diff --git a/python/timbal/server/voice.html b/python/timbal/server/voice.html index 50e50052..9f0195a3 100644 --- a/python/timbal/server/voice.html +++ b/python/timbal/server/voice.html @@ -38,17 +38,96 @@ -webkit-font-smoothing: antialiased; } .shell { - max-width: 480px; + max-width: 1160px; margin: 0 auto; height: 100%; max-height: 100dvh; min-height: 0; - padding: 1.5rem 1rem 2rem; + padding: 1.5rem 1rem 1.5rem; display: flex; flex-direction: column; gap: 1rem; overflow: hidden; } + .layout { + flex: 1; + display: flex; + gap: 1rem; + min-height: 0; + } + .panel { + flex: 0 0 250px; + background: var(--surface); + border: 1px solid var(--border); + border-radius: var(--radius); + box-shadow: var(--shadow); + padding: 1rem; + display: flex; + flex-direction: column; + gap: 0.85rem; + min-height: 0; + overflow-y: auto; + } + .panel-title { + font-size: 0.6875rem; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.06em; + color: var(--text-muted); + } + .field { + display: flex; + flex-direction: column; + gap: 0.3rem; + } + .field-label { + font-size: 0.6875rem; + font-weight: 600; + color: var(--text-muted); + } + .panel select { + width: 100%; + min-width: 0; + font: inherit; + font-size: 0.8125rem; + color: var(--text); + background: var(--surface); + border: 1px solid var(--border-strong); + border-radius: var(--radius-sm); + padding: 0.5rem 0.65rem; + outline: none; + cursor: pointer; + } + .panel select:focus { + border-color: var(--primary); + box-shadow: 0 0 0 3px rgba(0,0,0,.07); + } + .model-picker { + display: flex; + flex-direction: column; + gap: 0.55rem; + } + .model-count { + font-size: 0.65625rem; + color: var(--text-muted); + margin-top: -0.15rem; + } + .panel-note { + font-size: 0.6875rem; + color: var(--text-muted); + line-height: 1.45; + margin-top: auto; + padding-top: 0.65rem; + border-top: 1px solid var(--border); + } + @media (max-width: 900px) { + body { overflow: auto; } + .shell { max-height: none; height: auto; } + .layout { flex-direction: column; overflow: visible; } + .panel { flex: none; width: auto; overflow: visible; } + .card { order: -1; min-height: 60dvh; } + .panel-note { margin-top: 0; } + } .top { display: flex; align-items: center; @@ -100,9 +179,8 @@ flex-direction: column; align-items: flex-start; gap: 0.35rem; - padding: 0.75rem 1rem; + padding-bottom: 0.75rem; border-bottom: 1px solid var(--border); - background: linear-gradient(180deg, #fafafa 0%, var(--surface) 100%); } .conn-version { font-size: 0.75rem; @@ -164,35 +242,20 @@ 50% { opacity: 0.5; transform: scale(0.85); } } - #mic-select, #td-select { - flex: 1; - min-width: 0; - font: inherit; - font-size: 0.8125rem; - color: var(--text); - background: var(--surface); - border: 1px solid var(--border-strong); - border-radius: var(--radius-sm); - padding: 0.5rem 0.65rem; - outline: none; - cursor: pointer; - } - #mic-select:focus, #td-select:focus { - border-color: var(--primary); - box-shadow: 0 0 0 3px rgba(0,0,0,.07); - } - #td-select { - flex: 0 1 auto; - max-width: 46%; + .mic-row { + display: flex; + align-items: flex-end; + gap: 0.5rem; } + .mic-row .field { flex: 1; min-width: 0; } .mic-viz { display: flex; align-items: flex-end; justify-content: center; gap: 3px; height: 28px; - flex: 0 0 56px; - padding-bottom: 2px; + flex: 0 0 40px; + padding-bottom: 5px; } .mic-bar { width: 4px; @@ -238,30 +301,6 @@ pointer-events: none; } #voice-empty[hidden] { display: none !important; } - .orb-wrap { - width: 72px; - height: 72px; - margin-bottom: 1rem; - position: relative; - } - .orb { - position: absolute; - inset: 12px; - border-radius: 50%; - background: radial-gradient(circle at 30% 30%, #fafafa, #e4e4e7); - border: 1px solid var(--border); - } - .card.card--live #voice-empty .orb-wrap .ring { - position: absolute; - inset: 0; - border-radius: 50%; - border: 2px solid var(--success-soft); - animation: ring-pulse 2s ease-out infinite; - } - @keyframes ring-pulse { - 0% { transform: scale(0.85); opacity: 0.8; } - 100% { transform: scale(1.35); opacity: 0; } - } .empty-title { font-size: 0.9375rem; font-weight: 600; @@ -272,9 +311,21 @@ font-size: 0.8125rem; color: var(--text-muted); margin-top: 0.35rem; - max-width: 240px; + max-width: 280px; line-height: 1.4; } + .msg.status { + margin: 0.35rem auto 0.65rem; + max-width: 90%; + text-align: center; + color: var(--text-muted); + font-size: 0.75rem; + font-style: italic; + padding: 0.35rem 0.65rem; + border-radius: 999px; + background: rgba(0, 0, 0, 0.03); + border: 1px solid var(--border); + } .msg { margin-bottom: 0.65rem; max-width: 90%; } .msg.user { @@ -357,16 +408,6 @@ color: var(--text-muted); line-height: 1.35; } - .foot-row--mic { - display: flex; - align-items: center; - gap: 0.5rem; - min-width: 0; - } - .foot-row--mic #mic-select { - flex: 1; - min-width: 0; - } .foot-actions { display: flex; align-items: center; @@ -399,32 +440,48 @@ opacity: 0.4; cursor: not-allowed; } - .btn-stop { + .btn-session { flex-shrink: 0; width: 2.5rem; height: 2.5rem; padding: 0; border: none; border-radius: 10px; - background: var(--danger); display: inline-flex; align-items: center; justify-content: center; box-shadow: 0 1px 2px rgba(0, 0, 0, 0.1); } - .btn-stop:hover:not(:disabled) { - background: #b91c1c; - } - .btn-stop:focus-visible { + .btn-session:focus-visible { outline: 2px solid var(--primary); outline-offset: 2px; } - .btn-stop-icon { + .btn-session[data-mode="stop"] { + background: var(--danger); + } + .btn-session[data-mode="stop"]:hover:not(:disabled) { + background: #b91c1c; + } + .btn-session[data-mode="start"] { + background: var(--primary); + } + .btn-session[data-mode="start"]:hover:not(:disabled) { + opacity: 0.9; + } + .btn-session-icon--stop { width: 11px; height: 11px; border-radius: 2px; background: var(--primary-fg); } + .btn-session-icon--start { + width: 0; + height: 0; + margin-left: 2px; + border-style: solid; + border-width: 7px 0 7px 12px; + border-color: transparent transparent transparent var(--primary-fg); + } button.primary { background: var(--primary); color: var(--primary-fg); @@ -449,64 +506,111 @@ -
-
- -
+ + + diff --git a/python/timbal/server/voice.py b/python/timbal/server/voice.py index e32b783a..9df818e4 100644 --- a/python/timbal/server/voice.py +++ b/python/timbal/server/voice.py @@ -38,6 +38,7 @@ def default_voice_config_from_env() -> dict[str, Any]: """STT/TTS defaults for ``/voice/ws`` (ElevenLabs). Override with env or ``runnable.voice_config``.""" return { + "stt_provider": os.environ.get("TIMBAL_STT_PROVIDER", "elevenlabs"), "stt_model": os.environ.get("TIMBAL_STT_MODEL", "scribe_v2_realtime"), "tts_model": os.environ.get("TIMBAL_TTS_MODEL", "eleven_flash_v2_5"), "voice": (os.environ.get("ELEVENLABS_VOICE_ID") or os.environ.get("TIMBAL_VOICE_ID") or _DEFAULT_VOICE_ID), @@ -81,7 +82,7 @@ def merge_client_voice_overrides(server_defaults: dict[str, Any], client: dict[s return {**server_defaults, **{k: v for k, v in client.items() if v is not None}} -def runnable_meta_for_voice_page(runnable: Any, import_spec: str) -> dict[str, str]: +def runnable_meta_for_voice_page(runnable: Any, import_spec: str) -> dict[str, Any]: """Serializable identity for the voice UI (same object as ``/run``).""" name = str(getattr(runnable, "name", "") or "").strip() kind = "" @@ -90,7 +91,26 @@ def runnable_meta_for_voice_page(runnable: Any, import_spec: str) -> dict[str, s kind = str(md["type"]) if not kind: kind = type(runnable).__name__ - return {"name": name, "kind": kind, "import_spec": (import_spec or "").strip()} + model = getattr(runnable, "model", None) + model_s = str(model).strip() if isinstance(model, str) else "" + # Slim catalog for the playground model picker (from models.yaml via codegen). + from ..codegen.model_discovery import get_models + + models = [ + { + "id": m["id"], + "provider": m["provider"], + "display_name": m.get("display_name") or m["id"].split("/", 1)[-1], + } + for m in get_models() + ] + return { + "name": name, + "kind": kind, + "import_spec": (import_spec or "").strip(), + "model": model_s, + "models": models, + } _VOICE_HTML_META_TOKEN = "__TIMBAL_VOICE_RUNNABLE_META_JSON__" @@ -101,16 +121,15 @@ async def warmup_voice_stack(voice_config: dict[str, Any]) -> None: Two tiers, both best-effort: - * **Imports** (always): ``timbal.voice`` + the ElevenLabs adapter pull in - websockets/pydantic machinery, and the ``timbal[voice]`` extra adds - numpy + onnxruntime — together ~1s of import time that otherwise lands + * **Imports** (always): voice adapters (ElevenLabs + Deepgram) and the + ``timbal[voice]`` extra (numpy/onnxruntime) — ~1s that otherwise lands on the first WebSocket connection. - * **Models** (only when the *server-side* default ``turn_detector`` is a - "local" mode string): resolve the detector, load Smart Turn + Silero, - and run their warmup inference. Skipped otherwise — a client can still - pick "local" from the playground dropdown, in which case the load - happens at that session's startup (before "listening", off the - first-reply path). + * **Models**: load Smart Turn + Namo + Silero when the server default + turn detector is local (mode string **or** a ``LocalAudioTurnDetector`` + instance — demos often set the resolved instance on ``voice_config``). + Eager-loads those ONNX models whenever the voice extra is installed so + playground users who pick "Smart Turn" on first Start don't eat the + HuggingFace cold path mid-handshake. """ loop = asyncio.get_running_loop() @@ -118,8 +137,10 @@ def _import_stack() -> None: import importlib importlib.import_module("timbal.voice.elevenlabs") + importlib.import_module("timbal.voice.deepgram") try: importlib.import_module("timbal.voice.smart_turn") + importlib.import_module("timbal.voice.namo") importlib.import_module("timbal.voice.vad") except ImportError: pass # timbal[voice] extra not installed — heuristics only @@ -130,18 +151,26 @@ def _import_stack() -> None: logger.debug("voice_warmup_import_failed", error=str(e)) return - td = voice_config.get("turn_detector") - if not (isinstance(td, str) and td.strip().lower() in ("local", "audio", "smart_turn")): - return try: from ..voice.turn_detection import LocalAudioTurnDetector, resolve_turn_detector from ..voice.vad import SileroVad - detector = resolve_turn_detector(td) - if isinstance(detector, LocalAudioTurnDetector) and detector.audio_eou is not None: - await detector.audio_eou.start(sample_rate=16_000) - await SileroVad().start(sample_rate=16_000) - logger.info("voice_models_warmed") + td = voice_config.get("turn_detector") + detector = None + if isinstance(td, LocalAudioTurnDetector): + detector = td + elif isinstance(td, str) and td.strip().lower() in ("local", "audio", "smart_turn"): + detector = resolve_turn_detector(td) + elif td is None: + # Playground often switches to Smart Turn on first Start — warm it. + detector = resolve_turn_detector("local") + + if isinstance(detector, LocalAudioTurnDetector): + from ..voice.session import AudioInputConfig + + await detector.start(AudioInputConfig(sample_rate=16_000)) + await SileroVad().start(sample_rate=16_000) + logger.info("voice_models_warmed") except Exception as e: logger.debug("voice_warmup_models_failed", error=str(e)) @@ -169,6 +198,7 @@ async def voice_page(request: Request) -> HTMLResponse: async def voice_ws(ws: WebSocket) -> None: from ..core.agent import Agent from ..voice import ( + AgentStatus, AgentTextDelta, AgentTextDone, AudioInputConfig, @@ -184,7 +214,13 @@ async def voice_ws(ws: WebSocket) -> None: VoiceSession, VoiceSessionEvent, ) - from ..voice.elevenlabs import ElevenLabsRealtimeSTT, ElevenLabsStreamTTS + from ..voice.deepgram import ( + DeepgramFluxSTT, + effective_stt_model, + resolve_stt, + stt_provider_id, + ) + from ..voice.elevenlabs import ElevenLabsStreamTTS await ws.accept() logger.info("voice_ws_connected") @@ -237,15 +273,38 @@ async def voice_ws(ws: WebSocket) -> None: defaults: dict = getattr(ws.app.state, "voice_config", None) or {} merged = merge_client_voice_overrides(defaults, config) - stt = ElevenLabsRealtimeSTT() + stt_provider = merged.get("stt_provider") + stt_model_requested = merged.get("stt_model") + try: + stt = resolve_stt(stt_provider, model=stt_model_requested) + except ValueError as e: + logger.warning( + "voice_ws_bad_stt_provider", + error=str(e), + requested_provider=stt_provider, + requested_model=stt_model_requested, + ) + # Fallback must not keep a Flux/Nova model id on the ElevenLabs wire, + # or the client/config log will claim Deepgram while Scribe runs. + stt = resolve_stt("elevenlabs") + stt_model_requested = None + stt_is_flux = isinstance(stt, DeepgramFluxSTT) + # Config id for clients/logs (``deepgram-flux``), not the class name. + stt_provider = stt_provider_id(stt) + stt_model = effective_stt_model(stt, stt_model_requested) tts = ElevenLabsStreamTTS() + stt_extra = dict(merged.get("stt_extra", {})) + if stt_is_flux: + # Scribe-tuned VAD knobs don't apply to Flux's turn machine. + for k in ("commit_strategy", "min_speech_duration_ms", "vad_silence_threshold_secs", "vad_threshold"): + stt_extra.pop(k, None) audio_in = AudioInputConfig( - model=merged.get("stt_model"), + model=stt_model, language=merged.get("language"), sample_rate=merged.get("sample_rate", 16_000), encoding=merged.get("encoding", "pcm_s16le"), - extra=merged.get("stt_extra", {}), + extra=stt_extra, ) audio_out = AudioOutputConfig( model=merged.get("tts_model"), @@ -269,6 +328,19 @@ async def voice_ws(ws: WebSocket) -> None: raw_td = client_td elif client_td is not None and not isinstance(client_td, str): logger.warning("voice_ws_bad_turn_detector", error="client turn_detector must be a mode name string") + if stt_is_flux: + # Flux owns EOU (~260ms). Local/lexical run *after* EndOfTurn and add + # a second HOLD tax; they also disable the useful Provider path. Force + # provider unless the client explicitly picked heuristic/raw/provider. + td_mode = raw_td.strip().lower() if isinstance(raw_td, str) else None + if td_mode is None or td_mode in ("local", "audio", "smart_turn", "lexical"): + if td_mode is not None: + logger.info( + "voice_ws_flux_overrides_turn_detector", + requested=raw_td, + using="provider", + ) + raw_td = "provider" if raw_td is not None: try: # voice_config is process-wide; VoiceSession clones the resolved @@ -284,8 +356,28 @@ async def voice_ws(ws: WebSocket) -> None: vad_endpointing = merged.get("vad_endpointing") if not isinstance(vad_endpointing, bool): vad_endpointing = None + if stt_is_flux: + # Flux has no force-commit (commit() is a no-op); the Silero fast path + # would just burn CPU scoring audio it can never act on. + vad_endpointing = False + + # Playground / client may override the Agent's LLM for this session only. + raw_model = merged.get("model") + model_override = ( + raw_model.strip() + if isinstance(raw_model, str) and "/" in raw_model.strip() + else None + ) + llm_model = model_override or ( + str(runnable.model) if isinstance(getattr(runnable, "model", None), str) else None + ) logger.info( "voice_ws_session_config", + stt=type(stt).__name__, + stt_provider=stt_provider, + stt_model=stt_model, + stt_model_requested=merged.get("stt_model"), + model=llm_model, turn_detector=turn_detector_label, vad_endpointing="auto" if vad_endpointing is None else vad_endpointing, ) @@ -301,6 +393,7 @@ async def voice_ws(ws: WebSocket) -> None: audio_output=audio_out, turn_detector=turn_detector, vad_endpointing=vad_endpointing, + model=model_override, ) async def _recv_loop() -> None: @@ -374,6 +467,11 @@ async def _handle(event: VoiceSessionEvent) -> None: { "type": "session_started", "playback_acks": "recommended", + # Config id (``elevenlabs`` / ``deepgram-flux`` / …), not + # the STT class name — matches playground select values. + "stt_provider": stt_provider, + "stt_model": stt_model, + "model": llm_model, "turn_detector": turn_detector_label, # The endpointer arms during session startup (before this # event is emitted), so this reflects the real state — not @@ -384,7 +482,12 @@ async def _handle(event: VoiceSessionEvent) -> None: elif isinstance(event, TranscriptPartial): await _send_json({"type": "transcript_partial", "text": event.text}) elif isinstance(event, TranscriptCommitted): - await _send_json({"type": "transcript_committed", "text": event.text}) + payload: dict = {"type": "transcript_committed", "text": event.text} + if event.replace: + payload["replace"] = True + await _send_json(payload) + elif isinstance(event, AgentStatus): + await _send_json({"type": "agent_status", "text": event.text}) elif isinstance(event, AgentTextDelta): await _send_json({"type": "agent_text_delta", "text": event.text}) elif isinstance(event, AgentTextDone): diff --git a/python/timbal/utils/serialization.py b/python/timbal/utils/serialization.py index 8e78b656..21cdc2fc 100644 --- a/python/timbal/utils/serialization.py +++ b/python/timbal/utils/serialization.py @@ -225,20 +225,29 @@ def _next(): def coerce_to_dict(v: Any) -> dict[str, Any]: - """Utility function to convert LLM outputs into python objects.""" + """Utility function to convert LLM outputs into python objects. + + Providers often emit ``null`` / ``"null"`` / ``None`` for tools with no + parameters (Groq + OpenAI-compatible chat completions). Treat those as ``{}``. + """ + if v is None: + return {} if isinstance(v, dict): return v - elif isinstance(v, str): - if v.strip() == "": + if isinstance(v, str): + stripped = v.strip() + if not stripped or stripped.lower() in ("null", "none"): return {} try: - v = json.loads(v) - return v + parsed = json.loads(stripped) except Exception: try: - v = literal_eval(v) - return v + parsed = literal_eval(stripped) except Exception as e: raise ValueError(f"Cannot coerce value to dict: {v}") from e - else: + if parsed is None: + return {} + if isinstance(parsed, dict): + return parsed raise ValueError(f"Cannot coerce value to dict: {v}") + raise ValueError(f"Cannot coerce value to dict: {v}") diff --git a/python/timbal/voice/__init__.py b/python/timbal/voice/__init__.py index e035d59c..487c7c10 100644 --- a/python/timbal/voice/__init__.py +++ b/python/timbal/voice/__init__.py @@ -24,6 +24,7 @@ RealtimeSession, ) from .session import ( + AgentStatus, AgentTextDelta, AgentTextDone, AudioInputConfig, @@ -60,12 +61,25 @@ def __getattr__(name: str): - # Lazy: importing smart_turn / vad pulls numpy/onnxruntime (timbal[voice] - # extra), which must not be required just to import timbal.voice. + # Lazy: importing smart_turn / namo / vad pulls numpy/onnxruntime / + # transformers (timbal[voice] extra), which must not be required just to + # import timbal.voice. + if name in ("DeepgramFluxSTT", "DeepgramNovaSTT", "resolve_stt", "stt_provider_id"): + from . import deepgram + + return getattr(deepgram, name) + if name == "ElevenLabsRealtimeSTT": + from .elevenlabs import ElevenLabsRealtimeSTT + + return ElevenLabsRealtimeSTT if name == "SmartTurnEouModel": from .smart_turn import SmartTurnEouModel return SmartTurnEouModel + if name == "NamoTextEouPredictor": + from .namo import NamoTextEouPredictor + + return NamoTextEouPredictor if name == "SileroVad": from .vad import SileroVad @@ -74,6 +88,7 @@ def __getattr__(name: str): __all__ = [ + "AgentStatus", "AgentTextDelta", "AgentTextDone", "AudioEouModel", @@ -83,10 +98,14 @@ def __getattr__(name: str): "BufferedPlaybackTracker", "CommitAction", "CommitDecision", + "DeepgramFluxSTT", + "DeepgramNovaSTT", + "ElevenLabsRealtimeSTT", "EouPredictor", "HeuristicTurnDetector", "LexicalTurnDetector", "LocalAudioTurnDetector", + "NamoTextEouPredictor", "PartialDecision", "PlaybackTracker", "ProviderTurnDetector", @@ -118,5 +137,7 @@ def __getattr__(name: str): "VoiceSession", "VoiceSessionEvent", "endpointing_delay", + "resolve_stt", "resolve_turn_detector", + "stt_provider_id", ] diff --git a/python/timbal/voice/deepgram.py b/python/timbal/voice/deepgram.py new file mode 100644 index 00000000..a852795f --- /dev/null +++ b/python/timbal/voice/deepgram.py @@ -0,0 +1,490 @@ +"""Deepgram realtime STT for :class:`~timbal.voice.VoiceSession`. + +Two providers over raw WebSockets (no Deepgram SDK): + +* :class:`DeepgramFluxSTT` — ``wss://api.deepgram.com/v2/listen`` (Flux). + Model-native end-of-turn detection: ``TurnInfo`` events carry the whole + turn transcript plus an EOU confidence. ``EndOfTurn`` maps to a committed + transcript, so pair it with ``ProviderTurnDetector`` — Flux already did the + Smart Turn / Namo / HOLD work server-side (~260ms EOU, ``eot_threshold``). +* :class:`DeepgramNovaSTT` — ``wss://api.deepgram.com/v1/listen`` (Nova-3 & + friends). ASR only: interim results map to partials; ``is_final`` segments + are buffered and flushed as one committed utterance on ``speech_final`` + (Deepgram's documented recipe). Timbal turn detection stays fully engaged, + including the VAD endpointing fast path via ``Finalize``. + +Requires ``websockets`` (already a core dep) and ``DEEPGRAM_API_KEY``. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import json +import os +from collections.abc import AsyncIterator +from typing import Any +from urllib.parse import urlencode + +import structlog +from pydantic import SecretStr +from websockets.asyncio.client import connect as ws_connect +from websockets.exceptions import ConnectionClosed + +from .session import ( + AudioInputConfig, + SpeechToText, + TranscriptEvent, +) + +logger = structlog.get_logger("timbal.voice.deepgram") + +_DG_HOST = "api.deepgram.com" + +DEFAULT_FLUX_MODEL = "flux-general-multi" +DEFAULT_NOVA_MODEL = "nova-3" + +# Flux docs strongly recommend ~80ms chunks for model latency; Nova is happy +# with the same cadence (their examples use 100ms). Flush as soon as we have +# that much PCM16 mono @ 16 kHz — don't wait for a timer tick (timer-only +# flush made Flux Update partials feel "wait until you shut up"). +_AUDIO_FLUSH_INTERVAL = 0.08 +_AUDIO_FLUSH_BYTES = int(16_000 * _AUDIO_FLUSH_INTERVAL * 2) # 2560 +# Nova drops the socket with NET-0001 after ~10s without audio; send KeepAlive +# well inside that window (Deepgram recommends every 3-5s). +_NOVA_KEEPALIVE_INTERVAL = 5.0 + +# Query params Flux accepts (used to filter stt_extra passthrough). +_FLUX_QUERY_KEYS = frozenset( + { + "eot_threshold", + "eager_eot_threshold", + "eot_timeout_ms", + "keyterm", + "tag", + "mip_opt_out", + "profanity_filter", + "numerals", + "redact", + } +) + + +def _resolve_api_key(explicit: str | SecretStr | None) -> str: + if isinstance(explicit, SecretStr): + return explicit.get_secret_value() + if explicit: + return explicit + key = os.environ.get("DEEPGRAM_API_KEY") + if not key: + raise ValueError("Set DEEPGRAM_API_KEY or pass api_key to the provider.") + return key + + +def _encoding_param(config: AudioInputConfig) -> str: + """Timbal speaks PCM16LE mono end-to-end; Deepgram calls that ``linear16``.""" + if config.encoding in ("pcm_s16le", "linear16", ""): + return "linear16" + return config.encoding + + +class _DeepgramSTTBase(SpeechToText): + """Shared WS plumbing: buffered audio flusher, receiver task, event queue.""" + + def __init__(self, api_key: str | SecretStr | None = None) -> None: + self._api_key_explicit = api_key + self._api_key: str | None = None + self._ws: Any = None + self._buf = bytearray() + # Covers buffer mutation *and* ``_ws.send``: ``push_audio`` (threshold + # flush) and ``_flush_loop`` both drain PCM onto the socket; without a + # single lock those awaits interleave and Deepgram can see out-of-order + # or concurrent frames. Control frames (KeepAlive / Finalize / Close) + # share it too. + self._wire_lock = asyncio.Lock() + self._stop = asyncio.Event() + self._flusher: asyncio.Task[None] | None = None + self._receiver: asyncio.Task[None] | None = None + self._queue: asyncio.Queue[TranscriptEvent | None] = asyncio.Queue() + self._input_config: AudioInputConfig | None = None + + def _build_uri(self, config: AudioInputConfig) -> str: + raise NotImplementedError + + async def connect(self, config: AudioInputConfig) -> None: + self._api_key = _resolve_api_key(self._api_key_explicit) + self._input_config = config + uri = self._build_uri(config) + logger.debug("dg_stt_connecting", uri=uri[:160]) + self._ws = await ws_connect( + uri, + additional_headers={"Authorization": f"Token {self._api_key}"}, + ) + self._stop.clear() + self._flusher = asyncio.create_task(self._flush_loop()) + self._receiver = asyncio.create_task(self._receive_loop()) + + async def push_audio(self, chunk: bytes) -> None: + """Forward mic PCM to Deepgram with minimal buffering. + + Flux wants ~80ms frames for low-latency ``Update`` partials; a + timer-only flusher made captions feel commit-gated ("wait until you + shut up"). Send as soon as we have ≥80ms, and let the flush loop + drain any trailing remainder. + """ + if not chunk: + return + async with self._wire_lock: + self._buf.extend(chunk) + if len(self._buf) < _AUDIO_FLUSH_BYTES or self._ws is None: + return + raw = bytes(self._buf) + self._buf.clear() + try: + await self._ws.send(raw) + except ConnectionClosed: + pass + + async def _flush_audio(self) -> None: + async with self._wire_lock: + if self._ws is None: + return + raw = bytes(self._buf) + self._buf.clear() + if not raw: + return + try: + await self._ws.send(raw) + except ConnectionClosed: + pass + + async def _flush_loop(self) -> None: + try: + while not self._stop.is_set(): + await asyncio.sleep(_AUDIO_FLUSH_INTERVAL) + await self._flush_audio() + except asyncio.CancelledError: + raise + except ConnectionClosed: + pass + + async def _handle_message(self, msg: dict[str, Any]) -> None: + raise NotImplementedError + + async def _receive_loop(self) -> None: + assert self._ws is not None + try: + async for raw_msg in self._ws: + if isinstance(raw_msg, bytes): + continue + try: + msg = json.loads(raw_msg) + except ValueError: + continue + await self._handle_message(msg) + except ConnectionClosed as e: + logger.debug("dg_stt_ws_closed", error=str(e)) + # Deepgram closes normally after CloseStream; only surface abnormal + # closures as errors so the session tears down loudly. + if not self._stop.is_set() and e.rcvd is not None and e.rcvd.code not in (1000, 1001): + await self._queue.put( + TranscriptEvent(type="error", text=f"STT connection closed: {e}") + ) + except Exception as e: + logger.error("dg_stt_receive_error", error=str(e), exc_info=True) + await self._queue.put(TranscriptEvent(type="error", text=f"STT receive error: {e}")) + finally: + await self._queue.put(None) + + async def events(self) -> AsyncIterator[TranscriptEvent]: + while True: + item = await self._queue.get() + if item is None: + break + if item.type == "error": + raise RuntimeError(item.text) + if item.text: + yield item + + async def _send_json(self, payload: dict[str, Any]) -> None: + async with self._wire_lock: + if self._ws is None: + return + with contextlib.suppress(Exception): + await self._ws.send(json.dumps(payload)) + + async def close(self) -> None: + self._stop.set() + if self._flusher and not self._flusher.done(): + self._flusher.cancel() + with contextlib.suppress(asyncio.CancelledError): + await self._flusher + self._flusher = None + with contextlib.suppress(Exception): + await self._flush_audio() + await self._send_json({"type": "CloseStream"}) + if self._ws is not None: + with contextlib.suppress(Exception): + await self._ws.close() + self._ws = None + if self._receiver and not self._receiver.done(): + with contextlib.suppress(asyncio.CancelledError, Exception): + await self._receiver + self._receiver = None + + +class DeepgramFluxSTT(_DeepgramSTTBase): + """Deepgram Flux (``/v2/listen``) — conversational STT with native EOU. + + ``TurnInfo`` mapping: + + * ``Update`` / ``StartOfTurn`` → ``partial`` (transcript is the whole turn + so far, matching how the session treats Scribe partials) + * ``EndOfTurn`` → ``committed`` + * ``EagerEndOfTurn`` / ``TurnResumed`` → logged only (speculative-LLM + lifecycle is a follow-up) + + ``commit()`` is a no-op: the Flux turn machine owns endpointing, there is + no client-side force-commit. Run with ``turn_detector="provider"`` and + VAD endpointing off (the server wires both automatically). + """ + + def _build_uri(self, config: AudioInputConfig) -> str: + extra = dict(config.extra) + host = str(extra.pop("stt_host", _DG_HOST)) + # Ignore foreign model ids (e.g. env default scribe_v2_realtime left in + # place when only TIMBAL_STT_PROVIDER was switched). + model = config.model if is_flux_model(config.model) else DEFAULT_FLUX_MODEL + + params: list[tuple[str, str]] = [ + ("model", model), + ("encoding", _encoding_param(config)), + ("sample_rate", str(config.sample_rate)), + ] + # Flux rejects `language`; multi accepts language_hint (repeatable). + if config.language and model.endswith("-multi"): + params.append(("language_hint", config.language)) + # Default 5s leaves stranded Update partials hanging in the UI until + # the session sweeper synthesizes a commit. Prefer Flux's own + # EndOfTurn; keep this under the session stale window (2.5s). + if "eot_timeout_ms" not in extra: + params.append(("eot_timeout_ms", "2000")) + for k, v in extra.items(): + if v is None or k.startswith("_") or k not in _FLUX_QUERY_KEYS: + continue + if isinstance(v, (list, tuple)): + params.extend((k, str(item)) for item in v) + else: + params.append((k, str(v).lower() if isinstance(v, bool) else str(v))) + return f"wss://{host}/v2/listen?{urlencode(params)}" + + async def commit(self) -> None: + """No-op: Flux's turn machine owns endpointing (no force-commit API).""" + + async def _handle_message(self, msg: dict[str, Any]) -> None: + mt = msg.get("type", "") + if mt == "Connected": + logger.info("dg_flux_session_started", request_id=msg.get("request_id")) + return + if mt == "TurnInfo": + event = msg.get("event", "") + text = (msg.get("transcript") or "").strip() + if event == "EndOfTurn": + logger.debug( + "dg_flux_end_of_turn", + eou_confidence=msg.get("end_of_turn_confidence"), + turn_index=msg.get("turn_index"), + ) + if text: + await self._queue.put(TranscriptEvent(type="committed", text=text)) + elif event in ("Update", "StartOfTurn"): + if text: + await self._queue.put(TranscriptEvent(type="partial", text=text)) + elif event in ("EagerEndOfTurn", "TurnResumed"): + # Speculative reply lifecycle not wired yet — visibility only. + logger.info( + "dg_flux_eager_event", + turn_event=event, + eou_confidence=msg.get("end_of_turn_confidence"), + text_preview=text[:80], + ) + return + if mt == "FatalError": + err = msg.get("description") or msg.get("message") or "Unknown Flux error" + logger.error("dg_flux_fatal", error=err, code=msg.get("code")) + await self._queue.put(TranscriptEvent(type="error", text=f"STT fatal: {err}")) + + +class DeepgramNovaSTT(_DeepgramSTTBase): + """Deepgram Nova (``/v1/listen``) — plain streaming ASR. + + Per Deepgram's endpointing guide, ``is_final`` segments are buffered and + concatenated; ``speech_final`` flushes the buffer as one committed + utterance. Interims map to partials (buffer + interim, so the session + sees the whole utterance so far, like Scribe). + + ``commit()`` sends ``Finalize`` — Deepgram flushes its audio buffer and + replies with a final (``from_finalize: true``) result, so the Timbal VAD + endpointing fast path works unchanged. + """ + + def __init__(self, api_key: str | SecretStr | None = None) -> None: + super().__init__(api_key) + self._segments: list[str] = [] + self._keepalive: asyncio.Task[None] | None = None + + def _build_uri(self, config: AudioInputConfig) -> str: + extra = dict(config.extra) + host = str(extra.pop("stt_host", _DG_HOST)) + + model = config.model or "" + if not model or is_flux_model(model) or model.startswith(("scribe", "eleven")): + model = DEFAULT_NOVA_MODEL + params: dict[str, Any] = { + "model": model, + "encoding": _encoding_param(config), + "sample_rate": str(config.sample_rate), + "channels": "1", + "interim_results": "true", + "smart_format": "true", + "punctuate": "true", + "endpointing": "300", + } + if config.language: + params["language"] = config.language + for k, v in extra.items(): + if v is not None and not k.startswith("_"): + params[k] = str(v).lower() if isinstance(v, bool) else str(v) + return f"wss://{host}/v1/listen?{urlencode(params)}" + + async def connect(self, config: AudioInputConfig) -> None: + await super().connect(config) + self._keepalive = asyncio.create_task(self._keepalive_loop()) + + async def _keepalive_loop(self) -> None: + try: + while not self._stop.is_set(): + await asyncio.sleep(_NOVA_KEEPALIVE_INTERVAL) + await self._send_json({"type": "KeepAlive"}) + except asyncio.CancelledError: + raise + + async def commit(self) -> None: + """Force-finalize whatever Deepgram is holding (VAD endpointing path).""" + await self._flush_audio() + await self._send_json({"type": "Finalize"}) + + async def _handle_message(self, msg: dict[str, Any]) -> None: + mt = msg.get("type", "") + if mt == "Results": + alternatives = (msg.get("channel") or {}).get("alternatives") or [{}] + text = (alternatives[0].get("transcript") or "").strip() + is_final = bool(msg.get("is_final")) + speech_final = bool(msg.get("speech_final")) + if not is_final: + if text: + partial = " ".join((*self._segments, text)) if self._segments else text + await self._queue.put(TranscriptEvent(type="partial", text=partial)) + return + if text: + self._segments.append(text) + # speech_final only rides on is_final frames; a Finalize response + # (from_finalize) also arrives as is_final and must flush too. + if (speech_final or msg.get("from_finalize")) and self._segments: + utterance = " ".join(self._segments) + self._segments = [] + await self._queue.put(TranscriptEvent(type="committed", text=utterance)) + return + if mt == "UtteranceEnd": + # Only reachable when the caller opted into utterance_end_ms. + if self._segments: + utterance = " ".join(self._segments) + self._segments = [] + await self._queue.put(TranscriptEvent(type="committed", text=utterance)) + return + if mt == "Metadata": + logger.info("dg_nova_session_started", request_id=msg.get("request_id")) + return + if mt == "Error": + err = msg.get("description") or msg.get("message") or "Unknown Nova error" + logger.error("dg_nova_error", error=err) + await self._queue.put(TranscriptEvent(type="error", text=f"STT error: {err}")) + + async def close(self) -> None: + if self._keepalive and not self._keepalive.done(): + self._keepalive.cancel() + with contextlib.suppress(asyncio.CancelledError): + await self._keepalive + self._keepalive = None + await super().close() + + +def is_flux_model(model: str | None) -> bool: + return bool(model) and model.strip().lower().startswith("flux") + + +def effective_stt_model(provider_instance: SpeechToText, requested: str | None) -> str | None: + """Model id actually sent to the provider (foreign leftovers swapped out).""" + if isinstance(provider_instance, DeepgramFluxSTT): + return requested if is_flux_model(requested) else DEFAULT_FLUX_MODEL + if isinstance(provider_instance, DeepgramNovaSTT): + m = requested or "" + if not m or is_flux_model(m) or m.startswith(("scribe", "eleven")): + return DEFAULT_NOVA_MODEL + return requested + # ElevenLabs (and any non-Deepgram STT): never pass flux/nova ids through — + # e.g. unknown-provider fallback keeps the merged model string otherwise. + m = (requested or "").strip().lower() + if not m or is_flux_model(m) or m.startswith("nova"): + return None + return requested + + +def stt_provider_id(provider_instance: SpeechToText) -> str: + """Config-style provider id for the running STT instance. + + Matches playground / ``voice_config`` values (``elevenlabs``, + ``deepgram-flux``, ``deepgram-nova``) — not the Python class name. + """ + if isinstance(provider_instance, DeepgramFluxSTT): + return "deepgram-flux" + if isinstance(provider_instance, DeepgramNovaSTT): + return "deepgram-nova" + return "elevenlabs" + + +def resolve_stt( + provider: str | None = None, + *, + model: str | None = None, + api_key: str | SecretStr | None = None, +) -> SpeechToText: + """STT factory for the voice server. + + ``provider`` is ``"elevenlabs"`` / ``"deepgram"`` (case-insensitive; also + accepts UI labels like ``"deepgram-flux"`` / ``"deepgram-nova"``). When + ``None``, inferred from the model id: ``flux-*`` / ``nova-*`` → Deepgram, + anything else (including ``scribe_*``) → ElevenLabs. + + Bare ``"deepgram"`` defaults to Flux (voice-agent native EOU). Only an + explicit ``nova`` in the provider label or a ``nova-*`` model selects Nova + — a leftover ``scribe_*`` env model must not silently route to Nova. + """ + p = (provider or "").strip().lower() + m = (model or "").strip().lower() + if not p: + p = "deepgram" if (m.startswith("flux") or m.startswith("nova")) else "elevenlabs" + if p in ("elevenlabs", "el", "11labs"): + from .elevenlabs import ElevenLabsRealtimeSTT + + return ElevenLabsRealtimeSTT(api_key=api_key) + if p.startswith("deepgram") or p == "dg": + # UI labels win. Bare "deepgram"/"dg" → Flux unless model is clearly nova-*. + if "nova" in p: + return DeepgramNovaSTT(api_key=api_key) + if "flux" in p: + return DeepgramFluxSTT(api_key=api_key) + if m.startswith("nova"): + return DeepgramNovaSTT(api_key=api_key) + return DeepgramFluxSTT(api_key=api_key) + raise ValueError(f"Unknown STT provider: {provider!r}") diff --git a/python/timbal/voice/endpointing.py b/python/timbal/voice/endpointing.py index 279bcc2e..3f80c78b 100644 --- a/python/timbal/voice/endpointing.py +++ b/python/timbal/voice/endpointing.py @@ -51,23 +51,26 @@ def endpointing_delay( p: float, *, - min_delay: float = 0.0, + min_delay: float = 0.45, max_delay: float = 3.0, curve: float = 2.0, ) -> float: """Map EOU probability to extra silence to wait before force-committing. ``delay = min + (max - min) * (1 - p) ** curve`` — smooth, monotonic - decreasing in ``p``. With the defaults (fired ~0.3s after speech stop: - 0.2s VAD silence + ~0.1s Smart Turn inference): + decreasing in ``p``. ``min_delay`` is a LiveKit-shaped floor: even a + confident-complete score still waits before force-committing, so a brief + thinking pause ("Uh, I don't know." → "tell me a story") can cancel the + pending endpoint via resumed speech. Defaults (fired ~0.3s after speech + stop: 0.2s VAD silence + ~0.1s Smart Turn inference): ======== ========= ========================================== p delay total silence before commit ======== ========= ========================================== - 0.95 ~0.01s ~0.3s (confident: LiveKit-class response) - 0.7 ~0.27s ~0.6s - 0.5 0.75s ~1.05s (unsure: still beats the provider) - <0.4 >1.1s provider debounce (~1.2s) commits first — + 0.95 0.45s ~0.75s (confident: LiveKit min_delay floor) + 0.7 ~0.68s ~1.0s + 0.5 ~1.1s ~1.4s (unsure) + <0.4 >1.4s provider debounce (~1.2s) often commits first — incomplete utterances fall through to the existing HOLD machinery untouched ======== ========= ========================================== @@ -108,12 +111,19 @@ class VadEndpointer: SPEECH_RESUME_SECS = 0.064 """Consecutive speech (2 frames) that cancels a pending endpoint — a single noise-spike frame must not kill a valid pending commit.""" - MIN_DELAY_SECS = 0.0 + MIN_DELAY_SECS = 0.45 + """LiveKit-shaped floor: never force-commit instantly on high-p scores.""" MAX_DELAY_SECS = 3.0 DELAY_CURVE = 2.0 """See :func:`endpointing_delay`.""" MIN_COMMIT_INTERVAL_SECS = 2.0 """Floor between force-commits (ElevenLabs throttles rapid commits).""" + TEXT_INCOMPLETE_DELAY_SECS = 1.2 + """When the latest STT partial looks incomplete (hedge / trailing-off), + bump the delay to at least this — same short tier as the commit-path + inverse HOLD. Gives the continuation time to cancel the pending endpoint.""" + TEXT_INCOMPLETE_THRESHOLD = 0.4 + """``P(complete)`` below this from the text scorer → treat as incomplete.""" def __init__( self, @@ -126,6 +136,7 @@ def __init__( max_delay_secs: float | None = None, delay_curve: float | None = None, min_commit_interval_secs: float | None = None, + text_incomplete_delay_secs: float | None = None, ) -> None: self._vad = vad self.speech_threshold = speech_threshold if speech_threshold is not None else self.SPEECH_THRESHOLD @@ -137,10 +148,16 @@ def __init__( self.min_commit_interval_secs = ( min_commit_interval_secs if min_commit_interval_secs is not None else self.MIN_COMMIT_INTERVAL_SECS ) + self.text_incomplete_delay_secs = ( + text_incomplete_delay_secs + if text_incomplete_delay_secs is not None + else self.TEXT_INCOMPLETE_DELAY_SECS + ) self._score: Callable[[], Awaitable[float | None]] | None = None self._commit: Callable[[], Awaitable[None]] | None = None self._should_commit: Callable[[], bool] | None = None + self._text_score: Callable[[], Awaitable[float | None]] | None = None self._started = False self._closed = False @@ -168,11 +185,20 @@ def bind( score: Callable[[], Awaitable[float | None]], commit: Callable[[], Awaitable[None]], should_commit: Callable[[], bool], + text_score: Callable[[], Awaitable[float | None]] | None = None, ) -> None: - """Attach the session callbacks. Must run before :meth:`start`.""" + """Attach the session callbacks. Must run before :meth:`start`. + + ``text_score`` is optional: when provided, returns ``P(complete)`` for + the latest STT partial (or ``None``). Used to bump the delay when the + transcript looks mid-thought even if the audio model is confident — + same LiveKit min/max shape, text as the confidence signal. Swappable + later for a DistilBERT-class text EOU without changing this wiring. + """ self._score = score self._commit = commit self._should_commit = should_commit + self._text_score = text_score async def start(self, *, sample_rate: int) -> None: """Load Silero (downloading/instantiating off the event loop) and arm. @@ -287,6 +313,18 @@ async def _run_endpoint(self) -> None: max_delay=self.max_delay_secs, curve=self.delay_curve, ) + text_bumped = False + p_text: float | None = None + if self._text_score is not None: + try: + p_text = await self._text_score() + except Exception as e: + logger.warning("vad_text_score_failed", error=str(e)) + p_text = None + if p_text is not None and p_text < self.TEXT_INCOMPLETE_THRESHOLD: + bumped = max(delay, self.text_incomplete_delay_secs) + text_bumped = bumped > delay + delay = bumped # INFO on purpose: fires once per speech-stop and is the whole # observable behaviour of the endpointing fast path. logger.info( @@ -294,6 +332,8 @@ async def _run_endpoint(self) -> None: p=round(p, 3), delay_secs=round(delay, 3), score_ms=round((time.monotonic() - t0) * 1000, 1), + p_text=round(p_text, 3) if p_text is not None else None, + text_incomplete_bump=text_bumped, ) remaining = delay - (time.monotonic() - t0) if remaining > 0: diff --git a/python/timbal/voice/eou.py b/python/timbal/voice/eou.py index ea6a5ec6..360b0221 100644 --- a/python/timbal/voice/eou.py +++ b/python/timbal/voice/eou.py @@ -95,30 +95,118 @@ async def predict_eou(self, text: str) -> float: } ) +# Thinking-pause / hedge phrases: often punctuated as complete by STT +# ("Uh, I don't know.") but the speaker continues. Short single-token hedges +# match only as the *entire* utterance; multi-word phrases also match as a +# trailing clause ("well i don't know"). Swappable later for a DistilBERT / +# Namo-class text EOU behind the same :class:`TextEouPredictor` interface. +_SHORT_HEDGES = frozenset( + { + "uh", "um", "uhm", "hmm", "mm", "mhm", "mmhmm", "well", "so", "like", + "maybe", "perhaps", "idk", "pues", "este", "eh", + } +) +_PHRASE_HEDGES = frozenset( + { + "i don't know", + "i dont know", + "i do not know", + "not sure", + "i'm not sure", + "im not sure", + "i am not sure", + "let me think", + "i mean", + "you know", + "i guess", + "no se", + "no sé", + "a ver", + "no lo se", + "no lo sé", + } +) +_PUNCT_STRIP_RE = re.compile(r"[.?!…。？！,;:—–\-]+") + + +def _normalize_utterance(text: str) -> str: + """Lowercase, strip punctuation to spaces, collapse whitespace, unify apostrophes.""" + t = text.lower().strip() + for a in ("'", "'", "`"): + t = t.replace(a, "'") + t = _PUNCT_STRIP_RE.sub(" ", t) + return re.sub(r"\s+", " ", t).strip() + + +def _prefix_is_filler(prefix: str) -> bool: + """True when ``prefix`` is only short hedges / dangling fillers (or empty).""" + if not prefix: + return True + return all(w in _SHORT_HEDGES or w in _DANGLING_TOKENS for w in prefix.split()) + + +def _looks_like_hedge(text: str) -> bool: + """True when the utterance is (or trails off into) a thinking-pause hedge. + + Whole-utterance hedges ("I don't know.", bare "uh") always match. + Trailing phrase hedges ("… I mean", "… you know") only match when the + prefix is filler — otherwise finished requests like "I want you to know." + / "Know what I mean?" would score incomplete after punctuation is stripped. + """ + norm = _normalize_utterance(text) + if not norm: + return False + if norm in _SHORT_HEDGES or norm in _PHRASE_HEDGES: + return True + for phrase in _PHRASE_HEDGES: + suffix = " " + phrase + if not norm.endswith(suffix): + continue + prefix = norm[: -len(suffix)].strip() + if _prefix_is_filler(prefix): + return True + return False + class PunctuationEouPredictor(TextEouPredictor): - """Zero-dep lexical EOU: terminal punctuation + trailing dangling tokens. + """Zero-dep lexical EOU: punctuation, dangling tokens, and hedges. Scores (tunable via subclass attributes): + * thinking-pause hedge ("i don't know", bare "uh"/"well", filler+"I mean") + → :attr:`P_HEDGE` — wins over terminal punctuation (STT writes + "Uh, I don't know.") but not over finished sentences that merely end + with the same words ("I want you to know.") * ends with terminal punctuation → :attr:`P_TERMINAL` - * ends with continuing punctuation → :attr:`P_CONTINUING` + * ends with continuing punctuation / ellipsis → :attr:`P_CONTINUING` * last word is a dangling conjunction/preposition/filler → :attr:`P_DANGLING` * otherwise → :attr:`P_NEUTRAL` (slightly complete-leaning; STT often drops the final period and over-merging is worse than under-merging) - English + Spanish dangling lists match the server's ``es`` default. + English + Spanish dangling lists match the server's ``es`` default. This + is the dumb baseline behind :class:`TextEouPredictor` — swap for a small + ONNX text classifier later without changing turn-detection wiring. """ P_TERMINAL: float = 0.95 P_CONTINUING: float = 0.15 P_DANGLING: float = 0.15 + P_HEDGE: float = 0.2 P_NEUTRAL: float = 0.60 async def predict_eou(self, text: str) -> float: stripped = text.strip() if not stripped: return 1.0 + # Hedges before terminal punct: "Uh, I don't know." must not score + # complete just because STT stuck a period on a mid-thought pause. + if _looks_like_hedge(stripped): + return self.P_HEDGE + # Ellipsis before the terminal check: STT writes "..." / "…" exactly + # when the speaker trails off mid-thought — the opposite of terminal, + # despite ending in ".". + if stripped.endswith("...") or stripped.endswith("…"): + return self.P_CONTINUING last_char = stripped[-1] if last_char in _TERMINAL_PUNCT: return self.P_TERMINAL diff --git a/python/timbal/voice/namo.py b/python/timbal/voice/namo.py new file mode 100644 index 00000000..d70c4d36 --- /dev/null +++ b/python/timbal/voice/namo.py @@ -0,0 +1,247 @@ +"""Namo Turn Detector v1 — DistilBERT text EOU behind :class:`TextEouPredictor`. + +Runs VideoSDK's open +`Namo-Turn-Detector-v1-English `_ +(Apache-2.0, DistilBERT, quantized ONNX, ~135MB, <11ms CPU) on the STT +transcript and returns ``P(complete)``. Complements Smart Turn (audio) with a +semantic text signal — the missing half for mid-thought hedges like +"Uh, I don't know." that audio alone over-scores as complete. + +The multilingual mmBERT checkpoint is available via ``repo_id`` / +``TIMBAL_NAMO_REPO_ID``, but measured English polarity is mushy on that +graph (card examples like "so that's all I have for today" score incomplete); +prefer the English specialist until VideoSDK ships a sharper multilingual. + +Requires the ``timbal[voice]`` extra (``onnxruntime`` + ``huggingface_hub`` + +``transformers``). Import fails without it; +:func:`~timbal.voice.resolve_turn_detector` falls back to +:class:`~timbal.voice.PunctuationEouPredictor`. + +Usage:: + + from timbal.voice import LocalAudioTurnDetector + from timbal.voice.namo import NamoTextEouPredictor + + detector = LocalAudioTurnDetector( + audio_eou=..., + fallback_text_eou=NamoTextEouPredictor(), + ) + +Or ``turn_detector="local"`` — the resolver injects Namo automatically when +the extra is installed. + +Label contract (per the model card): class ``1`` = end of turn, ``0`` = not +end of turn. We expose ``softmax(logits)[1]`` as ``P(complete)``. +""" + +from __future__ import annotations + +import asyncio +import os +import re +from functools import lru_cache, partial + +import numpy as np +import onnxruntime as ort +import structlog +from transformers import AutoTokenizer + +from .eou import TextEouPredictor +from .smart_turn import _hf_download_cached_first + +logger = structlog.get_logger("timbal.voice.namo") + +# Default: English DistilBERT specialist — decisive on measured hedges +# ("Uh, I don't know." → ~0.0 complete). Multilingual mmBERT is opt-in. +DEFAULT_REPO_ID = "videosdk-live/Namo-Turn-Detector-v1-English" +ENGLISH_REPO_ID = DEFAULT_REPO_ID +MULTILINGUAL_REPO_ID = "videosdk-live/Namo-Turn-Detector-v1-Multilingual" +DEFAULT_FILENAME = "model_quant.onnx" +# English DistilBERT uses 512; multilingual card uses 8192. Cap per-repo. +_MAX_LENGTH_BY_REPO = { + DEFAULT_REPO_ID: 512, + MULTILINGUAL_REPO_ID: 8192, +} +_DEFAULT_MAX_LENGTH = 512 +# Class index for "end of turn" / complete (model card). +_COMPLETE_LABEL = 1 + +_WORD_RE = re.compile(r"[^\W\d_]+", re.UNICODE) +_TRAILING_TERMINAL_RE = re.compile(r"[.!？。…]+$") +# Short backchannels Namo (and STT punctuation) mis-score as incomplete — +# measured: "Yeah."→0.04, "Yeah"→0.92, "Okay."/"Okay"→~0.00. Force complete. +_FORCE_COMPLETE_ACKS = frozenset( + { + "yeah", + "yep", + "yup", + "yes", + "ok", + "okay", + "sure", + "bye", + "goodbye", + "thanks", + "thank you", + "hi", + "hello", + "hey", + "no", + "nah", + "nope", + "alright", + "right", + "cool", + "fine", + "got it", + "mhmm", + "mmhmm", + "mhm", + } +) +_ACK_FORCE_P = 0.95 + + +def _prepare_text_for_namo(text: str) -> tuple[str, float | None]: + """Normalize short STT acks; optionally force ``P(complete)``. + + Returns ``(text_for_model, override_or_None)``. Empty → complete. + """ + stripped = text.strip() + if not stripped: + return "", 1.0 + words = _WORD_RE.findall(stripped) + if len(words) <= 3: + stripped = _TRAILING_TERMINAL_RE.sub("", stripped).strip() + key = " ".join(w.lower() for w in _WORD_RE.findall(stripped)) + if key in _FORCE_COMPLETE_ACKS: + return stripped, _ACK_FORCE_P + return stripped, None + + +@lru_cache(maxsize=4) +def _load_bundle( + repo_id: str, filename: str, cpu_count: int, max_length: int +) -> tuple[ort.InferenceSession, object]: + """One (ort session, tokenizer) per repo — shared process-wide.""" + path = _hf_download_cached_first(repo_id, filename) + logger.debug("namo_loading_model", path=path, repo_id=repo_id) + so = ort.SessionOptions() + so.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL + so.inter_op_num_threads = 1 + so.intra_op_num_threads = cpu_count + so.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL + session = ort.InferenceSession(path, sess_options=so) + tokenizer = AutoTokenizer.from_pretrained(repo_id) + # Warm the graph + tokenizer so the first live score isn't the cold path. + inputs = tokenizer( + "warmup", + truncation=True, + max_length=max_length, + return_tensors="np", + ) + session.run( + None, + { + "input_ids": inputs["input_ids"], + "attention_mask": inputs["attention_mask"], + }, + ) + logger.info("namo_text_eou_loaded", path=path, repo_id=repo_id) + return session, tokenizer + + +class NamoTextEouPredictor(TextEouPredictor): + """Local text end-of-turn scoring with the Namo DistilBERT ONNX checkpoint. + + Parameters + ---------- + repo_id: + Hugging Face repo. Defaults to the English DistilBERT specialist + (:data:`DEFAULT_REPO_ID`); pass :data:`MULTILINGUAL_REPO_ID` (or set + ``TIMBAL_NAMO_REPO_ID``) for mmBERT multilingual weights. Constructor + wins over the env var. + model_path: + Optional local ``.onnx`` path (skips the Hub download for the graph; + the tokenizer still loads from ``repo_id``). + cpu_count: + ``intra_op_num_threads`` for the ONNX session (1 is plenty). + + ``start()`` loads off the event loop; ``predict_eou()`` runs tokenize + + inference on the default executor. Instances are safe to share across + sessions. + """ + + def __init__( + self, + repo_id: str | None = None, + *, + model_path: str | None = None, + cpu_count: int = 1, + ) -> None: + self.repo_id = ( + repo_id + or os.environ.get("TIMBAL_NAMO_REPO_ID") + or DEFAULT_REPO_ID + ) + self.model_path = model_path + self.cpu_count = cpu_count + self.max_length = _MAX_LENGTH_BY_REPO.get(self.repo_id, _DEFAULT_MAX_LENGTH) + self._session: ort.InferenceSession | None = None + self._tokenizer: object | None = None + self._load_lock = asyncio.Lock() + + async def start(self) -> None: + async with self._load_lock: + if self._session is not None: + return + loop = asyncio.get_running_loop() + self._session, self._tokenizer = await loop.run_in_executor(None, self._load) + + async def close(self) -> None: + # Shared process-wide; keep alive. + pass + + def _load(self) -> tuple[ort.InferenceSession, object]: + if self.model_path is not None: + # Custom graph path: still need the matching tokenizer from the repo. + so = ort.SessionOptions() + so.intra_op_num_threads = self.cpu_count + session = ort.InferenceSession(self.model_path, sess_options=so) + tokenizer = AutoTokenizer.from_pretrained(self.repo_id) + return session, tokenizer + return _load_bundle(self.repo_id, DEFAULT_FILENAME, self.cpu_count, self.max_length) + + async def predict_eou(self, text: str) -> float: + if self._session is None: + await self.start() + loop = asyncio.get_running_loop() + return await loop.run_in_executor(None, partial(self._predict_sync, text)) + + def _predict_sync(self, text: str) -> float: + stripped, override = _prepare_text_for_namo(text) + if override is not None: + return override + if not stripped: + return 1.0 + inputs = self._tokenizer( + stripped, + truncation=True, + max_length=self.max_length, + return_tensors="np", + ) + outputs = self._session.run( + None, + { + "input_ids": inputs["input_ids"].astype(np.int64), + "attention_mask": inputs["attention_mask"].astype(np.int64), + }, + ) + logits = np.asarray(outputs[0][0], dtype=np.float64) + # Stable softmax → P(complete) = P(label=1). + shifted = logits - np.max(logits) + exp = np.exp(shifted) + probs = exp / np.sum(exp) + if probs.shape[0] <= _COMPLETE_LABEL: + return float(probs[-1]) + return float(probs[_COMPLETE_LABEL]) diff --git a/python/timbal/voice/session.py b/python/timbal/voice/session.py index 5e0f2ac4..983801c0 100644 --- a/python/timbal/voice/session.py +++ b/python/timbal/voice/session.py @@ -32,9 +32,9 @@ from ..core.agent import Agent from ..state import get_run_context, set_run_context from ..state.context import RunContext -from ..types.content import TextContent +from ..types.content import TextContent, ToolUseContent from ..types.events import OutputEvent -from ..types.events.delta import DeltaEvent, Text, TextDelta +from ..types.events.delta import DeltaEvent, Text, TextDelta, ToolUse from ..types.message import Message from .playback import BufferedPlaybackTracker, PlaybackTracker, map_played_bytes_to_text from .turn_detection import ( @@ -42,6 +42,7 @@ PartialDecision, TurnDetector, TurnState, + _is_same_user_utterance_refinement, resolve_turn_detector, ) @@ -167,6 +168,8 @@ class TranscriptPartial(VoiceSessionEvent): class TranscriptCommitted(VoiceSessionEvent): type: Literal["transcript_committed"] = "transcript_committed" text: str + # True → rewrite last user bubble (CONTINUE_TURN merge), don't append another. + replace: bool = False class AgentTextDelta(VoiceSessionEvent): @@ -179,6 +182,13 @@ class AgentTextDone(VoiceSessionEvent): text: str +class AgentStatus(VoiceSessionEvent): + """Non-transcript status for the UI (e.g. tool calls while the mic is idle).""" + + type: Literal["agent_status"] = "agent_status" + text: str + + class AudioOutput(VoiceSessionEvent): model_config = ConfigDict(arbitrary_types_allowed=True) type: Literal["audio_output"] = "audio_output" @@ -372,10 +382,14 @@ def __init__( record_audio: bool = False, hold_timeout_secs: float = 1.5, vad_endpointing: bool | VadEndpointer | None = None, + model: str | None = None, ): self.agent = agent self.stt = stt self.tts = tts + # Optional per-session LLM override (playground model picker). Passed + # through to ``agent(prompt=..., model=...)`` — does not mutate the Agent. + self.model = model.strip() if isinstance(model, str) and model.strip() else None # Always clone: the session owns the detector's start/push_audio/close # lifecycle, and the spec may be a shared instance (server voice_config) # or a factory returning a singleton. Inspect ``session.turn_detector``, @@ -431,11 +445,29 @@ def __init__( self._last_commit_at: float = 0.0 self._partials_since_last_commit: int = 0 self._last_partial_at: float = 0.0 + # Latest non-empty STT partial text — fed to the VAD endpointer's + # optional text_score so a mid-thought hedge can bump the delay even + # before the provider commits. + self._latest_partial_text: str = "" # A pending HOLD must not expire while the user is audibly mid-utterance # (recent STT partial): the upcoming commit merges with / supersedes the # hold. Must exceed the STT VAD silence threshold (~1.2s default) so the # commit always lands before the extended expiry re-fires. self._hold_partial_grace_secs = 2.0 + # When the last STT commit event arrived. Anchors the hold-expiry + # extension: only partials *newer* than the commit mean the user + # resumed speaking — the committed fragment's own trailing partial + # refinements must not stretch the hold. + self._commit_event_at: float = 0.0 + # Watchdog for transcripts the provider never commits: STT can emit a + # partial (e.g. quiet speech ducked by AEC during assistant playback) + # whose VAD never registers an utterance — no commit ever fires and the + # words hang as a "…" caption forever. After this much silence past the + # last partial (comfortably beyond the provider's ~1.2s debounce, so it + # only fires when the provider clearly won't), force ``stt.commit()``. + self._stale_partial_commit_secs = 2.5 + self._stale_partial_poll_secs = 0.5 + self._stale_commit_sent_at = 0.0 # Serial TTS runs off the agent ``async for`` critical path so we keep pulling # LLM/Agent events (and emit trace OUTPUT) while audio still synthesizes. @@ -527,6 +559,7 @@ async def run(self, audio_in: AsyncIterable[bytes]) -> AsyncIterator[VoiceSessio audio_task = asyncio.create_task(self._forward_audio(audio_in)) stt_task = asyncio.create_task(self._process_stt_events()) + sweep_task = asyncio.create_task(self._sweep_stale_partials()) try: while True: @@ -535,10 +568,10 @@ async def run(self, audio_in: AsyncIterable[bytes]) -> AsyncIterator[VoiceSessio break yield event finally: - for task in (audio_task, stt_task): + for task in (audio_task, stt_task, sweep_task): if not task.done(): task.cancel() - await asyncio.gather(audio_task, stt_task, return_exceptions=True) + await asyncio.gather(audio_task, stt_task, sweep_task, return_exceptions=True) except Exception as e: logger.error("voice_session_error", error=str(e), exc_info=True) @@ -603,6 +636,11 @@ async def interrupt(self, *, truncate_completed: bool = True) -> None: await self._current_turn_task except (asyncio.CancelledError, Exception): pass + # Long replies synthesize faster than the WS can drain: dozens of + # AudioOutput frames sit in ``_event_queue``. Drop them *before* + # SessionInterrupted so the client is not still scheduling backlog PCM + # for seconds after a barge-in. + dropped_audio = self._drop_queued_audio_output() if was_active else 0 if ( was_active and self._turn_finalized_ok @@ -651,6 +689,7 @@ async def interrupt(self, *, truncate_completed: bool = True) -> None: "session_interrupt_emitted", heard_text_preview=(heard[:120] if heard else heard), heard_bytes=self._turn_heard_bytes, + dropped_queued_audio=dropped_audio, **_trace_debug_fields(), ) @@ -674,7 +713,10 @@ def _start_llm_warmup(self) -> None: OpenAI). Only applies to string model specs — a ``TestModel`` (or any custom model object) has no provider connection to warm. """ - model = getattr(self.agent, "model", None) + # Prefer the per-session override (playground model picker) — turns use + # ``self.model`` the same way. Warming ``agent.model`` alone misses the + # provider the first reply will actually hit. + model = self.model or getattr(self.agent, "model", None) if not (isinstance(model, str) and "/" in model): return @@ -716,6 +758,7 @@ async def _maybe_start_endpointer(self) -> None: score=score_fn, commit=self._endpoint_commit, should_commit=self._endpoint_should_commit, + text_score=self._endpoint_text_score, ) try: await endpointer.start(sample_rate=self.audio_input.sample_rate) @@ -748,6 +791,27 @@ def _endpoint_should_commit(self) -> bool: return False return self._last_partial_at > self._last_commit_at + async def _endpoint_text_score(self) -> float | None: + """``P(complete)`` for the latest STT partial, for VAD delay bumping. + + Prefers :meth:`~timbal.voice.LocalAudioTurnDetector.effective_text_eou` + (Namo blended with lexical) so finished questions don't inflate the + incomplete-text delay when the model under-scores. Falls back to raw + ``fallback_text_eou`` / punctuation baseline. Returns ``None`` when + there is no fresh partial — endpointer keeps the audio-only delay. + """ + if self._last_partial_at <= self._last_commit_at or not self._latest_partial_text: + return None + effective = getattr(self.turn_detector, "effective_text_eou", None) + if callable(effective): + return await effective(self._latest_partial_text) + text_eou = getattr(self.turn_detector, "fallback_text_eou", None) + if text_eou is None: + from .eou import PunctuationEouPredictor + + text_eou = PunctuationEouPredictor() + return await text_eou.predict_eou(self._latest_partial_text) + async def _endpoint_commit(self) -> None: self._endpoint_commit_sent_at = time.monotonic() await self.stt.commit() @@ -787,6 +851,37 @@ def _vad_vetoes_barge_in(self, text: str) -> bool: ) return True + def _vad_contradicts_recent_partial(self) -> bool: + """True when the local VAD is healthy and saw no real speech energy + recently — a fresh STT partial is then a hallucination, not the user + mid-utterance. Conservative on missing evidence (no endpointer / + starved VAD → ``False``), mirroring :meth:`_vad_vetoes_barge_in`. + """ + if self._endpointer is None: + return False + speech_secs = self._endpointer.speech_secs_in_window(self.BARGE_IN_VAD_WINDOW_SECS) + return speech_secs is not None and speech_secs < self.MIN_BARGE_IN_VAD_SPEECH_SECS + + def _vad_confirms_speech_since(self, since_monotonic: float) -> bool: + """True when Silero saw real speech *after* ``since_monotonic``. + + HOLD expiry used to call :meth:`_vad_contradicts_recent_partial`, whose + 2s lookback still contains the utterance that armed the hold — so any + late STT refinement after commit "confirmed" speech and floored every + short text-complete HOLD at the 2s grace window (live: armed 0.35s, + expired ~2.0s). Missing VAD → ``False`` (don't stretch; a real resume + still supersedes via COMMIT). No endpointer → ``True`` so unit tests + without Silero keep the partial-extends-hold behavior. + """ + if self._endpointer is None: + return True + window = time.monotonic() - since_monotonic + if window <= 0: + return False + window = min(window, self.BARGE_IN_VAD_WINDOW_SECS) + speech_secs = self._endpointer.speech_secs_in_window(window) + return speech_secs is not None and speech_secs >= self.MIN_BARGE_IN_VAD_SPEECH_SECS + # -- Internal: audio → STT --------------------------------------------- async def _forward_audio(self, audio_in: AsyncIterable[bytes]) -> None: @@ -804,6 +899,63 @@ async def _forward_audio(self, audio_in: AsyncIterable[bytes]) -> None: logger.error("audio_forward_error", error=str(e), exc_info=True) await self._emit(SessionError(message=f"Audio input error: {e}")) + async def _sweep_stale_partials(self) -> None: + """Watchdog: force-commit transcripts the provider never commits. + + Failure mode (seen live): the user speaks quietly while the assistant + is playing — AEC ducks the near-end audio, the STT still transcribes a + partial, but neither the provider VAD nor Silero registers an + utterance. No commit ever fires and the words hang as a "…" caption + until the session dies. When a partial has gone + ``_stale_partial_commit_secs`` with no commit and no newer partial, + force ``stt.commit()`` so the transcript flows through the normal + ``_handle_committed`` path. + + Providers whose ``commit()`` is a no-op (Deepgram Flux) never answer + that nudge. After a short grace for Finalize-capable providers, we + synthesize a committed event from the stranded partial text so the + caption cannot hang forever. + """ + try: + while not self._closed: + await asyncio.sleep(self._stale_partial_poll_secs) + if self._closed: + return + if self._last_partial_at <= self._last_commit_at: + continue + # One forced commit per stranded partial: an IGNOREd commit + # does not bump _last_commit_at, so without this guard a noise + # partial would retrigger a commit every poll. + if self._last_partial_at <= self._stale_commit_sent_at: + continue + stale_secs = time.monotonic() - self._last_partial_at + if stale_secs < self._stale_partial_commit_secs: + continue + self._stale_commit_sent_at = time.monotonic() + stranded = self._latest_partial_text + # INFO on purpose: this is the only trace that a transcript was + # rescued from a provider that silently refused to commit. + logger.info("stt_stale_partial_commit", stale_secs=round(stale_secs, 1)) + try: + await self.stt.commit() + except Exception as e: + logger.warning("stt_stale_partial_commit_failed", error=str(e)) + # Give Finalize-capable providers a beat to emit committed. + await asyncio.sleep(0.4) + if self._closed or not stranded: + continue + if ( + self._latest_partial_text == stranded + and self._last_partial_at > self._last_commit_at + ): + logger.info( + "stt_stale_partial_synthesized", + text_preview=stranded[:80], + ) + await self._handle_committed(stranded) + except asyncio.CancelledError: + return + # -- Internal: STT → turns --------------------------------------------- async def _process_stt_events(self) -> None: @@ -814,10 +966,15 @@ async def _process_stt_events(self) -> None: self._partials_since_last_commit += 1 if text: self._last_partial_at = time.monotonic() - await self._emit(TranscriptPartial(text=text)) + self._latest_partial_text = text decision = await self.turn_detector.on_partial(text, self._turn_state()) if decision is PartialDecision.BARGE_IN and self._vad_vetoes_barge_in(text): + # Hallucinated multi-word partials during TTS (no mic + # energy) — do not flash them in the playground caption. + # (_vad_vetoes_barge_in already logs stt_partial_barge_in_vetoed.) decision = PartialDecision.IGNORE + else: + await self._emit(TranscriptPartial(text=text)) if decision is PartialDecision.BARGE_IN: # INFO on purpose: a barge-in cancels TTS and truncates the # committed reply — when debugging "the agent went silent / @@ -876,18 +1033,36 @@ async def _arm_hold(self, text: str, timeout_secs: float) -> None: self._hold_armed_timeout_secs = timeout_secs self._last_commit_at = time.monotonic() + # Anchor for "user resumed speaking": partials older than the commit + # event are the held fragment's own trailing refinements and must not + # stretch the hold (they otherwise floor every expiry at the grace + # window instead of ``timeout_secs``). + anchor = self._commit_event_at if self._commit_event_at > 0 else time.monotonic() + async def _expire() -> None: me = asyncio.current_task() try: remaining = timeout_secs while True: await asyncio.sleep(remaining) - # Never fire mid-utterance: a recent STT partial means the - # user resumed speaking, and their commit is about to merge - # with / supersede this hold. Fire only after real silence. + # Never fire mid-utterance: a *new* STT partial since the + # commit means the user resumed speaking, and their commit + # is about to merge with / supersede this hold. Require + # post-commit mic energy — not "any speech in the last 2s", + # which still includes the held utterance itself and used + # to stretch every short HOLD out to the grace window. since_partial = time.monotonic() - self._last_partial_at - if since_partial < self._hold_partial_grace_secs: + if ( + self._last_partial_at > anchor + and since_partial < self._hold_partial_grace_secs + and self._vad_confirms_speech_since(anchor) + ): remaining = self._hold_partial_grace_secs - since_partial + logger.debug( + "stt_hold_extended", + remaining=round(remaining, 3), + timeout_secs=timeout_secs, + ) continue break except asyncio.CancelledError: @@ -901,6 +1076,22 @@ async def _expire() -> None: if self._hold_task is me: self._hold_task = None if held and not self._closed: + # A HOLD exists because the fragment looked incomplete. If it is + # still just a dangling token ("I", "the", "and") when the timer + # fires, promoting it to a user turn invents ghost replies + # (live: force-committed "I" → "The capital of France is Paris."). + from .eou import _DANGLING_TOKENS, _WORD_RE + + words = _WORD_RE.findall(held) + if len(words) == 1 and words[0].lower() in _DANGLING_TOKENS: + logger.info( + "stt_hold_expired_dropped", + text_preview=held[:120], + timeout_secs=timeout_secs, + reason="dangling_token", + **_trace_debug_fields(), + ) + return logger.info( "stt_hold_expired", text_preview=held[:120], @@ -941,7 +1132,8 @@ async def _begin_user_turn( self._transcript[-1] = TranscriptEntry(role="user", text=final_text) else: self._transcript.append(TranscriptEntry(role="user", text=final_text)) - await self._emit(TranscriptCommitted(text=final_text)) + replace_user_entry = False + await self._emit(TranscriptCommitted(text=final_text, replace=replace_user_entry)) self._active_turn_user_text = final_text self._turn_eou_at = time.monotonic() self._current_turn_task = asyncio.create_task(self._run_turn(final_text)) @@ -954,6 +1146,7 @@ async def _begin_user_turn( async def _handle_committed(self, text: str) -> None: if self._closed: return + self._commit_event_at = time.monotonic() # Any commit (endpointer-forced or provider debounce) makes a pending # VAD endpoint stale — the STT segment it targeted is already closed. if self._endpointer is not None: @@ -971,6 +1164,28 @@ async def _handle_committed(self, text: str) -> None: text_preview=text[:80], ) self._endpoint_commit_sent_at = None + # Late twin of a commit we already accepted (Flux EndOfTurn after a + # session-synthesized stale rescue, or provider double-final). The + # active-turn refinement gate misses this once the reply has finished + # and ``_active_turn_user_text`` is cleared — only then should this + # fire. Mid-turn / HOLD commits that look like refinements + # ("hello can" → "hello can you help…") must reach the detector so + # CONTINUE_TURN / merge can run. + if ( + not self._active_turn_user_text + and self._held_user_text is None + and self._transcript + and self._transcript[-1].role == "user" + and time.monotonic() - self._last_commit_at < 3.0 + and _is_same_user_utterance_refinement(self._transcript[-1].text, text) + ): + logger.info( + "stt_commit_ignored", + reason="late_duplicate", + text_preview=text[:160], + ) + self._partials_since_last_commit = 0 + return state = self._turn_state() self._partials_since_last_commit = 0 # Cancel the hold *timer* before awaiting the detector. Local audio EOU @@ -1032,11 +1247,19 @@ async def _handle_committed(self, text: str) -> None: ) if decision.action is CommitAction.HOLD: - # Stop any still-playing TTS (common right after agent_text_done). - # NEW_TURN / CONTINUE_TURN always interrupt first; HOLD must too or - # assistant audio keeps talking over the deferred user fragment. - await self.interrupt() - self._cancel_turn.clear() + # HOLD = "not sure yet" — do NOT chop an audible reply for a + # deferred fragment (echo-ish "Hello, hello." mid-TTS was wiping + # greetings). Partials that meant barge-in already interrupted. + # NEW_TURN / hold expiry / CONTINUE interrupt when the turn starts. + if self._assistant_audio_playing: + logger.info( + "stt_hold_defer_during_tts", + text_preview=final_text[:80], + reason=decision.reason, + ) + else: + await self.interrupt() + self._cancel_turn.clear() # Detector returns the full utterance to hold (refine/merge already applied). timeout = ( decision.hold_timeout_secs if decision.hold_timeout_secs is not None else self.hold_timeout_secs @@ -1119,7 +1342,10 @@ async def _run_turn(self, user_text: str) -> None: turn_phase = "creating_agent_generator" msg = Message(role="user", content=[TextContent(text=user_text)]) - agen = self.agent(prompt=msg) + agent_kwargs: dict[str, Any] = {"prompt": msg} + if self.model: + agent_kwargs["model"] = self.model + agen = self.agent(**agent_kwargs) async for event in agen: turn_phase = "awaiting_agent_event" if self._cancel_turn.is_set(): @@ -1132,6 +1358,12 @@ async def _run_turn(self, user_text: str) -> None: ) break + if isinstance(event, DeltaEvent) and isinstance(event.item, ToolUse) and event.item.name: + # Dead air while a tool runs — surface it so the playground + # doesn't look hung (live: get_datetime slept 3–5s with no UI). + await self._emit(AgentStatus(text=f"Calling {event.item.name}…")) + continue + if isinstance(event, DeltaEvent) and isinstance(event.item, TextDelta | Text): # Google (and others) often emit a full ``Text`` block first, then ``TextDelta`` tails. chunk = event.item.text if isinstance(event.item, Text) else event.item.text_delta @@ -1235,6 +1467,12 @@ async def _run_turn(self, user_text: str) -> None: # `out.stop_reason == "tool_use"` — second filler # hook point, after the text handling above so any # spoken text this turn suppresses the filler. + # Backup UI status when the provider only surfaces + # tool calls on the final Message (no ToolUse delta). + for block in out.content: + if isinstance(block, ToolUseContent) and block.name: + await self._emit(AgentStatus(text=f"Calling {block.name}…")) + break if not self._cancel_turn.is_set(): # Prefer API ``Message`` text, then streamed ``full_response``, so a # Unicode/stream mismatch does not drop ``_pending_tts_after_scheduled``. @@ -1817,6 +2055,29 @@ def _attach_metrics_to_trace(self, metrics: TurnMetrics) -> None: async def _emit(self, event: VoiceSessionEvent | None) -> None: await self._event_queue.put(event) + def _drop_queued_audio_output(self) -> int: + """Remove pending :class:`AudioOutput` frames so interrupt is not delayed. + + TTS can enqueue megabytes of PCM before the consumer (WS send) catches + up. Barge-in must surface ``SessionInterrupted`` immediately; unplayed + queued audio is discarded the same way the client clears its buffer. + Non-audio events stay in order. + """ + kept: list[VoiceSessionEvent | None] = [] + dropped = 0 + while True: + try: + event = self._event_queue.get_nowait() + except asyncio.QueueEmpty: + break + if isinstance(event, AudioOutput): + dropped += 1 + continue + kept.append(event) + for event in kept: + self._event_queue.put_nowait(event) + return dropped + async def _cleanup(self) -> None: self._cancel_hold() self._held_user_text = None diff --git a/python/timbal/voice/turn_detection.py b/python/timbal/voice/turn_detection.py index 5af95573..263fa3ab 100644 --- a/python/timbal/voice/turn_detection.py +++ b/python/timbal/voice/turn_detection.py @@ -282,6 +282,24 @@ def _looks_like_fresh_hold_utterance(text: str) -> bool: return True +_TERMINAL_END_CHARS = frozenset(".?!。？！") + + +def _looks_like_finished_utterance(text: str) -> bool: + """True when ``text`` ends with terminal punctuation (not an ellipsis trail-off). + + Used to refuse CONTINUE-merge of short STT crumbs onto an already-complete + turn ("…killer." + "No." → cancel+restart). Ellipsis / bare mid-thought + fragments stay "unfinished" so VAD splits can still CONTINUE. + """ + stripped = text.strip() + if not stripped: + return False + if stripped.endswith("...") or stripped.endswith("…"): + return False + return stripped[-1] in _TERMINAL_END_CHARS + + class HeuristicTurnDetector(TurnDetector): """Default detector: the regex/similarity heuristics tuned for ElevenLabs Scribe. @@ -304,6 +322,12 @@ class HeuristicTurnDetector(TurnDetector): EARLY_DUPLICATE_RATIO = 0.58 CONTINUATION_WINDOW_SECS = 3.0 CONTINUATION_MAX_CHARS = 30 + # After a finished commit has already started a turn, Scribe often emits a + # trailing crumb ("No.", "He", "Okay.") within tens of ms. CONTINUE-merging + # those cancels the LLM for a frankenstein — IGNORE ultra-early crumbs; + # later short barge-ins ("Stop.") still fall through to NEW_TURN. + TRAILING_CRUMB_WINDOW_SECS = 0.4 + TRAILING_CRUMB_MAX_WORDS = 2 async def on_partial(self, text: str, state: TurnState) -> PartialDecision: if not state.audio_playing or not text: @@ -398,8 +422,40 @@ async def on_committed(self, text: str, state: TurnState) -> CommitDecision: state.seconds_since_last_commit < self.CONTINUATION_WINDOW_SECS and len(text) < self.CONTINUATION_MAX_CHARS ): - combined = state.active_user_text.rstrip(", ") + " " + text - return CommitDecision(action=CommitAction.CONTINUE_TURN, text=combined, reason="continuation") + # Active already looks finished + trailer looks like its own + # utterance → not a VAD mid-phrase split. Ultra-early crumbs + # are STT ghosts (live: "…killer." then "No." at +55ms); + # anything later is a real short barge-in → NEW_TURN below. + if _looks_like_finished_utterance(state.active_user_text) and _looks_like_fresh_hold_utterance( + text + ): + words = _WORD_RE.findall(text) + if ( + state.seconds_since_last_commit < self.TRAILING_CRUMB_WINDOW_SECS + and len(words) <= self.TRAILING_CRUMB_MAX_WORDS + ): + # Question-final trailers after a finished active are usually + # VAD splits ("¿qué tal?" + "estás?"), not Scribe ghosts + # ("…killer." + "No."). Ghost crumbs are statement/ack-shaped. + if text.rstrip().endswith(("?", "？")): + combined = state.active_user_text.rstrip(", ") + " " + text + return CommitDecision( + action=CommitAction.CONTINUE_TURN, + text=combined, + reason="continuation", + ) + return CommitDecision( + action=CommitAction.IGNORE, + text=text, + reason="trailing_crumb", + ) + else: + combined = state.active_user_text.rstrip(", ") + " " + text + return CommitDecision( + action=CommitAction.CONTINUE_TURN, + text=combined, + reason="continuation", + ) return CommitDecision(action=CommitAction.NEW_TURN, text=text, reason="new_turn") @@ -583,11 +639,36 @@ class LocalAudioTurnDetector(HeuristicTurnDetector): completion_threshold: float = 0.5 # Grace window after an incomplete-scored commit before the fragment runs - # anyway. LiveKit's equivalent (max_endpointing_delay) defaults to 6.0s of - # *total* silence after speech when their EOU model says "not done". The - # HOLD only arms after the STT VAD silence (~1.2s with the server default), - # so 4.8s here reproduces that 6s total thinking budget. - DEFAULT_HOLD_TIMEOUT_SECS = 4.8 + # anyway. Reference points: LiveKit's max_endpointing_delay is 6.0s of + # *total* silence when their EOU model says "not done"; Pipecat's + # smart-turn fallback (stop_secs) is 3.0s of continued silence. The HOLD + # only arms after the STT VAD silence (~1.2s with the server default), so + # 3.0s here gives ~4.2s total — between the two. This is also the full + # price of a *wrong* "incomplete" score (e.g. Smart Turn on a bare + # "Thank you."), and no re-score can rescue those mid-hold: the backend + # trims trailing silence before scoring, so waiting longer reproduces the + # same window and the same score. + DEFAULT_HOLD_TIMEOUT_SECS = 3.0 + # Confidence tier for the HOLD (LiveKit's min/max endpointing delay shape, + # with the transcript as the confidence signal): when the audio model says + # "incomplete" but the text looks finished (terminal punctuation — Smart + # Turn systematically under-scores short closers like "Thank you." / + # "I am David." / "Quite good."), the hold shrinks to this. Keep it short: + # the VAD endpointer has usually already paid ~0.5–3s of silence delay + # before the commit, so a second 1.2s tax felt like dead air on every + # finished utterance Smart Turn under-scored. Both-signals-incomplete + # keeps the full timeout. + TEXT_COMPLETE_HOLD_TIMEOUT_SECS = 0.35 + # The tier needs *confidently* finished text (terminal punctuation scores + # P_TERMINAL=0.95), not the predictor's complete-leaning neutral (0.60) — + # unpunctuated text must not shorten the hold. + TEXT_COMPLETE_TIER_THRESHOLD = 0.9 + # Inverse tier: audio says complete but text looks mid-thought (hedges + # score P_HEDGE=0.2, dangling/continuing ~0.15). Don't NEW_TURN — short + # HOLD so a continuation ("…tell me a story") can merge. Neutral (0.60) + # must NOT trigger this, or every unpunctuated complete fires a hold. + TEXT_INCOMPLETE_TIER_THRESHOLD = 0.4 + TEXT_INCOMPLETE_HOLD_TIMEOUT_SECS = 1.2 # The model consumes the last 8s of *speech*; the EOU backend trims the # trailing silence (STT commit debounce, hold pauses) before scoring, so # buffer extra raw PCM to keep a full 8s of signal after the trim. @@ -600,6 +681,8 @@ def __init__( *, completion_threshold: float | None = None, hold_timeout_secs: float | None = None, + text_complete_hold_timeout_secs: float | None = None, + text_incomplete_hold_timeout_secs: float | None = None, fallback_text_eou: TextEouPredictor | None = None, ) -> None: self.audio_eou = audio_eou @@ -608,16 +691,48 @@ def __init__( self.hold_timeout_secs = ( hold_timeout_secs if hold_timeout_secs is not None else self.DEFAULT_HOLD_TIMEOUT_SECS ) - # Used only when the buffered PCM is too short to score (fast commit at - # session start / mic audio not routed): a zero-dep lexical check so an - # obviously unfinished utterance still HOLDs instead of inheriting the - # heuristic NEW_TURN. + self.text_complete_hold_timeout_secs = ( + text_complete_hold_timeout_secs + if text_complete_hold_timeout_secs is not None + else self.TEXT_COMPLETE_HOLD_TIMEOUT_SECS + ) + self.text_incomplete_hold_timeout_secs = ( + text_incomplete_hold_timeout_secs + if text_incomplete_hold_timeout_secs is not None + else self.TEXT_INCOMPLETE_HOLD_TIMEOUT_SECS + ) + # Used when the buffered PCM is too short to score, and as the text + # confidence signal for both HOLD tiers (complete-text shortens; + # incomplete-text delays an audio-complete commit). Default is the + # zero-dep lexical baseline; ``resolve_turn_detector("local")`` injects + # Namo when ``timbal[voice]`` is installed. self.fallback_text_eou = fallback_text_eou or PunctuationEouPredictor() + # Namo under-scores many finished questions (~0.0 on "How are you?" / + # "What's two plus two?"). Lexical gate corroborates so we only pay the + # incomplete-text tax when both agree the user is mid-thought. + self._lexical_gate = PunctuationEouPredictor() self._sample_rate = 16_000 self._pcm: deque[bytes] = deque() self._pcm_bytes = 0 self._max_pcm_bytes = 0 + async def effective_text_eou(self, text: str) -> float: + """``P(complete)`` for HOLD tiers / VAD delay — Namo with lexical rescue. + + Namo under-scores many finished questions (~0.0 on "How are you?"). + Only lift the score when the lexical baseline is *confidently* complete + (terminal punct ≥ :attr:`TEXT_COMPLETE_TIER_THRESHOLD`) — a blunt + ``max(namo, lexical)`` would also promote neutral unpunctuated + mid-thoughts (lexical ~0.60) over the incomplete tier and neuter Namo. + """ + p = await self.fallback_text_eou.predict_eou(text) + if isinstance(self.fallback_text_eou, PunctuationEouPredictor): + return p + p_lex = await self._lexical_gate.predict_eou(text) + if p_lex >= self.TEXT_COMPLETE_TIER_THRESHOLD: + return max(p, p_lex) + return p + async def start(self, config: AudioInputConfig) -> None: self._sample_rate = int(getattr(config, "sample_rate", None) or 16_000) # PCM16 mono: 2 bytes/sample. @@ -626,10 +741,12 @@ async def start(self, config: AudioInputConfig) -> None: self._pcm_bytes = 0 if self.audio_eou is not None: await self.audio_eou.start(sample_rate=self._sample_rate) + await self.fallback_text_eou.start() async def close(self) -> None: if self.audio_eou is not None: await self.audio_eou.close() + await self.fallback_text_eou.close() self._pcm.clear() self._pcm_bytes = 0 @@ -639,6 +756,8 @@ def clone(self) -> LocalAudioTurnDetector: audio_eou=self.audio_eou, completion_threshold=self.completion_threshold, hold_timeout_secs=self.hold_timeout_secs, + text_complete_hold_timeout_secs=self.text_complete_hold_timeout_secs, + text_incomplete_hold_timeout_secs=self.text_incomplete_hold_timeout_secs, fallback_text_eou=self.fallback_text_eou, ) @@ -682,6 +801,17 @@ async def on_committed(self, text: str, state: TurnState) -> CommitDecision: return decision if decision.action is CommitAction.CONTINUE_TURN: return decision + # Mid-reply barge-in the parent already classified as NEW_TURN (finished + # active + fresh trailer past the crumb window). Do not rescore into HOLD + # — session defers HOLD during TTS, which would mute the interrupt. + if ( + decision.action is CommitAction.NEW_TURN + and state.assistant_active + and state.active_user_text + and _looks_like_finished_utterance(state.active_user_text) + and _looks_like_fresh_hold_utterance(text) + ): + return decision # Parent hold_merge of a self-contained commit → start the turn now # instead of re-holding, but keep the held fragment in the turn text # (never drop transcribed user speech; see parent hold_supersede). @@ -726,7 +856,7 @@ async def on_committed(self, text: str, state: TurnState) -> CommitDecision: # score the text lexically so an incomplete fast commit still HOLDs. if decision.action is not CommitAction.NEW_TURN: return decision - p_text = await self.fallback_text_eou.predict_eou(candidate) + p_text = await self.effective_text_eou(candidate) if p_text >= self.completion_threshold: return decision return CommitDecision( @@ -751,24 +881,79 @@ async def on_committed(self, text: str, state: TurnState) -> CommitDecision: text_preview=candidate[:80], ) if p >= self.completion_threshold: + # Inverse tier (see TEXT_INCOMPLETE_HOLD_TIMEOUT_SECS): audio says + # done but the transcript looks mid-thought — hold short so a + # continuation can merge. Never overrule into IGNORE; just delay. + try: + p_text = await self.effective_text_eou(candidate) + except Exception as e: + logger.warning("text_eou_predict_failed", error=str(e)) + p_text = None + if p_text is not None and p_text < self.TEXT_INCOMPLETE_TIER_THRESHOLD: + # Mid-agent barge-in that looks incomplete still merges via + # CONTINUE rather than parking a HOLD over the reply — but not + # onto a finished active + fresh trailer (STT crumb / barge-in). + if ( + state.active_user_text + and state.assistant_active + and not ( + _looks_like_finished_utterance(state.active_user_text) + and _looks_like_fresh_hold_utterance(text) + ) + ): + combined = state.active_user_text.rstrip(", ") + " " + text + return CommitDecision( + action=CommitAction.CONTINUE_TURN, + text=combined, + reason="audio_complete_text_incomplete_continue", + ) + return CommitDecision( + action=CommitAction.HOLD, + text=candidate, + reason="audio_complete_text_incomplete", + hold_timeout_secs=min( + self.text_incomplete_hold_timeout_secs, self.hold_timeout_secs + ), + ) return CommitDecision( action=CommitAction.NEW_TURN, text=candidate, reason="audio_complete" if state.holding else (decision.reason or "new_turn"), ) # Incomplete: mid-agent-turn → merge+restart; else HOLD (session debounce). - if state.active_user_text and state.assistant_active: + # Skip merge when the active turn already looks finished and the trailer + # is a fresh utterance — Heuristic's trailing_crumb / NEW_TURN path. + if ( + state.active_user_text + and state.assistant_active + and not ( + _looks_like_finished_utterance(state.active_user_text) + and _looks_like_fresh_hold_utterance(text) + ) + ): combined = state.active_user_text.rstrip(", ") + " " + text return CommitDecision( action=CommitAction.CONTINUE_TURN, text=combined, reason="audio_continuation", ) + # Confidence tier (see TEXT_COMPLETE_HOLD_TIMEOUT_SECS): a transcript + # that reads finished disagrees with the audio score — hold, but short. + timeout = self.hold_timeout_secs + reason = "audio_hold" + try: + p_text = await self.effective_text_eou(candidate) + except Exception as e: + logger.warning("text_eou_predict_failed", error=str(e)) + p_text = None + if p_text is not None and p_text >= self.TEXT_COMPLETE_TIER_THRESHOLD: + timeout = min(self.text_complete_hold_timeout_secs, self.hold_timeout_secs) + reason = "audio_hold_text_complete" return CommitDecision( action=CommitAction.HOLD, text=candidate, - reason="audio_hold", - hold_timeout_secs=self.hold_timeout_secs, + reason=reason, + hold_timeout_secs=timeout, ) @@ -796,8 +981,31 @@ def _default_audio_eou() -> AudioEouModel | None: return _DEFAULT_AUDIO_EOU +def _default_text_eou() -> TextEouPredictor: + """Namo English DistilBERT when ``timbal[voice]`` is installed, else punctuation. + + Shared process-wide (same rationale as :func:`_default_audio_eou`). + """ + global _DEFAULT_TEXT_EOU + if _DEFAULT_TEXT_EOU is not _TEXT_EOU_UNSET: + return _DEFAULT_TEXT_EOU + try: + from .namo import NamoTextEouPredictor + except ImportError: + logger.debug( + "namo_text_eou_unavailable", + hint="falling back to PunctuationEouPredictor; install timbal[voice] for Namo", + ) + _DEFAULT_TEXT_EOU = PunctuationEouPredictor() + return _DEFAULT_TEXT_EOU + _DEFAULT_TEXT_EOU = NamoTextEouPredictor() + return _DEFAULT_TEXT_EOU + + _AUDIO_EOU_UNSET: Any = object() _DEFAULT_AUDIO_EOU: AudioEouModel | None = _AUDIO_EOU_UNSET +_TEXT_EOU_UNSET: Any = object() +_DEFAULT_TEXT_EOU: TextEouPredictor | Any = _TEXT_EOU_UNSET def resolve_turn_detector(spec: Any = None) -> TurnDetector: @@ -805,11 +1013,12 @@ def resolve_turn_detector(spec: Any = None) -> TurnDetector: Accepted names: ``heuristic`` (default), ``provider``, ``local``, ``lexical``, ``raw`` (debug: no silence/noise/echo filtering at all). - ``local`` auto-loads the Smart Turn v3 :class:`~timbal.voice.eou.AudioEouModel` - when the ``timbal[voice]`` extra is installed (heuristic degradation otherwise). - Instances are returned unchanged — callers that reuse one spec across - concurrent sessions (server ``voice_config``) must :meth:`~TurnDetector.clone` - per session, or pass a zero-arg factory callable instead. + ``local`` auto-loads Smart Turn (audio) + Namo (text) when the + ``timbal[voice]`` extra is installed (heuristic / punctuation degradation + otherwise). Instances are returned unchanged — callers that reuse one spec + across concurrent sessions (server ``voice_config``) must + :meth:`~TurnDetector.clone` per session, or pass a zero-arg factory + callable instead. """ if spec is None: return HeuristicTurnDetector() @@ -822,9 +1031,12 @@ def resolve_turn_detector(spec: Any = None) -> TurnDetector: if key in ("provider", "stt"): return ProviderTurnDetector() if key in ("local", "audio", "smart_turn"): - return LocalAudioTurnDetector(audio_eou=_default_audio_eou()) + return LocalAudioTurnDetector( + audio_eou=_default_audio_eou(), + fallback_text_eou=_default_text_eou(), + ) if key in ("lexical", "semantic", "punctuation"): - return LexicalTurnDetector() + return LexicalTurnDetector(text_eou=_default_text_eou()) if key in ("raw", "none", "off"): return RawTurnDetector() raise ValueError( diff --git a/uv.lock b/uv.lock index 6bc225a2..a88ac520 100644 --- a/uv.lock +++ b/uv.lock @@ -1836,6 +1836,110 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, ] +[[package]] +name = "regex" +version = "2026.7.19" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/20/98/04b13f1ddfb63158025291c02e03eb42fbb7acb51d091d541050eb4e35e8/regex-2026.7.19.tar.gz", hash = "sha256:7e77b324909c1617cbb4c668677e2c6ae13f44d7c1de0d4f15f2e3c10f3315b5", size = 416440, upload-time = "2026-07-19T00:19:48.923Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/e5/cef4de2bac939280b68d32adc659478845238a8274f2f79c465063f590ad/regex-2026.7.19-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ac777001cdfc28b72477d93c8564bb7583081ea8fb45cdca3d568e0a4f87183c", size = 494012, upload-time = "2026-07-19T00:16:39.927Z" }, + { url = "https://files.pythonhosted.org/packages/ff/87/e86f51eb117457bb7803132ffe5cb6e2841e2b5bea4cc85d397f3c6e257d/regex-2026.7.19-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:59787bd5f8c70aa339084e961d2996b53fbdeab4d5393bba5c1fe1fc32e02bae", size = 295281, upload-time = "2026-07-19T00:16:41.433Z" }, + { url = "https://files.pythonhosted.org/packages/41/2e/2360c41d8080a3d9ec7e5c90fad6eab3b50192869d10e9a5609e48c8177b/regex-2026.7.19-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:90c633e7e8d6bf4e992b8b36ce69e018f834b641dd6de8cea6d78c06ffa119c5", size = 290615, upload-time = "2026-07-19T00:16:43.058Z" }, + { url = "https://files.pythonhosted.org/packages/cf/69/b65ba4344efbc771b28fe5dde84cbbb6c8f9551165952fe78def5b9dde6a/regex-2026.7.19-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:87ccab0db8d5f4fbb0272642113c1adb2ffc698c16d3a0944580222331fa7a20", size = 791804, upload-time = "2026-07-19T00:16:44.662Z" }, + { url = "https://files.pythonhosted.org/packages/81/b6/a40dfa0dc6224b36f620c00296eacc830489cbf8c2837b6750dfe6170375/regex-2026.7.19-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9e50d748a32da622f256e8d505867f5d3c43a837c6a9f0efb149655fadd1042a", size = 861723, upload-time = "2026-07-19T00:16:46.412Z" }, + { url = "https://files.pythonhosted.org/packages/e3/02/735991dee71abd83196a7962f7ed8bf5aa05720ff06e2d3ff896a85e2bbb/regex-2026.7.19-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bf1516fe58fc104f39b2d1dbe2d5e27d0cd45c4be2e42ba6ee0cc763701ec3c7", size = 905932, upload-time = "2026-07-19T00:16:47.956Z" }, + { url = "https://files.pythonhosted.org/packages/45/6c/e7098d8b846ccdbf431d8c081b61e496526a27a28094ed09e0dce21b3f54/regex-2026.7.19-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:09f3e5287f94f17b709dc9a9e70865855feee835c861613be144218ce4ca82cc", size = 801407, upload-time = "2026-07-19T00:16:49.43Z" }, + { url = "https://files.pythonhosted.org/packages/8a/18/34b69274e2649bcc7d9b089c2b2983fb2632d8ecf667e359593be9072e79/regex-2026.7.19-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6383cd2ed53a646c659ba1fe65727db76437fdaa069e697a0b44a51d5843d864", size = 774448, upload-time = "2026-07-19T00:16:51.352Z" }, + { url = "https://files.pythonhosted.org/packages/bb/e6/0a72247d025585fd3800b98e040b84d562a88af6303347100484849f4f01/regex-2026.7.19-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:09d3007fc76249a83cdd33de160d50e6cb77f54e09d8fa9e7148e10607ce24af", size = 783297, upload-time = "2026-07-19T00:16:53.071Z" }, + { url = "https://files.pythonhosted.org/packages/b1/aa/c4f65ae7dd02a36b323a70c4cff326e1f3442361aaebc9311100a130d54f/regex-2026.7.19-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6f8c6e7a1cfa3dc9d0ee2de0e65e834537fa29992cc3976ffec914afc35c5dd5", size = 854736, upload-time = "2026-07-19T00:16:54.607Z" }, + { url = "https://files.pythonhosted.org/packages/62/c3/668082bcc817b9e694189b84997aeba7385b7779faa6711788679c482e35/regex-2026.7.19-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:b2ea4a3e8357be8849e833beeae757ac3c7a6b3fc055c03c808a53c91ad30d82", size = 763298, upload-time = "2026-07-19T00:16:56.289Z" }, + { url = "https://files.pythonhosted.org/packages/4b/fb/2d07ad555e7af88aa5f867fdafa47a8d945ee237c20af3ebceb46a820835/regex-2026.7.19-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:80115dd39481fd3a4b4080220799dbcacb921a844de4b827264ececacbe17c78", size = 844430, upload-time = "2026-07-19T00:16:57.933Z" }, + { url = "https://files.pythonhosted.org/packages/51/15/c82a471fe3dce56f03745635b43aa456c40dc0db089e07ef148b331507d1/regex-2026.7.19-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d6ce43a0269d68cee79a7d1ade7def53c20f8f2a047b92d7b5d5bcc73ae88327", size = 789683, upload-time = "2026-07-19T00:16:59.583Z" }, + { url = "https://files.pythonhosted.org/packages/b5/f4/7532a2c59d56f5398902c20de60f0c9a5d1cd364e42a051b48e1b210be7b/regex-2026.7.19-cp311-cp311-win32.whl", hash = "sha256:9be2a6647740dd3cca6acb24e87f03d7632cd280dbce9bbe40c26353a215a45d", size = 266778, upload-time = "2026-07-19T00:17:01.032Z" }, + { url = "https://files.pythonhosted.org/packages/83/2b/cf1bc631db154eb95520d9d5dbc2371ff77a0f014bbf7d748fed8496aa63/regex-2026.7.19-cp311-cp311-win_amd64.whl", hash = "sha256:8d3469c91dd92ee41b7c95280edbd975ef1ba9195086686623a1c6e8935ce965", size = 277983, upload-time = "2026-07-19T00:17:02.571Z" }, + { url = "https://files.pythonhosted.org/packages/8d/bd/56ceaf170e875d5a6761bf2bfd0d040f1cacc896850d5e40cb29b11bbd06/regex-2026.7.19-cp311-cp311-win_arm64.whl", hash = "sha256:36aacfb15faaff3ced55afbf35ec72f50d4aee22082c4f7fe0573a33e2fca92e", size = 276961, upload-time = "2026-07-19T00:17:04.135Z" }, + { url = "https://files.pythonhosted.org/packages/3b/b9/d11d7e501ac8fd7d617684423ebb9561e0b998481c1e4cbc0cb212c5d74a/regex-2026.7.19-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2cc3460cedf7579948486eab03bc9ad7089df4d7281c0f47f4afe03e8d13f02d", size = 496778, upload-time = "2026-07-19T00:17:05.677Z" }, + { url = "https://files.pythonhosted.org/packages/3f/a9/a5ab6f312f24318019170dc485d5421fe4f89e43a98640da50d95a8a7041/regex-2026.7.19-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0e9554c8785eac5cffe6300f69a91f58ba72bc88a5f8d661235ad7c6aa5b8ccd", size = 297122, upload-time = "2026-07-19T00:17:07.59Z" }, + { url = "https://files.pythonhosted.org/packages/b3/63/4cab4d7f2d384a144d420b763d97674cb70619c878ea6fcd7640d0e62143/regex-2026.7.19-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d7da47a0f248977f08e2cb659ff3c17ddc13a4d39b3a7baa0a81bf5b415430f6", size = 292009, upload-time = "2026-07-19T00:17:09.648Z" }, + { url = "https://files.pythonhosted.org/packages/22/85/102a81b218298957d4ea7d2f084fae537a71add9d6ff93c8e67284c5f45e/regex-2026.7.19-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93db40c8de0815baab96a06e08a984bac71f989d13bab789e382158c5d426797", size = 796708, upload-time = "2026-07-19T00:17:11.542Z" }, + { url = "https://files.pythonhosted.org/packages/78/b5/dc136af5629938a037cd2b304c12240e132ec92f38be8ff9cc89af2a1f2d/regex-2026.7.19-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:66bd62c59a5427746e8c44becae1d9b99d22fb13f30f492083dfb9ad7c45cc18", size = 865651, upload-time = "2026-07-19T00:17:13.312Z" }, + { url = "https://files.pythonhosted.org/packages/e0/75/67402ae3cd9c8c988a4c805d15ee3eef015e7ca4cb112cf3e640fc1f4153/regex-2026.7.19-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1649eb39fcc9ea80c4d2f110fde2b8ab2aef3877b98f02ab9b14e961f418c511", size = 911756, upload-time = "2026-07-19T00:17:15.015Z" }, + { url = "https://files.pythonhosted.org/packages/2a/8e/096d00c7c480ef2ff4265349b14e2261d4ab787ba1f74e2e80d1c58079c3/regex-2026.7.19-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9dce8ec9695f531a1b8a6f314fd4b393adcccf2ea861db480cdf97a301d01a68", size = 801798, upload-time = "2026-07-19T00:17:17.208Z" }, + { url = "https://files.pythonhosted.org/packages/f0/41/e7ecac6edb5722417f85cc67eaf386322fbe8acf6918ec2fdc37c20dd9d0/regex-2026.7.19-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3080a7fd38ef049bd489e01c970c97dd84ff446a885b0f1f6b26d9b1ad13ce11", size = 776933, upload-time = "2026-07-19T00:17:19.347Z" }, + { url = "https://files.pythonhosted.org/packages/6f/69/03c9b3f058d66403e0ca2c938696e81d51cd4c6d47ec5265f02f96948d9a/regex-2026.7.19-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1d793a7988e04fcb1e2e135567443d82173225d657419ec09414a9b5a145b986", size = 784338, upload-time = "2026-07-19T00:17:21.057Z" }, + { url = "https://files.pythonhosted.org/packages/f6/f7/b38ab3d43f284afbb618fcd15d0e77eb786ae461ce1f6bc7494619ddc0f2/regex-2026.7.19-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e8b0abe7d870f53ca5143895fef7d1041a0c831a140d3dc2c760dd7ba25d4a8b", size = 860452, upload-time = "2026-07-19T00:17:23.119Z" }, + { url = "https://files.pythonhosted.org/packages/15/5c/ff60ef0571121714f3cf9920bc183071e384a10b556d042e0fdb06cc07a5/regex-2026.7.19-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:4e5413bd5f13d3a4e3539ca98f70f75e7fca92518dd7f117f030ebedd10b60cb", size = 765958, upload-time = "2026-07-19T00:17:24.81Z" }, + { url = "https://files.pythonhosted.org/packages/aa/0f/bd34021162c0ab47f9a315bd56cd5642e920c8e5668a75ef6c6a6fca590d/regex-2026.7.19-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:73b133a9e6fb512858e7f065e96f1180aa46646bc74a83aea62f1d314f3dd035", size = 851765, upload-time = "2026-07-19T00:17:26.993Z" }, + { url = "https://files.pythonhosted.org/packages/2a/20/a2ca43edade0595cccfdc98636739f536d9e26898e7dbddc2b9e98898953/regex-2026.7.19-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dbe6493fbd27321b1d1f2dd4f5c7e5bd4d8b1d7cab7f32fd67db3d0b2ed8248a", size = 789714, upload-time = "2026-07-19T00:17:28.699Z" }, + { url = "https://files.pythonhosted.org/packages/5d/47/e02db4015d424fc83c00ea0ac8c5e5ec14397943de9abf909d5ce3a25931/regex-2026.7.19-cp312-cp312-win32.whl", hash = "sha256:ddd67571c10869f65a5d7dde536d1e066e306cc90de57d7de4d5f34802428bb5", size = 267157, upload-time = "2026-07-19T00:17:31.051Z" }, + { url = "https://files.pythonhosted.org/packages/08/8e/c780c131f79b42ed22d1bd7da4096c2c35f813e835acd02ef0f018bd892c/regex-2026.7.19-cp312-cp312-win_amd64.whl", hash = "sha256:e30d40268a28d54ce0437031750497004c22602b8e3ab891f759b795a003b312", size = 277777, upload-time = "2026-07-19T00:17:32.848Z" }, + { url = "https://files.pythonhosted.org/packages/3e/4c/e4d7e086449bdf379d89774bf1f89dc4a41943f3c5a6125a03905b34b5fb/regex-2026.7.19-cp312-cp312-win_arm64.whl", hash = "sha256:de9208bb427130c82a5dbfd104f92c8876fc9559278c880b3002755bbbe9c83d", size = 277136, upload-time = "2026-07-19T00:17:34.803Z" }, + { url = "https://files.pythonhosted.org/packages/5d/3d/84165e4299ff76f3a40fe1f2abf939e976f693383a08d2beea6af62bd2c1/regex-2026.7.19-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f035d9dc1d25eff9d361456572231c7d27b5ccd473ca7dc0adfce732bd006d40", size = 496552, upload-time = "2026-07-19T00:17:36.808Z" }, + { url = "https://files.pythonhosted.org/packages/02/a2/a65293e6e4cf28eb7ee1be5335a5386c40d6742e9f47fafc8fec785e16c7/regex-2026.7.19-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c42572142ed0b9d5d261ba727157c426510da78e20828b66bbb855098b8a4e38", size = 296983, upload-time = "2026-07-19T00:17:38.816Z" }, + { url = "https://files.pythonhosted.org/packages/95/47/2d0564e93d87bc48618360ddca232a2ca612bbdf53ce8465d45ca5ce14ee/regex-2026.7.19-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:40b34dd88658e4fedd2fddbf0275ac970d00614b731357f425722a3ed1983d11", size = 291832, upload-time = "2026-07-19T00:17:40.726Z" }, + { url = "https://files.pythonhosted.org/packages/07/cd/42dfbabff3dfc9603c501c0e2e2c5adbb09d127b267bf5348de0af338c15/regex-2026.7.19-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c41c63992bf1874cebb6e7f56fd7d3c007924659a604ae3d90e427d40d4fd13", size = 796775, upload-time = "2026-07-19T00:17:42.382Z" }, + { url = "https://files.pythonhosted.org/packages/df/5d/f6a4839f2b934e3eed5973fd07f5929ee97d4c98939fb275ea23c274ee16/regex-2026.7.19-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d3372064506b94dd2c67c845f2db8062e9e9ba84d04e33cb96d7d33c11fe1ae", size = 865687, upload-time = "2026-07-19T00:17:44.185Z" }, + { url = "https://files.pythonhosted.org/packages/14/b0/b47d6c36049bc59806a50bd4c86ced70bbe058d787f80281b1d7a9b0e024/regex-2026.7.19-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fce7760bf283405b2c7999cab3da4e72f7deca6396013115e3f7a955db9760da", size = 911962, upload-time = "2026-07-19T00:17:46.442Z" }, + { url = "https://files.pythonhosted.org/packages/2a/be/ff61f28f9273658cfe23acbbac5217221f6519960ed401e61dfdab12bc35/regex-2026.7.19-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0d702548d89d572b2929879bc883bb7a4c4709efafe4512cadee56c55c9bd15", size = 801817, upload-time = "2026-07-19T00:17:48.25Z" }, + { url = "https://files.pythonhosted.org/packages/c3/bb/8b4f7f26b333f9f79e1b453613c39bb4776f51d38ae66dd0ba31d6b354ca/regex-2026.7.19-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d446c6ac40bb6e05025ccee55b84d80fe9bf8e93010ffc4bb9484f13d498835f", size = 776908, upload-time = "2026-07-19T00:17:50.183Z" }, + { url = "https://files.pythonhosted.org/packages/09/13/610110fc5921d380516d03c26b652555f08aa0d23ea78a771231873c3638/regex-2026.7.19-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4c3501bfa814ab07b5580741f9bf78dfdfe146a04057f82df9e2402d2a975939", size = 784426, upload-time = "2026-07-19T00:17:52.454Z" }, + { url = "https://files.pythonhosted.org/packages/ca/f5/1ef9e2a83a5947c57ebff0b377cb5727c3d5ec1992317a320d035cd0dbb6/regex-2026.7.19-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c4585c3e64b4f9e583b4d2683f18f5d5d872b3d71dcf24594b74ecc23602fa96", size = 860600, upload-time = "2026-07-19T00:17:54.229Z" }, + { url = "https://files.pythonhosted.org/packages/a0/02/073af33a3ec149241d11c80acea91e722aa0adbf05addd50f251c4fe89c3/regex-2026.7.19-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:571fde9741eb0ccde23dd4e0c1d50fbae910e901fa7e629faf39b2dda740d220", size = 765950, upload-time = "2026-07-19T00:17:56.041Z" }, + { url = "https://files.pythonhosted.org/packages/81/a9/d1e9f819dc394a568ef370cd56cf25394e957a2235f8370f23b576e5a475/regex-2026.7.19-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:15b364b9b98d6d2fe1a85034c23a3180ff913f46caddc3895f6fd65186255ccc", size = 851794, upload-time = "2026-07-19T00:17:57.897Z" }, + { url = "https://files.pythonhosted.org/packages/03/3a/8ae83eda7579feacdf984e71fb9e70635fb6f832eeddca58427ec4fca926/regex-2026.7.19-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffd8893ccc1c2fce6e0d6ca402d716fe1b29db70c7132609a05955e31b2aa8f2", size = 789845, upload-time = "2026-07-19T00:17:59.97Z" }, + { url = "https://files.pythonhosted.org/packages/4b/23/c195cbfe5a75fdec64d8f6554fd15237b837919d2c61bdc141d7c807b08b/regex-2026.7.19-cp313-cp313-win32.whl", hash = "sha256:f0fa4fa9c3632d708742baf2282f2055c11d888a790362670a403cbf48a2c404", size = 267135, upload-time = "2026-07-19T00:18:01.958Z" }, + { url = "https://files.pythonhosted.org/packages/b2/80/a11de8404b7272b70acb45c1c05987cce60b45d5693da2e176f0e390d564/regex-2026.7.19-cp313-cp313-win_amd64.whl", hash = "sha256:d51ffd3427640fa2da6ade574ceba932f210ad095f65fcc450a2b0a0d454868e", size = 277747, upload-time = "2026-07-19T00:18:04.121Z" }, + { url = "https://files.pythonhosted.org/packages/d1/29/0f5c8eff1b4f1f3d83276d365fccecf666afcc7d947420943bf394d07adb/regex-2026.7.19-cp313-cp313-win_arm64.whl", hash = "sha256:c670fe7be5b6020b76bc6e8d2196074657e1327595bca93a389e1a76ab130ad8", size = 277129, upload-time = "2026-07-19T00:18:05.821Z" }, + { url = "https://files.pythonhosted.org/packages/dc/4c/44b74742052cedda40f9ae469532a037112f7311a36669a891fba8984bb0/regex-2026.7.19-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:db47b561c9afd884baa1f96f797c9ca369872c4b65912bc691cfa99e68340af2", size = 501134, upload-time = "2026-07-19T00:18:07.567Z" }, + { url = "https://files.pythonhosted.org/packages/f0/45/bbd038b5e39ee5613a5a689290145b40058cc152c41de9cc23639d2b9734/regex-2026.7.19-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:65dcd28d3eba2ab7c2fd906485cc301392b47cc2234790d27d4e4814e02cdfda", size = 299418, upload-time = "2026-07-19T00:18:09.38Z" }, + { url = "https://files.pythonhosted.org/packages/65/38/c5bde94b4cedfd5850d64c3f08222d8e1600e84f6ee71d9b44b4b8163f74/regex-2026.7.19-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f2e7f8e2ab6c2922be02c7ec45185aa5bd771e2e57b95455ee343a44d8130dff", size = 294486, upload-time = "2026-07-19T00:18:11.188Z" }, + { url = "https://files.pythonhosted.org/packages/d7/6a/2f5e107cb26c960b781967178899daf2787a7ab151844ed3c01d6fc95474/regex-2026.7.19-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe31f28c94402043161876a258a9c6f757cb485905c7614ce8d6cd40e6b7bdc1", size = 811643, upload-time = "2026-07-19T00:18:12.975Z" }, + { url = "https://files.pythonhosted.org/packages/37/d4/a2f963406d7d73a62eed84ba05a258afb6cad1b21aa4517443ce40506b78/regex-2026.7.19-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8f6fa298bb4f7f58a33334406218ba74716e68feddf5e4e54cd5d8082705abf", size = 871081, upload-time = "2026-07-19T00:18:14.733Z" }, + { url = "https://files.pythonhosted.org/packages/45/a3/44be546340bedb15f13063f5e7fe16793ea4d9ea2e805d09bd174ac27724/regex-2026.7.19-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cc1b2440423a851fad781309dd87843868f4f66a6bcd1ddb9225cf4ec2c84732", size = 917372, upload-time = "2026-07-19T00:18:16.724Z" }, + { url = "https://files.pythonhosted.org/packages/f8/f6/e0870b0fd2a40dba0074e4b76e514b21313d37946c9248453e34ec43923e/regex-2026.7.19-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ac59a0900474a52b7c04af8196affc22bd9842acb0950df12f7b813e983609a", size = 816089, upload-time = "2026-07-19T00:18:18.617Z" }, + { url = "https://files.pythonhosted.org/packages/ae/27/957e8e22690ad6634572b39b71f130a6105f4d0718bb16849eac00fff147/regex-2026.7.19-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4896db1f4ce0576765b8272aa922df324e0f5b9bb2c3d03044ff32a7234a9aba", size = 785206, upload-time = "2026-07-19T00:18:20.464Z" }, + { url = "https://files.pythonhosted.org/packages/76/a4/186e410941e731037c01166069ab86da9f65e8f8110c18009ccf4bd623ee/regex-2026.7.19-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4e6883a021db30511d9fb8cfb0f222ce1f2c369f7d4d8b0448f449a93ba0bdfc", size = 800431, upload-time = "2026-07-19T00:18:22.716Z" }, + { url = "https://files.pythonhosted.org/packages/73/9f/e4e10e023d291d64a33e246610b724493bf1ce98e0e59c9b7c837e5acfb7/regex-2026.7.19-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:09523a592938aa9f587fb74467c63ff0cf88fc3df14c82ab0f0517dcf76aaa62", size = 864906, upload-time = "2026-07-19T00:18:24.772Z" }, + { url = "https://files.pythonhosted.org/packages/24/57/ccb20b6be5f1f52a053d1ba2a8f7a077edb9d918248b8490d7506c6832b3/regex-2026.7.19-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:1ebac3474b8589fce2f9b225b650afd61448f7c73a5d0255a10cc6366471aed1", size = 773559, upload-time = "2026-07-19T00:18:27.008Z" }, + { url = "https://files.pythonhosted.org/packages/a3/82/f3b263cf8fad927dc102891da8502e718b7ff9d19af7a2a07c03865d7188/regex-2026.7.19-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4a0530bb1b8c1c985e7e2122e2b4d3aedd8a3c21c6bfddae6767c4405668b56e", size = 857739, upload-time = "2026-07-19T00:18:29.107Z" }, + { url = "https://files.pythonhosted.org/packages/47/2e/1687bd1b6c2aed5e672ccf845fc11557821fe7366d921b50889ea5ce57bf/regex-2026.7.19-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2ef7eeb108c47ce7bcc9513e51bcb1bf57e8f483d52fce68a8642e3527141ae0", size = 804522, upload-time = "2026-07-19T00:18:31.362Z" }, + { url = "https://files.pythonhosted.org/packages/76/7c/cc4e7655181b2d9235b704f2c5e19d8eff002bbc437bae59baee0e381aca/regex-2026.7.19-cp313-cp313t-win32.whl", hash = "sha256:64b6ca7391a1395c2638dd5c7456d67bea44fc6c5e8e92c5dc8aa6a8f23292b4", size = 269141, upload-time = "2026-07-19T00:18:33.479Z" }, + { url = "https://files.pythonhosted.org/packages/bb/14/961b4c7b05a2391c32dbc85e27773076671ef8f97f36cec70fe414734c02/regex-2026.7.19-cp313-cp313t-win_amd64.whl", hash = "sha256:f04b9f56b0e0614c0126be12c2c2d9f8850c1e57af302bd0a63bed379d4af974", size = 280036, upload-time = "2026-07-19T00:18:35.419Z" }, + { url = "https://files.pythonhosted.org/packages/ce/67/795644550d788ddbb6dc458c95895f8009978ea6d6ea76b005eb3f45e8c9/regex-2026.7.19-cp313-cp313t-win_arm64.whl", hash = "sha256:fcee38cd8e5089d6d4f048ba1233b3ad76e5954f545382180889112ff5cb712d", size = 279394, upload-time = "2026-07-19T00:18:37.454Z" }, + { url = "https://files.pythonhosted.org/packages/d2/25/0c4c452f8ef3efe456745b2f33195f5904b573fb4c2ff3f0cb9ec188461e/regex-2026.7.19-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:a81758ed242b861b72e778ba34d41366441a2e10b16b472784c88da2dea7e2dd", size = 496750, upload-time = "2026-07-19T00:18:39.633Z" }, + { url = "https://files.pythonhosted.org/packages/24/9e/b70ca6c1704f6c7cd32a9e143c86cc5968d10981eca284bad670c245ea7d/regex-2026.7.19-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4aa5435cdb3eb6f55fe98a171b05e3fbcd95fadaa4aa32acf62afd9b0cfdbcac", size = 297093, upload-time = "2026-07-19T00:18:41.583Z" }, + { url = "https://files.pythonhosted.org/packages/87/74/0b692da2520d51fbff19c88b83d97e4c702909dd02386c585998b7e2dbed/regex-2026.7.19-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:60be8693a1dadc210bbcbc0db3e26da5f7d01d1d5a3da594e99b4fa42df404f5", size = 292043, upload-time = "2026-07-19T00:18:43.347Z" }, + { url = "https://files.pythonhosted.org/packages/e3/a7/1d478e614016045a33feae57446215f9fd65b665a5ceb2f891fb3183bc52/regex-2026.7.19-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19662dbedbe783d323196312d38f5ba53cf56296378252171985da6899887d3", size = 797214, upload-time = "2026-07-19T00:18:45.362Z" }, + { url = "https://files.pythonhosted.org/packages/aa/ae/11b9c9411d92c30e3d2db32df5a31133e4a99a8fc397a604fd08f6c4bffb/regex-2026.7.19-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d15df07081d91b76ff20d43f94592ee110330152d617b730fdbe5ef9fb680053", size = 866433, upload-time = "2026-07-19T00:18:47.315Z" }, + { url = "https://files.pythonhosted.org/packages/b1/62/2b2efc4992f91d6d204b24c647c9f9412e85379d92b7c0ab9fdae622327e/regex-2026.7.19-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:56ad4d9f77df871a99e25c37091052a02528ec0eb059de928ee33956b854b45b", size = 911360, upload-time = "2026-07-19T00:18:49.588Z" }, + { url = "https://files.pythonhosted.org/packages/14/71/986ceea9aa3da548bf1357cad89b63915ec6d21ec957c8113b29ece567df/regex-2026.7.19-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7322ec6cc9fba9d49ab888bb82d67ac5625627aa168f0165139b17018df3fb8a", size = 801275, upload-time = "2026-07-19T00:18:51.767Z" }, + { url = "https://files.pythonhosted.org/packages/15/be/ce9d9534b2cda96eab32c548261224b9b4e220a4126f098f60f42ae7b4cd/regex-2026.7.19-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9c7472192ebfad53a6be7c4a8bfb2d64b81c0e93a1fc8c57e1dd0b638297b5d1", size = 777131, upload-time = "2026-07-19T00:18:54.053Z" }, + { url = "https://files.pythonhosted.org/packages/61/2b/58b5c710f2c3929515a25f3a1ca0dad0dcd4518d4fff3cf23bc7adb8dcd2/regex-2026.7.19-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c10b82c2634df08dfb13b1f04e38fe310d086ee092f4f69c0c8da234251e556e", size = 785020, upload-time = "2026-07-19T00:18:56.579Z" }, + { url = "https://files.pythonhosted.org/packages/84/03/5fe091935b74f15fe0f97998c215cae418d1c0413f6258c7d4d2e83aa37f/regex-2026.7.19-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:17ed5692f6acc4183e98331101a5f9e4f64d72fe58b753da4d444a2c77d05b12", size = 861263, upload-time = "2026-07-19T00:18:58.64Z" }, + { url = "https://files.pythonhosted.org/packages/d8/fa/d60bf82e10841eef62a9e32aac401468f05fddfbcb2942e342b1ba3d2433/regex-2026.7.19-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:22a992de9a0d91bda927bf02b94351d737a0302905432c88a53de7c4b9ce62e2", size = 766199, upload-time = "2026-07-19T00:19:00.705Z" }, + { url = "https://files.pythonhosted.org/packages/bf/5d/11e64d151b0662b81d6bf644c74dc118d461df85bdf2577fadbbf751788a/regex-2026.7.19-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:618a0aed532be87294c4477b0481f3aa0f1520f4014a4374dd4cf789b4cd2c97", size = 851317, upload-time = "2026-07-19T00:19:03.015Z" }, + { url = "https://files.pythonhosted.org/packages/7c/34/532efb87488d90807bae6a443d357ee5e2728a478c597619c8aaa17cc0bd/regex-2026.7.19-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ce9e679f776649746729b6c86382da519ef649c8e34cc41df0d2e5e0f6c36d4", size = 789557, upload-time = "2026-07-19T00:19:05.338Z" }, + { url = "https://files.pythonhosted.org/packages/d6/90/3a8d5ca977171ec3ae21a71207d2228b2663bde14d7f7ef0e6363ecf9290/regex-2026.7.19-cp314-cp314-win32.whl", hash = "sha256:73f272fba87b8ccfe70a137d02a54af386f6d27aa509fbffdd978f5947aae1aa", size = 272531, upload-time = "2026-07-19T00:19:07.487Z" }, + { url = "https://files.pythonhosted.org/packages/96/e1/8862885e70409de70e8c005f57fb2e7be8d9ef0317250d60f4c9660a300d/regex-2026.7.19-cp314-cp314-win_amd64.whl", hash = "sha256:d721e53758b2cca74990185eb0671dd466d7a388a1a45d0c6f4c13cef41a68ac", size = 280831, upload-time = "2026-07-19T00:19:09.46Z" }, + { url = "https://files.pythonhosted.org/packages/08/82/2693e53e29f9104d9de95d37ce4dd826bd32d5f9c0085d3aa6ac042675c4/regex-2026.7.19-cp314-cp314-win_arm64.whl", hash = "sha256:65fa6cb38ed5e9c3637e68e544f598b39c3b86b808ed0627a67b68320384b459", size = 281099, upload-time = "2026-07-19T00:19:11.398Z" }, + { url = "https://files.pythonhosted.org/packages/92/b7/9a01aa16461a18cde9d7b9c3ab21e501db2ce33725f53014342b91df2b0a/regex-2026.7.19-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:5a2721c8720e2cb3c209925dfb9200199b4b07361c9e01d321719404b21458b3", size = 501121, upload-time = "2026-07-19T00:19:13.425Z" }, + { url = "https://files.pythonhosted.org/packages/f3/5e/bbaeca815dc9191c424c94a4fdc5c87c75748a64a6271821212ebdd4e1a3/regex-2026.7.19-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:199535629f25caf89698039af3d1ad5fcae7f933e2112c73f1cdf49165c99518", size = 299415, upload-time = "2026-07-19T00:19:15.43Z" }, + { url = "https://files.pythonhosted.org/packages/cd/d6/0dd1a321afaab95eb7ff44aa0f637301786f1dc71c6b797b9ed236ed8890/regex-2026.7.19-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9b60d7814174f059e5de4ab98271cc5ba9259cfea55273a81544dceea32dc8d9", size = 294483, upload-time = "2026-07-19T00:19:17.879Z" }, + { url = "https://files.pythonhosted.org/packages/92/5f/40bacf91d0904f812e13bbbab3864604c463eced8afdc54aeaa50492ea95/regex-2026.7.19-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dbece16025afda5e3031af0c4059207e61dcf73ef13af844964f57f387d1c435", size = 811833, upload-time = "2026-07-19T00:19:20.102Z" }, + { url = "https://files.pythonhosted.org/packages/94/7c/4902744261f775aeede8b5627314b38482da29cf49a57b66a6fb753246c5/regex-2026.7.19-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d24ecb4f5e009ea0bd275ee37ad9953b32005e2e5e60f8bbae16da0dbbf0d3a0", size = 871270, upload-time = "2026-07-19T00:19:22.365Z" }, + { url = "https://files.pythonhosted.org/packages/16/70/6980c9be6bf21c0a60ed3e0aea39cf419ecf3b08d1d9947bc56e196ef186/regex-2026.7.19-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8cae6fd77a5b72dae505084b1a2ee0360139faf72fedbab667cd7cc65aae7a6a", size = 917534, upload-time = "2026-07-19T00:19:24.529Z" }, + { url = "https://files.pythonhosted.org/packages/52/92/8b2bd872782ce8c42691e39acb38eb8efe014e5ddb78ad7d943d6f197ce9/regex-2026.7.19-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9724e6cb5e478cd7d8cabf027826178739cb18cf0e117d0e32814d479fa02276", size = 816135, upload-time = "2026-07-19T00:19:26.919Z" }, + { url = "https://files.pythonhosted.org/packages/de/2d/33a602f657bdc4041f17d79f92ab18261d255d91a06117a6e29df023e5e2/regex-2026.7.19-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:572fc57b0009c735ee56c175ea021b637a15551a312f56734277f923d6fd0f6c", size = 785492, upload-time = "2026-07-19T00:19:29.192Z" }, + { url = "https://files.pythonhosted.org/packages/9e/36/0987cf4cb271680064a70d24a475873775a151d0b7058698a006cb0cae4a/regex-2026.7.19-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:20568e182eb82d39a6bf7cff3fd58566f14c75c6f74b2c8c96537eecf9010e3a", size = 800658, upload-time = "2026-07-19T00:19:31.392Z" }, + { url = "https://files.pythonhosted.org/packages/a8/24/c14f31c135e1ba55fa4f9a58ca98d0842512bf6188230763c31c8f449e3b/regex-2026.7.19-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1d58561843f0ff7dc78b4c28b5e2dc388f3eff94ebc8a232a3adba961fc00009", size = 865073, upload-time = "2026-07-19T00:19:33.485Z" }, + { url = "https://files.pythonhosted.org/packages/14/85/181a12211f22469f24d2de1ebddfe397d2396e2c29013b9a58134a91069a/regex-2026.7.19-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:61bb1bd45520aacd56dd80943bd34991fb5350afdd1f36f2282230fd5154a218", size = 773684, upload-time = "2026-07-19T00:19:35.599Z" }, + { url = "https://files.pythonhosted.org/packages/23/58/bd1a0c1a62251366f8d21f41b1ea3c76994962071b8b6ea42f72d505c0f0/regex-2026.7.19-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:cd3584591ea4429026cdb931b054342c2bcf189b44ff367f8d5c15bc092a2966", size = 857769, upload-time = "2026-07-19T00:19:37.738Z" }, + { url = "https://files.pythonhosted.org/packages/e4/4f/f7e2dad6756b2fe1fe75dd90a628c3b45f249d39f948dd90cd2476325417/regex-2026.7.19-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cc26a66e212fa5d6c6170c3a40d99d888db3020c6fdab1523250d4341382e44", size = 804546, upload-time = "2026-07-19T00:19:40.229Z" }, + { url = "https://files.pythonhosted.org/packages/2b/d7/01d31d5bdb09bc026fab77f59a371fdf8f9b292e4810546c56182ca70498/regex-2026.7.19-cp314-cp314t-win32.whl", hash = "sha256:2c4e61e2e1be56f63ec3cc618aa9e0de81ef6f43d177205451840022e24f5b78", size = 274526, upload-time = "2026-07-19T00:19:42.398Z" }, + { url = "https://files.pythonhosted.org/packages/52/0e/cea4ce73bc0a8247a0748228ae6669984c7e1f8134b6fa66e59c0572e0ea/regex-2026.7.19-cp314-cp314t-win_amd64.whl", hash = "sha256:c639ea314df70a7b2811e8020448c75af8c9445f5a60f8a4ced81c306a9380c2", size = 283763, upload-time = "2026-07-19T00:19:44.644Z" }, + { url = "https://files.pythonhosted.org/packages/6f/b6/26e41975febae63b7a6e3e02f32cff6cff2e4f10d19c929082f56aebf7c6/regex-2026.7.19-cp314-cp314t-win_arm64.whl", hash = "sha256:9a15e785f244f3e07847b984ce8773fc3da10a9f3c131cc49a4c5b4d672b4547", size = 283451, upload-time = "2026-07-19T00:19:46.639Z" }, +] + [[package]] name = "requests" version = "2.34.2" @@ -1997,6 +2101,39 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3e/0a/9e1be9035b37448ce2e68c978f0591da94389ade5a5abafa4cf99985d1b2/ruff-0.15.4-py3-none-win_arm64.whl", hash = "sha256:60d5177e8cfc70e51b9c5fad936c634872a74209f934c1e79107d11787ad5453", size = 10966776, upload-time = "2026-02-26T20:03:56.908Z" }, ] +[[package]] +name = "safetensors" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/45/06/f955dbbb1859e3bd23c8ac6141af5106e7ad5fedec4a3a6e3d60f94b7001/safetensors-0.8.0.tar.gz", hash = "sha256:fabaf3e0f18a6618d9b36560682562157f77c2b71fcffc7b432be2baed9d753d", size = 325846, upload-time = "2026-06-09T07:52:25.563Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/a0/f718cda65b05407d228f97602cf60dca269c979867aa5beb25410de26cd3/safetensors-0.8.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c554f85858e05226d3c2828e32395e677434685d6d94594a41643361c5e837f0", size = 473568, upload-time = "2026-06-09T07:52:18.829Z" }, + { url = "https://files.pythonhosted.org/packages/f5/b1/fa7c600e7dceae12e9606c7578cbc9ff1e1ed55844883ee5c92205e86226/safetensors-0.8.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:c80201d22cbf405b80647a60ada77bba06c8fba2da2743ba1e89cdcc39a81f25", size = 484562, upload-time = "2026-06-09T07:52:17.518Z" }, + { url = "https://files.pythonhosted.org/packages/09/7d/65a7de0af421317bb36a067241e4235fff194eed60b961ed6d3f59a3fc60/safetensors-0.8.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a46e5ff292c356d6991e60942ba7f79817682d3a2cef0702136448cb9c4d235", size = 502844, upload-time = "2026-06-09T07:52:07.624Z" }, + { url = "https://files.pythonhosted.org/packages/91/4f/3175c9d75634e0e0dda0082794193521035edd7c70a6f212bf33ca06ddf4/safetensors-0.8.0-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4124502b78f03534117c848f87a39b8f31e577b15eff423bf8bfb95f2a8c30d0", size = 511823, upload-time = "2026-06-09T07:52:09.565Z" }, + { url = "https://files.pythonhosted.org/packages/20/87/846c289e7aa2299eff406335717cf43ce8777194ece8aad75772e0411615/safetensors-0.8.0-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7bc0a787ba8a35be368ee3574edfa2b1ad389eebd0a72e482ae275490e3f6c98", size = 633461, upload-time = "2026-06-09T07:52:11.128Z" }, + { url = "https://files.pythonhosted.org/packages/76/22/8d64d9df2c45d5ded401df889d0ad90882804ca172d79ec4f0df8f727fe0/safetensors-0.8.0-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:040070828e36dc8e122178bbbd5830ff9e97920affb84cbe0f46442497bed358", size = 545148, upload-time = "2026-06-09T07:52:13.603Z" }, + { url = "https://files.pythonhosted.org/packages/28/50/f203ff3a3ddfe19308efc83c5a3a29ed02bf786732ec35e68bf9162f3365/safetensors-0.8.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd6f3f93c9a0a7cc2788ee63fb763353d4bd2e89b0751bc78fcf7dda00bea774", size = 516040, upload-time = "2026-06-09T07:52:16.29Z" }, + { url = "https://files.pythonhosted.org/packages/46/fb/cdaed17ceb2948784fd9c36b6fd3e951b608547cea81a48e8ee6f8cfdfcb/safetensors-0.8.0-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:fcdd41ec4628fee5799f807c73c353629130fbd942aa23d83c623dd6c9d52d78", size = 513832, upload-time = "2026-06-09T07:52:12.37Z" }, + { url = "https://files.pythonhosted.org/packages/0d/49/1e15de264dcc3b77943d2d0c56a95809956883b1c2d6d585c792523f180b/safetensors-0.8.0-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8e9f537aa183a38ace122d27303dcd986b26bd2a7591f9181d7f0c396f4677ca", size = 559930, upload-time = "2026-06-09T07:52:14.743Z" }, + { url = "https://files.pythonhosted.org/packages/2a/43/bf38443278eab4b1be1fce2931e2b012ad9cb7df52ada751d0aab8f7659a/safetensors-0.8.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:87eec7ffed2b809f05a398a8becb7d013f19f7837cd15d9748580d6cf30dbaf4", size = 678670, upload-time = "2026-06-09T07:52:20.032Z" }, + { url = "https://files.pythonhosted.org/packages/72/e3/68cd3fa5b48488e84add63e04cb12f3bc28ae4638c06d4508c6e88823d0e/safetensors-0.8.0-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:4a95ae2b05d7726d751da4ebf626a2ca782b706e101bd894c95bc2450b1cffcc", size = 786679, upload-time = "2026-06-09T07:52:21.322Z" }, + { url = "https://files.pythonhosted.org/packages/29/4b/1c19c509d56e01f4fbb3d0a2e597450f6cc04d1d56cf52defb0a62dfd715/safetensors-0.8.0-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:3ae091f16662658bdc019a4ff6cb4c085bb7d725eb5978b183ffd265863b6d2d", size = 765683, upload-time = "2026-06-09T07:52:22.594Z" }, + { url = "https://files.pythonhosted.org/packages/27/43/41c1621732edd934d868a00d1b891584c892a7b62a9aab82ea5a0a5623ee/safetensors-0.8.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8e080062fcde23be189565e1c3305d16751a218ecf9412c8601e64204eb6f846", size = 722361, upload-time = "2026-06-09T07:52:23.924Z" }, + { url = "https://files.pythonhosted.org/packages/8e/3f/73ccf82579412b4a71c4ca673f10b5f1f888d7cf5af7fe24f27d30307be4/safetensors-0.8.0-cp310-abi3-win32.whl", hash = "sha256:2ddf52eac562eda224f99acfa7889d02968c1fd59a5b011ae7d8137c37e9c02d", size = 342401, upload-time = "2026-06-09T07:52:28.895Z" }, + { url = "https://files.pythonhosted.org/packages/1b/6d/3fba214c1e5e0f69991677ec3bc17023f0421776975e1de0c682dca475e2/safetensors-0.8.0-cp310-abi3-win_amd64.whl", hash = "sha256:096ec1a98435df7beb08853bb5aa9081a84f23d0adc67ed1a0a10550f608373f", size = 355540, upload-time = "2026-06-09T07:52:27.832Z" }, + { url = "https://files.pythonhosted.org/packages/8d/fc/7eedc3510d97878876e32774eebbeb61c43f148a96e915c84229a3e967aa/safetensors-0.8.0-cp310-abi3-win_arm64.whl", hash = "sha256:f7838e5135a406ad3e02efdcb8cf2e5397d368b0154537c4fec682dbc544d452", size = 340500, upload-time = "2026-06-09T07:52:26.745Z" }, +] + +[[package]] +name = "shellingham" +version = "1.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310, upload-time = "2023-10-24T04:13:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, +] + [[package]] name = "six" version = "1.17.0" @@ -2086,6 +2223,7 @@ all = [ { name = "python-docx", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "rich", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "ruff", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "transformers", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "uvicorn", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "websockets", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] @@ -2109,6 +2247,7 @@ server = [ voice = [ { name = "huggingface-hub", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "onnxruntime", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "transformers", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] [package.dev-dependencies] @@ -2149,6 +2288,8 @@ requires-dist = [ { name = "ruff", marker = "extra == 'all'", specifier = ">=0.9.3" }, { name = "ruff", marker = "extra == 'codegen'", specifier = ">=0.9.3" }, { name = "structlog", specifier = ">=25.1.0" }, + { name = "transformers", marker = "extra == 'all'", specifier = ">=4.40.0" }, + { name = "transformers", marker = "extra == 'voice'", specifier = ">=4.40.0" }, { name = "typing-extensions", specifier = ">=4.14.1" }, { name = "uuid7", specifier = ">=0.1.0" }, { name = "uvicorn", marker = "extra == 'all'", specifier = ">=0.34.0" }, @@ -2168,6 +2309,32 @@ dev = [ { name = "yappi", specifier = ">=1.7.6" }, ] +[[package]] +name = "tokenizers" +version = "0.22.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/73/6f/f80cfef4a312e1fb34baf7d85c72d4411afde10978d4657f8cdd811d3ccc/tokenizers-0.22.2.tar.gz", hash = "sha256:473b83b915e547aa366d1eee11806deaf419e17be16310ac0a14077f1e28f917", size = 372115, upload-time = "2026-01-05T10:45:15.988Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/92/97/5dbfabf04c7e348e655e907ed27913e03db0923abb5dfdd120d7b25630e1/tokenizers-0.22.2-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:544dd704ae7238755d790de45ba8da072e9af3eea688f698b137915ae959281c", size = 3100275, upload-time = "2026-01-05T10:41:02.158Z" }, + { url = "https://files.pythonhosted.org/packages/2e/47/174dca0502ef88b28f1c9e06b73ce33500eedfac7a7692108aec220464e7/tokenizers-0.22.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:1e418a55456beedca4621dbab65a318981467a2b188e982a23e117f115ce5001", size = 2981472, upload-time = "2026-01-05T10:41:00.276Z" }, + { url = "https://files.pythonhosted.org/packages/d6/84/7990e799f1309a8b87af6b948f31edaa12a3ed22d11b352eaf4f4b2e5753/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2249487018adec45d6e3554c71d46eb39fa8ea67156c640f7513eb26f318cec7", size = 3290736, upload-time = "2026-01-05T10:40:32.165Z" }, + { url = "https://files.pythonhosted.org/packages/78/59/09d0d9ba94dcd5f4f1368d4858d24546b4bdc0231c2354aa31d6199f0399/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:25b85325d0815e86e0bac263506dd114578953b7b53d7de09a6485e4a160a7dd", size = 3168835, upload-time = "2026-01-05T10:40:38.847Z" }, + { url = "https://files.pythonhosted.org/packages/47/50/b3ebb4243e7160bda8d34b731e54dd8ab8b133e50775872e7a434e524c28/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bfb88f22a209ff7b40a576d5324bf8286b519d7358663db21d6246fb17eea2d5", size = 3521673, upload-time = "2026-01-05T10:40:56.614Z" }, + { url = "https://files.pythonhosted.org/packages/e0/fa/89f4cb9e08df770b57adb96f8cbb7e22695a4cb6c2bd5f0c4f0ebcf33b66/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1c774b1276f71e1ef716e5486f21e76333464f47bece56bbd554485982a9e03e", size = 3724818, upload-time = "2026-01-05T10:40:44.507Z" }, + { url = "https://files.pythonhosted.org/packages/64/04/ca2363f0bfbe3b3d36e95bf67e56a4c88c8e3362b658e616d1ac185d47f2/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:df6c4265b289083bf710dff49bc51ef252f9d5be33a45ee2bed151114a56207b", size = 3379195, upload-time = "2026-01-05T10:40:51.139Z" }, + { url = "https://files.pythonhosted.org/packages/2e/76/932be4b50ef6ccedf9d3c6639b056a967a86258c6d9200643f01269211ca/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:369cc9fc8cc10cb24143873a0d95438bb8ee257bb80c71989e3ee290e8d72c67", size = 3274982, upload-time = "2026-01-05T10:40:58.331Z" }, + { url = "https://files.pythonhosted.org/packages/1d/28/5f9f5a4cc211b69e89420980e483831bcc29dade307955cc9dc858a40f01/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:29c30b83d8dcd061078b05ae0cb94d3c710555fbb44861139f9f83dcca3dc3e4", size = 9478245, upload-time = "2026-01-05T10:41:04.053Z" }, + { url = "https://files.pythonhosted.org/packages/6c/fb/66e2da4704d6aadebf8cb39f1d6d1957df667ab24cff2326b77cda0dcb85/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:37ae80a28c1d3265bb1f22464c856bd23c02a05bb211e56d0c5301a435be6c1a", size = 9560069, upload-time = "2026-01-05T10:45:10.673Z" }, + { url = "https://files.pythonhosted.org/packages/16/04/fed398b05caa87ce9b1a1bb5166645e38196081b225059a6edaff6440fac/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:791135ee325f2336f498590eb2f11dc5c295232f288e75c99a36c5dbce63088a", size = 9899263, upload-time = "2026-01-05T10:45:12.559Z" }, + { url = "https://files.pythonhosted.org/packages/05/a1/d62dfe7376beaaf1394917e0f8e93ee5f67fea8fcf4107501db35996586b/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:38337540fbbddff8e999d59970f3c6f35a82de10053206a7562f1ea02d046fa5", size = 10033429, upload-time = "2026-01-05T10:45:14.333Z" }, + { url = "https://files.pythonhosted.org/packages/fd/18/a545c4ea42af3df6effd7d13d250ba77a0a86fb20393143bbb9a92e434d4/tokenizers-0.22.2-cp39-abi3-win32.whl", hash = "sha256:a6bf3f88c554a2b653af81f3204491c818ae2ac6fbc09e76ef4773351292bc92", size = 2502363, upload-time = "2026-01-05T10:45:20.593Z" }, + { url = "https://files.pythonhosted.org/packages/65/71/0670843133a43d43070abeb1949abfdef12a86d490bea9cd9e18e37c5ff7/tokenizers-0.22.2-cp39-abi3-win_amd64.whl", hash = "sha256:c9ea31edff2968b44a88f97d784c2f16dc0729b8b143ed004699ebca91f05c48", size = 2747786, upload-time = "2026-01-05T10:45:18.411Z" }, + { url = "https://files.pythonhosted.org/packages/72/f4/0de46cfa12cdcbcd464cc59fde36912af405696f687e53a091fb432f694c/tokenizers-0.22.2-cp39-abi3-win_arm64.whl", hash = "sha256:9ce725d22864a1e965217204946f830c37876eee3b2ba6fc6255e8e903d5fcbc", size = 2612133, upload-time = "2026-01-05T10:45:17.232Z" }, +] + [[package]] name = "tomli" version = "2.4.0" @@ -2234,6 +2401,41 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl", hash = "sha256:ee1e4c0e59148062281c49d80b25b67771a127c85fc9676d3be5f243206826bf", size = 78374, upload-time = "2026-02-03T17:35:50.982Z" }, ] +[[package]] +name = "transformers" +version = "5.14.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "numpy", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "packaging", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "pyyaml", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "regex", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "safetensors", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "tokenizers", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "tqdm", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "typer", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5a/fb/2a2ba88f325e68a921d8b69ff63b477830b2e73ade9a3c8c8cab2f06d741/transformers-5.14.1.tar.gz", hash = "sha256:60d196c27781eacf8637e2b533f517582907ad6f9ae142046d6b69431a5b2173", size = 9295927, upload-time = "2026-07-16T09:41:57.773Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6f/67/8d85ca2323233ae3c0365a659c4e52ee1f587b440e4bc577e7d8e4416d0f/transformers-5.14.1-py3-none-any.whl", hash = "sha256:9db974c4079ede2d1a3ea7ca5a240df33f2cc26fc2b36ba64c5f2a4f43b6e725", size = 11625234, upload-time = "2026-07-16T09:41:54.143Z" }, +] + +[[package]] +name = "typer" +version = "0.27.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "rich", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "shellingham", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/37/78/fda3361b56efc27944f24225f6ecd13d96d6fcfe37bd0eb34e2f4c63f9fc/typer-0.27.0.tar.gz", hash = "sha256:629bd12ea5d13a17148125d9a264f949eb171fb3f120f9b04d85873cab054fa5", size = 203430, upload-time = "2026-07-15T19:21:07.007Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/03/26a383c9e58c213199d1aad1c3d353cfc22d4444ec6d2c0bf8ad02523843/typer-0.27.0-py3-none-any.whl", hash = "sha256:6f4b27631e47f077871b7dc30e933ec0131c1390fbe0e387ea5574b5bac9ccf1", size = 122716, upload-time = "2026-07-15T19:21:05.553Z" }, +] + [[package]] name = "typing-extensions" version = "4.15.0"