diff --git a/api/services/pipecat/service_factory.py b/api/services/pipecat/service_factory.py index f6fc6b821..de85c689c 100644 --- a/api/services/pipecat/service_factory.py +++ b/api/services/pipecat/service_factory.py @@ -933,6 +933,24 @@ def _migrate_deprecated_google_model(model: str) -> str: @_report_service_factory_failures(ErrorSource.LLM, provider_argument=0) +def _google_thinking_for_model(model: str): + """Return a thinking config compatible with the resolved Google model. + + Gemini 2.5 uses ``thinking_budget``; Gemini 3 uses ``thinking_level`` — the + two must not be mixed. We match on the model-family *prefix* and return + ``None`` for any id that isn't a recognized Gemini 2.5/3 family (custom, + legacy, or future, including ids that merely embed the family name), so an + unsupported thinking config is never sent; the raised ``max_tokens`` ceiling + alone then guards against truncation. + """ + ml = (model or "").strip().lower() + if ml.startswith("gemini-2.5") or ml.startswith("gemini-2-5"): + return GoogleLLMService.ThinkingConfig(thinking_budget=4096) + if ml.startswith("gemini-3"): + return GoogleLLMService.ThinkingConfig(thinking_level="low") + return None + + def create_llm_service_from_provider( provider: str, model: str, @@ -948,6 +966,7 @@ def create_llm_service_from_provider( location: str | None = None, credentials: str | None = None, temperature: float | None = None, + max_tokens: int | None = None, bill_to: str | None = None, usage_context: str | None = None, ): @@ -1000,16 +1019,35 @@ def create_llm_service_from_provider( ) elif provider == ServiceProviders.GOOGLE.value: model = _migrate_deprecated_google_model(model) + google_settings_kwargs: dict = {"model": model, "temperature": 0.1} + if max_tokens is not None: + # Give the grader a large explicit output ceiling and bound "thinking" + # so the reasoning budget can't crowd out the answer (Gemini's default + # 4096, shared with dynamic thinking, truncated long QA-grader JSON). + # Pick the thinking knob that matches the model family (2.5 -> budget, + # 3 -> level; omit for unknown/legacy) via _google_thinking_for_model. + google_settings_kwargs["max_tokens"] = max_tokens + thinking = _google_thinking_for_model(model) + if thinking is not None: + google_settings_kwargs["thinking"] = thinking return DograhGoogleLLMService( api_key=api_key, - settings=GoogleLLMSettings(model=model, temperature=0.1), + settings=GoogleLLMSettings(**google_settings_kwargs), ) elif provider == ServiceProviders.GOOGLE_VERTEX.value: + vertex_settings_kwargs: dict = {"model": model, "temperature": 0.1} + if max_tokens is not None: + # Same truncation guard as the Google branch: raise the output + # ceiling and bound thinking with the model-family-appropriate knob. + vertex_settings_kwargs["max_tokens"] = max_tokens + thinking = _google_thinking_for_model(model) + if thinking is not None: + vertex_settings_kwargs["thinking"] = thinking return DograhGoogleVertexLLMService( credentials=credentials, project_id=project_id, location=location or "us-east4", - settings=GoogleVertexLLMSettings(model=model, temperature=0.1), + settings=GoogleVertexLLMSettings(**vertex_settings_kwargs), ) elif provider == ServiceProviders.AZURE.value: if endpoint: @@ -1282,6 +1320,7 @@ def create_llm_service( user_config, correlation_id: str | None = None, usage_context: str | None = None, + max_tokens: int | None = None, ): """Create and return appropriate LLM service based on user configuration.""" provider = user_config.llm.provider @@ -1323,6 +1362,7 @@ def create_llm_service( api_key, correlation_id=correlation_id, usage_context=usage_context, + max_tokens=max_tokens, **kwargs, ) @@ -1332,6 +1372,7 @@ def create_llm_service_with_model_override( model_override: str | None, correlation_id: str | None = None, usage_context: str | None = None, + max_tokens: int | None = None, ): """Create an LLM service with an optional model override. @@ -1343,6 +1384,7 @@ def create_llm_service_with_model_override( user_config, correlation_id=correlation_id, usage_context=usage_context, + max_tokens=max_tokens, ) if user_config.llm is None: @@ -1354,4 +1396,5 @@ def create_llm_service_with_model_override( overridden_config, correlation_id=correlation_id, usage_context=usage_context, + max_tokens=max_tokens, ) diff --git a/api/services/workflow/qa/llm_config.py b/api/services/workflow/qa/llm_config.py index 9b5911a6d..d43d43738 100644 --- a/api/services/workflow/qa/llm_config.py +++ b/api/services/workflow/qa/llm_config.py @@ -14,6 +14,10 @@ from api.services.workflow.dto import QANodeData QA_USAGE_CONTEXT = "qa_analysis" +# Explicit output ceiling for QA grader inference. Google/Anthropic default to +# 4096 output tokens (shared with dynamic thinking on Gemini), truncating long +# grading JSON. A generous ceiling + bounded thinking lets a full grade complete. +QA_MAX_OUTPUT_TOKENS = 16384 async def create_qa_llm_service( @@ -47,6 +51,7 @@ async def create_qa_llm_service( api_key, correlation_id=correlation_id, usage_context=QA_USAGE_CONTEXT, + max_tokens=QA_MAX_OUTPUT_TOKENS, **kwargs, ) return llm, model @@ -75,6 +80,7 @@ async def create_qa_llm_service( model_override, correlation_id=correlation_id, usage_context=QA_USAGE_CONTEXT, + max_tokens=QA_MAX_OUTPUT_TOKENS, ) return llm, model diff --git a/api/tests/test_google_thinking_config.py b/api/tests/test_google_thinking_config.py new file mode 100644 index 000000000..51a2ef0b7 --- /dev/null +++ b/api/tests/test_google_thinking_config.py @@ -0,0 +1,74 @@ +"""Regression test: QA-grader thinking config must match the Google model family. + +``thinking_budget`` is only valid for Gemini 2.5; Gemini 3 uses ``thinking_level``, +and the two must not be mixed. Any id that isn't a recognized 2.5/3 family (custom, +legacy, future, or one that merely embeds the family name) must get no thinking +config, so an unsupported setting can't make the grader reject the request — the +raised ``max_tokens`` ceiling alone still guards against truncation. + +The last test exercises the real wiring: the QA ``max_tokens`` plus the guarded +thinking config must build valid Google *and* Vertex settings. +""" + +from api.services.pipecat.service_factory import ( + GoogleLLMSettings, + GoogleVertexLLMSettings, + _google_thinking_for_model, +) + +QA_MAX_TOKENS = 16384 + + +def test_gemini_25_uses_thinking_budget(): + tc = _google_thinking_for_model("gemini-2.5-flash") + assert tc is not None + assert tc.thinking_budget == 4096 + assert getattr(tc, "thinking_level", None) is None + + +def test_gemini_25_revision_still_matches(): + # dated/preview revisions still start with the family prefix + assert _google_thinking_for_model("gemini-2.5-flash-002").thinking_budget == 4096 + + +def test_gemini_3_uses_thinking_level(): + tc = _google_thinking_for_model("gemini-3-pro-preview") + assert tc is not None + assert tc.thinking_level == "low" + assert tc.thinking_budget is None + + +def test_unknown_legacy_or_embedded_substring_omits_thinking(): + # legacy family, a custom id, ids that only *contain* the family name, empty + for m in ( + "gemini-1.5-pro", + "some-custom-model", + "acme-gemini-2.5-wrapper", + "x-gemini-3-y", + "", + ): + assert _google_thinking_for_model(m) is None, m + + +def test_qa_settings_accept_max_tokens_and_family_thinking(): + """The real wiring: QA max_tokens + guarded thinking build valid settings.""" + for Settings in (GoogleLLMSettings, GoogleVertexLLMSettings): + tc = _google_thinking_for_model("gemini-2.5-flash") + s = Settings( + model="gemini-2.5-flash", + temperature=0.1, + max_tokens=QA_MAX_TOKENS, + thinking=tc, + ) + assert s.max_tokens == QA_MAX_TOKENS + assert s.thinking is not None and s.thinking.thinking_budget == 4096 + + # legacy model: max_tokens still applied, no thinking config attached + legacy = Settings( + model="gemini-1.5-pro", + temperature=0.1, + max_tokens=QA_MAX_TOKENS, + thinking=_google_thinking_for_model("gemini-1.5-pro"), + ) + assert legacy.max_tokens == QA_MAX_TOKENS + assert legacy.thinking is None