fix(qa): prevent grader truncation

api/services/pipecat/service_factory.py

@@ -933,6 +933,24 @@ def _migrate_deprecated_google_model(model: str) -> str:
 
 
 @_report_service_factory_failures(ErrorSource.LLM, provider_argument=0)
+def _google_thinking_for_model(model: str):
+    """Return a thinking config compatible with the resolved Google model.
+
+    Gemini 2.5 uses ``thinking_budget``; Gemini 3 uses ``thinking_level`` — the
+    two must not be mixed. We match on the model-family *prefix* and return
+    ``None`` for any id that isn't a recognized Gemini 2.5/3 family (custom,
+    legacy, or future, including ids that merely embed the family name), so an
+    unsupported thinking config is never sent; the raised ``max_tokens`` ceiling
+    alone then guards against truncation.
+    """
+    ml = (model or "").strip().lower()
+    if ml.startswith("gemini-2.5") or ml.startswith("gemini-2-5"):
+        return GoogleLLMService.ThinkingConfig(thinking_budget=4096)
+    if ml.startswith("gemini-3"):
+        return GoogleLLMService.ThinkingConfig(thinking_level="low")
+    return None
+
+
 def create_llm_service_from_provider(
     provider: str,
     model: str,
@@ -948,6 +966,7 @@ def create_llm_service_from_provider(
     location: str | None = None,
     credentials: str | None = None,
     temperature: float | None = None,
+    max_tokens: int | None = None,
     bill_to: str | None = None,
     usage_context: str | None = None,
 ):
@@ -1000,16 +1019,35 @@ def create_llm_service_from_provider(
         )
     elif provider == ServiceProviders.GOOGLE.value:
         model = _migrate_deprecated_google_model(model)
+        google_settings_kwargs: dict = {"model": model, "temperature": 0.1}
+        if max_tokens is not None:
+            # Give the grader a large explicit output ceiling and bound "thinking"
+            # so the reasoning budget can't crowd out the answer (Gemini's default
+            # 4096, shared with dynamic thinking, truncated long QA-grader JSON).
+            # Pick the thinking knob that matches the model family (2.5 -> budget,
+            # 3 -> level; omit for unknown/legacy) via _google_thinking_for_model.
+            google_settings_kwargs["max_tokens"] = max_tokens
+            thinking = _google_thinking_for_model(model)
+            if thinking is not None:
+                google_settings_kwargs["thinking"] = thinking
         return DograhGoogleLLMService(
             api_key=api_key,
-            settings=GoogleLLMSettings(model=model, temperature=0.1),
+            settings=GoogleLLMSettings(**google_settings_kwargs),
         )
     elif provider == ServiceProviders.GOOGLE_VERTEX.value:
+        vertex_settings_kwargs: dict = {"model": model, "temperature": 0.1}
+        if max_tokens is not None:
+            # Same truncation guard as the Google branch: raise the output
+            # ceiling and bound thinking with the model-family-appropriate knob.
+            vertex_settings_kwargs["max_tokens"] = max_tokens
+            thinking = _google_thinking_for_model(model)
+            if thinking is not None:
+                vertex_settings_kwargs["thinking"] = thinking
         return DograhGoogleVertexLLMService(
             credentials=credentials,
             project_id=project_id,
             location=location or "us-east4",
-            settings=GoogleVertexLLMSettings(model=model, temperature=0.1),
+            settings=GoogleVertexLLMSettings(**vertex_settings_kwargs),
         )
     elif provider == ServiceProviders.AZURE.value:
         if endpoint:
@@ -1282,6 +1320,7 @@ def create_llm_service(
     user_config,
     correlation_id: str | None = None,
     usage_context: str | None = None,
+    max_tokens: int | None = None,
 ):
     """Create and return appropriate LLM service based on user configuration."""
     provider = user_config.llm.provider
@@ -1323,6 +1362,7 @@ def create_llm_service(
         api_key,
         correlation_id=correlation_id,
         usage_context=usage_context,
+        max_tokens=max_tokens,
         **kwargs,
     )
 
@@ -1332,6 +1372,7 @@ def create_llm_service_with_model_override(
     model_override: str | None,
     correlation_id: str | None = None,
     usage_context: str | None = None,
+    max_tokens: int | None = None,
 ):
     """Create an LLM service with an optional model override.
 
@@ -1343,6 +1384,7 @@ def create_llm_service_with_model_override(
             user_config,
             correlation_id=correlation_id,
             usage_context=usage_context,
+            max_tokens=max_tokens,
         )
 
     if user_config.llm is None:
@@ -1354,4 +1396,5 @@ def create_llm_service_with_model_override(
         overridden_config,
         correlation_id=correlation_id,
         usage_context=usage_context,
+        max_tokens=max_tokens,
     )

api/services/workflow/qa/llm_config.py

@@ -14,6 +14,10 @@
 from api.services.workflow.dto import QANodeData
 
 QA_USAGE_CONTEXT = "qa_analysis"
+# Explicit output ceiling for QA grader inference. Google/Anthropic default to
+# 4096 output tokens (shared with dynamic thinking on Gemini), truncating long
+# grading JSON. A generous ceiling + bounded thinking lets a full grade complete.
+QA_MAX_OUTPUT_TOKENS = 16384
 
 
 async def create_qa_llm_service(
@@ -47,6 +51,7 @@ async def create_qa_llm_service(
             api_key,
             correlation_id=correlation_id,
             usage_context=QA_USAGE_CONTEXT,
+            max_tokens=QA_MAX_OUTPUT_TOKENS,
             **kwargs,
         )
         return llm, model
@@ -75,6 +80,7 @@ async def create_qa_llm_service(
         model_override,
         correlation_id=correlation_id,
         usage_context=QA_USAGE_CONTEXT,
+        max_tokens=QA_MAX_OUTPUT_TOKENS,
     )
     return llm, model
 

api/tests/test_google_thinking_config.py

@@ -0,0 +1,74 @@
+"""Regression test: QA-grader thinking config must match the Google model family.
+
+``thinking_budget`` is only valid for Gemini 2.5; Gemini 3 uses ``thinking_level``,
+and the two must not be mixed. Any id that isn't a recognized 2.5/3 family (custom,
+legacy, future, or one that merely embeds the family name) must get no thinking
+config, so an unsupported setting can't make the grader reject the request — the
+raised ``max_tokens`` ceiling alone still guards against truncation.
+
+The last test exercises the real wiring: the QA ``max_tokens`` plus the guarded
+thinking config must build valid Google *and* Vertex settings.
+"""
+
+from api.services.pipecat.service_factory import (
+    GoogleLLMSettings,
+    GoogleVertexLLMSettings,
+    _google_thinking_for_model,
+)
+
+QA_MAX_TOKENS = 16384
+
+
+def test_gemini_25_uses_thinking_budget():
+    tc = _google_thinking_for_model("gemini-2.5-flash")
+    assert tc is not None
+    assert tc.thinking_budget == 4096
+    assert getattr(tc, "thinking_level", None) is None
+
+
+def test_gemini_25_revision_still_matches():
+    # dated/preview revisions still start with the family prefix
+    assert _google_thinking_for_model("gemini-2.5-flash-002").thinking_budget == 4096
+
+
+def test_gemini_3_uses_thinking_level():
+    tc = _google_thinking_for_model("gemini-3-pro-preview")
+    assert tc is not None
+    assert tc.thinking_level == "low"
+    assert tc.thinking_budget is None
+
+
+def test_unknown_legacy_or_embedded_substring_omits_thinking():
+    # legacy family, a custom id, ids that only *contain* the family name, empty
+    for m in (
+        "gemini-1.5-pro",
+        "some-custom-model",
+        "acme-gemini-2.5-wrapper",
+        "x-gemini-3-y",
+        "",
+    ):
+        assert _google_thinking_for_model(m) is None, m
+
+
+def test_qa_settings_accept_max_tokens_and_family_thinking():
+    """The real wiring: QA max_tokens + guarded thinking build valid settings."""
+    for Settings in (GoogleLLMSettings, GoogleVertexLLMSettings):
+        tc = _google_thinking_for_model("gemini-2.5-flash")
+        s = Settings(
+            model="gemini-2.5-flash",
+            temperature=0.1,
+            max_tokens=QA_MAX_TOKENS,
+            thinking=tc,
+        )
+        assert s.max_tokens == QA_MAX_TOKENS
+        assert s.thinking is not None and s.thinking.thinking_budget == 4096
+
+        # legacy model: max_tokens still applied, no thinking config attached
+        legacy = Settings(
+            model="gemini-1.5-pro",
+            temperature=0.1,
+            max_tokens=QA_MAX_TOKENS,
+            thinking=_google_thinking_for_model("gemini-1.5-pro"),
+        )
+        assert legacy.max_tokens == QA_MAX_TOKENS
+        assert legacy.thinking is None