Merge branch 'main' into fix/openapi-tool-timeout

Author: fuki01

pyproject.toml

@@ -56,7 +56,7 @@ dependencies = [
   "opentelemetry-resourcedetector-gcp>=1.9.0a0, <2.0.0",
   "opentelemetry-sdk>=1.36.0, <1.40.0",
   "pyarrow>=14.0.0",
-  "pydantic>=2.0, <3.0.0",                                   # For data validation/models
+  "pydantic>=2.7.0, <3.0.0",                                 # For data validation/models
   "python-dateutil>=2.9.0.post0, <3.0.0",                    # For Vertext AI Session Service
   "python-dotenv>=1.0.0, <2.0.0",                            # To manage environment variables
   "requests>=2.32.4, <3.0.0",
@@ -108,6 +108,7 @@ community = [
 
 eval = [
   # go/keep-sorted start
+  "Jinja2>=3.1.4,<4.0.0",                         # For eval template rendering
   "google-cloud-aiplatform[evaluation]>=1.100.0",
   "pandas>=2.2.3",
   "rouge-score>=0.1.2",

src/google/adk/evaluation/conversation_scenarios.py

@@ -14,9 +14,14 @@
 
 from __future__ import annotations
 
+from typing import Optional
+
 from pydantic import Field
+from pydantic import field_validator
 
 from .common import EvalBaseModel
+from .simulation.pre_built_personas import get_default_persona_registry
+from .simulation.user_simulator_personas import UserPersona
 
 
 class ConversationScenario(EvalBaseModel):
@@ -48,6 +53,18 @@ class ConversationScenario(EvalBaseModel):
   your overall goal is complete.
   """
 
+  user_persona: Optional[UserPersona] = Field(default=None)
+  """User persona that the user simulator should adopt. If a persona id is specified instead, we will try to use one of our default personas."""
+
+  @field_validator("user_persona", mode="before")
+  @classmethod
+  def validate_user_persona(
+      cls, value: Optional[UserPersona | str]
+  ) -> Optional[UserPersona]:
+    if value is not None and isinstance(value, str):
+      return get_default_persona_registry().get_persona(value)
+    return value
+
 
 class ConversationScenarios(EvalBaseModel):
   """A simple container for the list of ConversationScenario.

src/google/adk/evaluation/simulation/llm_backed_user_simulator.py

@@ -31,6 +31,8 @@
 from .._retry_options_utils import add_default_retry_options_if_not_present
 from ..conversation_scenarios import ConversationScenario
 from ..evaluator import Evaluator
+from .llm_backed_user_simulator_prompts import get_llm_backed_user_simulator_prompt
+from .llm_backed_user_simulator_prompts import is_valid_user_simulator_template
 from .user_simulator import BaseUserSimulatorConfig
 from .user_simulator import NextUserMessage
 from .user_simulator import Status
@@ -41,63 +43,6 @@
 _AUTHOR_USER = "user"
 _STOP_SIGNAL = "</finished>"
 
-_DEFAULT_USER_AGENT_INSTRUCTIONS = """You are a Simulated User designed to test an AI Agent.
-
-Your single most important job is to react logically to the Agent's last message.
-The Conversation Plan is your canonical grounding, not a script; your response MUST be dictated by what the Agent just said.
-
-# Primary Operating Loop
-
-You MUST follow this three-step process while thinking:
-
-Step 1: Analyze what the Agent just said or did. Specifically, is the Agent asking you a question, reporting a successful or unsuccessful operation, or saying something incorrect or unexpected?
-
-Step 2: Choose one action based on your analysis:
-* ANSWER any questions the Agent asked.
-* ADVANCE to the next request as per the Conversation Plan if the Agent succeeds in satisfying your current request.
-* INTERVENE if the Agent is yet to complete your current request and the Conversation Plan requires you to modify it.
-* CORRECT the Agent if it is making a mistake or failing.
-* END the conversation if any of the below stopping conditions are met:
-  - The Agent has completed all your requests from the Conversation Plan.
-  - The Agent has failed to fulfill a request *more than once*.
-  - The Agent has performed an incorrect operation and informs you that it is unable to correct it.
-  - The Agent ends the conversation on its own by transferring you to a *human/live agent* (NOT another AI Agent).
-
-Step 3: Formulate a response based on the chosen action and the below Action Protocols and output it.
-
-# Action Protocols
-
-**PROTOCOL: ANSWER**
-* Only answer the Agent's questions using information from the Conversation Plan.
-* Do NOT provide any additional information the Agent did not explicitly ask for.
-* If you do not have the information requested by the Agent, inform the Agent. Do NOT make up information that is not in the Conversation Plan.
-* Do NOT advance to the next request in the Conversation Plan.
-
-**PROTOCOL: ADVANCE**
-* Make the next request from the Conversation Plan.
-* Skip redundant requests already fulfilled by the Agent.
-
-**PROTOCOL: INTERVENE**
-* Change your current request as directed by the Conversation Plan with natural phrasing.
-
-**PROTOCOL: CORRECT**
-* Challenge illogical or incorrect statements made by the Agent.
-* If the Agent did an incorrect operation, ask the Agent to fix it.
-* If this is the FIRST time the Agent failed to satisfy your request, ask the Agent to try again.
-
-**PROTOCOL: END**
-* End the conversation only when any of the stopping conditions are met; do NOT end prematurely.
-* Output `{stop_signal}` to indicate that the conversation with the AI Agents is over.
-
-# Conversation Plan
-
-{conversation_plan}
-
-# Conversation History
-
-{conversation_history}
-"""
-
 
 class LlmBackedUserSimulatorConfig(BaseUserSimulatorConfig):
   """Contains configurations required by an LLM backed user simulator."""
@@ -130,32 +75,34 @@ class LlmBackedUserSimulatorConfig(BaseUserSimulatorConfig):
   custom_instructions: Optional[str] = Field(
       default=None,
       description="""Custom instructions for the LlmBackedUserSimulator. The
-instructions must contain the following formatting placeholders:
-* {stop_signal} : text to be generated when the user simulator decides that the
+instructions must contain the following formatting placeholders following Jinja syntax:
+* {{ stop_signal }} : text to be generated when the user simulator decides that the
   conversation is over.
-* {conversation_plan} : the overall plan for the conversation that the user
+* {{ conversation_plan }} : the overall plan for the conversation that the user
   simulator must follow.
-* {conversation_history} : the conversation between the user and the agent so
-  far.""",
+* {{ conversation_history }} : the conversation between the user and the agent so
+  far.
+* {{ persona }} : Only needed if specifying user_persona in the conversation scenario.
+""",
   )
 
   @field_validator("custom_instructions")
   @classmethod
   def validate_custom_instructions(cls, value: Optional[str]) -> Optional[str]:
     if value is None:
       return value
-    if not all(
-        placeholder in value
-        for placeholder in [
-            "{stop_signal}",
-            "{conversation_plan}",
-            "{conversation_history}",
-        ]
+    if not is_valid_user_simulator_template(
+        value,
+        required_params=[
+            "stop_signal",
+            "conversation_plan",
+            "conversation_history",
+        ],
     ):
       raise ValueError(
           "custom_instructions must contain each of the following formatting"
-          " placeholders:"
-          " {stop_signal}, {conversation_plan}, {conversation_history}"
+          " placeholders using Jinja syntax: {{ stop_signal }}, {{"
+          " conversation_plan }}, {{ conversation_history }}"
       )
     return value
 
@@ -180,11 +127,7 @@ def __init__(
     llm_registry = LLMRegistry()
     llm_class = llm_registry.resolve(self._config.model)
     self._llm = llm_class(model=self._config.model)
-    self._instructions = (
-        self._config.custom_instructions
-        if self._config.custom_instructions
-        else _DEFAULT_USER_AGENT_INSTRUCTIONS
-    )
+    self._user_persona = self._conversation_scenario.user_persona
 
   @classmethod
   def _summarize_conversation(
@@ -221,10 +164,12 @@ async def _get_llm_response(
       # first invocation - send the static starting prompt
       return self._conversation_scenario.starting_prompt
 
-    user_agent_instructions = self._instructions.format(
-        stop_signal=_STOP_SIGNAL,
+    user_agent_instructions = get_llm_backed_user_simulator_prompt(
         conversation_plan=self._conversation_scenario.conversation_plan,
         conversation_history=rewritten_dialogue,
+        stop_signal=_STOP_SIGNAL,
+        custom_instructions=self._config.custom_instructions,
+        user_persona=self._user_persona,
     )
 
     llm_request = LlmRequest(