From 51870e803bf44f2f2a8e36fdecc6c62af49558ba Mon Sep 17 00:00:00 2001 From: Nagkumar Arkalgud Date: Thu, 19 Dec 2024 11:32:33 -0800 Subject: [PATCH] Simulator task free mode (#38858) * Update task_query_response.prompty remove required keys * Update task_simulate.prompty * Update task_query_response.prompty * Update task_simulate.prompty * Fix the api_key needed * Update for release * Black fix for file * Add original text in global context * Update test * Update the indirect attack simulator * Black suggested fixes * Update simulator prompty * Update adversarial scenario enum to exclude XPIA * Update changelog * Black fixes * Remove duplicate import * Fix the mypy error * Mypy please be happy * Updates to non adv simulator * accept context from assistant messages, exclude them when using them for conversation * update changelog * pylint fixes * pylint fixes * remove redundant quotes * Fix typo * pylint fix * Update broken tests * Include the grounding json in the manifest * Fix typo * Come on package * Release 1.0.0b5 * Notice from Chang * Remove adv_conv template parameters from the outputs * Update chanagelog * Experimental tags on adv scenarios * Readme fix onbreaking change * Add the category and both user and assistant context to the response of qr_json_lines * Update changelog * Rename _kwargs to _options * _options as prefix * update troubleshooting for simulator * Rename according to suggestions * Clean up readme * more links * Bugfix: zip_longest created null parameters * Updated changelog * zip does the job * remove ununsed import * Update simulator to work with task free mode * Update changelog * black fixes --------- Co-authored-by: Nagkumar Arkalgud Co-authored-by: Nagkumar Arkalgud Co-authored-by: Nagkumar Arkalgud --- .../azure-ai-evaluation/CHANGELOG.md | 3 ++ .../ai/evaluation/simulator/_simulator.py | 34 ++++++++++++------- 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md index 3614c102ef56..758e9c980987 100644 --- a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md +++ b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md @@ -7,6 +7,9 @@ ### Breaking Changes ### Bugs Fixed +- Removed `[remote]` extra. This is no longer needed when tracking results in Azure AI Studio. +- Fixed `AttributeError: 'NoneType' object has no attribute 'get'` while running simulator with 1000+ results +- Fixed the non adversarial simulator to run in task-free mode ### Other Changes diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_simulator.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_simulator.py index 2b66e10db36d..ed6b5547d0bd 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_simulator.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_simulator.py @@ -157,7 +157,7 @@ async def __call__( f"You have specified 'num_queries' < len('tasks') ({num_queries} < {len(tasks)}). " f"Only the first {num_queries} lines of the specified tasks will be simulated." ) - num_queries = min(num_queries, len(tasks)) + max_conversation_turns *= 2 # account for both user and assistant turns prompty_model_config = self.model_config @@ -586,7 +586,10 @@ async def _create_conversations_from_query_responses( for i, query_response_pair in enumerate(query_responses): query = query_response_pair["q"] response = query_response_pair["r"] - task = tasks[i] + try: + task = tasks[i] + except IndexError: + task = None conversation = await self._complete_conversation( conversation_starter=query, @@ -621,7 +624,7 @@ async def _complete_conversation( *, conversation_starter: str, max_conversation_turns: int, - task: str, + task: Optional[str], user_simulator_prompty: Optional[str], user_simulator_prompty_options: Dict[str, Any], target: Callable, @@ -659,16 +662,21 @@ async def _complete_conversation( user_simulator_prompty_options=user_simulator_prompty_options, ) if len(conversation_history) == 0: - conversation_starter_from_simulated_user = await user_flow( - task=task, - conversation_history=[ - { - "role": "assistant", - "content": conversation_starter, - } - ], - action="rewrite the assistant's message as you have to accomplish the task by asking the right questions. Make sure the original question is not lost in your rewrite.", - ) + if task: + conversation_starter_from_simulated_user = await user_flow( + task=task, + conversation_history=[ + { + "role": "assistant", + "content": conversation_starter, + } + ], + action="rewrite the assistant's message as you have to accomplish the task by asking the right questions. Make sure the original question is not lost in your rewrite.", + ) + else: + conversation_starter_from_simulated_user = { + "content": conversation_starter, + } else: conversation_starter_from_simulated_user = await user_flow( task=task,