Add simulator.

.cspell.json

@@ -102,101 +102,115 @@
     "uvicorn"
   ],
   "ignoreWords": [
+    "prompty",
+    "AADSTS700082",
+    "punc",
+    "mltable",
+    "tiktoken",
+    "Mobius",
+    "evals",
+    "nlines",
     "openmpi",
-    "ipynb",
-    "xdist",
-    "pydash",
-    "tqdm",
     "rtype",
-    "epocs",
-    "fout",
-    "funcs",
-    "todos",
-    "fstring",
-    "creds",
-    "zipp",
-    "gmtime",
-    "pyjwt",
-    "nbconvert",
-    "nbformat",
-    "pypandoc",
-    "dotenv",
-    "miniconda",
-    "datas",
-    "tcgetpgrp",
+    "Hwfoxydrg",
+    "thead",
     "yamls",
-    "fmt",
-    "serpapi",
+    "runsv",
+    "continuumio",
+    "datas",
+    "pydash",
     "genutils",
-    "metadatas",
-    "tiktoken",
-    "bfnrt",
-    "orelse",
-    "thead",
+    "werkzeug",
+    "fmt",
+    "httpx",
+    "pkill",
+    "levelno",
+    "nohup",
+    "pyjwt",
+    "wscript",
     "sympy",
-    "ghactions",
+    "endofprompt",
+    "miniconda",
+    "getch",
     "esac",
-    "MSRC",
-    "pycln",
+    "LANCZOS",
+    "cmpl",
+    "spawnve",
+    "metadatas",
+    "llmops",
+    "unlabel",
+    "llms",
     "strictyaml",
-    "psutil",
-    "getch",
-    "tcgetattr",
-    "TCSADRAIN",
-    "stringio",
-    "jsonify",
-    "werkzeug",
-    "continuumio",
-    "pydantic",
-    "iterrows",
+    "uqkvl",
+    "nologo",
     "dtype",
-    "fillna",
-    "nlines",
+    "gunicorn",
+    "Pyinstaller",
     "aggr",
+    "pycln",
+    "jsonify",
+    "MSRC",
+    "metagenai",
     "tcsetattr",
-    "pysqlite",
-    "AADSTS700082",
-    "Pyinstaller",
-    "runsvdir",
-    "runsv",
-    "levelno",
-    "LANCZOS",
-    "Mobius",
-    "ruamel",
-    "gunicorn",
-    "pkill",
-    "pgrep",
-    "Hwfoxydrg",
-    "llms",
+    "xdist",
+    "addrs",
+    "tcgetpgrp",
+    "setenv",
+    "epocs",
     "vcrpy",
-    "uionly",
-    "llmops",
-    "Abhishek",
-    "restx",
-    "httpx",
+    "TCSADRAIN",
+    "nbconvert",
+    "dotenv",
+    "fout",
+    "STARTF",
+    "otel",
+    "fillna",
+    "tqdm",
+    "gmtime",
+    "ghactions",
+    "runsvdir",
+    "pywin",
+    "serpapi",
+    "pysqlite",
+    "zipp",
     "tiiuae",
-    "nohup",
-    "metagenai",
-    "WBITS",
-    "laddr",
+    "aiohttp",
     "nrows",
-    "Dumpable",
-    "XCLASS",
-    "otel",
+    "funcs",
+    "orelse",
+    "Unlabel",
+    "ipynb",
     "OTLP",
+    "pgrep",
+    "pypandoc",
+    "nbformat",
+    "psutil",
     "spawnv",
-    "spawnve",
-    "addrs",
-    "pywin",
-    "STARTF",
-    "mltable",
-    "prompty",
-    "Prompty",
-    "setenv",
-    "cscript",
-    "nologo",
-    "wscript",
+    "fstring",
+    "pydantic",
     "raisvc",
+    "restx",
+    "WBITS",
+    "cscript",
+    "bfnrt",
+    "ruamel",
+    "Prompty",
+    "todos",
+    "uionly",
+    "tcgetattr",
+    "stringio",
+    "Dumpable",
+    "XCLASS",
+    "iterrows",
+    "laddr",
+    "unbatched",
+    "creds",
+    "pybase",
+    "tkey",
+    "tparam",
+    "ncols",
+    "wohdjewodhfjevwdjfywlemfhe",
+    "dontcare",
     "evals",
     "dcid"
   ],

.github/workflows/promptflow-evals-unit-test.yml

@@ -102,4 +102,4 @@ jobs:
           format: markdown
           hide_complexity: true
           output: both
-          thresholds: 40 60
+          thresholds: 40 60

src/promptflow-evals/promptflow/evals/synthetic/simulator/__init__.py

@@ -0,0 +1,11 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+import os
+
+_template_dir = os.path.join(os.path.dirname(__file__), "templates")
+
+# from .simulator.simulator import Simulator  # pylint: disable=wrong-import-position
+
+
+# __all__ = ["Simulator"]

src/promptflow-evals/promptflow/evals/synthetic/simulator/_conversation/__init__.py

@@ -0,0 +1,44 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+# pylint: disable=unused-import
+from .al_conversation_bot import AugLoopConversationBot
+
+# pylint: disable=unused-import
+from .augloop_client import AugLoopParams
+
+# pylint: disable=unused-import
+from .constants import ConversationRole
+
+# pylint: disable=unused-import
+from .conversation import debug_conversation, play_conversation, simulate_conversation
+
+# pylint: disable=unused-import
+from .conversation_bot import ConversationBot
+
+# pylint: disable=unused-import
+from .conversation_request import ConversationRequest
+
+# pylint: disable=unused-import
+from .conversation_turn import ConversationTurn
+
+# pylint: disable=unused-import
+from .conversation_writer import ConversationWriter
+
+# pylint: disable=unused-import
+from .dummy_conversation_bot import DummyConversationBot
+
+__all__ = [
+    "AugLoopConversationBot",
+    "AugLoopParams",
+    "ConversationRole",
+    "debug_conversation",
+    "play_conversation",
+    "simulate_conversation",
+    "ConversationBot",
+    "ConversationRequest",
+    "ConversationTurn",
+    "ConversationWriter",
+    "DummyConversationBot",
+]

src/promptflow-evals/promptflow/evals/synthetic/simulator/_conversation/al_conversation_bot.py

@@ -0,0 +1,96 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+import logging
+import time
+from typing import Dict, List, Tuple
+
+from promptflow.evals.synthetic.simulator._model_tools import RetryClient
+
+from .augloop_client import AugLoopClient, AugLoopParams
+from .constants import ConversationRole
+from .conversation_bot import ConversationBot
+from .conversation_turn import ConversationTurn
+
+
+class AugLoopConversationBot(ConversationBot):
+    def __init__(  # pylint: disable=super-init-not-called
+        self,
+        role: ConversationRole,
+        augLoopParams: AugLoopParams,
+        instantiation_parameters: Dict[str, str],
+    ):
+        """
+        Create an AugLoop ConversationBot with specific name,
+        persona and a sentence that can be used as a conversation starter.
+
+        Parameters
+        ----------
+        role: The role of the bot in the conversation, either USER or ASSISTANT
+        augLoopParams: The augloop params to use for connecting to augloop
+        conversation_template: A jinja2 template that describes the conversation,
+        this is used to generate the prompt for the LLM
+        instantiation_parameters: A dictionary of parameters that are used to instantiate the conversation template
+        """
+        if role == ConversationRole.USER:
+            raise Exception("AugLoop conversation Bot is not enabled for USER role")
+
+        self.role = role
+        self.augLoopParams = augLoopParams
+
+        self.persona_template_args = instantiation_parameters
+        self.name = (
+            self.persona_template_args.get("chatbot_name", role.value) or f"Augloop_{augLoopParams.workflowName}"
+        )
+
+        self.logger = logging.getLogger(repr(self))
+
+        self.augLoopClient = AugLoopClient(augLoopParams)
+
+    async def generate_response(
+        self,
+        session: RetryClient,
+        conversation_history: List[ConversationTurn],
+        max_history: int,
+        turn_number: int = 0,
+    ) -> Tuple[dict, dict, int, dict]:
+        """
+        Prompt the ConversationBot for a response.
+
+        :param session: The aiohttp session to use for the request.
+        :type session: RetryClient
+        :param conversation_history: The turns in the conversation so far.
+        :type conversation_history: List[ConversationTurn]
+        :param max_history: Parameters used to query GPT-4 model.
+        :type max_history: int
+        :param turn_number: Parameters used to query GPT-4 model.
+        :type turn_number: int
+        :return: The response from the ConversationBot.
+        :rtype: Tuple[dict, dict, int, dict]
+        """
+
+        messageToSend = conversation_history[-1].message
+
+        time_start = time.time()
+
+        # send message
+        response_data = self.augLoopClient.send_signal_and_wait_for_annotation(messageToSend)
+
+        time_taken = time.time() - time_start
+
+        if not response_data["success"]:
+            raise Exception("Unexpected result from Augloop")
+
+        parsed_response = {
+            "samples": response_data["messages"],
+            "id": response_data["id"],
+        }
+
+        messages = [{"role": "system", "content": messageToSend}]
+        request = {"messages": messages}
+
+        return parsed_response, request, int(time_taken), response_data["full_message"]
+
+    def __repr__(self):
+        return f"Bot(name={self.name}, role={self.role.name}, model=Augloop)"