Review comments

Author: gjwoods

docs/reference/tools-reference/open_source_llm_tool.md

@@ -24,13 +24,18 @@ Setup connections to provisioned resources in prompt flow.
 |-------------|----------|----------|----------|-------------|
 | CustomConnection | Required | Required | -        | -           |
 
-Instructions to create a Custom Connection [can be found here.](https://learn.microsoft.com/en-us/azure/machine-learning/prompt-flow/how-to-integrate-with-langchain?view=azureml-api-2#create-a-connection)
+Instructions to create a Custom Connection [can be found here.](https://microsoft.github.io/promptflow/how-to-guides/manage-connections.html#create-a-connection)
 
 The keys to set are:
 
-1. endpoint_url
-2. endpoint_api_key
-3. model_family
+1. **endpoint_url**
+   - This value can be found at the previously created Inferencing endpoint.
+2. **endpoint_api_key**
+   - Ensure to set this as a secret value.
+   - This value can be found at the previously created Inferencing endpoint.
+3. **model_family**
+   - Supported values: LLAMA, DOLLY, GPT2, or FALCON
+   - This value is dependent on the type of deployment you are targetting.
 
 *These values can be found at the previously created Inferencing endpoint.*
 
@@ -56,4 +61,4 @@ The keys to set are:
 1. Choose a Model from the catalog and deploy.
 2. Setup and select the connections to model deployment.
 3. Configure the model api and its parameters
-4. Prepare the Prompt with [guidance](./prompt-tool.md#how-to-write-prompt).
+4. Prepare the Prompt with [guidance](prompt-tool.md#how-to-write-prompt).

src/promptflow-tools/promptflow/tools/open_source_llm.py

@@ -59,20 +59,20 @@ def format_generic_response_payload(output: bytes, response_key: str) -> str:
     except KeyError as e:
         if response_key is None:
             message = f"""Expected the response to fit the following schema:
-                `[
-                    <text>
-                ]`
-                Instead, received {response_json} and access failed at key `{e}`.
-                """
+`[
+    <text>
+]`
+Instead, received {response_json} and access failed at key `{e}`.
+"""
         else:
             message = f"""Expected the response to fit the following schema:
-                `[
-                    {{
-                        "{response_key}": <text>
-                    }}
-                ]`
-                Instead, received {response_json} and access failed at key `{e}`.
-                """
+`[
+    {{
+        "{response_key}": <text>
+    }}
+]`
+Instead, received {response_json} and access failed at key `{e}`.
+"""
         raise OpenSourceLLMUserError(message=message)
 
 
@@ -364,22 +364,22 @@ def __init__(self, connection: CustomConnection):
                 accepted_keys = ",".join([key for key in self.REQUIRED_CONFIG_KEYS])
                 raise OpenSourceLLMKeyValidationError(
                     message=f"""Required key `{key}` not found in given custom connection.
-                      Required keys are: {accepted_keys}."""
+Required keys are: {accepted_keys}."""
                 )
         for key in self.REQUIRED_SECRET_KEYS:
             if key not in conn_dict:
                 accepted_keys = ",".join([key for key in self.REQUIRED_SECRET_KEYS])
                 raise OpenSourceLLMKeyValidationError(
                     message=f"""Required secret key `{key}` not found in given custom connection.
-                      Required keys are: {accepted_keys}."""
+Required keys are: {accepted_keys}."""
                 )
         try:
             self.model_family = ModelFamily[connection.configs['model_family']]
         except KeyError:
             accepted_models = ",".join([model.name for model in ModelFamily])
             raise OpenSourceLLMKeyValidationError(
                 message=f"""Given model_family '{connection.configs['model_family']}' not recognized.
-                  Supported models are: {accepted_models}."""
+Supported models are: {accepted_models}."""
             )
         self.connection = connection
 

src/promptflow-tools/tests/conftest.py

@@ -8,7 +8,7 @@
 # Avoid circular dependencies: Use import 'from promptflow._internal' instead of 'from promptflow'
 # since the code here is in promptflow namespace as well
 from promptflow._internal import ConnectionManager
-from promptflow.connections import CustomConnection, OpenAIConnection
+from promptflow.connections import CustomConnection, OpenAIConnection, SerpConnection
 from promptflow.tools.aoai import AzureOpenAI
 
 PROMOTFLOW_ROOT = Path(__file__).absolute().parents[1]
@@ -57,7 +57,7 @@ def skip_if_no_key(request, mocker):
         conn_name = request.node.get_closest_marker('skip_if_no_key').args[0]
         connection = request.getfixturevalue(conn_name)
         # if dummy placeholder key, skip.
-        if isinstance(connection, OpenAIConnection):
+        if isinstance(connection, OpenAIConnection) or isinstance(connection, SerpConnection):
             if "-api-key" in connection.api_key:
                 pytest.skip('skipped because no key')
         elif isinstance(connection, CustomConnection):

src/promptflow-tools/tests/test_open_source_llm.py

@@ -59,6 +59,7 @@ def test_open_source_llm_con_url_chat(self, gpt2_custom_connection):
             os.call(self.chat_prompt, API.CHAT)
         assert exc_info.value.message == """Required key `endpoint_url` not found in given custom connection.
 Required keys are: endpoint_url,model_family."""
+        assert exc_info.value.error_codes == "UserError/ToolValidationError/OpenSourceLLMKeyValidationError".split("/")
 
     @pytest.mark.skip_if_no_key("gpt2_custom_connection")
     def test_open_source_llm_con_key_chat(self, gpt2_custom_connection):
@@ -70,6 +71,7 @@ def test_open_source_llm_con_key_chat(self, gpt2_custom_connection):
             "Required secret key `endpoint_api_key` "
             + """not found in given custom connection.
 Required keys are: endpoint_api_key.""")
+        assert exc_info.value.error_codes == "UserError/ToolValidationError/OpenSourceLLMKeyValidationError".split("/")
 
     @pytest.mark.skip_if_no_key("gpt2_custom_connection")
     def test_open_source_llm_con_model_chat(self, gpt2_custom_connection):
@@ -79,6 +81,7 @@ def test_open_source_llm_con_model_chat(self, gpt2_custom_connection):
             os.call(self.completion_prompt, API.COMPLETION)
         assert exc_info.value.message == """Required key `model_family` not found in given custom connection.
 Required keys are: endpoint_url,model_family."""
+        assert exc_info.value.error_codes == "UserError/ToolValidationError/OpenSourceLLMKeyValidationError".split("/")
 
     def test_open_source_llm_escape_chat(self):
         danger = r"The quick \brown fox\tjumped\\over \the \\boy\r\n"
@@ -117,6 +120,7 @@ def test_open_source_llm_llama_parse_system_not_accepted(self):
             + "API, please select the appropriate API type and deployment name. If you do intend to use the Chat "
             + "API, please refer to the guideline at https://aka.ms/pfdoc/chat-prompt or view the samples in our "
             + "gallery that contain 'Chat' in the name.")
+        assert exc_info.value.error_codes == "UserError/OpenSourceLLMUserError".split("/")
 
     def test_open_source_llm_llama_parse_ignore_whitespace(self):
         bad_chat_prompt = f"""system:
@@ -135,6 +139,7 @@ def test_open_source_llm_llama_parse_ignore_whitespace(self):
             + "appropriate API type and deployment name. If you do intend to use the Chat API, please refer to the "
             + "guideline at https://aka.ms/pfdoc/chat-prompt or view the samples in our gallery that contain 'Chat' "
             + "in the name.")
+        assert exc_info.value.error_codes == "UserError/OpenSourceLLMUserError".split("/")
 
     def test_open_source_llm_llama_parse_chat_with_comp(self):
         with pytest.raises(OpenSourceLLMUserError) as exc_info:
@@ -146,6 +151,7 @@ def test_open_source_llm_llama_parse_chat_with_comp(self):
             + "intend to use the Completion API, please select the appropriate API type and deployment name. If you do "
             + "intend to use the Chat API, please refer to the guideline at https://aka.ms/pfdoc/chat-prompt or view "
             + "the samples in our gallery that contain 'Chat' in the name.")
+        assert exc_info.value.error_codes == "UserError/OpenSourceLLMUserError".split("/")
 
     @pytest.mark.skip_if_no_key("gpt2_custom_connection")
     def test_open_source_llm_llama_endpoint_miss(self, gpt2_custom_connection):
@@ -158,6 +164,7 @@ def test_open_source_llm_llama_endpoint_miss(self, gpt2_custom_connection):
         assert exc_info.value.message == (
             "Exception hit calling Oneline Endpoint: "
             + "HTTPError: HTTP Error 424: Failed Dependency")
+        assert exc_info.value.error_codes == "UserError/OpenSourceLLMOnlineEndpointError".split("/")
 
     @pytest.mark.skip_if_no_key("gpt2_custom_connection")
     def test_open_source_llm_llama_deployment_miss(self, gpt2_custom_connection):
@@ -170,3 +177,4 @@ def test_open_source_llm_llama_deployment_miss(self, gpt2_custom_connection):
         assert exc_info.value.message == (
             "Exception hit calling Oneline Endpoint: "
             + "HTTPError: HTTP Error 404: Not Found")
+        assert exc_info.value.error_codes == "UserError/OpenSourceLLMOnlineEndpointError".split("/")