Merge branch 'main' into brynn/refine-connection-provider-usage

Signed-off-by: Brynn Yin <[email protected]>

.github/workflows/promptflow-evals-e2e-test.yml

@@ -31,7 +31,7 @@ jobs:
     needs: build
     strategy:
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
+        os: [ubuntu-latest, windows-latest, macos-13]
         python-version: ['3.8', '3.9', '3.10', '3.11']
       fail-fast: false
     # snok/install-poetry need this to support Windows

.github/workflows/promptflow-evals-unit-test.yml

@@ -31,7 +31,7 @@ jobs:
     needs: build
     strategy:
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
+        os: [ubuntu-latest, windows-latest, macos-13]
         python-version: ['3.8', '3.9', '3.10', '3.11']
       fail-fast: false
     # snok/install-poetry need this to support Windows

.github/workflows/promptflow-release-testing-matrix.yml

@@ -75,7 +75,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
+        os: [ubuntu-latest, windows-latest, macos-13]
         pythonVersion: ['3.8', '3.9', '3.10', '3.11']
     defaults:
       run:
@@ -129,7 +129,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
+        os: [ubuntu-latest, windows-latest, macos-13]
         pythonVersion: ['3.8', '3.9', '3.10', '3.11']
     # snok/install-poetry need this to support Windows
     defaults:
@@ -160,13 +160,11 @@ jobs:
         run: |
           poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_tracing-*.whl', recursive=True)[0])")
           poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_core-*.whl', recursive=True)[0])")
+          poetry run pip install -e ../promptflow-recording
         working-directory: ${{ env.WORKING_DIRECTORY }}
       - name: install test dependency group
         run: poetry install --only test
         working-directory: ${{ env.WORKING_DIRECTORY }}
-      - name: install recording
-        run: poetry install
-        working-directory: ${{ env.RECORD_DIRECTORY }}
       - name: run core tests
         run: poetry run pytest ./tests/core
         working-directory: ${{ env.WORKING_DIRECTORY }}
@@ -186,7 +184,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
+        os: [ubuntu-latest, windows-latest, macos-13]
         pythonVersion: ['3.8', '3.9', '3.10', '3.11']
     # snok/install-poetry need this to support Windows
     defaults:
@@ -209,13 +207,11 @@ jobs:
         run: |
           poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_tracing-*.whl', recursive=True)[0])")
           poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_core-*.whl', recursive=True)[0]+'[azureml-serving]')")
+          poetry run pip install -e ../promptflow-recording
         working-directory: ${{ env.WORKING_DIRECTORY }}
       - name: install test dependency group
         run: poetry install --only test
         working-directory: ${{ env.WORKING_DIRECTORY }}
-      - name: install recording
-        run: poetry install
-        working-directory: ${{ env.RECORD_DIRECTORY }}
       - name: run azureml-serving tests
         run: poetry run pytest ./tests/azureml-serving
         working-directory: ${{ env.WORKING_DIRECTORY }}
@@ -236,7 +232,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
+        os: [ubuntu-latest, windows-latest, macos-13]
         pythonVersion: ['3.8', '3.9', '3.10', '3.11']
     # snok/install-poetry need this to support Windows
     defaults:
@@ -274,9 +270,6 @@ jobs:
       - name: install test dependency group
         run: poetry install --only test
         working-directory: ${{ env.WORKING_DIRECTORY }}
-      - name: install recording
-        run: poetry install
-        working-directory: ${{ env.RECORD_DIRECTORY }}
       - name: run devkit tests
         run: poetry run pytest ./tests/sdk_cli_test ./tests/sdk_pfs_test -n auto -m "unittest or e2etest"
         working-directory: ${{ env.WORKING_DIRECTORY }}
@@ -293,7 +286,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
+        os: [ubuntu-latest, windows-latest, macos-13]
         pythonVersion: ['3.8', '3.9', '3.10', '3.11']
     env:
       PROMPT_FLOW_TEST_MODE: "live"
@@ -356,7 +349,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
+        os: [ubuntu-latest, windows-latest, macos-13]
         pythonVersion: ['3.8', '3.9', '3.10', '3.11']
     runs-on: ${{ matrix.os }}
     steps:
@@ -401,18 +394,30 @@ jobs:
         gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[azure,executor-service]"}}
         gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)"}}
         pip freeze
-    - name: Run Executor Test
+    - name: Run Executor Unit Test
+      shell: pwsh
+      working-directory: ${{ github.workspace }}
+      run: |
+        pip install langchain
+        pip install numexpr
+        python scripts/building/run_coverage_tests.py `
+          -p ${{ github.workspace }}/src/promptflow/promptflow `
+          -t ${{ github.workspace }}/src/promptflow/tests/executor/unittests `
+          -l eastus `
+          -m "all" `
+          -o "${{ github.workspace }}/test-results-executor-unit.xml"
+    - name: Run Executor E2E Test
       shell: pwsh
       working-directory: ${{ github.workspace }}
       run: |
         pip install langchain
         pip install numexpr
         python scripts/building/run_coverage_tests.py `
           -p ${{ github.workspace }}/src/promptflow/promptflow `
-          -t ${{ github.workspace }}/src/promptflow/tests/executor/e2etests ${{ github.workspace }}/src/promptflow/tests/executor/unittests `
+          -t ${{ github.workspace }}/src/promptflow/tests/executor/e2etests `
           -l eastus `
           -m "all" `
-          -o "${{ github.workspace }}/test-results-executor.xml"
+          -o "${{ github.workspace }}/test-results-executor-e2e.xml"
     - name: Upload pytest test results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }})
       if: ${{ always() }}
       uses: actions/upload-artifact@v3

.github/workflows/promptflow-tracing-e2e-test.yml

@@ -32,7 +32,7 @@ jobs:
     needs: build
     strategy:
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
+        os: [ubuntu-latest, windows-latest, macos-13]
         python-version: ['3.8', '3.9', '3.10', '3.11']
       fail-fast: false
     # snok/install-poetry need this to support Windows

.github/workflows/promptflow-tracing-unit-test.yml

@@ -32,7 +32,7 @@ jobs:
     needs: build
     strategy:
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
+        os: [ubuntu-latest, windows-latest, macos-13]
         python-version: ['3.8', '3.9', '3.10', '3.11']
       fail-fast: false
     # snok/install-poetry need this to support Windows

.github/workflows/sdk-cli-perf-monitor-test.yml

@@ -35,7 +35,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        os: [ubuntu-latest, macos-13, windows-latest]
     defaults:
       run:
         shell: bash

src/promptflow-azure/tests/sdk_cli_azure_test/conftest.py

@@ -25,7 +25,7 @@
 from mock import MagicMock, mock
 from pytest_mock import MockerFixture
 
-from promptflow._sdk._constants import FlowType, RunStatus
+from promptflow._sdk._constants import FLOW_TOOLS_JSON, PROMPT_FLOW_DIR_NAME, FlowType, RunStatus
 from promptflow._sdk.entities import Run
 from promptflow._utils.user_agent_utils import ClientUserAgentUtil
 from promptflow.azure import PFClient
@@ -450,6 +450,9 @@ def created_flow(pf: PFClient, randstr: Callable[[str], str], variable_recorder)
     """Create a flow for test."""
     flow_display_name = randstr("flow_display_name")
     flow_source = FLOWS_DIR / "simple_hello_world"
+    tool_json_path = f"{flow_source}/{PROMPT_FLOW_DIR_NAME}/{FLOW_TOOLS_JSON}"
+    if os.path.isfile(tool_json_path):
+        os.remove(tool_json_path)
     description = "test flow description"
     tags = {"owner": "sdk-test"}
     result = pf.flows.create_or_update(

src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_flow_operations.py

@@ -6,6 +6,7 @@
 import pytest
 from sdk_cli_azure_test.conftest import FLOWS_DIR
 
+from promptflow._sdk._constants import FLOW_TOOLS_JSON, PROMPT_FLOW_DIR_NAME
 from promptflow.azure._entities._flow import Flow
 from promptflow.exceptions import UserErrorException
 
@@ -24,6 +25,8 @@ class TestFlow:
     def test_create_flow(self, created_flow: Flow):
         # most of the assertions are in the fixture itself
         assert isinstance(created_flow, Flow)
+        flow_tools_json_path = FLOWS_DIR / "simple_hello_world" / PROMPT_FLOW_DIR_NAME / FLOW_TOOLS_JSON
+        assert not flow_tools_json_path.exists()
 
     def test_get_flow(self, pf, created_flow: Flow):
         result = pf.flows.get(name=created_flow.name)

src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_run_operations.py

@@ -22,7 +22,7 @@
 from sdk_cli_azure_test.conftest import DATAS_DIR, FLOWS_DIR
 
 from promptflow._constants import FLOW_FLEX_YAML
-from promptflow._sdk._constants import DownloadedRun, RunStatus
+from promptflow._sdk._constants import FLOW_TOOLS_JSON, PROMPT_FLOW_DIR_NAME, DownloadedRun, RunStatus
 from promptflow._sdk._errors import InvalidRunError, InvalidRunStatusError, RunNotFoundError
 from promptflow._sdk._load_functions import load_run
 from promptflow._sdk.entities import Run
@@ -79,6 +79,23 @@ def test_run_bulk(self, pf, runtime: str, randstr: Callable[[str], str]):
         assert isinstance(run, Run)
         assert run.name == name
 
+    @pytest.mark.skipif(not is_live(), reason="Recording issue.")
+    def test_run_without_generate_tools_json(self, pf, runtime: str, randstr: Callable[[str], str]):
+        name = randstr("name")
+        flow_dir = f"{FLOWS_DIR}/simple_hello_world"
+        tools_json_path = Path(flow_dir) / PROMPT_FLOW_DIR_NAME / FLOW_TOOLS_JSON
+        if tools_json_path.exists():
+            tools_json_path.unlink()
+        run = pf.run(
+            flow=flow_dir,
+            data=f"{DATAS_DIR}/simple_hello_world.jsonl",
+            column_mapping={"name": "${data.name}"},
+            name=name,
+        )
+        assert isinstance(run, Run)
+        assert run.name == name
+        assert not tools_json_path.exists()
+
     def test_run_resume(self, pf: PFClient, randstr: Callable[[str], str]):
         # Note: Use fixed run name here to ensure resume call has same body then can be recorded.
         name = "resume_from_run_using_automatic_runtime"

src/promptflow-core/promptflow/core/_connection_provider/_dict_connection_provider.py

@@ -99,11 +99,14 @@ def list(self):
         return [c for c in self._connections.values()]
 
     def get(self, name: str) -> Any:
-        if isinstance(name, str):
-            return self._connections.get(name)
-        elif ConnectionType.is_connection_value(name):
+        if ConnectionType.is_connection_value(name):
             return name
-        raise ConnectionNotFound(
-            f"Connection {name!r} not found in dict connection provider."
-            f"Available keys are {list(self._connections.keys())}."
-        )
+        connection = None
+        if isinstance(name, str):
+            connection = self._connections.get(name)
+        if not connection:
+            raise ConnectionNotFound(
+                f"Connection {name!r} not found in dict connection provider."
+                f"Available keys are {list(self._connections.keys())}."
+            )
+        return connection

src/promptflow-devkit/promptflow/_proxy/_base_inspector_proxy.py

@@ -55,7 +55,7 @@ def prepare_metadata(
         3) before flow upload.
 
         For dag flow, it will generate flow.tools.json;
-        For python flex flow, it will do nothing;
-        For csharp flex flow, it will generate metadata based on a dotnet command.
+        For flex flow, it will generate metadata based on a dotnet command.
+        For python flow, we have a runtime to gather metadata in both local and cloud, so we don't prepare anything
         """
         return

src/promptflow-devkit/promptflow/_proxy/_csharp_inspector_proxy.py

@@ -14,13 +14,17 @@
 from promptflow._constants import FlowEntryRegex
 from promptflow._sdk._constants import ALL_CONNECTION_TYPES, FLOW_META_JSON, FLOW_TOOLS_JSON, PROMPT_FLOW_DIR_NAME
 from promptflow._utils.flow_utils import is_flex_flow, read_json_content
+from promptflow._utils.logger_utils import get_cli_sdk_logger
 from promptflow._utils.yaml_utils import load_yaml
 from promptflow.exceptions import UserErrorException
 
 from ._base_inspector_proxy import AbstractInspectorProxy
 
 EXECUTOR_SERVICE_DLL = "Promptflow.dll"
 
+# inspector proxy is mainly used in preparation stage instead of execution stage, so we use cli sdk logger here
+logger = get_cli_sdk_logger()
+
 
 class CSharpInspectorProxy(AbstractInspectorProxy):
     def __init__(self):
@@ -113,16 +117,23 @@ def prepare_metadata(
                 cwd=working_dir,
             )
         except subprocess.CalledProcessError as e:
+            if is_flex_flow(flow_path=flow_file):
+                meta_path = working_dir / PROMPT_FLOW_DIR_NAME / FLOW_META_JSON
+            else:
+                meta_path = working_dir / PROMPT_FLOW_DIR_NAME / FLOW_TOOLS_JSON
+
+            logger.warning(
+                f"Failed to generate flow meta for csharp flow. "
+                f"Command: {' '.join(command)} "
+                f"Working directory: {working_dir.as_posix()} "
+                f"Return code: {e.returncode} "
+                f"Output: {e.output}"
+            )
+            if meta_path.is_file():
+                logger.warning(f"Will try to use generated flow meta at {meta_path.as_posix()}.")
             raise UserErrorException(
-                message_format="Failed to generate flow meta for csharp flow.\n"
-                "Command: {command}\n"
-                "Working directory: {working_directory}\n"
-                "Return code: {return_code}\n"
-                "Output: {output}",
-                command=" ".join(command),
-                working_directory=working_dir.as_posix(),
-                return_code=e.returncode,
-                output=e.output,
+                "Failed to generate flow meta for csharp flow and not generated flow meta "
+                f"found at {meta_path.as_posix()}. Please check log for more details."
             )
         finally:
             if temp_init_kwargs_file:

src/promptflow-devkit/promptflow/_proxy/_python_inspector_proxy.py

@@ -5,7 +5,7 @@
 from promptflow._constants import FlowEntryRegex
 from promptflow._core.entry_meta_generator import _generate_flow_meta
 from promptflow._sdk._constants import FLOW_META_JSON_GEN_TIMEOUT
-from promptflow._utils.flow_utils import is_flex_flow, resolve_python_entry_file
+from promptflow._utils.flow_utils import resolve_python_entry_file
 
 from ._base_inspector_proxy import AbstractInspectorProxy
 
@@ -53,11 +53,6 @@ def prepare_metadata(
         working_dir: Path,
         **kwargs,
     ) -> None:
-        if not is_flex_flow(flow_path=flow_file, working_dir=working_dir):
-            from promptflow._sdk._utils import generate_flow_tools_json
-
-            generate_flow_tools_json(
-                flow_directory=working_dir,
-                dump=True,
-                used_packages_only=True,
-            )
+        # for python, we have a runtime to gather metadata in both local and cloud, so we don't prepare anything
+        # here so that people may submit the flow to cloud without local runtime
+        pass

src/promptflow-devkit/promptflow/_sdk/operations/_local_azure_connection_operations.py

@@ -105,7 +105,17 @@ def get(self, name: str, **kwargs) -> _Connection:
 
         :param name: Name of the connection.
         :type name: str
-        :return: connection object retrieved from the database.
+        :return: connection object retrieved from Azure.
+        :rtype: ~promptflow.sdk.entities._connection._Connection
+        """
+        return self._get(name, **kwargs)
+
+    def _get(self, name: str, **kwargs) -> _Connection:
+        """Get a connection entity.
+
+        :param name: Name of the connection.
+        :type name: str
+        :return: connection object retrieved from Azure.
         :rtype: ~promptflow.sdk.entities._connection._Connection
         """
         with_secrets = kwargs.get("with_secrets", False)

src/promptflow-devkit/tests/sdk_cli_global_config_test/e2etests/test_global_config.py

@@ -5,12 +5,14 @@
 from promptflow._sdk._load_functions import load_flow
 from promptflow._sdk.entities._flows._flow_context_resolver import FlowContextResolver
 from promptflow.contracts.run_info import Status
+from promptflow.core import Prompty
 from promptflow.core._connection_provider._workspace_connection_provider import WorkspaceConnectionProvider
 from promptflow.executor._script_executor import ScriptExecutor
 
 TEST_CONFIG_DIR = PROMPTFLOW_ROOT / "tests" / "test_configs"
 FLOWS_DIR = TEST_CONFIG_DIR / "flows"
 DATAS_DIR = TEST_CONFIG_DIR / "datas"
+PROMPTY_DIR = TEST_CONFIG_DIR / "prompty"
 EAGER_FLOW_ROOT = TEST_CONFIG_DIR / "eager_flows"
 
 
@@ -50,6 +52,12 @@ def assert_client(mock_self, provider, **kwargs):
         with mock.patch("promptflow.core._serving.flow_invoker.FlowInvoker.resolve_connections", assert_client):
             FlowContextResolver.resolve(flow=flow)
 
+    def test_prompty_callable(self, pf):
+        # Test prompty callable with global config ws connection
+        prompty = Prompty.load(source=f"{PROMPTY_DIR}/prompty_example.prompty")
+        result = prompty(question="what is the result of 1+1?")
+        assert "2" in result
+
     def test_flex_flow_run_with_openai_chat(self, pf):
         # Test flex flow run successfully with global config ws connection
         flow_file = EAGER_FLOW_ROOT / "callable_class_with_openai" / "flow.flex.yaml"