PF Recording/Replaying part for sdk_cli

.github/actions/step_sdk_setup/action.yml

@@ -1,4 +1,4 @@
-name: step_sdk_setup_win
+name: step_sdk_setup
 inputs:
   scriptPath:
     required: false
@@ -30,7 +30,7 @@ runs:
     if: inputs.setupType == 'promptflow_with_extra'
     shell: pwsh
     run: |
-      Set-PSDebug -Trace 2
+      Set-PSDebug -Trace 1
       pip install -r ./dev_requirements.txt
       echo "########### pip list (Before) ###########"
       pip list
@@ -44,7 +44,7 @@ runs:
     if: inputs.setupType == 'promptflow_dev'
     shell: pwsh
     run: |
-      Set-PSDebug -Trace 2
+      Set-PSDebug -Trace 1
       pip install -r ./dev_requirements.txt
       python ./setup.py bdist_wheel
       $package = Get-ChildItem ./dist | ? { $_.Name.Contains('.whl')}

.github/workflows/promptflow-replay-test.yml

@@ -0,0 +1,94 @@
+name: promptflow-replay-test
+on:
+  pull_request:
+    paths:
+      - src/promptflow/**
+      - scripts/**
+      - .github/workflows/promptflow-replay-test.yml
+  workflow_dispatch:
+env:
+  testWorkingDirectory: ${{ github.workspace }}/src/promptflow
+  PYTHONPATH: ${{ github.workspace }}/src/promptflow
+  IS_IN_CI_PIPELINE: "true"
+  PF_RECORDING_MODE: "replay"
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: checkout
+      uses: actions/checkout@v4
+    - name: Display and Set Environment Variables
+      run: |
+        export pyVersion="3.9";
+        env | sort >> $GITHUB_OUTPUT
+      id: display_env
+      shell: bash -el {0}
+    - name: Python Setup - ${{ matrix.os }} - Python Version ${{ steps.display_env.outputs.pyVersion }}
+      uses: "./.github/actions/step_create_python_environment"
+      with:
+        pythonVersion: ${{ steps.display_env.outputs.pyVersion }}
+    - name: Build wheel
+      uses: "./.github/actions/step_sdk_setup"
+      with:
+        setupType: promptflow_with_extra
+        scriptPath: ${{ env.testWorkingDirectory }}
+    - name: Upload Wheel
+      if: always()
+      uses: actions/upload-artifact@v3
+      with:
+        name: wheel
+        path: |
+          ${{ github.workspace }}/src/promptflow/dist/*.whl
+          ${{ github.workspace }}/src/promptflow-tools/dist/*.whl
+  sdk_cli_tests:
+    needs: build
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: checkout
+      uses: actions/checkout@v4
+    - name: Display and Set Environment Variables
+      run: |
+        export pyVersion="3.9";
+        env | sort >> $GITHUB_OUTPUT
+      id: display_env
+      shell: bash -el {0}
+    - name: Python Setup - ${{ matrix.os }} - Python Version ${{ steps.display_env.outputs.pyVersion }}
+      uses: "./.github/actions/step_create_python_environment"
+      with:
+        pythonVersion: ${{ steps.display_env.outputs.pyVersion }}
+    - name: Download Artifacts
+      uses: actions/download-artifact@v3
+      with:
+        name: wheel
+        path: artifacts
+    - name: Install wheel
+      shell: pwsh
+      working-directory: artifacts
+      run: |
+        Set-PSDebug -Trace 1
+        pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
+        gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[azure]"}}
+        gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}}
+        pip freeze
+    - name: Run Test
+      shell: pwsh
+      working-directory: ${{ env.testWorkingDirectory }}
+      run: |
+        gci env:* | sort-object name
+        $env:PYTHONPATH=""
+        echo '{}' > ${{ github.workspace }}/src/promptflow/connections.json
+        python "../../scripts/building/run_coverage_tests.py" `
+        -p promptflow `
+        -t ${{ github.workspace }}/src/promptflow/tests/sdk_cli_test `
+        -l eastus `
+        -m "unittest or e2etest" `
+        --coverage-config ${{ github.workspace }}/src/promptflow/tests/sdk_cli_test/.coveragerc
+

scripts/building/run_coverage_tests.py

@@ -16,7 +16,7 @@
         help="Pytest marker to identify the tests to run",
         default="all",
     )
-    parser.add_argument("-n", help="Pytest number of process to run the tests", default="15")
+    parser.add_argument("-n", help="Pytest number of process to run the tests", default="auto")
     parser.add_argument(
         "--model-name",
         help="The model file name to run the tests",
@@ -72,7 +72,7 @@
         "-n",
         args.n,
         "--dist",
-        "loadgroup",
+        "loadfile",
         "--log-level=info",
         "--log-format=%(asctime)s %(levelname)s %(message)s",
         "--log-date-format=[%Y-%m-%d %H:%M:%S]",

src/promptflow/dev_requirements.txt

@@ -11,7 +11,6 @@ pytest-nunit
 pytest
 pydash
 wheel
-azure-mgmt-storage==21.0.0
 # Note: This is possibly-insecure, do not add this to setup.
 keyrings.alt
 

src/promptflow/promptflow/_internal/__init__.py

@@ -36,11 +36,11 @@
     ToolsManager,
     builtins,
     collect_package_tools,
+    gen_dynamic_list,
     gen_tool_by_source,
     register_apis,
     register_builtins,
     register_connections,
-    gen_dynamic_list,
 )
 from promptflow._core.tracer import Tracer
 from promptflow._sdk._constants import LOCAL_MGMT_DB_PATH
@@ -83,6 +83,7 @@
     logger,
     update_log_path,
 )
+from promptflow._utils.multimedia_utils import _create_image_from_file, is_multimedia_dict
 from promptflow._utils.utils import (
     AttrDict,
     camel_to_snake,
@@ -96,12 +97,11 @@
 from promptflow.executor._errors import InputNotFound
 from promptflow.executor._tool_invoker import DefaultToolInvoker
 from promptflow.storage._cache_storage import LocalCacheStorage
+from promptflow.storage._run_storage import DefaultRunStorage
 from promptflow.storage._sqlite_client import (
     INDEX,
     PRIMARY_KEY,
     DuplicatedPrimaryKeyException,
     NotFoundException,
     SqliteClient,
 )
-from promptflow.storage._run_storage import DefaultRunStorage
-from promptflow._utils.multimedia_utils import is_multimedia_dict, _create_image_from_file

src/promptflow/promptflow/_sdk/_utils.py

@@ -639,7 +639,6 @@ def _gen_dynamic_list(function_config: Dict) -> List:
     from promptflow._cli._utils import get_workspace_triad_from_local
 
     workspace_triad = get_workspace_triad_from_local()
-
     if workspace_triad.subscription_id and workspace_triad.resource_group_name and workspace_triad.workspace_name:
         return gen_dynamic_list(func_path, func_kwargs, workspace_triad._asdict())
     # if no workspace triple available, just skip.

src/promptflow/tests/conftest.py

@@ -11,6 +11,7 @@
 from _pytest.monkeypatch import MonkeyPatch
 from filelock import FileLock
 from pytest_mock import MockerFixture
+from sdk_cli_test.recording_utilities import is_replaying
 
 from promptflow._cli._utils import AzureMLWorkspaceTriad
 from promptflow._constants import PROMPTFLOW_CONNECTIONS
@@ -68,6 +69,11 @@ def env_with_secrets_config_file():
 
 @pytest.fixture
 def azure_open_ai_connection() -> AzureOpenAIConnection:
+    if is_replaying():
+        return AzureOpenAIConnection(
+            api_key="dummy_key",
+            api_base="dummy_base",
+        )
     return ConnectionManager().get("azure_open_ai_connection")
 
 

src/promptflow/tests/sdk_cli_test/conftest.py

@@ -13,8 +13,20 @@
 from promptflow._telemetry.telemetry import TELEMETRY_ENABLED
 from promptflow._utils.utils import environment_variable_overwrite
 
+from .recording_utilities import (
+    is_recording,
+    is_replaying,
+    mock_bulkresult_get_openai_metrics,
+    mock_flowoperations_test,
+    mock_get_local_connections_from_executable,
+    mock_persist_node_run,
+    mock_toolresolver_resolve_tool_by_node,
+    mock_update_run_func,
+)
+
 PROMOTFLOW_ROOT = Path(__file__) / "../../.."
 RUNTIME_TEST_CONFIGS_ROOT = Path(PROMOTFLOW_ROOT / "tests/test_configs/runtime")
+RECORDINGS_TEST_CONFIGS_ROOT = Path(PROMOTFLOW_ROOT / "tests/test_configs/node_recordings").resolve()
 CONNECTION_FILE = (PROMOTFLOW_ROOT / "connections.json").resolve().absolute().as_posix()
 MODEL_ROOT = Path(PROMOTFLOW_ROOT / "tests/test_configs/flows")
 
@@ -71,7 +83,7 @@ def local_custom_connection(local_client, azure_open_ai_connection):
 @pytest.fixture
 def setup_local_connection(local_client):
     global _connection_setup
-    if _connection_setup:
+    if is_replaying():
         return
     connection_dict = json.loads(open(CONNECTION_FILE, "r").read())
     for name, _dct in connection_dict.items():
@@ -130,6 +142,45 @@ def serving_client_python_stream_tools(mocker: MockerFixture):
     return create_client_by_model("python_stream_tools", mocker)
 
 
+@pytest.fixture
+def mock_for_recordings(request: pytest.FixtureRequest, mocker: MockerFixture) -> None:
+    """
+    mock_for_recordings This is the entrance method of recording/replaying mode.
+    environment variables: PF_RECORDING_MODE is the key env var to control this test feature.
+    Record: is_recording() will return True, is_replaying() will return False.
+        Get node run info (currently llm node), and save the info in the following key value pair
+        Key: Ordered dict of all inputs => sha1 hash value
+        Value: base64 of output value.
+    Replay: is_recording() will return False, is_replaying() will return True.
+        hijack all llm nodes with customized tool, it calculate the hash of inputs, and get outputs.
+    """
+    recording_file: Path = RECORDINGS_TEST_CONFIGS_ROOT / f"{str(request.cls.__name__).lower()}_storage_record.json"
+    if is_recording():
+        RECORDINGS_TEST_CONFIGS_ROOT.mkdir(parents=True, exist_ok=True)
+        mocker.patch(
+            "promptflow._sdk.operations._local_storage_operations.LocalStorageOperations.persist_node_run",
+            mock_persist_node_run(recording_file),
+        )
+        mocker.patch(
+            "promptflow._sdk.operations._flow_operations.FlowOperations._test",
+            mock_flowoperations_test(recording_file),
+        )
+
+    if is_replaying():
+        mocker.patch(
+            "promptflow._core.run_tracker.RunTracker._update_flow_run_info_with_node_runs", mock_update_run_func
+        )
+        mocker.patch("promptflow.executor._result.BulkResult.get_openai_metrics", mock_bulkresult_get_openai_metrics)
+
+        mocker.patch(
+            "promptflow.executor._tool_resolver.ToolResolver.resolve_tool_by_node",
+            mock_toolresolver_resolve_tool_by_node(recording_file),
+        )
+        mocker.patch(
+            "promptflow._sdk._utils.get_local_connections_from_executable", mock_get_local_connections_from_executable
+        )
+
+
 @pytest.fixture
 def sample_image():
     image_path = (Path(MODEL_ROOT) / "python_tool_with_simple_image" / "logo.jpg").resolve()