Fix

.github/workflows/sdk-cli-azure-test-production.yml

@@ -1,66 +1,47 @@
 # execute tests in src/promptflow/tests/sdk_cli_azure_test with live mode
 
-name: sdk-cli-azure-test-production
+name: sdk-cli-azure-test-anypath
 
 on:
   schedule:
     - cron: "30 20 * * *"  # 4:30 Beijing Time (GMT+8)
 
   workflow_dispatch:
+    inputs:
+      # can leave empty when trigger manually
+      # GitHub Actions API for trigger does not return workflow run id
+      # there we reference below Stack Overflow solution:
+      # https://stackoverflow.com/a/69500478
+      # which adds an identifier in workflow run jobs and can be used for filter
+      filepath:
+        description: file or path you want to trigger a test
+        required: true
+        default: "./tests/sdk_cli_azure_test"
+        type: string
+
 
 
 env:
-  packageSetupType: promptflow_with_extra
-  testWorkingDirectory: ${{ github.workspace }}/src/promptflow
-  PYTHONPATH: ${{ github.workspace }}/src/promptflow
   IS_IN_CI_PIPELINE: "true"
   PROMPT_FLOW_TEST_MODE: "live"
   PROMPT_FLOW_WORKSPACE_NAME: "promptflow-eastus"
+  TRACING_DIRECTORY: ${{ github.workspace }}/src/promptflow-tracing
+  WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-azure
+  CORE_DIRECTORY: ${{ github.workspace }}/src/promptflow-core
+  DEVKIT_DIRECTORY: ${{ github.workspace }}/src/promptflow-devkit
+  PROMPTFLOW_DIRECTORY: ${{ github.workspace }}/src/promptflow
+  TOOL_DIRECTORY: ${{ github.workspace }}/src/promptflow-tools
+  RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
 
 
 jobs:
-  build:
-    strategy:
-      fail-fast: false
-    runs-on: ubuntu-latest
-    steps:
-    - name: checkout
-      uses: actions/checkout@v4
-    - name: Display and Set Environment Variables
-      run: |
-        env | sort >> $GITHUB_OUTPUT
-      id: display_env
-      shell: bash -el {0}
-    - name: Python Setup - ubuntu-latest - Python Version 3.9
-      uses: "./.github/actions/step_create_python_environment"
-      with:
-        pythonVersion: 3.9
-    - name: Build wheel
-      uses: "./.github/actions/step_sdk_setup"
-      with:
-        setupType: promptflow_with_extra
-        scriptPath: ${{ env.testWorkingDirectory }}
-    - name: Upload Wheel
-      if: always()
-      uses: actions/upload-artifact@v3
-      with:
-        name: wheel
-        path: |
-          ${{ github.workspace }}/src/promptflow/dist/*.whl
-          ${{ github.workspace }}/src/promptflow-tools/dist/*.whl
-
   sdk_cli_azure_test_live:
-    needs: build
     strategy:
       fail-fast: false
       matrix:
-        # too many live tests will lead to 429 when create run
-        include:
-          - os: ubuntu-latest
-            pythonVersion: '3.9'
-          - os: windows-latest
-            pythonVersion: '3.10'
+        # replay tests can cover more combinations
         os: [ubuntu-latest]
+        pythonVersion: ['3.8', '3.9', '3.10', '3.11']
 
     runs-on: ${{ matrix.os }}
     steps:
@@ -70,81 +51,33 @@ jobs:
       - name: Display and Set Environment Variables
         run: env | sort >> $GITHUB_OUTPUT
 
-      - name: Python Setup - ${{ matrix.os }} - Python Version ${{ matrix.pythonVersion }}
-        uses: "./.github/actions/step_create_python_environment"
+      - uses: actions/setup-python@v5
         with:
-          pythonVersion: ${{ matrix.pythonVersion }}
-
-      - name: Download Artifacts
-        uses: actions/download-artifact@v3
-        with:
-          name: wheel
-          path: artifacts
-
-      - name: Install wheel
-        shell: pwsh
-        working-directory: artifacts
-        run: |
-          Set-PSDebug -Trace 1
-          pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
-          pip install ${{ github.workspace }}/src/promptflow-tracing
-          pip install ${{ github.workspace }}/src/promptflow-core
-          pip install ${{ github.workspace }}/src/promptflow-devkit
-          pip install ${{ github.workspace }}/src/promptflow-azure
-          gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[azure]"}}
-          gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}}
-          pip freeze
-
-      - name: Azure login
+          python-version: ${{ matrix.pythonVersion }}
+      - uses: snok/install-poetry@v1
+      - name: Azure Login
         uses: azure/login@v1
         with:
           creds: ${{ secrets.AZURE_CREDENTIALS }}
-
-      - name: generate live test resources
+      - name: Generate Configs
         uses: "./.github/actions/step_generate_configs"
         with:
-          targetFolder: ${{ env.testWorkingDirectory }}
-
-      - name: Run SDK CLI Azure Test (live mode)
+          targetFolder: ${{ env.PROMPTFLOW_DIRECTORY }}
+      - name: install promptflow-devkit from wheel
+        # wildcard expansion (*) does not work in Windows, so leverage python to find and install
+        run: |
+          poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_tracing-*.whl', recursive=True)[0])")
+          poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_core-*.whl', recursive=True)[0]+'[azureml-serving]')")
+          poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_devkit-*.whl', recursive=True)[0]+'[executable]')")
+          poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_azure-*.whl', recursive=True)[0])")
+          poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow-*.whl', recursive=True)[0])")
+          poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_tools-*.whl', recursive=True)[0])")
+          poetry run pip install -e ../promptflow-recording
+
+      - name: Run SDK CLI Azure Test
         shell: pwsh
-        working-directory: ${{ env.testWorkingDirectory }}
+        working-directory: ${{ env.WORKING_DIRECTORY }}
         run: |
-          python "../../scripts/building/run_coverage_tests.py" `
-            -p promptflow `
-            -t ${{ github.workspace }}/src/promptflow/tests/sdk_cli_azure_test `
-            -l eastus `
-            -m "unittest or e2etest" `
-            --coverage-config ${{ github.workspace }}/src/promptflow/tests/sdk_cli_test/.coveragerc
+          poetry run pytest ${{ inputs.filepath }} -p promptflow --cov=promptflow --cov-config=pyproject.toml `
+          --cov-report=term --cov-report=html --cov-report=xml -n auto -m "unittest or e2etest"
 
-      - name: Upload Test Results
-        if: always()
-        uses: actions/upload-artifact@v3
-        with:
-          name: Test Results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }})
-          path: |
-            ${{ env.testWorkingDirectory }}/*.xml
-            ${{ env.testWorkingDirectory }}/htmlcov/
-
-  publish-test-results-sdk-cli-azure-test:
-    needs: sdk_cli_azure_test_live
-    if: always()
-
-    runs-on: ubuntu-latest
-    permissions:
-      checks: write
-      pull-requests: write
-      contents: read
-      issues: read
-
-    steps:
-    - name: checkout
-      uses: actions/checkout@v4
-    - name: Publish Test Results
-      uses: "./.github/actions/step_publish_test_results"
-      with:
-        testActionFileName: promptflow-sdk-cli-azure-e2e-test.yml
-        testResultTitle: promptflow SDK CLI Azure E2E Test Result
-        osVersion: ubuntu-latest
-        pythonVersion: 3.9
-        coverageThreshold: 40
-        context: test/sdk_cli

src/promptflow-azure/pyproject.toml

@@ -50,6 +50,11 @@ promptflow-devkit = "<2.0.0"
 [tool.poetry.group.dev.dependencies]
 pre-commit = "*"
 import-linter = "*"
+promptflow-tracing = {path = "../promptflow-tracing", develop = true}
+promptflow-core = {path = "../promptflow-core", develop = true}
+promptflow-devkit = {path = "../promptflow-devkit", develop = true}
+promptflow = {path = "../promptflow", develop = false}
+promptflow-tools = {path = "../promptflow-tools", develop = false}
 
 [tool.poetry.group.test.dependencies]
 pytest = "*"

src/promptflow-azure/tests/sdk_cli_azure_test/e2etests/test_run_operations.py

@@ -987,7 +987,7 @@ def test_upload_flex_flow_run_with_yaml(self, pf: PFClient, randstr: Callable[[s
             description="test sdk local to cloud",
         )
         assert run.status == "Completed"
-        assert "error" not in run._to_dict()
+        assert "error" not in str(run._to_dict())
 
         # check the run is uploaded to cloud
         Local2CloudTestHelper.check_local_to_cloud_run(pf, run)

src/promptflow-devkit/promptflow/_sdk/_utils.py

@@ -64,8 +64,6 @@
     UnsecureConnectionError,
 )
 from promptflow._sdk._vendor import IgnoreFile, get_ignore_file, get_upload_files_from_folder
-from promptflow._sdk.entities._flows.base import FlowBase
-from promptflow._sdk.entities._flows.dag import Flow as DAGFlow
 from promptflow._utils.flow_utils import is_flex_flow, resolve_flow_path
 from promptflow._utils.logger_utils import get_cli_sdk_logger
 from promptflow._utils.user_agent_utils import ClientUserAgentUtil
@@ -1077,9 +1075,12 @@ def is_flex_run(run: "Run") -> bool:
 update_dict_value_with_connections = update_dict_value_with_connections
 
 
-def get_flow_name(flow: Union[FlowBase, Path]) -> str:
+def get_flow_name(flow) -> str:
     if isinstance(flow, Path):
         return flow.resolve().name
+
+    from promptflow._sdk.entities._flows.dag import Flow as DAGFlow
+
     if isinstance(flow, DAGFlow):
         return flow.name
     # others: flex flow, prompty, etc.