diff --git a/.github/workflows/promptflow-release-testing-matrix-alter.yml b/.github/workflows/promptflow-release-testing-matrix-alter.yml
deleted file mode 100644
index a74e4545060..00000000000
--- a/.github/workflows/promptflow-release-testing-matrix-alter.yml
+++ /dev/null
@@ -1,266 +0,0 @@
-name: promptflow-release-testing-matrix
-on:
-  workflow_call:
-  workflow_dispatch:
-    inputs:
-      # can leave empty when trigger manually
-      # GitHub Actions API for trigger does not return workflow run id
-      # there we reference below Stack Overflow solution:
-      # https://stackoverflow.com/a/69500478
-      # which adds an identifier in workflow run jobs and can be used for filter
-      id:
-        description: Identifier for the workflow run
-        required: false
-        type: string
-env:
-  testWorkingDirectory: src/promptflow
-  PYTHONPATH: ${{ github.workspace }}/src/promptflow
-  IS_IN_CI_PIPELINE: "true"
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-    - name: workflow run id - ${{ inputs.id }}
-      run: |
-        echo "workflow run id: ${{ inputs.id }}"
-    - name: checkout
-      uses: actions/checkout@v4
-    - name: Python Setup - 3.9
-      uses: "./.github/actions/step_create_python_environment"
-      with:
-        pythonVersion: 3.9
-    - name: Build wheel
-      uses: "./.github/actions/step_sdk_setup"
-      with:
-        setupType: promptflow_with_extra
-        scriptPath: ${{ env.testWorkingDirectory }}
-    - name: Upload Wheel
-      uses: actions/upload-artifact@v3
-      with:
-        name: wheel
-        path: |
-          ${{ github.workspace }}/src/promptflow/dist/*.whl
-          ${{ github.workspace }}/src/promptflow-tools/dist/*.whl
-  promptflow_sdk_cli_tests:
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' || github.event_name == 'pull_request' }}
-    needs: build
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-latest]
-        pythonVersion: ['3.8', '3.9', '3.10']
-    runs-on: ${{ matrix.os }}
-    steps:
-    - name: set test mode
-      shell: pwsh
-      run: |
-        # Windows support of multiprocessing is different from other platforms, mock injections is not working properly in replay mode.
-        if ("${{ github.event_name }}" -eq "pull_request" -and (-not "${{ matrix.os }}" -eq "windows-latest")) {
-          echo "PROMPT_FLOW_TEST_MODE=replay"  | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append
-        } else {
-          echo "PROMPT_FLOW_TEST_MODE=live"  | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append
-        }
-    - name: checkout
-      uses: actions/checkout@v3
-    - name: Display and Set Environment Variables
-      run:
-        env | sort >> $GITHUB_OUTPUT
-      shell: bash -el {0}
-    - name: Python Env Setup - ${{ matrix.os }} - Python Version ${{ matrix.pythonVersion }}
-      uses: "./.github/actions/step_create_python_environment"
-      with:
-        pythonVersion: ${{ matrix.pythonVersion }}
-    - name: Download Artifacts
-      uses: actions/download-artifact@v3
-      with:
-        name: wheel
-        path: artifacts
-    - name: Azure Login
-      uses: azure/login@v1
-      with:
-        creds: ${{ secrets.AZURE_CREDENTIALS }}
-    - name: Generate Configs
-      uses: "./.github/actions/step_generate_configs"
-      with:
-        targetFolder: ${{ env.testWorkingDirectory }}
-    - name: Install pf
-      shell: pwsh
-      working-directory: artifacts
-      run: |
-        pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
-        gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)"}}
-        gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}}
-        pip freeze
-    - name: Run SDK CLI Test
-      shell: pwsh
-      working-directory: ${{ env.testWorkingDirectory }}
-      run: |
-        python "../../scripts/building/run_coverage_tests.py" `
-          -p promptflow `
-          -t ${{ github.workspace }}/src/promptflow/tests/sdk_cli_test `
-          -l eastus `
-          -m "unittest or e2etest" `
-          -o "${{ github.workspace }}/test-results-sdk-cli.xml" `
-          --ignore-glob ${{ github.workspace }}/src/promptflow/tests/sdk_cli_test/e2etests/test_executable.py
-    - name: Install pf executable
-      shell: pwsh
-      working-directory: artifacts
-      run: |
-        Set-PSDebug -Trace 1
-        gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[executable]"}}
-        pip freeze
-    - name: Run SDK CLI Executable Test
-      shell: pwsh
-      working-directory: ${{ env.testWorkingDirectory }}
-      run: |
-        python "../../scripts/building/run_coverage_tests.py" `
-          -p promptflow `
-          -t ${{ github.workspace }}/src/promptflow/tests/sdk_cli_test/e2etests/test_executable.py `
-          -l eastus `
-          -m "unittest or e2etest" `
-          -o "${{ github.workspace }}/test-results-sdk-cli-executable.xml"
-    - name: Upload pytest test results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }})
-      if: ${{ always() }}
-      uses: actions/upload-artifact@v3
-      with:
-        name: Test Results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }})
-        path: ${{ github.workspace }}/*.xml
-  promptflow_sdk_cli_azure_tests:
-    needs: build
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-latest]
-        pythonVersion: ['3.8', '3.9', '3.10']
-    runs-on: ${{ matrix.os }}
-    steps:
-    - name: checkout
-      uses: actions/checkout@v3
-    - name: Display and Set Environment Variables
-      run:
-        env | sort >> $GITHUB_OUTPUT
-      shell: bash -el {0}
-    - name: Python Env Setup - ${{ matrix.os }} - Python Version ${{ matrix.pythonVersion }}
-      uses: "./.github/actions/step_create_python_environment"
-      with:
-        pythonVersion: ${{ matrix.pythonVersion }}
-    - name: Download Artifacts
-      uses: actions/download-artifact@v3
-      with:
-        name: wheel
-        path: artifacts
-    - name: Azure Login
-      uses: azure/login@v1
-      with:
-        creds: ${{ secrets.AZURE_CREDENTIALS }}
-    - name: Generate Configs
-      uses: "./.github/actions/step_generate_configs"
-      with:
-        targetFolder: ${{ env.testWorkingDirectory }}
-    - name: Install pf azure
-      shell: pwsh
-      working-directory: artifacts
-      run: |
-        pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
-        gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[azure]"}}
-        gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)"}}
-        pip freeze
-    - name: Run SDK CLI Azure Test
-      shell: pwsh
-      working-directory: ${{ env.testWorkingDirectory }}
-      env:
-        PROMPT_FLOW_TEST_MODE: "live"
-      run: |
-        python "../../scripts/building/run_coverage_tests.py" `
-          -p promptflow `
-          -t ${{ github.workspace }}/src/promptflow/tests/sdk_cli_azure_test `
-          -l eastus `
-          -m "unittest or e2etest" `
-          -o "${{ github.workspace }}/test-results-sdk-cli-azure.xml"
-    - name: Upload pytest test results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }})
-      if: ${{ always() }}
-      uses: actions/upload-artifact@v3
-      with:
-        name: Test Results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }})
-        path: ${{ github.workspace }}/*.xml
-  promptflow_executor_tests:
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' || github.event_name == 'pull_request' }}
-    needs: build
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-latest]
-        pythonVersion: ['3.8', '3.9', '3.10']
-    runs-on: ${{ matrix.os }}
-    steps:
-    - name: checkout
-      uses: actions/checkout@v3
-    - name: Display and Set Environment Variables
-      run:
-        env | sort >> $GITHUB_OUTPUT
-      shell: bash -el {0}
-    - name: Python Env Setup - ${{ matrix.os }} - Python Version ${{ matrix.pythonVersion }}
-      uses: "./.github/actions/step_create_python_environment"
-      with:
-        pythonVersion: ${{ matrix.pythonVersion }}
-    - name: Download Artifacts
-      uses: actions/download-artifact@v3
-      with:
-        name: wheel
-        path: artifacts
-    - name: Azure Login
-      uses: azure/login@v1
-      with:
-        creds: ${{ secrets.AZURE_CREDENTIALS }}
-    - name: Generate Configs
-      uses: "./.github/actions/step_generate_configs"
-      with:
-        targetFolder: ${{ env.testWorkingDirectory }}
-    - name: Install pf
-      shell: pwsh
-      working-directory: artifacts
-      run: |
-        pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
-        gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[azure]"}}
-        gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}}
-        pip freeze
-    - name: Run Executor Test
-      shell: pwsh
-      working-directory: ${{ github.workspace }}
-      run: |
-        gci env:* | sort-object name
-        az account show
-        pip install langchain
-        pip install numexpr
-        python scripts/building/run_coverage_tests.py `
-          -p ${{ github.workspace }}/src/promptflow/promptflow `
-          -t ${{ github.workspace }}/src/promptflow/tests/executor/e2etests ${{ github.workspace }}/src/promptflow/tests/executor/unittests `
-          -l eastus `
-          -m "all" `
-          -o "${{ github.workspace }}/test-results-executor.xml"
-    - name: Upload pytest test results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }})
-      if: ${{ always() }}
-      uses: actions/upload-artifact@v3
-      with:
-        name: Test Results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }})
-        path: ${{ github.workspace }}/*.xml
-  publish-test-results:
-    name: "Publish Tests Results"
-    needs: [ promptflow_sdk_cli_tests, promptflow_sdk_cli_azure_tests, promptflow_executor_tests ]
-    runs-on: ubuntu-latest
-    permissions:
-      checks: write
-      pull-requests: write
-      contents: read
-      issues: read
-    if: always()
-
-    steps:
-      - name: Download Artifacts
-        uses: actions/download-artifact@v3
-        with:
-          path: artifacts
-      - name: Publish Test Results
-        uses: EnricoMi/publish-unit-test-result-action@v2
-        with:
-          files: "artifacts/**/test-*.xml"
diff --git a/.github/workflows/promptflow-release-testing-matrix.yml b/.github/workflows/promptflow-release-testing-matrix.yml
index 945a3caf525..9d63907fdca 100644
--- a/.github/workflows/promptflow-release-testing-matrix.yml
+++ b/.github/workflows/promptflow-release-testing-matrix.yml
@@ -13,7 +13,6 @@ on:
         required: false
         type: string
 env:
-  packageSetupType: promptflow_with_extra
   testWorkingDirectory: src/promptflow
   PYTHONPATH: ${{ github.workspace }}/src/promptflow
   IS_IN_CI_PIPELINE: "true"
@@ -24,12 +23,36 @@ jobs:
       - name: workflow run id - ${{ inputs.id }}
         run: |
           echo "workflow run id: ${{ inputs.id }}"
-  all_tests:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - name: checkout
+      uses: actions/checkout@v4
+    - name: Python Setup - 3.9
+      uses: "./.github/actions/step_create_python_environment"
+      with:
+        pythonVersion: 3.9
+    - name: Build wheel
+      uses: "./.github/actions/step_sdk_setup"
+      with:
+        setupType: promptflow_with_extra
+        scriptPath: ${{ env.testWorkingDirectory }}
+    - name: Upload Wheel
+      uses: actions/upload-artifact@v3
+      with:
+        name: wheel
+        path: |
+          ${{ github.workspace }}/src/promptflow/dist/*.whl
+          ${{ github.workspace }}/src/promptflow-tools/dist/*.whl
+  promptflow_sdk_cli_tests:
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' || github.event_name == 'pull_request' }}
+    needs: build
+    env:
+      PROMPT_FLOW_TEST_MODE: "live"
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
-        testType: [sdk-cli, executor]
+        os: [ubuntu-latest, windows-latest, macos-latest]
         pythonVersion: ['3.8', '3.9', '3.10', '3.11']
     runs-on: ${{ matrix.os }}
     steps:
@@ -43,43 +66,174 @@ jobs:
       uses: "./.github/actions/step_create_python_environment"
       with:
         pythonVersion: ${{ matrix.pythonVersion }}
-    - name: Build wheel
-      uses: "./.github/actions/step_sdk_setup"
+    - name: Download Artifacts
+      uses: actions/download-artifact@v3
       with:
-        setupType: promptflow_with_extra
-        scriptPath: ${{ env.testWorkingDirectory }}
+        name: wheel
+        path: artifacts
     - name: Azure Login
       uses: azure/login@v1
+      if: env.PROMPT_FLOW_TEST_MODE == 'live'
       with:
         creds: ${{ secrets.AZURE_CREDENTIALS }}
     - name: Generate Configs
       uses: "./.github/actions/step_generate_configs"
+      if: env.PROMPT_FLOW_TEST_MODE == 'live'
       with:
         targetFolder: ${{ env.testWorkingDirectory }}
-    - name: Get number of CPU cores
-      uses: SimenB/github-actions-cpu-cores@v1
-      id: cpu-cores
+    - name: generate test resources placeholder
+      if: env.PROMPT_FLOW_TEST_MODE != 'live'
+      shell: pwsh
+      working-directory: ${{ env.testWorkingDirectory }}
+      run: |
+        cp ${{ github.workspace }}/src/promptflow/dev-connections.json.example ${{ github.workspace }}/src/promptflow/connections.json
+    - name: Install pf
+      shell: pwsh
+      working-directory: artifacts
+      run: |
+        pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
+        gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)"}}
+        gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}}
+        pip freeze
     - name: Run SDK CLI Test
-      if : ${{ matrix.testType == 'sdk-cli' }}
       shell: pwsh
       working-directory: ${{ env.testWorkingDirectory }}
       run: |
-        gci env:* | sort-object name
-        az account show
         python "../../scripts/building/run_coverage_tests.py" `
           -p promptflow `
-          -t ${{ github.workspace }}/src/promptflow/tests/sdk_cli_azure_test ${{ github.workspace }}/src/promptflow/tests/sdk_cli_test `
+          -t ${{ github.workspace }}/src/promptflow/tests/sdk_cli_test `
           -l eastus `
           -m "unittest or e2etest" `
-          -n ${{ steps.cpu-cores.outputs.count }}
-        mv ${{ github.workspace }}/${{ env.testWorkingDirectory }}/test-results.xml ${{ github.workspace }}/test-results-sdk-cli.xml
+          -o "${{ github.workspace }}/test-results-sdk-cli.xml" `
+          --ignore-glob ${{ github.workspace }}/src/promptflow/tests/sdk_cli_test/e2etests/test_executable.py
+    - name: Install pf executable
+      shell: pwsh
+      working-directory: artifacts
+      run: |
+        Set-PSDebug -Trace 1
+        gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[executable]"}}
+        pip freeze
+    - name: Run SDK CLI Executable Test
+      shell: pwsh
+      working-directory: ${{ env.testWorkingDirectory }}
+      run: |
+        python "../../scripts/building/run_coverage_tests.py" `
+          -p promptflow `
+          -t ${{ github.workspace }}/src/promptflow/tests/sdk_cli_test/e2etests/test_executable.py `
+          -l eastus `
+          -m "unittest or e2etest" `
+          -o "${{ github.workspace }}/test-results-sdk-cli-executable.xml"
+    - name: Upload pytest test results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }})
+      if: ${{ always() }}
+      uses: actions/upload-artifact@v3
+      with:
+        name: promptflow_sdk_cli_tests Test Results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }})
+        path: ${{ github.workspace }}/*.xml
+  promptflow_sdk_cli_azure_tests:
+    needs: build
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        pythonVersion: ['3.8', '3.9', '3.10', '3.11']
+    env:
+      PROMPT_FLOW_TEST_MODE: "live"
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: checkout
+      uses: actions/checkout@v4
+    - name: Display and Set Environment Variables
+      run:
+        env | sort >> $GITHUB_OUTPUT
+      shell: bash -el {0}
+    - name: Python Env Setup - ${{ matrix.os }} - Python Version ${{ matrix.pythonVersion }}
+      uses: "./.github/actions/step_create_python_environment"
+      with:
+        pythonVersion: ${{ matrix.pythonVersion }}
+    - name: Download Artifacts
+      uses: actions/download-artifact@v3
+      with:
+        name: wheel
+        path: artifacts
+    - name: Azure Login
+      uses: azure/login@v1
+      if: env.PROMPT_FLOW_TEST_MODE == 'live'
+      with:
+        creds: ${{ secrets.AZURE_CREDENTIALS }}
+    - name: Generate Configs
+      uses: "./.github/actions/step_generate_configs"
+      if: env.PROMPT_FLOW_TEST_MODE == 'live'
+      with:
+        targetFolder: ${{ env.testWorkingDirectory }}
+    - name: Install pf azure
+      shell: pwsh
+      working-directory: artifacts
+      run: |
+        pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
+        gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[azure]"}}
+        gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)"}}
+        pip freeze
+    - name: Run SDK CLI Azure Test
+      shell: pwsh
+      working-directory: ${{ env.testWorkingDirectory }}
+      run: |
+        python "../../scripts/building/run_coverage_tests.py" `
+          -p promptflow `
+          -t ${{ github.workspace }}/src/promptflow/tests/sdk_cli_azure_test `
+          -l eastus `
+          -m "unittest or e2etest" `
+          -o "${{ github.workspace }}/test-results-sdk-cli-azure.xml"
+    - name: Upload pytest test results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }})
+      if: ${{ always() }}
+      uses: actions/upload-artifact@v3
+      with:
+        name: promptflow_sdk_cli_azure_tests Test Results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }})
+        path: ${{ github.workspace }}/*.xml
+  promptflow_executor_tests:
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' || github.event_name == 'pull_request' }}
+    needs: build
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        pythonVersion: ['3.8', '3.9', '3.10', '3.11']
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: checkout
+      uses: actions/checkout@v4
+    - name: Display and Set Environment Variables
+      run:
+        env | sort >> $GITHUB_OUTPUT
+      shell: bash -el {0}
+    - name: Python Env Setup - ${{ matrix.os }} - Python Version ${{ matrix.pythonVersion }}
+      uses: "./.github/actions/step_create_python_environment"
+      with:
+        pythonVersion: ${{ matrix.pythonVersion }}
+    - name: Download Artifacts
+      uses: actions/download-artifact@v3
+      with:
+        name: wheel
+        path: artifacts
+    - name: Azure Login
+      uses: azure/login@v1
+      with:
+        creds: ${{ secrets.AZURE_CREDENTIALS }}
+    - name: Generate Configs
+      uses: "./.github/actions/step_generate_configs"
+      with:
+        targetFolder: ${{ env.testWorkingDirectory }}
+    - name: Install pf
+      shell: pwsh
+      working-directory: artifacts
+      run: |
+        pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
+        gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[azure]"}}
+        gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}}
+        pip freeze
     - name: Run Executor Test
-      if : ${{ matrix.testType == 'executor' }}
       shell: pwsh
       working-directory: ${{ github.workspace }}
       run: |
-        gci env:* | sort-object name
-        az account show
         pip install langchain
         pip install numexpr
         python scripts/building/run_coverage_tests.py `
@@ -87,29 +241,30 @@ jobs:
           -t ${{ github.workspace }}/src/promptflow/tests/executor/e2etests ${{ github.workspace }}/src/promptflow/tests/executor/unittests `
           -l eastus `
           -m "all" `
-          -n ${{ steps.cpu-cores.outputs.count }}
-        mv ./test-results.xml ./test-results-executor.xml
+          -o "${{ github.workspace }}/test-results-executor.xml"
     - name: Upload pytest test results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }})
       if: ${{ always() }}
       uses: actions/upload-artifact@v3
       with:
-        name: Test Results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }})
+        name: promptflow_executor_tests Test Results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }})
         path: ${{ github.workspace }}/*.xml
   publish-test-results:
     name: "Publish Tests Results"
-    needs:
-      - all_tests
+    needs: [ promptflow_sdk_cli_tests, promptflow_sdk_cli_azure_tests, promptflow_executor_tests ]
     runs-on: ubuntu-latest
-    permissions: write-all
+    permissions:
+      checks: write
+      pull-requests: write
+      contents: read
+      issues: read
     if: always()
 
     steps:
-    - name: checkout
-      uses: actions/checkout@v4
-    - name: Publish Test Results
-      uses: "./.github/actions/step_publish_test_results"
-      with:
-        testActionFileName: promptflow-release-testing-matrix.yml
-        testResultTitle: Release Test Matrix
-        token: ${{ secrets.GITHUB_TOKEN }}
-        coverageThreshold: 0
+      - name: Download Artifacts
+        uses: actions/download-artifact@v3
+        with:
+          path: artifacts
+      - name: Publish Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        with:
+          files: "artifacts/**/test-*.xml"
diff --git a/scripts/building/run_coverage_tests.py b/scripts/building/run_coverage_tests.py
index 15ec2f98483..96781561fd1 100644
--- a/scripts/building/run_coverage_tests.py
+++ b/scripts/building/run_coverage_tests.py
@@ -69,7 +69,7 @@
     run_command(["pip", "list"])
     run_command(["pip", "show", "promptflow", "promptflow-sdk"])
 
-    pytest_command = ["pytest", "--junitxml=test-results.xml"]
+    pytest_command = ["pytest", f"--junitxml={args.o}"]
     pytest_command += test_paths_list
     if args.coverage_config:
         if args.p:
diff --git a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_azure_cli_activity_name.py b/src/promptflow/tests/sdk_cli_azure_test/unittests/test_azure_cli_activity_name.py
index 7fc2752afb3..9b9a7eb731f 100644
--- a/src/promptflow/tests/sdk_cli_azure_test/unittests/test_azure_cli_activity_name.py
+++ b/src/promptflow/tests/sdk_cli_azure_test/unittests/test_azure_cli_activity_name.py
@@ -1,4 +1,5 @@
 import pytest
+
 from promptflow._cli._pf_azure.entry import get_parser_args
 from promptflow._cli._utils import _get_cli_activity_name
 
@@ -11,37 +12,30 @@ def get_cli_activity_name(cmd):
 @pytest.mark.unittest
 class TestAzureCliTimeConsume:
     def test_pfazure_run_create(self, activity_name="pfazure.run.create"):
-        assert get_cli_activity_name(
-            cmd=(
-                "pfazure",
-                "run",
-                "create",
-                "--flow",
-                "print_input_flow",
-                "--data",
-                "print_input_flow.jsonl"
-            )) == activity_name
+        assert (
+            get_cli_activity_name(
+                cmd=("pfazure", "run", "create", "--flow", "print_input_flow", "--data", "print_input_flow.jsonl")
+            )
+            == activity_name
+        )
 
     def test_pfazure_run_update(self, activity_name="pfazure.run.update"):
-        assert get_cli_activity_name(
-            cmd=(
-                "pfazure",
-                "run",
-                "update",
-                "--name",
-                "test_run",
-                "--set",
-                "display_name=test_run",
-                "description='test_description'",
-                "tags.key1=value1"
-            )) == activity_name
+        assert (
+            get_cli_activity_name(
+                cmd=(
+                    "pfazure",
+                    "run",
+                    "update",
+                    "--name",
+                    "test_run",
+                    "--set",
+                    "display_name=test_run",
+                    "description='test_description'",
+                    "tags.key1=value1",
+                )
+            )
+            == activity_name
+        )
 
     def test_run_restore(self, activity_name="pfazure.run.restore"):
-        assert get_cli_activity_name(
-            cmd=(
-                "pfazure",
-                "run",
-                "restore",
-                "--name",
-                "test_run"
-            )) == activity_name
+        assert get_cli_activity_name(cmd=("pfazure", "run", "restore", "--name", "test_run")) == activity_name
diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/test_executable.py b/src/promptflow/tests/sdk_cli_test/e2etests/test_executable.py
index 5d0b28a6140..e10597382e7 100644
--- a/src/promptflow/tests/sdk_cli_test/e2etests/test_executable.py
+++ b/src/promptflow/tests/sdk_cli_test/e2etests/test_executable.py
@@ -18,7 +18,10 @@
 @pytest.mark.cli_test
 @pytest.mark.e2etest
 class TestExecutable:
-    @pytest.mark.skipif(sys.platform == "win32", reason="Raise Exception: Process terminated with exit code 4294967295")
+    @pytest.mark.skipif(
+        sys.platform == "win32" or sys.platform == "darwin",
+        reason="Raise Exception: Process terminated with exit code 4294967295",
+    )
     def test_flow_build_executable(self):
         source = f"{FLOWS_DIR}/web_classification/flow.dag.yaml"
         target = "promptflow._sdk.operations._flow_operations.FlowOperations._run_pyinstaller"
diff --git a/src/promptflow/tests/sdk_pfs_test/conftest.py b/src/promptflow/tests/sdk_pfs_test/conftest.py
index 17147455d9e..30dc613db4f 100644
--- a/src/promptflow/tests/sdk_pfs_test/conftest.py
+++ b/src/promptflow/tests/sdk_pfs_test/conftest.py
@@ -6,13 +6,14 @@
 from flask.app import Flask
 
 from promptflow import PFClient
-from promptflow._sdk._service.app import create_app
 
 from .utils import PFSOperations
 
 
 @pytest.fixture
 def app() -> Flask:
+    from promptflow._sdk._service.app import create_app
+
     app, _ = create_app()
     app.config.update({"TESTING": True})
     yield app