Merge branch 'main' into chenyin/auto_pyinstaller

.github/workflows/promptflow-tracing-e2e-test.yml

@@ -0,0 +1,157 @@
+name: promptflow-tracing-e2e-test
+on:
+  schedule:
+    - cron: "40 17 * * *" # Every day starting at 1:40 BJT
+  pull_request_target:
+    paths:
+      - src/promptflow/**
+      - scripts/building/**
+      - .github/workflows/promptflow-tracing-e2e-test.yml
+  workflow_dispatch:
+env:
+  packageSetupType: promptflow_with_extra
+  testWorkingDirectory: ${{ github.workspace }}/src/promptflow
+  PYTHONPATH: ${{ github.workspace }}/src/promptflow
+  IS_IN_CI_PIPELINE: "true"
+jobs:
+  authorize:
+    environment:
+      # forked prs from pull_request_target will be run in external environment, domain prs will be run in internal environment
+      ${{ github.event_name == 'pull_request_target' &&
+      github.event.pull_request.head.repo.full_name != github.repository &&
+      'external' || 'internal' }}
+    runs-on: ubuntu-latest
+    steps:
+      - run: true
+  build:
+    needs: authorize
+    strategy:
+      fail-fast: false
+    runs-on: ubuntu-latest
+    steps:
+    - name: checkout
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.sha || github.ref }}
+        fetch-depth: 0
+    - name: merge main to current branch
+      uses: "./.github/actions/step_merge_main"
+    - name: Display and Set Environment Variables
+      run: |
+        env | sort >> $GITHUB_OUTPUT
+      id: display_env
+      shell: bash -el {0}
+    - name: Python Setup - ubuntu-latest - Python Version 3.9
+      uses: "./.github/actions/step_create_python_environment"
+      with:
+        pythonVersion: 3.9
+    - name: Build wheel
+      uses: "./.github/actions/step_sdk_setup"
+      with:
+        setupType: promptflow_with_extra
+        scriptPath: ${{ env.testWorkingDirectory }}
+    - name: Upload Wheel
+      if: always()
+      uses: actions/upload-artifact@v3
+      with:
+        name: wheel
+        path: |
+          ${{ github.workspace }}/src/promptflow/dist/*.whl
+          ${{ github.workspace }}/src/promptflow-tools/dist/*.whl
+  tracing_tests:
+    needs: build
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        pythonVersion: ['3.8', '3.9', '3.10', '3.11']
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: checkout
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.sha || github.ref }}
+        fetch-depth: 0
+    - name: merge main to current branch
+      uses: "./.github/actions/step_merge_main"
+    - name: Display and Set Environment Variables
+      run: |
+        env | sort >> $GITHUB_OUTPUT
+      id: display_env
+      shell: bash -el {0}
+    - name: Python Setup - ${{ matrix.os }} - Python Version ${{ matrix.pythonVersion }}
+      uses: "./.github/actions/step_create_python_environment"
+      with:
+        pythonVersion: ${{ matrix.pythonVersion }}
+    - name: Download Artifacts
+      uses: actions/download-artifact@v3
+      with:
+        name: wheel
+        path: artifacts
+    - name: Install wheel
+      shell: pwsh
+      working-directory: artifacts
+      run: |
+        Set-PSDebug -Trace 1
+        pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
+        gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)"}}
+        gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}}
+        pip freeze
+    - name: Azure Login
+      uses: azure/login@v1
+      with:
+        creds: ${{ secrets.AZURE_CREDENTIALS }}
+    - name: Generate Configs
+      uses: "./.github/actions/step_generate_configs"
+      with:
+        targetFolder: ${{ env.testWorkingDirectory }}
+    - name: Get number of CPU cores
+      uses: SimenB/github-actions-cpu-cores@v1
+      id: cpu-cores
+    - name: run promptflow-tracing test
+      shell: pwsh
+      working-directory: ${{ env.testWorkingDirectory }}
+      run: |
+        python "../../scripts/building/run_coverage_tests.py" `
+          -p promptflow `
+          -t ${{ github.workspace }}/src/promptflow/tests/tracing_test/e2etests `
+          -l eastus `
+          -m "e2etest" `
+          -n ${{ steps.cpu-cores.outputs.count }} `
+          --coverage-config ${{ github.workspace }}/src/promptflow/tests/tracing_test/.coveragerc `
+          -o "${{ env.testWorkingDirectory }}/test-results-tracing.xml"
+    - name: Upload Test Results
+      if: always()
+      uses: actions/upload-artifact@v3
+      with:
+        name: Test Results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }})
+        path: |
+          ${{ env.testWorkingDirectory }}/*.xml
+          ${{ env.testWorkingDirectory }}/htmlcov/
+  publish-test-results-tracing-test:
+    name: "Publish Tests Results"
+    needs: tracing_tests
+    if: always()
+    runs-on: ubuntu-latest
+    permissions:
+      checks: write
+      pull-requests: write
+      contents: read
+      issues: read
+    steps:
+    - name: checkout
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.sha || github.ref }}
+        fetch-depth: 0
+    - name: merge main to current branch
+      uses: "./.github/actions/step_merge_main"
+    - name: Publish Test Results
+      uses: "./.github/actions/step_publish_test_results"
+      with:
+        testActionFileName: promptflow-tracing-e2e-test.yml
+        testResultTitle: promptflow-tracing e2e test result
+        osVersion: ubuntu-latest
+        pythonVersion: 3.9
+        coverageThreshold: 40
+        context: test/tracing

.github/workflows/promptflow-tracing-unit-test.yml

@@ -113,6 +113,7 @@ jobs:
 
 
   publish-test-results-tracing-test:
+    name: "Publish Tests Results"
     needs: tracing_tests
     if: always()
 

docs/cloud/azureai/runtime-change-log.md

@@ -14,6 +14,16 @@ You can check the runtime image version from the flow execution log:
 
 ## Change log
 Default runtime image is continuously updated, and here we record the new features and fixed bugs of each image version.
+
+### 20240228.v3
+
+#### New features
+- Support async flow test for long running jobs.
+
+#### Bugs fixed
+- Fix bug when collecting package tools.
+
+
 ### 20240222.v3
 
 #### New features
@@ -40,7 +50,7 @@ NA
 
 #### Bugs fixed
 - Fix the bug that exception raised during preparing data is not set in run history.
-- Fix the bug that unexpected exception is raised when executor process crushes. 
+- Fix the bug that unexpected exception is raised when executor process crushes.
 
 ### 20240116.v1
 

docs/how-to-guides/develop-a-tool/create-dynamic-list-tool-input.md

@@ -125,14 +125,21 @@ pip install my-tools-package>=0.0.8
 ### I'm a tool author, and want to dynamically list Azure resources in my tool input. What should I pay attention to?
 1. Clarify azure workspace triple "subscription_id", "resource_group_name", "workspace_name" in the list function signature. System helps append workspace triple to function input parameters if they are in function signature. See [list_endpoint_names](https://github.com/microsoft/promptflow/blob/main/examples/tools/tool-package-quickstart/my_tool_package/tools/tool_with_dynamic_list_input.py) as an example.
 ```python
-def list_endpoint_names(subscription_id, resource_group_name, workspace_name, prefix: str = "") -> List[Dict[str, str]]:
+def list_endpoint_names(subscription_id: str = None,
+                        resource_group_name: str = None, 
+                        workspace_name: str = None,
+                        prefix: str = "") -> List[Dict[str, str]]:
     """This is an example to show how to get Azure ML resource in tool input list function.
 
     :param subscription_id: Azure subscription id.
     :param resource_group_name: Azure resource group name.
     :param workspace_name: Azure ML workspace name.
     :param prefix: prefix to add to each item.
     """
+    # return an empty list if workspace triad is not available.
+    if not subscription_id or not resource_group_name or not workspace_name:
+        return []
+
     from azure.ai.ml import MLClient
     from azure.identity import DefaultAzureCredential
 
@@ -185,4 +192,13 @@ If you are unable to see any options in a dynamic list tool input, you may see a
 If this occurs, follow these troubleshooting steps:
 
 - Note the exact error message shown. This provides details on why the dynamic list failed to populate.
+- Check the tool documentation for any prerequisites or special instructions. For example, if the dynamic list function requires Azure credentials, ensure you have installed azure dependencies, logged in and set the default workspace.
+  ```sh
+  pip install azure-ai-ml
+  ```
+  ```sh
+  az login
+  az account set --subscription <subscription_id>
+  az configure --defaults group=<resource_group_name> workspace=<workspace_name>
+  ```
 - Contact the tool author/support team and report the issue. Provide the error message so they can investigate the root cause. 

docs/how-to-guides/manage-runs.md

@@ -154,7 +154,7 @@ print(run)
 Get run details with TABLE format.
 
 ```bash
-pf run show --name <run-name>
+pf run show-details --name <run-name>
 ```
 
 ![img](../media/how-to-guides/run_show_details.png)

examples/tools/tool-package-quickstart/my_tool_package/tools/tool_with_dynamic_list_input.py

@@ -31,14 +31,21 @@ def my_list_func(prefix: str = "", size: int = 10, **kwargs) -> List[Dict[str, U
     return result
 
 
-def list_endpoint_names(subscription_id, resource_group_name, workspace_name, prefix: str = "") -> List[Dict[str, str]]:
+def list_endpoint_names(subscription_id: str = None,
+                        resource_group_name: str = None,
+                        workspace_name: str = None,
+                        prefix: str = "") -> List[Dict[str, str]]:
     """This is an example to show how to get Azure ML resource in tool input list function.
 
     :param subscription_id: Azure subscription id.
     :param resource_group_name: Azure resource group name.
     :param workspace_name: Azure ML workspace name.
     :param prefix: prefix to add to each item.
     """
+    # return an empty list if workspace triad is not available.
+    if not subscription_id or not resource_group_name or not workspace_name:
+        return []
+
     from azure.ai.ml import MLClient
     from azure.identity import DefaultAzureCredential
 

scripts/docs/doc_generation.ps1

@@ -62,6 +62,16 @@ Write-Host "Copy doc to: $TempDocPath"
 ROBOCOPY $DocPath $TempDocPath /S /NFL /NDL /XD "*.git" [System.IO.Path]::Combine($DocPath, "_scripts\_build")
 ProcessFiles
 
+function ForceOverwrite {
+    param (
+        [string] $Module
+    )
+    $FileName = "promptflow.{0}.rst" -f $Module
+    $TargetRst = [System.IO.Path]::Combine($RepoRootPath, ("scripts\docs\{0}" -f $FileName))
+    $AutoGenConnectionRst = [System.IO.Path]::Combine($RefDocPath, $FileName)
+    Copy-Item -Path $TargetRst -Destination $AutoGenConnectionRst -Force
+}
+
 if($WithReferenceDoc){
     $RefDocRelativePath = "reference\python-library-reference"
     $RefDocPath = [System.IO.Path]::Combine($TempDocPath, $RefDocRelativePath)
@@ -76,9 +86,9 @@ if($WithReferenceDoc){
     Write-Host "=============== Overwrite promptflow.connections.rst ==============="
     # We are doing this overwrite because the connection entities are also defined in the promptflow.entities module
     # and it will raise duplicate object description error if we don't do so when we run sphinx-build later.
-    $ConnectionRst = [System.IO.Path]::Combine($RepoRootPath, "scripts\docs\promptflow.connections.rst")
-    $AutoGenConnectionRst = [System.IO.Path]::Combine($RefDocPath, "promptflow.connections.rst")
-    Copy-Item -Path $ConnectionRst -Destination $AutoGenConnectionRst -Force
+    ForceOverwrite "connections"
+    ForceOverwrite "core"
+    ForceOverwrite "client"
 }
 
 

scripts/docs/promptflow.client.rst

@@ -0,0 +1,14 @@
+promptflow.client package
+==============================
+
+.. autoclass:: promptflow.client.PFClient
+   :members:
+   :undoc-members:
+   :show-inheritance:
+   :noindex:
+
+.. autoclass:: promptflow.client.load_run
+   :members:
+   :undoc-members:
+   :show-inheritance:
+   :noindex:

scripts/docs/promptflow.core.rst

@@ -0,0 +1,20 @@
+promptflow.core package
+==============================
+
+.. autoclass:: promptflow.core.log_metric
+   :members:
+   :undoc-members:
+   :show-inheritance:
+   :noindex:
+
+.. autoclass:: promptflow.core.ToolProvider
+   :members:
+   :undoc-members:
+   :show-inheritance:
+   :noindex:
+
+.. autoclass:: promptflow.core.tool
+   :members:
+   :undoc-members:
+   :show-inheritance:
+   :noindex:

src/promptflow-tools/CHANGELOG.md

@@ -1,7 +1,47 @@
 # Release History
 
+## 1.4.0 (Upcoming)
+
+### Features Added
+- Enable token based auth in azure openai connection for tools.
+- Add "seed" to LLM and GPT-Vision tool inputs.
+
+## 1.3.0 (2024.03.01)
+
+### Improvements
+- Disable openai built-in retry mechanism for better debuggability and real-time status updates.
+- Refine the retry-after interval for openai retry error.
+
+## 1.2.0 (2024.02.07)
+
+### Features Added
+- Support tool "Azure OpenAI GPT-4 Turbo with Vision" to auto list "deployment_name".
+
+### Improvements
+- Match the promptflow-tools setup to promptflow setup.
+
+## 1.1.0 (2024.01.23)
+
+### Features Added
+- Use ruamel.yaml instead of pyyaml in promptflow.
+
+## 1.0.3 (2024.01.08)
+
+### Features Added
+- Add new tool "Azure OpenAI GPT-4 Turbo with Vision".
+- Support "response_format" for azure openai chat api in LLM tool.
+
+## 1.0.1 (2023.12.13)
+
+### Features Added
+- Support "response_format" for openai chat api in LLM tool.
+- Add "allow_manual_entry" for embedding tool.
+
+### Improvements
+- Handle all OpenSource\HuggingFace Models with the same MIR request format in 'Open Model LLM' tool.
+
 ## 1.0.0 (2023.11.30)
 
 ### Features Added
-- Support openai 1.x in promptflow-tools
-- Add new tool "OpenAI GPT-4V"
+- Support openai 1.x in promptflow-tools.
+- Add new tool "OpenAI GPT-4V".

src/promptflow-tools/connections.json.example

@@ -9,6 +9,16 @@
     },
     "module": "promptflow.connections"
   },
+  "azure_open_ai_connection_meid": {
+    "type": "AzureOpenAIConnection",
+    "value": {
+      "api_base": "aoai-api-endpoint",
+      "api_type": "azure",
+      "api_version": "2023-07-01-preview",
+      "auth_mode": "meid_token"
+    },
+    "module": "promptflow.connections"
+  },
   "serp_connection": {
     "type": "SerpConnection",
     "value": {