Merge branch 'main' into dev/linatang/support-connection-provider-in-…

…script-executor

.github/workflows/promptflow-evals-e2e-test.yml

@@ -31,7 +31,7 @@ jobs:
     needs: build
     strategy:
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
+        os: [ubuntu-latest, windows-latest, macos-13]
         python-version: ['3.8', '3.9', '3.10', '3.11']
       fail-fast: false
     # snok/install-poetry need this to support Windows

.github/workflows/promptflow-evals-unit-test.yml

@@ -31,7 +31,7 @@ jobs:
     needs: build
     strategy:
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
+        os: [ubuntu-latest, windows-latest, macos-13]
         python-version: ['3.8', '3.9', '3.10', '3.11']
       fail-fast: false
     # snok/install-poetry need this to support Windows

.github/workflows/promptflow-release-testing-matrix.yml

@@ -75,7 +75,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
+        os: [ubuntu-latest, windows-latest, macos-13]
         pythonVersion: ['3.8', '3.9', '3.10', '3.11']
     defaults:
       run:
@@ -129,7 +129,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
+        os: [ubuntu-latest, windows-latest, macos-13]
         pythonVersion: ['3.8', '3.9', '3.10', '3.11']
     # snok/install-poetry need this to support Windows
     defaults:
@@ -160,13 +160,11 @@ jobs:
         run: |
           poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_tracing-*.whl', recursive=True)[0])")
           poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_core-*.whl', recursive=True)[0])")
+          poetry run pip install -e ../promptflow-recording
         working-directory: ${{ env.WORKING_DIRECTORY }}
       - name: install test dependency group
         run: poetry install --only test
         working-directory: ${{ env.WORKING_DIRECTORY }}
-      - name: install recording
-        run: poetry install
-        working-directory: ${{ env.RECORD_DIRECTORY }}
       - name: run core tests
         run: poetry run pytest ./tests/core
         working-directory: ${{ env.WORKING_DIRECTORY }}
@@ -186,7 +184,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
+        os: [ubuntu-latest, windows-latest, macos-13]
         pythonVersion: ['3.8', '3.9', '3.10', '3.11']
     # snok/install-poetry need this to support Windows
     defaults:
@@ -209,13 +207,11 @@ jobs:
         run: |
           poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_tracing-*.whl', recursive=True)[0])")
           poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_core-*.whl', recursive=True)[0]+'[azureml-serving]')")
+          poetry run pip install -e ../promptflow-recording
         working-directory: ${{ env.WORKING_DIRECTORY }}
       - name: install test dependency group
         run: poetry install --only test
         working-directory: ${{ env.WORKING_DIRECTORY }}
-      - name: install recording
-        run: poetry install
-        working-directory: ${{ env.RECORD_DIRECTORY }}
       - name: run azureml-serving tests
         run: poetry run pytest ./tests/azureml-serving
         working-directory: ${{ env.WORKING_DIRECTORY }}
@@ -236,7 +232,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
+        os: [ubuntu-latest, windows-latest, macos-13]
         pythonVersion: ['3.8', '3.9', '3.10', '3.11']
     # snok/install-poetry need this to support Windows
     defaults:
@@ -274,9 +270,6 @@ jobs:
       - name: install test dependency group
         run: poetry install --only test
         working-directory: ${{ env.WORKING_DIRECTORY }}
-      - name: install recording
-        run: poetry install
-        working-directory: ${{ env.RECORD_DIRECTORY }}
       - name: run devkit tests
         run: poetry run pytest ./tests/sdk_cli_test ./tests/sdk_pfs_test -n auto -m "unittest or e2etest"
         working-directory: ${{ env.WORKING_DIRECTORY }}
@@ -293,7 +286,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
+        os: [ubuntu-latest, windows-latest, macos-13]
         pythonVersion: ['3.8', '3.9', '3.10', '3.11']
     env:
       PROMPT_FLOW_TEST_MODE: "live"
@@ -356,7 +349,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
+        os: [ubuntu-latest, windows-latest, macos-13]
         pythonVersion: ['3.8', '3.9', '3.10', '3.11']
     runs-on: ${{ matrix.os }}
     steps:
@@ -401,18 +394,30 @@ jobs:
         gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[azure,executor-service]"}}
         gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)"}}
         pip freeze
-    - name: Run Executor Test
+    - name: Run Executor Unit Test
+      shell: pwsh
+      working-directory: ${{ github.workspace }}
+      run: |
+        pip install langchain
+        pip install numexpr
+        python scripts/building/run_coverage_tests.py `
+          -p ${{ github.workspace }}/src/promptflow/promptflow `
+          -t ${{ github.workspace }}/src/promptflow/tests/executor/unittests `
+          -l eastus `
+          -m "all" `
+          -o "${{ github.workspace }}/test-results-executor-unit.xml"
+    - name: Run Executor E2E Test
       shell: pwsh
       working-directory: ${{ github.workspace }}
       run: |
         pip install langchain
         pip install numexpr
         python scripts/building/run_coverage_tests.py `
           -p ${{ github.workspace }}/src/promptflow/promptflow `
-          -t ${{ github.workspace }}/src/promptflow/tests/executor/e2etests ${{ github.workspace }}/src/promptflow/tests/executor/unittests `
+          -t ${{ github.workspace }}/src/promptflow/tests/executor/e2etests `
           -l eastus `
           -m "all" `
-          -o "${{ github.workspace }}/test-results-executor.xml"
+          -o "${{ github.workspace }}/test-results-executor-e2e.xml"
     - name: Upload pytest test results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }})
       if: ${{ always() }}
       uses: actions/upload-artifact@v3

.github/workflows/promptflow-tracing-e2e-test.yml

@@ -32,7 +32,7 @@ jobs:
     needs: build
     strategy:
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
+        os: [ubuntu-latest, windows-latest, macos-13]
         python-version: ['3.8', '3.9', '3.10', '3.11']
       fail-fast: false
     # snok/install-poetry need this to support Windows

.github/workflows/promptflow-tracing-unit-test.yml

@@ -32,7 +32,7 @@ jobs:
     needs: build
     strategy:
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
+        os: [ubuntu-latest, windows-latest, macos-13]
         python-version: ['3.8', '3.9', '3.10', '3.11']
       fail-fast: false
     # snok/install-poetry need this to support Windows

.github/workflows/sdk-cli-perf-monitor-test.yml

@@ -35,7 +35,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        os: [ubuntu-latest, macos-13, windows-latest]
     defaults:
       run:
         shell: bash

docs/README.md

@@ -10,7 +10,7 @@ Below is a table of important doc pages.
 |----------------|----------------|
 |Quick start|[Getting started with prompt flow](./how-to-guides/quick-start.md)|
 |Concepts|[Flows](./concepts/concept-flows.md)<br> [Tools](./concepts/concept-tools.md)<br> [Connections](./concepts/concept-connections.md)<br> [Variants](./concepts/concept-variants.md)<br> |
-|How-to guides|[How to initialize and test a flow](./how-to-guides/develop-a-flow/init-and-test-a-flow.md) <br>[How to run and evaluate a flow](./how-to-guides/run-and-evaluate-a-flow/index.md)<br> [How to tune prompts using variants](./how-to-guides/tune-prompts-with-variants.md)<br>[How to deploy a flow](./how-to-guides/deploy-a-flow/index.md)<br>[How to create and use your own tool package](./how-to-guides/develop-a-tool/create-and-use-tool-package.md)|
+|How-to guides|[How to initialize and test a flow](./how-to-guides/develop-a-dag-flow/init-and-test-a-flow.md) <br>[How to run and evaluate a flow](./how-to-guides/run-and-evaluate-a-flow/index.md)<br> [How to tune prompts using variants](./how-to-guides/tune-prompts-with-variants.md)<br>[How to deploy a flow](./how-to-guides/deploy-a-flow/index.md)<br>[How to create and use your own tool package](./how-to-guides/develop-a-tool/create-and-use-tool-package.md)|
 |Tools reference|[LLM tool](./reference/tools-reference/llm-tool.md)<br> [Prompt tool](./reference/tools-reference/prompt-tool.md)<br> [Python tool](./reference/tools-reference/python-tool.md)<br> [Embedding tool](./reference/tools-reference/embedding_tool.md)<br>[SERP API tool](./reference/tools-reference/serp-api-tool.md) ||
 
 

docs/cloud/azureai/run-promptflow-in-azure-ai.md

@@ -155,7 +155,7 @@ At the end of stream logs, you can find the `portal_url` of the submitted run, c
 
 ### Run snapshot of the flow with additional includes
 
-Flows that enabled [additional include](../../how-to-guides/develop-a-flow/referencing-external-files-or-folders-in-a-flow.md) files can also be submitted for execution in the workspace. Please note that the specific additional include files or folders will be uploaded and organized within the **Files** folder of the run snapshot in the cloud.
+Flows that enabled [additional include](../../how-to-guides/develop-a-dag-flow/referencing-external-files-or-folders-in-a-flow.md) files can also be submitted for execution in the workspace. Please note that the specific additional include files or folders will be uploaded and organized within the **Files** folder of the run snapshot in the cloud.
 
 ![img](../../media/cloud/azureml/run-with-additional-includes.png)
 

docs/cloud/azureai/use-flow-in-azure-ml-pipeline.md

@@ -1,7 +1,7 @@
 # Use flow in Azure ML pipeline job
 In practical scenarios, flows fulfill various functions. For example, consider an offline flow specifically designed to assess the relevance score for communication sessions between humans and agents. This flow is triggered nightly and processes a substantial amount of session data. In such a context, Parallel component and AzureML pipeline emerge as the optimal choices for handling large-scale, highly resilient, and efficient offline batch requirements. 
 
-Once you’ve developed and thoroughly tested your flow using the guidelines in the [init and test a flow](../../how-to-guides/develop-a-flow/init-and-test-a-flow.md) section, this guide will walk you through utilizing your flow as a parallel component within an AzureML pipeline job.
+Once you’ve developed and thoroughly tested your flow, this guide will walk you through utilizing your flow as a parallel component within an AzureML pipeline job.
 
 :::{admonition} Pre-requirements
 To enable this feature, customer need to:
@@ -329,7 +329,7 @@ Given above, if your flow has logic relying on identity or environment variable,
 | key         | source | type                   | description                                                  |
 | ----------- | ------ | ---------------------- | ------------------------------------------------------------ |
 | data        | fixed  | uri_folder or uri_file | required; to pass in input data. Supported format includes [`mltable`](https://learn.microsoft.com/en-us/azure/machine-learning/how-to-mltable?view=azureml-api-2&tabs=cli#authoring-mltable-files) and list of jsonl files. |
-| run_outputs | fixed  | uri_folder             | optional; to pass in output of a standard flow for [an evaluation flow](../../how-to-guides/develop-a-flow/develop-evaluation-flow.md). Should be linked to a `flow_outputs` of a previous flow node in the pipeline. |
+| run_outputs | fixed  | uri_folder             | optional; to pass in output of a standard flow for [an evaluation flow](../../how-to-guides/develop-a-dag-flow/develop-evaluation-flow.md). Should be linked to a `flow_outputs` of a previous flow node in the pipeline. |
 
 ### Output ports
 

docs/concepts/concept-flows.md

@@ -12,14 +12,25 @@ Our [examples](https://github.com/microsoft/promptflow/tree/main/examples/flex-f
 
 Thus LLM apps can be defined as Directed Acyclic Graphs (DAGs) of function calls. These DAGs are flows in prompt flow.
 
-A flow in prompt flow is a DAG of functions (we call them [tools](./concept-tools.md)). These functions/tools connected via input/output dependencies and executed based on the topology by prompt flow executor.
+A `DAG flow` in prompt flow is a DAG of functions (we call them [tools](./concept-tools.md)). These functions/tools connected via input/output dependencies and executed based on the topology by prompt flow executor.
 
 A flow is represented as a YAML file and can be visualized with our [Prompt flow for VS Code extension](https://marketplace.visualstudio.com/items?itemName=prompt-flow.prompt-flow). Here is an example `flow.dag.yaml`:
 
 ![flow_dag](../media/how-to-guides/quick-start/flow_dag.png)
 
 Please refer to our [examples](https://github.com/microsoft/promptflow/tree/main/examples/flows) to learn how to write a `DAG flow`. 
 
+## When to use Flex or DAG flow
+
+`Dag flow` provides a UI-friendly way to develop your LLM app, which has the following benifits:
+- **Low code**: user can drag-and-drop in UI to create a LLM app.
+- **DAG Visualization**: user can easily understand the logic structure of the app with DAG view.
+
+`Flex flow` provides a code-friendly way to develop your LLM app, which has the following benifits:
+- **Quick start**: Users can quickly test with a simple prompt, then customize with python code with Tracing visualization UI.
+- **More advanced orchestration**: Users can write complex flow with Python built-in control operators (if-else, foreach) or other 3rd party / open-source library. 
+- **Easy onboard from other platforms**: user might already onboard platforms like `langchain` and `sematic kernel` with existing code. User can easily onboard promptflow with a few code changes.
+
 ## Flow types
 
 Prompt flow examples organize flows by three categories:
@@ -42,6 +53,6 @@ DAG flow [examples](https://github.com/microsoft/promptflow/tree/main/examples/f
 ## Next steps
 
 - [Quick start](../how-to-guides/quick-start.md)
-- [Initialize and test a flow](../how-to-guides/develop-a-flow/init-and-test-a-flow.md)
+- [Initialize and test a flow](../how-to-guides/develop-a-dag-flow/init-and-test-a-flow.md)
 - [Run and evaluate a flow](../how-to-guides/run-and-evaluate-a-flow/index.md)
 - [Tune prompts using variants](../how-to-guides/tune-prompts-with-variants.md)

docs/how-to-guides/develop-a-dag-flow/index.md

@@ -0,0 +1,26 @@
+# Develop a dag flow
+
+LLM apps can be defined as Directed Acyclic Graphs (DAGs) of function calls. These DAGs are flows in prompt flow.
+
+A `DAG flow` in prompt flow is a DAG of functions (we call them [tools](../../concepts//concept-tools.md)). These functions/tools connected via input/output dependencies and executed based on the topology by prompt flow executor.
+
+A flow is represented as a YAML file and can be visualized with our [Prompt flow for VS Code extension](https://marketplace.visualstudio.com/items?itemName=prompt-flow.prompt-flow). Here is an example `flow.dag.yaml`:
+
+![flow_dag](../../media/how-to-guides/quick-start/flow_dag.png)
+
+Please refer to our [examples](https://github.com/microsoft/promptflow/tree/main/examples/flows) and guides in this section to learn how to write a `DAG flow`. 
+
+Note: 
+- promptflow also support user develop a a flow using code. learn more on comparasion of these two [flow concepts](../../concepts/concept-flows.md).
+
+```{toctree}
+:maxdepth: 1
+
+init-and-test-a-flow
+develop-standard-flow
+develop-chat-flow
+develop-evaluation-flow
+add-conditional-control-to-a-flow
+process-image-in-flow
+referencing-external-files-or-folders-in-a-flow
+```