Merge branch 'main' into shimin/add_flex_resume_test

.cspell.json

@@ -215,10 +215,12 @@
     "mpnet",
     "wargs",
     "dcid",
+    "piezo",
+    "Piezo",
     "cmpop"
   ],
   "flagWords": [
     "Prompt Flow"
   ],
   "allowCompoundWords": true
-}
+}

.github/workflows/promptflow-core-test.yml

@@ -43,6 +43,24 @@ jobs:
       - name: generate end-to-end test config from secret
         run: echo '${{ secrets.PF_TRACING_E2E_TEST_CONFIG }}' >> connections.json
         working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: set test mode
+        run: |
+          echo "PROMPT_FLOW_TEST_MODE=$(if [[ "${{ github.event_name }}" == "pull_request" ]]; then echo replay; else echo live; fi)" >> $GITHUB_ENV
+      - name: Azure login (non pull_request workflow)
+        if: github.event_name != 'pull_request'
+        uses: azure/login@v1
+        with:
+          creds: ${{ secrets.AZURE_CREDENTIALS }}
+      - name: generate live test resources (non pull_request workflow)
+        if: github.event_name != 'pull_request'
+        uses: "./.github/actions/step_generate_configs"
+        with:
+          targetFolder: ${{ env.PROMPTFLOW_DIRECTORY }}
+      - name: generate live test resources (pull_request workflow)
+        if: github.event_name == 'pull_request'
+        working-directory: ${{ env.PROMPTFLOW_DIRECTORY }}
+        run: |
+          cp ${{ github.workspace }}/src/promptflow/dev-connections.json.example ${{ github.workspace }}/src/promptflow/connections.json
       - name: run core tests
         run: poetry run pytest ./tests/core --cov=promptflow --cov-config=pyproject.toml --cov-report=term --cov-report=html --cov-report=xml
         working-directory: ${{ env.WORKING_DIRECTORY }}
@@ -77,6 +95,24 @@ jobs:
           poetry run pip show promptflow-tracing
           poetry run pip show promptflow-core
         working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: set test mode
+        run: |
+          echo "PROMPT_FLOW_TEST_MODE=$(if [[ "${{ github.event_name }}" == "pull_request" ]]; then echo replay; else echo live; fi)" >> $GITHUB_ENV
+      - name: Azure login (non pull_request workflow)
+        if: github.event_name != 'pull_request'
+        uses: azure/login@v1
+        with:
+          creds: ${{ secrets.AZURE_CREDENTIALS }}
+      - name: generate live test resources (non pull_request workflow)
+        if: github.event_name != 'pull_request'
+        uses: "./.github/actions/step_generate_configs"
+        with:
+          targetFolder: ${{ env.PROMPTFLOW_DIRECTORY }}
+      - name: generate live test resources (pull_request workflow)
+        if: github.event_name == 'pull_request'
+        working-directory: ${{ env.PROMPTFLOW_DIRECTORY }}
+        run: |
+          cp ${{ github.workspace }}/src/promptflow/dev-connections.json.example ${{ github.workspace }}/src/promptflow/connections.json
       - name: run azureml-serving tests
         run: poetry run pytest ./tests/azureml-serving --cov=promptflow --cov-config=pyproject.toml --cov-report=term --cov-report=html --cov-report=xml
         working-directory: ${{ env.WORKING_DIRECTORY }}

.github/workflows/samples_tracing_llm_tracellm.yml

@@ -0,0 +1,64 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_tracing_llm_tracellm
+on:
+  schedule:
+    - cron: "36 21 * * *" # Every day starting at 5:36 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/tutorials/tracing/llm/**, .github/workflows/samples_tracing_llm_tracellm.yml, examples/requirements.txt, examples/connections/azure_openai.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+jobs:
+  samples_tracing_llm_tracellm:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          creds: ${{ secrets.AZURE_CREDENTIALS }}
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r ${{ github.workspace }}/examples/requirements.txt
+          pip install -r ${{ github.workspace }}/examples/dev_requirements.txt
+      - name: setup .env file
+        working-directory: examples/tutorials/tracing/llm
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create Aoai Connection
+        run: pf connection create -f ${{ github.workspace }}/examples/connections/azure_openai.yml --set api_key="${{ secrets.AOAI_API_KEY_TEST }}" api_base="${{ secrets.AOAI_API_ENDPOINT_TEST }}"
+      - name: Test Notebook
+        working-directory: examples/tutorials/tracing/llm
+        run: |
+          papermill -k python trace-llm.ipynb trace-llm.output.ipynb
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: artifact
+          path: examples/tutorials/tracing/llm

.github/workflows/sdk-cli-azure-test-production.yml

@@ -73,7 +73,7 @@ jobs:
         run: |
           set -xe
           poetry install --with ci,test
-  
+
           poetry run pip show promptflow-tracing
           poetry run pip show promptflow-core
           poetry run pip show promptflow-devkit
@@ -85,3 +85,11 @@ jobs:
         working-directory: ${{ env.WORKING_DIRECTORY }}
         run: |
           poetry run pytest ${{ inputs.filepath }} -n auto -m "unittest or e2etest"
+
+      - name: Upload Test Results
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: Test Results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }})
+          path: |
+            ${{ env.WORKING_DIRECTORY }}/tests/sdk_cli_azure_test/count.json

docs/README.md

@@ -10,7 +10,7 @@ Below is a table of important doc pages.
 |----------------|----------------|
 |Quick start|[Getting started with prompt flow](./how-to-guides/quick-start.md)|
 |Concepts|[Flows](./concepts/concept-flows.md)<br> [Tools](./concepts/concept-tools.md)<br> [Connections](./concepts/concept-connections.md)<br> [Variants](./concepts/concept-variants.md)<br> |
-|How-to guides|[How to initialize and test a flow](./how-to-guides/develop-a-flow/init-and-test-a-flow.md) <br>[How to run and evaluate a flow](./how-to-guides/run-and-evaluate-a-flow/index.md)<br> [How to tune prompts using variants](./how-to-guides/tune-prompts-with-variants.md)<br>[How to deploy a flow](./how-to-guides/deploy-a-flow/index.md)<br>[How to create and use your own tool package](./how-to-guides/develop-a-tool/create-and-use-tool-package.md)|
+|How-to guides|[How to initialize and test a flow](./how-to-guides/develop-a-dag-flow/init-and-test-a-flow.md) <br>[How to run and evaluate a flow](./how-to-guides/run-and-evaluate-a-flow/index.md)<br> [How to tune prompts using variants](./how-to-guides/tune-prompts-with-variants.md)<br>[How to deploy a flow](./how-to-guides/deploy-a-flow/index.md)<br>[How to create and use your own tool package](./how-to-guides/develop-a-tool/create-and-use-tool-package.md)|
 |Tools reference|[LLM tool](./reference/tools-reference/llm-tool.md)<br> [Prompt tool](./reference/tools-reference/prompt-tool.md)<br> [Python tool](./reference/tools-reference/python-tool.md)<br> [Embedding tool](./reference/tools-reference/embedding_tool.md)<br>[SERP API tool](./reference/tools-reference/serp-api-tool.md) ||
 
 

docs/cloud/azureai/consume-connections-from-azure-ai.md

@@ -43,13 +43,13 @@ Note:
 
 Currently, we support three types of connections:
 
-|Connection provider|Type|Description|Provider Specification|Use Case|
-|---|---|---|---|---|
-| Local Connections| Local| Enables consume the connections created and locally and stored in local sqlite. |NA| Ideal when connections need to be stored and managed locally.|
-|Azure AI connection - For current working directory| Cloud provider| Enables the consumption of connections from a cloud provider, such as a specific Azure Machine Learning workspace or Azure AI project.| Specify the resource ID in a `config.json` file placed in the project folder. <br> [Click here for more details](../../how-to-guides/set-global-configs.md#azureml)| A dynamic approach for consuming connections from different providers in specific projects. Allows for setting different provider configurations for different flows by updating the `config.json` in the project folder.|
-|Azure AI connection - For this machine| Cloud| Enables the consumption of connections from a cloud provider, such as a specific Azure Machine Learning workspace or Azure AI project. | Use a `connection string` to specify a cloud resource as the provider on your local machine. <br> [Click here for more details](../../how-to-guides/set-global-configs.md#full-azure-machine-learning-workspace-resource-id)|A global provider setting that applies across all working directories on your machine.|
+|Connection provider|Type|Description| Provider Specification                                                                                                                                                                                                           |Use Case|
+|---|---|---|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---|
+| Local Connections| Local| Enables consume the connections created and locally and stored in local sqlite. | NA                                                                                                                                                                                                                               | Ideal when connections need to be stored and managed locally.|
+|Azure AI connection - For current working directory| Cloud provider| Enables the consumption of connections from a cloud provider, such as a specific Azure Machine Learning workspace or Azure AI project.| Specify the resource ID in a `config.json` file placed in the project folder. <br> [Click here for more details](../../how-to-guides/set-promptflow-configs.md#azureml)                                                          | A dynamic approach for consuming connections from different providers in specific projects. Allows for setting different provider configurations for different flows by updating the `config.json` in the project folder.|
+|Azure AI connection - For this machine| Cloud| Enables the consumption of connections from a cloud provider, such as a specific Azure Machine Learning workspace or Azure AI project. | Use a `connection string` to specify a cloud resource as the provider on your local machine. <br> [Click here for more details](../../how-to-guides/set-promptflow-configs.md#full-azure-machine-learning-workspace-resource-id) |A global provider setting that applies across all working directories on your machine.|
 
 ## Next steps
 
-- Set global configs on [connection.provider](../../how-to-guides/set-global-configs.md#connectionprovider).
+- Set global configs on [connection.provider](../../how-to-guides/set-promptflow-configs.md#connectionprovider).
 - [Manage connections on local](../../how-to-guides/manage-connections.md).

docs/cloud/azureai/run-promptflow-in-azure-ai.md

@@ -155,7 +155,7 @@ At the end of stream logs, you can find the `portal_url` of the submitted run, c
 
 ### Run snapshot of the flow with additional includes
 
-Flows that enabled [additional include](../../how-to-guides/develop-a-flow/referencing-external-files-or-folders-in-a-flow.md) files can also be submitted for execution in the workspace. Please note that the specific additional include files or folders will be uploaded and organized within the **Files** folder of the run snapshot in the cloud.
+Flows that enabled [additional include](../../how-to-guides/develop-a-dag-flow/referencing-external-files-or-folders-in-a-flow.md) files can also be submitted for execution in the workspace. Please note that the specific additional include files or folders will be uploaded and organized within the **Files** folder of the run snapshot in the cloud.
 
 ![img](../../media/cloud/azureml/run-with-additional-includes.png)
 

docs/cloud/azureai/use-flow-in-azure-ml-pipeline.md

@@ -1,7 +1,7 @@
 # Use flow in Azure ML pipeline job
 In practical scenarios, flows fulfill various functions. For example, consider an offline flow specifically designed to assess the relevance score for communication sessions between humans and agents. This flow is triggered nightly and processes a substantial amount of session data. In such a context, Parallel component and AzureML pipeline emerge as the optimal choices for handling large-scale, highly resilient, and efficient offline batch requirements. 
 
-Once you’ve developed and thoroughly tested your flow using the guidelines in the [init and test a flow](../../how-to-guides/develop-a-flow/init-and-test-a-flow.md) section, this guide will walk you through utilizing your flow as a parallel component within an AzureML pipeline job.
+Once you’ve developed and thoroughly tested your flow, this guide will walk you through utilizing your flow as a parallel component within an AzureML pipeline job.
 
 :::{admonition} Pre-requirements
 To enable this feature, customer need to:
@@ -329,7 +329,7 @@ Given above, if your flow has logic relying on identity or environment variable,
 | key         | source | type                   | description                                                  |
 | ----------- | ------ | ---------------------- | ------------------------------------------------------------ |
 | data        | fixed  | uri_folder or uri_file | required; to pass in input data. Supported format includes [`mltable`](https://learn.microsoft.com/en-us/azure/machine-learning/how-to-mltable?view=azureml-api-2&tabs=cli#authoring-mltable-files) and list of jsonl files. |
-| run_outputs | fixed  | uri_folder             | optional; to pass in output of a standard flow for [an evaluation flow](../../how-to-guides/develop-a-flow/develop-evaluation-flow.md). Should be linked to a `flow_outputs` of a previous flow node in the pipeline. |
+| run_outputs | fixed  | uri_folder             | optional; to pass in output of a standard flow for [an evaluation flow](../../how-to-guides/develop-a-dag-flow/develop-evaluation-flow.md). Should be linked to a `flow_outputs` of a previous flow node in the pipeline. |
 
 ### Output ports
 

docs/concepts/concept-flows.md

@@ -12,14 +12,25 @@ Our [examples](https://github.com/microsoft/promptflow/tree/main/examples/flex-f
 
 Thus LLM apps can be defined as Directed Acyclic Graphs (DAGs) of function calls. These DAGs are flows in prompt flow.
 
-A flow in prompt flow is a DAG of functions (we call them [tools](./concept-tools.md)). These functions/tools connected via input/output dependencies and executed based on the topology by prompt flow executor.
+A `DAG flow` in prompt flow is a DAG of functions (we call them [tools](./concept-tools.md)). These functions/tools connected via input/output dependencies and executed based on the topology by prompt flow executor.
 
 A flow is represented as a YAML file and can be visualized with our [Prompt flow for VS Code extension](https://marketplace.visualstudio.com/items?itemName=prompt-flow.prompt-flow). Here is an example `flow.dag.yaml`:
 
 ![flow_dag](../media/how-to-guides/quick-start/flow_dag.png)
 
 Please refer to our [examples](https://github.com/microsoft/promptflow/tree/main/examples/flows) to learn how to write a `DAG flow`. 
 
+## When to use Flex or DAG flow
+
+`Dag flow` provides a UI-friendly way to develop your LLM app, which has the following benifits:
+- **Low code**: user can drag-and-drop in UI to create a LLM app.
+- **DAG Visualization**: user can easily understand the logic structure of the app with DAG view.
+
+`Flex flow` provides a code-friendly way to develop your LLM app, which has the following benifits:
+- **Quick start**: Users can quickly test with a simple prompt, then customize with python code with Tracing visualization UI.
+- **More advanced orchestration**: Users can write complex flow with Python built-in control operators (if-else, foreach) or other 3rd party / open-source library. 
+- **Easy onboard from other platforms**: user might already onboard platforms like `langchain` and `sematic kernel` with existing code. User can easily onboard promptflow with a few code changes.
+
 ## Flow types
 
 Prompt flow examples organize flows by three categories:
@@ -42,6 +53,6 @@ DAG flow [examples](https://github.com/microsoft/promptflow/tree/main/examples/f
 ## Next steps
 
 - [Quick start](../how-to-guides/quick-start.md)
-- [Initialize and test a flow](../how-to-guides/develop-a-flow/init-and-test-a-flow.md)
+- [Initialize and test a flow](../how-to-guides/develop-a-dag-flow/init-and-test-a-flow.md)
 - [Run and evaluate a flow](../how-to-guides/run-and-evaluate-a-flow/index.md)
 - [Tune prompts using variants](../how-to-guides/tune-prompts-with-variants.md)