Merge branch 'main' into zhrua/pfazure_prompty

.github/pipelines/promptflow-csharp-e2e-test-tests.yml

@@ -31,17 +31,16 @@ steps:
     AZURE_OPENAI_ENDPOINT: $(azureOpenAiApiBase)
     IS_IN_CI_PIPELINE: true
 
-# enable this after we fix recording issue
-#- task: PowerShell@2
-#  displayName: 'Run azure sdk cli tests'
-#  inputs:
-#    targetType: 'inline'
-#    script: |
-#      pytest tests/sdk_cli_azure_test/e2etests/test_csharp_sdk.py
-#    workingDirectory: $(Build.SourcesDirectory)/src/promptflow-azure
-#  env:
-#    CSHARP_TEST_PROJECTS_ROOT: $(Build.SourcesDirectory)/$(flowProjectRelativePath)
-#    AZURE_OPENAI_API_KEY: $(azureOpenAiApiKey)
-#    AZURE_OPENAI_ENDPOINT: $(azureOpenAiApiBase)
-#    PROMPT_FLOW_TEST_MODE: "replay"
-#    IS_IN_CI_PIPELINE: true
+- task: PowerShell@2
+  displayName: 'Run azure sdk cli tests'
+  inputs:
+    targetType: 'inline'
+    script: |
+      pytest tests/sdk_cli_azure_test/e2etests/test_csharp_sdk.py
+    workingDirectory: $(Build.SourcesDirectory)/src/promptflow-azure
+  env:
+    CSHARP_TEST_PROJECTS_ROOT: $(Build.SourcesDirectory)/$(flowProjectRelativePath)
+    AZURE_OPENAI_API_KEY: $(azureOpenAiApiKey)
+    AZURE_OPENAI_ENDPOINT: $(azureOpenAiApiBase)
+    PROMPT_FLOW_TEST_MODE: "replay"
+    IS_IN_CI_PIPELINE: true

.github/workflows/build_doc_ci.yml

@@ -8,6 +8,7 @@ on:
     paths:
       - 'README.md'
       - 'docs/**'
+      - 'examples/**.ipynb'
       - 'scripts/docs/**'
       - '.github/workflows/build_doc_ci.yml'
       - 'src/promptflow-tracing/promptflow/**'

.github/workflows/promptflow-evals-e2e-test.yml

@@ -12,7 +12,6 @@ on:
 env:
   IS_IN_CI_PIPELINE: "true"
   WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-evals
-  RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
 
 jobs:
   build:
@@ -70,8 +69,8 @@ jobs:
         run: poetry install --only test
         working-directory: ${{ env.WORKING_DIRECTORY }}
       - name: install recording
-        run: poetry install
-        working-directory: ${{ env.RECORD_DIRECTORY }}
+        run: poetry run pip install -e ../promptflow-recording
+        working-directory: ${{ env.WORKING_DIRECTORY }}
       - name: generate end-to-end test config from secret
         run: echo '${{ secrets.PF_EVALS_E2E_TEST_CONFIG }}' >> connections.json
         working-directory: ${{ env.WORKING_DIRECTORY }}

.github/workflows/promptflow-evals-unit-test.yml

@@ -12,7 +12,6 @@ on:
 env:
   IS_IN_CI_PIPELINE: "true"
   WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-evals
-  RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
 
 jobs:
   build:
@@ -66,8 +65,8 @@ jobs:
         run: poetry install --only test
         working-directory: ${{ env.WORKING_DIRECTORY }}
       - name: install recording
-        run: poetry install
-        working-directory: ${{ env.RECORD_DIRECTORY }}
+        run: poetry run pip install -e ../promptflow-recording
+        working-directory: ${{ env.WORKING_DIRECTORY }}
       - name: run unit tests
         run: poetry run pytest -m unittest --cov=promptflow --cov-config=pyproject.toml --cov-report=term --cov-report=html --cov-report=xml
         working-directory: ${{ env.WORKING_DIRECTORY }}

.github/workflows/publish_doc.yml

@@ -9,6 +9,7 @@ on:
     paths:
       - 'README.md'
       - 'docs/**'
+      - 'examples/**.ipynb'
       - 'scripts/docs/**'
       - '.github/workflows/publish_doc.yml'
       - 'src/promptflow-tracing/promptflow/**'

.gitignore

@@ -71,7 +71,8 @@ instance/
 .scrapy
 
 # Sphinx documentation
-docs/_build/
+scripts/docs/_build/
+scripts/docs/jupyter_execute/
 
 # PyBuilder
 .pybuilder/
@@ -175,8 +176,6 @@ hello-world-proj/**
 # secrets
 **/connections.json
 
-# doc build
-scripts/docs/_build/
 **/bash_script.sh
 
 .index

docs/how-to-guides/tracing/index.md

@@ -117,4 +117,47 @@ pf run create --flow . --data "./data/bert-paper-qna.jsonl" --column-mapping cha
 ```
 Then you will get a run related trace URL, e.g. http://localhost:<port>/v1.0/ui/traces?run=chat_with_pdf_20240226_181222_219335
 
-![batch_run_record](../../media/trace/batch_run_record.png)
+![batch_run_record](../../media/trace/batch_run_record.png)
+
+### Search
+
+Trace UI supports simple Python expression for search experience, which is demonstrated in below GIF:
+
+![advanced_search](../../media/trace/advanced-search.gif)
+
+Currently it supports bool operator `and` and `or`, compare operator `==`, `!=`, `>`, `>=`, `<`, `<=`; and the fields that are searchable: `name`, `kind`, `status`, `start_time`, `cumulative_token_count.total`, `cumulative_token_count.prompt` and `cumulative_token_count.completion`. You can find the hints by clicking the button right to the search edit box.
+
+![search_hint](../../media/trace/trace-ui-search-hint.png)
+
+## Local trace management
+
+### Delete
+
+Prompt flow provides capability to delete traces in local storage, user can delete traces by collection, time range or prompt flow run with both CLI and SDK:
+
+::::{tab-set}
+:::{tab-item} CLI
+:sync: CLI
+
+```bash
+pf trace delete --collection <collection-name>  # delete specific collection
+pf trace delete --collection <collection-name> --started-before '2024-03-01T16:00:00.123456'  # delete traces started before the time in specific collection
+pf trace delete --run <run-name>  # delete traces originated from specific prompt flow run
+```
+:::
+
+:::{tab-item} SDK
+:sync: SDK
+
+```python
+from promptflow.client import PFClient
+
+pf = PFClient()
+pf.traces.delete(collection="<collection-name>")  # delete specific collection
+pf.traces.delete(collection="<collection-name>", started_before="2024-03-01T16:00:00.123456")  # delete traces started before the time in specific collection
+pf.traces.delete(run="<run-name>")  # delete traces originated from specific prompt flow run
+```
+
+:::
+
+::::

docs/reference/pf-command-reference.md

@@ -908,7 +908,7 @@ Manage prompt flow service.
 
 ### pf service start
 
-Start prompt flow service.
+Start the prompt flow service.
 
 ```bash
 pf service start [--port]
@@ -917,17 +917,25 @@ pf service start [--port]
 ```
 
 #### Examples
-Start prompt flow service.
+Prompt flow will try to start the service on the default port 23333. If the port is already taken, prompt flow will 
+sequentially probe new ports, incrementing by one each time. Prompt flow retains the port number for future reference 
+and will utilize it for subsequent service startups.
+
 ```bash
 pf service start
 ```
 
-Force restart promptflow service.
+Forcefully start the prompt flow service. If the port is already in use, the existing service will be terminated and 
+restart a new service
+
 ```bash
 pf service start --force
 ```
 
-Start promptflow service with specific port.
+Start the prompt flow service with a specified port. If the port is already taken, prompt flow will raise an error 
+unless forcefully start the service with the `--force` flag. Upon availability, prompt flow retains the port number for 
+future reference and will utilize it for subsequent service startups.
+
 ```bash
 pf service start --port 65553
 ```
@@ -941,7 +949,7 @@ pf service start --debug
 
 `--port -p`
 
-Port of the promptflow service.
+The designated port of the prompt flow service and port number will be remembered if port is available.
 
 `--force`
 

examples/flex-flows/basic/flex-flow-quickstart-azure.ipynb

@@ -45,7 +45,7 @@
     "We are using `DefaultAzureCredential` to get access to workspace. \n",
     "`DefaultAzureCredential` should be capable of handling most Azure SDK authentication scenarios. \n",
     "\n",
-    "Reference for more available credentials if it does not work for you: [configure credential example](../../configuration.ipynb), [azure-identity reference doc](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity?view=azure-python)."
+    "Reference for more available credentials if it does not work for you: [configure credential example](https://github.com/microsoft/promptflow/blob/main/examples/configuration.ipynb), [azure-identity reference doc](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity?view=azure-python)."
    ]
   },
   {
@@ -71,7 +71,7 @@
    "source": [
     "### Get a handle to the workspace\n",
     "\n",
-    "We use config file to connect to a workspace. The Azure ML workspace should be configured with computer cluster. [Check this notebook for configure a workspace](../../configuration.ipynb)"
+    "We use config file to connect to a workspace. The Azure ML workspace should be configured with computer cluster. [Check this notebook for configure a workspace](https://github.com/microsoft/promptflow/blob/main/examples/configuration.ipynb)"
    ]
   },
   {
@@ -107,7 +107,7 @@
    "source": [
     "## 2. Batch run the function as flow with multi-line data\n",
     "\n",
-    "Create a [flow.flex.yaml](flow.flex.yaml) file to define a flow which entry pointing to the python function we defined.\n"
+    "Create a `flow.flex.yaml` file to define a flow which entry pointing to the python function we defined.\n"
    ]
   },
   {
@@ -233,11 +233,15 @@
     "By now you've successfully run your first prompt flow and even did evaluation on it. That's great!\n",
     "\n",
     "You can check out more examples:\n",
-    "- [Basic Chat](../chat-basic/README.md): demonstrates how to create a chatbot that can remember previous interactions and use the conversation history to generate next message."
+    "- [Basic Chat](https://github.com/microsoft/promptflow/tree/main/examples/flex-flows/chat-basic): demonstrates how to create a chatbot that can remember previous interactions and use the conversation history to generate next message."
    ]
   }
  ],
  "metadata": {
+  "build_doc": {
+   "category": "azure",
+   "section": "Flow"
+  },
   "description": "A quickstart tutorial to run a flex flow and evaluate it in azure.",
   "kernelspec": {
    "display_name": "prompt_flow",
@@ -254,7 +258,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.5"
+   "version": "3.9.18"
   },
   "resources": "examples/requirements-azure.txt, examples/flex-flows/basic, examples/flex-flows/eval-code-quality"
  },

examples/flex-flows/basic/flex-flow-quickstart.ipynb

@@ -52,7 +52,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "Note: before running below cell, please configure required environment variable `AZURE_OPENAI_API_KEY`, `AZURE_OPENAI_ENDPOINT` by create an `.env` file. Please refer to [.env.example](.env.example) as an template."
+    "Note: before running below cell, please configure required environment variable `AZURE_OPENAI_API_KEY`, `AZURE_OPENAI_ENDPOINT` by create an `.env` file. Please refer to `../.env.example` as an template."
    ]
   },
   {
@@ -112,7 +112,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "Now, let's add another layer of function call. In [programmer.py](programmer.py) there is a function called `write_simple_program`, which calls a new function called `load_prompt` and previous `my_llm_tool` function."
+    "Now, let's add another layer of function call. In `programmer.py` there is a function called `write_simple_program`, which calls a new function called `load_prompt` and previous `my_llm_tool` function."
    ]
   },
   {
@@ -168,7 +168,7 @@
    "source": [
     "## 2. Batch run the function as flow with multi-line data\n",
     "\n",
-    "Create a [flow.flex.yaml](flow.flex.yaml) file to define a flow which entry pointing to the python function we defined.\n"
+    "Create a [flow.flex.yaml](https://github.com/microsoft/promptflow/blob/main/examples/flex-flows/basic/flow.flex.yaml) file to define a flow which entry pointing to the python function we defined.\n"
    ]
   },
   {
@@ -304,11 +304,15 @@
     "By now you've successfully run your first prompt flow and even did evaluation on it. That's great!\n",
     "\n",
     "You can check out more examples:\n",
-    "- [Basic Chat](../chat-basic/README.md): demonstrates how to create a chatbot that can remember previous interactions and use the conversation history to generate next message."
+    "- [Basic Chat](https://github.com/microsoft/promptflow/tree/main/examples/flex-flows/chat-basic): demonstrates how to create a chatbot that can remember previous interactions and use the conversation history to generate next message."
    ]
   }
  ],
  "metadata": {
+  "build_doc": {
+   "category": "local",
+   "section": "Flow"
+  },
   "description": "A quickstart tutorial to run a flex flow and evaluate it.",
   "kernelspec": {
    "display_name": "prompt_flow",

examples/flows/chat/chat-with-pdf/chat-with-pdf-azure.ipynb

@@ -4,7 +4,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "# Chat with PDF using Azure AI\n",
+    "# Chat with PDF in Azure\n",
     "\n",
     "This is a simple flow that allow you to ask questions about the content of a PDF file and get answers.\n",
     "You can run the flow with a URL to a PDF file and question as argument.\n",
@@ -161,7 +161,7 @@
    "metadata": {},
    "source": [
     "# 3. Evaluate the \"groundedness\"\n",
-    "The [eval-groundedness flow](../../evaluation/eval-groundedness/) is using ChatGPT/GPT4 model to grade the answers generated by chat-with-pdf flow."
+    "The `eval-groundedness flow` is using ChatGPT/GPT4 model to grade the answers generated by chat-with-pdf flow."
    ]
   },
   {
@@ -289,6 +289,10 @@
   }
  ],
  "metadata": {
+  "build_doc": {
+   "category": "azure",
+   "section": "Flow"
+  },
   "description": "A tutorial of chat-with-pdf flow that executes in Azure AI",
   "kernelspec": {
    "display_name": "prompt-flow",

examples/flows/chat/chat-with-pdf/chat-with-pdf.ipynb

@@ -93,7 +93,7 @@
    "source": [
     "## 2. Test the flow\n",
     "\n",
-    "**Note**: this sample uses [predownloaded PDFs](./chat_with_pdf/.pdfs/) and [prebuilt FAISS Index](./chat_with_pdf/.index/) to speed up execution time.\n",
+    "**Note**: this sample uses `predownloaded PDFs` and `prebuilt FAISS Index` to speed up execution time.\n",
     "You can remove the folders to start a fresh run."
    ]
   },
@@ -103,6 +103,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "# ./chat_with_pdf/.pdfs/ stores predownloaded PDFs\n",
+    "# ./chat_with_pdf/.index/ stores prebuilt index files\n",
+    "\n",
     "output = pf.flows.test(\n",
     "    \".\",\n",
     "    inputs={\n",
@@ -167,7 +170,7 @@
    "metadata": {},
    "source": [
     "# 4. Evaluate the \"groundedness\"\n",
-    "The [eval-groundedness flow](../../evaluation/eval-groundedness/) is using ChatGPT/GPT4 model to grade the answers generated by chat-with-pdf flow."
+    "The `eval-groundedness flow` is using ChatGPT/GPT4 model to grade the answers generated by chat-with-pdf flow."
    ]
   },
   {
@@ -224,7 +227,7 @@
    "metadata": {},
    "source": [
     "You will see a web page like this. It gives you detail about how each row is graded and even the details how the evaluation run executes:\n",
-    "![pf-visualize-screenshot](./assets/pf-visualize-screenshot.png)"
+    "![pf-visualize-screenshot](./media/chat-with-pdf/pf-visualize-screenshot.png)"
    ]
   },
   {
@@ -299,6 +302,10 @@
   }
  ],
  "metadata": {
+  "build_doc": {
+   "category": "local",
+   "section": "Flow"
+  },
   "description": "A tutorial of chat-with-pdf flow that allows user ask questions about the content of a PDF file and get answers",
   "kernelspec": {
    "display_name": "prompt-flow",
@@ -315,7 +322,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.12"
+   "version": "3.9.17"
   }
  },
  "nbformat": 4,