Add how to build copilot tutorial

.cspell.json

@@ -106,6 +106,7 @@
     "meid",
     "Entra",
     "uvicorn",
+    "attribited",
     "MistralAI"
   ],
   "ignoreWords": [
@@ -208,6 +209,8 @@
     "wscript",
     "raisvc",
     "evals",
+    "setenv",
+    "pypdf",
     "redoc",
     "starlette",
     "mlindex",

.github/workflows/samples_flows_evaluation_eval_multi_turn_metrics.yml

@@ -0,0 +1,110 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flows_evaluation_eval_multi_turn_metrics
+on:
+  schedule:
+    - cron: "7 21 * * *" # Every day starting at 5:7 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flows/evaluation/eval-multi-turn-metrics/**, examples/*requirements.txt, .github/workflows/samples_flows_evaluation_eval_multi_turn_metrics.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+jobs:
+  samples_flows_evaluation_eval_multi_turn_metrics:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flows/evaluation/eval-multi-turn-metrics
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/flows/evaluation/eval-multi-turn-metrics
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          creds: ${{ secrets.AZURE_CREDENTIALS }}
+      - name: Extract Steps examples/flows/evaluation/eval-multi-turn-metrics/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flows/evaluation/eval-multi-turn-metrics/README.md -o examples/flows/evaluation/eval-multi-turn-metrics
+      - name: Cat script
+        working-directory: examples/flows/evaluation/eval-multi-turn-metrics
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flows/evaluation/eval-multi-turn-metrics
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flows/evaluation/eval-multi-turn-metrics
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flows/evaluation/eval-multi-turn-metrics
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: artifact
+          path: examples/flows/evaluation/eval-multi-turn-metrics/bash_script.sh

.github/workflows/samples_flows_evaluation_eval_single_turn_metrics.yml

@@ -0,0 +1,110 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flows_evaluation_eval_single_turn_metrics
+on:
+  schedule:
+    - cron: "32 19 * * *" # Every day starting at 3:32 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flows/evaluation/eval-single-turn-metrics/**, examples/*requirements.txt, .github/workflows/samples_flows_evaluation_eval_single_turn_metrics.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+jobs:
+  samples_flows_evaluation_eval_single_turn_metrics:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flows/evaluation/eval-single-turn-metrics
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/flows/evaluation/eval-single-turn-metrics
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          creds: ${{ secrets.AZURE_CREDENTIALS }}
+      - name: Extract Steps examples/flows/evaluation/eval-single-turn-metrics/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flows/evaluation/eval-single-turn-metrics/README.md -o examples/flows/evaluation/eval-single-turn-metrics
+      - name: Cat script
+        working-directory: examples/flows/evaluation/eval-single-turn-metrics
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flows/evaluation/eval-single-turn-metrics
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flows/evaluation/eval-single-turn-metrics
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flows/evaluation/eval-single-turn-metrics
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: artifact
+          path: examples/flows/evaluation/eval-single-turn-metrics/bash_script.sh

.github/workflows/samples_flows_standard_question_simulation.yml

@@ -0,0 +1,110 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flows_standard_question_simulation
+on:
+  schedule:
+    - cron: "37 19 * * *" # Every day starting at 3:37 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flows/standard/question-simulation/**, examples/*requirements.txt, .github/workflows/samples_flows_standard_question_simulation.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+jobs:
+  samples_flows_standard_question_simulation:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flows/standard/question-simulation
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/flows/standard/question-simulation
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          creds: ${{ secrets.AZURE_CREDENTIALS }}
+      - name: Extract Steps examples/flows/standard/question-simulation/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flows/standard/question-simulation/README.md -o examples/flows/standard/question-simulation
+      - name: Cat script
+        working-directory: examples/flows/standard/question-simulation
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flows/standard/question-simulation
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flows/standard/question-simulation
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flows/standard/question-simulation
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: artifact
+          path: examples/flows/standard/question-simulation/bash_script.sh

.gitignore

@@ -173,6 +173,8 @@ connection.json
 .azureml
 # dummy custom tool package example
 hello-world-proj/**
+# gen test data config
+examples/gen_test_data/config.yml
 # secrets
 **/connections.json
 

docs/cloud/azureai/generate-test-data-cloud.md

@@ -0,0 +1,36 @@
+# How to generate test data in cloud based on documents
+This guide will help you learn how to generate test data on Azure AI, so that you can integrate the created flow and process a large amount of data.
+
+
+## Prerequisites
+
+1. Go through [local test data generation guide](https://github.com/microsoft/promptflow/blob/53a685dbff920e891ef61cacb5f2f19e761ee809/docs/how-to-guides/generate-test-data.md) and prepare your [test data generation flow](https://github.com/microsoft/promptflow/blob/53a685dbff920e891ef61cacb5f2f19e761ee809/examples/gen_test_data/example_flow/).
+2. Go to the [example_gen_test_data](https://github.com/microsoft/promptflow/blob/53a685dbff920e891ef61cacb5f2f19e761ee809/examples/gen_test_data) folder and run command `pip install -r requirements_cloud.txt` to prepare local environment.
+3. Prepare cloud environment.
+    - Navigate to file [conda.yml](https://github.com/microsoft/promptflow/blob/53a685dbff920e891ef61cacb5f2f19e761ee809/examples/gen_test_data/conda.yml).
+    - For specific document file types, you may need to install extra packages:
+      - .docx - `pip install docx2txt`
+      - .pdf - `pip install pypdf`
+      - .ipynb - `pip install nbconvert`
+      > !Note: We use llama index `SimpleDirectoryReader` to load documents. For the latest information on required packages, please check [here](https://docs.llamaindex.ai/en/stable/examples/data_connectors/simple_directory_reader.html).
+
+4. Prepare Azure AI resources in cloud.
+    - An Azure AI ML workspace - [Create workspace resources you need to get started with Azure AI](https://learn.microsoft.com/en-us/azure/machine-learning/quickstart-create-resources?view=azureml-api-2).
+    - A compute target - [Learn more about compute cluster](https://learn.microsoft.com/en-us/azure/machine-learning/concept-compute-target?view=azureml-api-2).
+5. [Create cloud AzureOpenAI or OpenAI connection](https://microsoft.github.io/promptflow/cloud/azureai/run-promptflow-in-azure-ai.html#create-necessary-connections)
+
+6. Prepare test data generation setting.
+    - Navigate to [example_gen_test_data](https://github.com/microsoft/promptflow/blob/53a685dbff920e891ef61cacb5f2f19e761ee809/examples/gen_test_data) folder.
+    - Prepare `config.yml` by copying [`config.yml.example`](https://github.com/microsoft/promptflow/blob/53a685dbff920e891ef61cacb5f2f19e761ee809/examples/gen_test_data/config.yml.example).
+    - Fill in configurations in the `config.yml` by following inline comment instructions.
+
+
+## Generate test data at cloud
+For handling larger test data, you can leverage the PRS component to run flow in cloud.
+- Navigate to [example_gen_test_data](https://github.com/microsoft/promptflow/blob/53a685dbff920e891ef61cacb5f2f19e761ee809/examples/gen_test_data) folder.
+- After configuration, run the following command to generate the test data set:
+  ```bash
+  python -m gen_test_data.run --cloud
+  ```
+
+- The generated test data will be a data asset which can be found in the output of the last node. You can register this data asset for future use.