Merge branch 'main' into feat/argcomplete

microsoft · Mar 29, 2024 · e9bf0f4 · e9bf0f4
2 parents d818b4c + ef87727
commit e9bf0f4
Show file tree

Hide file tree

Showing 760 changed files with 6,041 additions and 2,010 deletions.
diff --git a/.cspell.json b/.cspell.json
@@ -25,17 +25,21 @@
     "**/*.txt",
     ".gitignore",
     "scripts/docs/_build/**",
-    "src/promptflow/promptflow/azure/_restclient/flow/**",
-    "src/promptflow/promptflow/azure/_restclient/swagger.json",
-    "src/promptflow/promptflow/azure/_models/**",
-    "src/promptflow/promptflow/core/_connection_provider/_models/**",
+    "src/promptflow-azure/promptflow/azure/_restclient/flow/**",
+    "src/promptflow-azure/promptflow/azure/_restclient/swagger.json",
+    "src/promptflow-azure/promptflow/azure/_models/**",
+    "src/promptflow-azure/tests/**",
+    "src/promptflow-core/promptflow/core/_connection_provider/_models/**",
     "src/promptflow/tests/**",
+    "src/promptflow-recording/**",
     "src/promptflow-tools/tests/**",
     "**/flow.dag.yaml",
+    "**/pyproject.toml",
     "**/setup.py",
     "scripts/installer/curl_install_pypi/**",
     "scripts/installer/windows/**",
     "src/promptflow/promptflow/_sdk/_service/pfsvc.py",
+    "src/promptflow-devkit/promptflow/_sdk/_service/pfsvc.py",
     ".github/workflows/**",
     ".github/actions/**",
     ".github/pipelines/**"

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -1,18 +1,22 @@
 * @microsoft/prompt-flow-approvers
 
-/src/promptflow/promptflow/_core/ @microsoft/promptflow-execution
-/src/promptflow/promptflow/_internal/ @microsoft/promptflow-execution
-/src/promptflow/promptflow/batch/ @microsoft/promptflow-execution
-/src/promptflow/promptflow/executor/ @microsoft/promptflow-execution
-/src/promptflow/promptflow/integrations/ @microsoft/promptflow-execution
-/src/promptflow/promptflow/storage/ @microsoft/promptflow-execution
+/src/promptflow-tracing/promptflow/tracing/_start_trace.py @microsoft/promptflow-sdk
+
+/src/promptflow-tracing/promptflow/tracing/ @microsoft/promptflow-execution
+/src/promptflow-core/promptflow/_core/ @microsoft/promptflow-execution
+/src/promptflow-devkit/promptflow/_internal/ @microsoft/promptflow-execution
+/src/promptflow-devkit/promptflow/batch/ @microsoft/promptflow-execution
+/src/promptflow-core/promptflow/executor/ @microsoft/promptflow-execution
+/src/promptflow-core/promptflow/integrations/ @microsoft/promptflow-execution
+/src/promptflow-core/promptflow/storage/ @microsoft/promptflow-execution
 /src/promptflow/tests/executor/ @microsoft/promptflow-execution
 
-/src/promptflow/promptflow/_cli/ @microsoft/promptflow-sdk
-/src/promptflow/promptflow/_sdk/ @microsoft/promptflow-sdk
-/src/promptflow/promptflow/azure/ @microsoft/promptflow-sdk
-/src/promptflow/promptflow/entities/ @microsoft/promptflow-sdk
-/src/promptflow/promptflow/operations/ @microsoft/promptflow-sdk
+/src/promptflow-core/promptflow/core/ @microsoft/promptflow-sdk
+/src/promptflow-devkit/promptflow/_cli/ @microsoft/promptflow-sdk
+/src/promptflow-devkit/promptflow/_sdk/ @microsoft/promptflow-sdk
+/src/promptflow-devkit/promptflow/entities/ @microsoft/promptflow-sdk
+/src/promptflow-devkit/promptflow/operations/ @microsoft/promptflow-sdk
+/src/promptflow-azure/promptflow/azure/ @microsoft/promptflow-sdk
 /src/promptflow/tests/sdk_cli_test/ @microsoft/promptflow-sdk
 /src/promptflow/tests/sdk_cli_azure_test/ @microsoft/promptflow-sdk
 /src/promptflow/tests/sdk_cli_global_config_test/ @microsoft/promptflow-sdk

diff --git a/.github/actions/step_create_python_environment/action.yml b/.github/actions/step_create_python_environment/action.yml
@@ -12,7 +12,7 @@ runs:
   using: composite
   steps:
   - name: Set up Python
-    uses: actions/setup-python@v4
+    uses: actions/setup-python@v5
     with:
       python-version: ${{ inputs.pythonVersion }}
   - uses: snok/install-poetry@v1

diff --git a/.github/labeler.yml b/.github/labeler.yml
@@ -3,21 +3,23 @@ examples:
 documentation:
 - docs/**
 cli:
-- src/promptflow/promptflow/_cli/**
+- src/promptflow-devkit/promptflow/_cli/**
 sdk:
-- src/promptflow/promptflow/_sdk/**
-- src/promptflow/promptflow/azure/**
+- src/promptflow-devkit/promptflow/_sdk/**
+- src/promptflow-azure/promptflow/azure/**
 promptflow:
-- src/promptflow/**
+- src/promptflow-core/**
+promptflow-tracing:
+- src/promptflow-tracing/**
 promptflow-tools:
 - src/promptflow-tools/**
 executor:
-- src/promptflow/promptflow/batch/**
-- src/promptflow/promptflow/executor/**
-- src/promptflow/promptflow/_core/**
-- src/promptflow/promptflow/_internal/**
-- src/promptflow/promptflow/integrations/**
-- src/promptflow/promptflow/storage/**
+- src/promptflow-devkit/promptflow/batch/**
+- src/promptflow-core/promptflow/executor/**
+- src/promptflow-core/promptflow/_core/**
+- src/promptflow-devkit/promptflow/_internal/**
+- src/promptflow-core/promptflow/integrations/**
+- src/promptflow-core/promptflow/storage/**
 - src/promptflow/tests/executor/**
 fundamental:
 - scripts/**

diff --git a/.github/workflows/build_doc_ci.yml b/.github/workflows/build_doc_ci.yml
@@ -5,13 +5,15 @@ on:
   pull_request:
     branches:
       - main
-      - preview/docs
     paths:
       - 'README.md'
       - 'docs/**'
       - 'scripts/docs/**'
       - '.github/workflows/build_doc_ci.yml'
-      - 'src/promptflow/promptflow/**'
+      - 'src/promptflow-tracing/promptflow/**'
+      - 'src/promptflow-core/promptflow/**'
+      - 'src/promptflow-devkit/promptflow/**'
+      - 'src/promptflow-azure/promptflow/**'
 
 env:
   packageSetupType: promptflow_with_extra
@@ -30,17 +32,15 @@ jobs:
       - name: Python Setup
         uses: "./.github/actions/step_create_python_environment"
 
-      - name: Dev setup
-        uses: "./.github/actions/step_sdk_setup"
-        with:
-          setupType: ${{ env.packageSetupType }}
-          scriptPath: ${{ env.testWorkingDirectory }}
-
-      - name: Install tracing
+      - name: Install packages
         shell: pwsh
+        # Note: Use -e to avoid duplicate object warning when build apidoc.
         run: |
-          pip uninstall -y promptflow-tracing
+          pip uninstall -y promptflow-tracing promptflow-core promptflow-devkit promptflow-azure
           pip install -e ${{ github.workspace }}/src/promptflow-tracing
+          pip install -e ${{ github.workspace }}/src/promptflow-core
+          pip install -e ${{ github.workspace }}/src/promptflow-devkit
+          pip install -e ${{ github.workspace }}/src/promptflow-azure
           pip freeze
 
       - name: Build doc with reference doc
@@ -60,7 +60,23 @@ jobs:
         with:
           submodules: true
 
+      - name: Python Setup
+        uses: "./.github/actions/step_create_python_environment"
+
+      - name: Install packages
+        shell: pwsh
+        # Note: Use -e to avoid duplicate object warning when build apidoc.
+        run: |
+          pip uninstall -y promptflow-tracing promptflow-core promptflow-devkit promptflow-azure
+          pip install -e ${{ github.workspace }}/src/promptflow-tracing
+          pip install -e ${{ github.workspace }}/src/promptflow-core
+          pip install -e ${{ github.workspace }}/src/promptflow-devkit
+          pip install -e ${{ github.workspace }}/src/promptflow-azure
+          pip freeze
+
       - name: Build LinkCheck
         shell: powershell
         working-directory: scripts/docs/
-        run: ./doc_generation.ps1 -BuildLinkCheck -WarningAsError:$true
+        run: |-
+          pip install langchain
+          ./doc_generation.ps1 -WithReferenceDoc:$true -WarningAsError:$true -BuildLinkCheck
diff --git a/.github/workflows/flowdag_schema_check.yml b/.github/workflows/flowdag_schema_check.yml
@@ -30,6 +30,6 @@ jobs:
           PYTHONPATH: ${{ github.workspace }}/src/promptflow
         run: |
           cd ${{ github.workspace }}/src
-          pip install -e promptflow[azure]
-          pip install -e promptflow-tools
+          pip install ./promptflow[azure]
+          pip install ./promptflow-tools
           python ${{ github.workspace }}/scripts/readme/schema_checker.py
diff --git a/.github/workflows/promptflow-executor-e2e-test.yml b/.github/workflows/promptflow-executor-e2e-test.yml
@@ -4,6 +4,7 @@ on:
     - cron: "40 20 * * *" # Every day starting at 4:40 BJT
   pull_request_target:
     paths:
+      - src/promptflow-core/*
       - src/promptflow/*
       - src/promptflow/promptflow/*
       - src/promptflow/promptflow/_core/**
@@ -17,6 +18,7 @@ on:
       - src/promptflow/promptflow/storage/**
       - src/promptflow/tests/*
       - src/promptflow/tests/executor/**
+      - src/promptflow-tracing/promptflow/**
       - scripts/building/**
       - .github/workflows/promptflow-executor-e2e-test.yml
   workflow_dispatch:
@@ -25,6 +27,7 @@ env:
   testWorkingDirectory: ${{ github.workspace }}/src/promptflow
   PYTHONPATH: ${{ github.workspace }}/src/promptflow
   IS_IN_CI_PIPELINE: "true"
+  RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
 jobs:
   authorize:
     environment:
@@ -102,11 +105,18 @@ jobs:
       run: |
         Set-PSDebug -Trace 1
         pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
-        gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[executor-service]"}}
+        pip install ${{ github.workspace }}/src/promptflow-tracing
+        pip install ${{ github.workspace }}/src/promptflow-core[executor-service]
+        pip install ${{ github.workspace }}/src/promptflow-devkit
+        pip install ${{ github.workspace }}/src/promptflow-azure
+        gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)"}}
         gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}}
-        pip uninstall -y promptflow-tracing
-        pip install -e ${{ github.workspace }}/src/promptflow-tracing
         pip freeze
+    - name: install recording
+      run: |
+        pip install vcrpy
+        pip install .
+      working-directory: ${{ env.RECORD_DIRECTORY }}
     - name: Azure Login
       uses: azure/login@v1
       with:

diff --git a/.github/workflows/promptflow-executor-unit-test.yml b/.github/workflows/promptflow-executor-unit-test.yml
@@ -4,6 +4,7 @@ on:
     - cron: "40 19 * * *" # Every day starting at 3:40 BJT
   pull_request_target:
     paths:
+      - src/promptflow-core/*
       - src/promptflow/*
       - src/promptflow/promptflow/*
       - src/promptflow/promptflow/_core/**
@@ -25,6 +26,7 @@ env:
   testWorkingDirectory: ${{ github.workspace }}/src/promptflow
   PYTHONPATH: ${{ github.workspace }}/src/promptflow
   IS_IN_CI_PIPELINE: "true"
+  RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
 jobs:
   authorize:
     environment:
@@ -107,11 +109,18 @@ jobs:
       run: |
         Set-PSDebug -Trace 1
         pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
-        gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[executor-service]"}}
+        pip install ${{ github.workspace }}/src/promptflow-tracing
+        pip install ${{ github.workspace }}/src/promptflow-core[executor-service]
+        pip install ${{ github.workspace }}/src/promptflow-devkit
+        pip install ${{ github.workspace }}/src/promptflow-azure
+        gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)"}}
         gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}}
-        pip uninstall -y promptflow-tracing
-        pip install -e ${{ github.workspace }}/src/promptflow-tracing
         pip freeze
+    - name: install recording
+      run: |
+        pip install vcrpy
+        pip install .
+      working-directory: ${{ env.RECORD_DIRECTORY }}
     - name: Azure Login
       uses: azure/login@v1
       with:

diff --git a/.github/workflows/promptflow-global-config-test.yml b/.github/workflows/promptflow-global-config-test.yml
@@ -4,6 +4,7 @@ on:
     - cron: "40 18 * * *" # Every day starting at 2:40 BJT
   pull_request_target:
     paths:
+      - src/promptflow-core/*
       - src/promptflow/**
       - scripts/building/**
       - .github/workflows/promptflow-global-config-test.yml
@@ -60,11 +61,18 @@ jobs:
       with:
         setupType: ${{ env.packageSetupType }}
         scriptPath: ${{ env.testWorkingDirectory }}
-    - name: Install tracing
+    - name: Install dependency
       shell: pwsh
       run: |
         pip uninstall -y promptflow-tracing
-        pip install -e ${{ github.workspace }}/src/promptflow-tracing
+        pip install ${{ github.workspace }}/src/promptflow-tracing
+        echo "Installed promptflow-tracing"
+        pip uninstall -y promptflow-core
+        pip install ${{ github.workspace }}/src/promptflow-core
+        pip uninstall -y promptflow-devkit
+        pip install ${{ github.workspace }}/src/promptflow-devkit
+        pip uninstall -y promptflow-azure
+        pip install ${{ github.workspace }}/src/promptflow-azure
         pip freeze
     - name: Azure Login
       uses: azure/login@v1

diff --git a/.github/workflows/promptflow-import-linter.yml b/.github/workflows/promptflow-import-linter.yml
@@ -18,20 +18,20 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.11
     - uses: snok/install-poetry@v1
     - name: Install all packages
       run: |
         cd ${{ env.WORKING_DIRECTORY }}/src/promptflow-tracing
-        touch ./promptflow/__init__.py
         poetry install --with dev
         cd ${{ env.WORKING_DIRECTORY }}/src/promptflow-core
-        touch ./promptflow/__init__.py
         poetry install --with dev
         cd ${{ env.WORKING_DIRECTORY }}/src/promptflow-devkit
-        touch ./promptflow/__init__.py
         poetry install --with dev
         cd ${{ env.WORKING_DIRECTORY }}/src/promptflow-azure
-        touch ./promptflow/__init__.py
         poetry install --with dev
       working-directory: ${{ env.WORKING_DIRECTORY }}
     - name: import lint

diff --git a/.github/workflows/promptflow-release-testing-matrix.yml b/.github/workflows/promptflow-release-testing-matrix.yml
@@ -16,6 +16,7 @@ env:
   testWorkingDirectory: src/promptflow
   PYTHONPATH: ${{ github.workspace }}/src/promptflow
   IS_IN_CI_PIPELINE: "true"
+  RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
   TRACING_PATH: ${{ github.workspace }}/src/promptflow-tracing
 jobs:
   id:
@@ -73,6 +74,11 @@ jobs:
       with:
         name: wheel
         path: artifacts
+    - name: install recording
+      run:
+        pip install vcrpy
+        pip install -e .
+      working-directory: ${{ env.RECORD_DIRECTORY }}
     - name: Azure Login
       uses: azure/login@v1
       if: env.PROMPT_FLOW_TEST_MODE == 'live'
@@ -97,11 +103,6 @@ jobs:
         gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)"}}
         gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}}
         pip freeze
-    - name: Install tracing
-      shell: pwsh
-      working-directory: ${{ env.TRACING_PATH }}
-      run: |
-        poetry install --only main
     - name: Run SDK CLI Test
       shell: pwsh
       working-directory: ${{ env.testWorkingDirectory }}
@@ -162,6 +163,11 @@ jobs:
       with:
         name: wheel
         path: artifacts
+    - name: install recording
+      run: |
+        pip install vcrpy
+        pip install -e .
+      working-directory: ${{ env.RECORD_DIRECTORY }}
     - name: Azure Login
       uses: azure/login@v1
       if: env.PROMPT_FLOW_TEST_MODE == 'live'
@@ -180,11 +186,6 @@ jobs:
         gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[azure]"}}
         gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)"}}
         pip freeze
-    - name: Install tracing
-      shell: pwsh
-      working-directory: ${{ env.TRACING_PATH }}
-      run: |
-        poetry install --only main
     - name: Run SDK CLI Azure Test
       shell: pwsh
       working-directory: ${{ env.testWorkingDirectory }}
@@ -226,6 +227,11 @@ jobs:
       with:
         name: wheel
         path: artifacts
+    - name: install recording
+      run: |
+        pip install vcrpy
+        pip install -e .
+      working-directory: ${{ env.RECORD_DIRECTORY }}
     - name: Azure Login
       uses: azure/login@v1
       with:
@@ -242,11 +248,6 @@ jobs:
         gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[azure,executor-service]"}}
         gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}}
         pip freeze
-    - name: Install tracing
-      shell: pwsh
-      working-directory: ${{ env.TRACING_PATH }}
-      run: |
-        poetry install --only main
     - name: Run Executor Test
       shell: pwsh
       working-directory: ${{ github.workspace }}