promptflow-release-testing-matrix.yml

name: promptflow-release-testing-matrix
on:
  workflow_call:
  workflow_dispatch:
    inputs:
      # can leave empty when trigger manually
      # GitHub Actions API for trigger does not return workflow run id
      # there we reference below Stack Overflow solution:
      # https://stackoverflow.com/a/69500478
      # which adds an identifier in workflow run jobs and can be used for filter
      id:
        description: Identifier for the workflow run
        required: false
        type: string
env:
  IS_IN_CI_PIPELINE: "true"
  TRACING_DIRECTORY: ${{ github.workspace }}/src/promptflow-tracing
  AZURE_DIRECTORY: ${{ github.workspace }}/src/promptflow-azure
  CORE_DIRECTORY: ${{ github.workspace }}/src/promptflow-core
  DEVKIT_DIRECTORY: ${{ github.workspace }}/src/promptflow-devkit
  PROMPTFLOW_DIRECTORY: ${{ github.workspace }}/src/promptflow
  TOOL_DIRECTORY: ${{ github.workspace }}/src/promptflow-tools
  RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
  PROMPT_FLOW_WORKSPACE_NAME: "promptflow-eastus"

permissions:
  id-token: write
  contents: read

jobs:
  id:
    runs-on: ubuntu-latest
    steps:
      - name: workflow run id - ${{ inputs.id }}
        run: |
          echo "workflow run id: ${{ inputs.id }}"
  build:
    runs-on: ubuntu-latest
    steps:
    - name: checkout
      uses: actions/checkout@v4
    - uses: actions/setup-python@v5
      with:
        python-version: "3.9"
    - uses: snok/install-poetry@v1
    - working-directory: ${{ env.TRACING_DIRECTORY }}
      run: poetry build -f wheel
    - working-directory: ${{ env.CORE_DIRECTORY }}
      run: poetry build -f wheel
    - working-directory: ${{ env.DEVKIT_DIRECTORY }}
      run: poetry build -f wheel
    - working-directory: ${{ env.AZURE_DIRECTORY }}
      run: poetry build -f wheel
    - working-directory: ${{ env.PROMPTFLOW_DIRECTORY }}
      run: |
        pip install -r ./dev_requirements.txt
        python ./setup.py bdist_wheel
    - working-directory: ${{ env.TOOL_DIRECTORY }}
      run: python ./setup.py bdist_wheel

    - name: Upload Wheel
      uses: actions/upload-artifact@v3
      with:
        name: wheel
        path: |
          ${{ github.workspace }}/src/promptflow/dist/*.whl
          ${{ github.workspace }}/src/promptflow-tracing/dist/*.whl
          ${{ github.workspace }}/src/promptflow-core/dist/*.whl
          ${{ github.workspace }}/src/promptflow-devkit/dist/*.whl
          ${{ github.workspace }}/src/promptflow-azure/dist/*.whl
          ${{ github.workspace }}/src/promptflow-tools/dist/*.whl

  promptflow_tracing_tests:
    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' || github.event_name == 'pull_request' }}
    needs: build
    env:
      PROMPT_FLOW_TEST_MODE: "live"
      WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-tracing
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, windows-latest, macos-13]
        pythonVersion: ['3.9', '3.10', '3.11']
    defaults:
      run:
        shell: bash
    runs-on: ${{ matrix.os }}
    steps:
    - name: checkout
      uses: actions/checkout@v4
    - name: Display and Set Environment Variables
      run:
        env | sort >> $GITHUB_OUTPUT
    - uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.pythonVersion }}
    - uses: snok/install-poetry@v1
    - name: Download Artifacts
      uses: actions/download-artifact@v3
      with:
        name: wheel
        path: ${{ env.WORKING_DIRECTORY }}/artifacts
    - name: install promptflow-tracing from wheel
      # wildcard expansion (*) does not work in Windows, so leverage python to find and install
      run: poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_tracing-*.whl', recursive=True)[0])")
      working-directory: ${{ env.WORKING_DIRECTORY }}
    - name: install test dependency group
      run: poetry install --only test
      working-directory: ${{ env.WORKING_DIRECTORY }}
    - name: install recording
      run: poetry install
      working-directory: ${{ env.RECORD_DIRECTORY }}
    - name: generate end-to-end test config from secret
      run: echo '${{ secrets.PF_TRACING_E2E_TEST_CONFIG }}' >> connections.json
      working-directory: ${{ env.WORKING_DIRECTORY }}
    - name: run e2e tests
      run: poetry run pytest -m e2etest --tb=short
      working-directory: ${{ env.WORKING_DIRECTORY }}
    - name: upload coverage report
      uses: actions/upload-artifact@v4
      with:
        name: promptflow_tracing_tests report-${{ matrix.os }}-py${{ matrix.pythonVersion }}
        path: |
          ${{ env.WORKING_DIRECTORY }}/*.xml


  promptflow_core_tests:
    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' || github.event_name == 'pull_request' }}
    needs: build
    environment:
      internal
    env:
      PROMPT_FLOW_TEST_MODE: "live"
      WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-core
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, windows-latest, macos-13]
        pythonVersion: ['3.9', '3.10', '3.11']
    # snok/install-poetry need this to support Windows
    defaults:
      run:
        shell: bash
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.pythonVersion }}
      - uses: snok/install-poetry@v1
      - name: Download Artifacts
        uses: actions/download-artifact@v3
        with:
          name: wheel
          path: ${{ env.WORKING_DIRECTORY }}/artifacts
      - name: Azure Login
        uses: azure/login@v1
        with:
          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
          tenant-id: ${{secrets.AZURE_TENANT_ID}}
          client-id: ${{secrets.AZURE_CLIENT_ID}}
      - name: Generate Configs
        uses: "./.github/actions/step_generate_configs"
        with:
          targetFolder: ${{ env.PROMPTFLOW_DIRECTORY }}
      - name: install promptflow-core from wheel
        # wildcard expansion (*) does not work in Windows, so leverage python to find and install
        run: |
          poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_tracing-*.whl', recursive=True)[0])")
          poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_core-*.whl', recursive=True)[0])")
          poetry run pip install -e ../promptflow-recording
        working-directory: ${{ env.WORKING_DIRECTORY }}
      - name: install test dependency group
        run: poetry install --only test
        working-directory: ${{ env.WORKING_DIRECTORY }}
      - name: run core tests
        run: poetry run pytest ./tests/core --tb=short
        working-directory: ${{ env.WORKING_DIRECTORY }}
      - name: upload coverage report
        uses: actions/upload-artifact@v4
        with:
          name: promptflow_core_tests report-${{ matrix.os }}-py${{ matrix.pythonVersion }}
          path: |
            ${{ env.WORKING_DIRECTORY }}/*.xml

  promptflow_core_azureml_serving_tests:
    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' || github.event_name == 'pull_request' }}
    needs: build
    env:
      PROMPT_FLOW_TEST_MODE: "live"
      WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-core
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, windows-latest, macos-13]
        pythonVersion: ['3.9', '3.10', '3.11']
    # snok/install-poetry need this to support Windows
    defaults:
      run:
        shell: bash
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.pythonVersion }}
      - uses: snok/install-poetry@v1
      - name: Download Artifacts
        uses: actions/download-artifact@v3
        with:
          name: wheel
          path: ${{ env.WORKING_DIRECTORY }}/artifacts
      - name: install promptflow-core from wheel
        # wildcard expansion (*) does not work in Windows, so leverage python to find and install
        run: |
          poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_tracing-*.whl', recursive=True)[0])")
          poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_core-*.whl', recursive=True)[0]+'[azureml-serving]')")
          poetry run pip install -e ../promptflow-recording
        working-directory: ${{ env.WORKING_DIRECTORY }}
      - name: install test dependency group
        run: poetry install --only test
        working-directory: ${{ env.WORKING_DIRECTORY }}
      - name: run azureml-serving tests
        run: poetry run pytest ./tests/azureml-serving --tb=short
        working-directory: ${{ env.WORKING_DIRECTORY }}
      - name: upload report
        uses: actions/upload-artifact@v4
        with:
          name: promptflow_core_azureml_serving_tests report-${{ matrix.os }}-py${{ matrix.pythonVersion }}
          path: |
            ${{ env.WORKING_DIRECTORY }}/*.xml


  promptflow_devkit_tests:
    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' || github.event_name == 'pull_request' }}
    needs: build
    environment:
      internal
    env:
      PROMPT_FLOW_TEST_MODE: "live"
      WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-devkit
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, windows-latest, macos-13]
        pythonVersion: ['3.9', '3.10', '3.11']
    # snok/install-poetry need this to support Windows
    defaults:
      run:
        shell: bash
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.pythonVersion }}
      - uses: snok/install-poetry@v1
      - name: Download Artifacts
        uses: actions/download-artifact@v3
        with:
          name: wheel
          path: ${{ env.WORKING_DIRECTORY }}/artifacts
      - name: Azure Login
        uses: azure/login@v1
        with:
          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
          tenant-id: ${{secrets.AZURE_TENANT_ID}}
          client-id: ${{secrets.AZURE_CLIENT_ID}}
      - name: Generate Configs
        uses: "./.github/actions/step_generate_configs"
        with:
          targetFolder: ${{ env.PROMPTFLOW_DIRECTORY }}
      - name: install promptflow-devkit from wheel
        # wildcard expansion (*) does not work in Windows, so leverage python to find and install
        run: |
          poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_tracing-*.whl', recursive=True)[0])")
          poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_core-*.whl', recursive=True)[0]+'[azureml-serving]')")
          poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_devkit-*.whl', recursive=True)[0]+'[executable]')")
          poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow-*.whl', recursive=True)[0])")
          poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_tools-*.whl', recursive=True)[0])")
        working-directory: ${{ env.WORKING_DIRECTORY }}
      - name: install test dependency group
        run: poetry install --only test
        working-directory: ${{ env.WORKING_DIRECTORY }}
      - name: run devkit tests
        run: poetry run pytest ./tests/sdk_cli_test ./tests/sdk_pfs_test -n auto -m "unittest or e2etest" --tb=short
        working-directory: ${{ env.WORKING_DIRECTORY }}
      - name: upload coverage report
        uses: actions/upload-artifact@v4
        with:
          name: promptflow_devkit_tests report-${{ matrix.os }}-py${{ matrix.pythonVersion }}
          path: |
            ${{ env.WORKING_DIRECTORY }}/*.xml


  promptflow_azure_tests:
    needs: build
    environment:
      internal
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, windows-latest, macos-13]
        pythonVersion: ['3.9', '3.10', '3.11']
    env:
      PROMPT_FLOW_TEST_MODE: "live"
      WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-azure
      PROMPT_FLOW_WORKSPACE_NAME: "promptflow-eastus"
    # snok/install-poetry need this to support Windows
    defaults:
      run:
        shell: bash
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.pythonVersion }}
      - uses: snok/install-poetry@v1
      - name: Download Artifacts
        uses: actions/download-artifact@v3
        with:
          name: wheel
          path: ${{ env.WORKING_DIRECTORY }}/artifacts
      - name: Azure Login
        uses: azure/login@v1
        with:
          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
          tenant-id: ${{secrets.AZURE_TENANT_ID}}
          client-id: ${{secrets.AZURE_CLIENT_ID}}
      - name: Generate Configs
        uses: "./.github/actions/step_generate_configs"
        with:
          targetFolder: ${{ env.PROMPTFLOW_DIRECTORY }}
      - name: install promptflow-azure from wheel
        # wildcard expansion (*) does not work in Windows, so leverage python to find and install
        run: |
          poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_tracing-*.whl', recursive=True)[0])")
          poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_core-*.whl', recursive=True)[0]+'[azureml-serving]')")
          poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_devkit-*.whl', recursive=True)[0]+'[executable]')")
          poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_azure-*.whl', recursive=True)[0])")
          poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow-*.whl', recursive=True)[0])")
          poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_tools-*.whl', recursive=True)[0])")
          poetry run pip install -e ../promptflow-recording
        working-directory: ${{ env.WORKING_DIRECTORY }}
      - name: install test dependency group
        run: poetry install --only test
        working-directory: ${{ env.WORKING_DIRECTORY }}
      - name: run azure tests
        run: poetry run pytest ./tests/sdk_cli_azure_test -n auto -m "unittest or e2etest" --tb=short
        working-directory: ${{ env.WORKING_DIRECTORY }}
      - name: upload coverage report
        uses: actions/upload-artifact@v4
        with:
          name: promptflow_azure_tests report-${{ matrix.os }}-py${{ matrix.pythonVersion }}
          path: |
            ${{ env.WORKING_DIRECTORY }}/*.xml


  promptflow_executor_tests:
    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' || github.event_name == 'pull_request' }}
    needs: build
    environment:
      internal
    env:
      testWorkingDirectory: src/promptflow
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, windows-latest, macos-13]
        pythonVersion: ['3.9', '3.10', '3.11']
    runs-on: ${{ matrix.os }}
    steps:
    - name: checkout
      uses: actions/checkout@v4
    - name: Display and Set Environment Variables
      run:
        env | sort >> $GITHUB_OUTPUT
      shell: bash -el {0}
    - name: Python Env Setup - ${{ matrix.os }} - Python Version ${{ matrix.pythonVersion }}
      uses: "./.github/actions/step_create_python_environment"
      with:
        pythonVersion: ${{ matrix.pythonVersion }}
    - name: Download Artifacts
      uses: actions/download-artifact@v3
      with:
        name: wheel
        path: artifacts
    - name: install recording
      run: |
        pip install vcrpy
        pip install -e .
      working-directory: ${{ env.RECORD_DIRECTORY }}
    - name: Azure Login
      uses: azure/login@v1
      with:
        subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
        tenant-id: ${{secrets.AZURE_TENANT_ID}}
        client-id: ${{secrets.AZURE_CLIENT_ID}}
    - name: Generate Configs
      uses: "./.github/actions/step_generate_configs"
      with:
        targetFolder: ${{ env.testWorkingDirectory }}
    - name: Install pf
      shell: pwsh
      working-directory: artifacts
      run: |
        pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
        pip uninstall -y promptflow-core promptflow-devkit promptflow-tracing
        pip install ${{ github.workspace }}/src/promptflow-tracing
        pip install ${{ github.workspace }}/src/promptflow-core
        pip install ${{ github.workspace }}/src/promptflow-devkit[pyarrow]
        pip install ${{ github.workspace }}/src/promptflow-azure
        gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[azure,executor-service]"}}
        gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)"}}
        pip freeze
    - name: Run Executor Unit Test
      shell: pwsh
      working-directory: ${{ github.workspace }}
      run: |
        pip install langchain
        pip install numexpr
        python scripts/building/run_coverage_tests.py `
          -p ${{ github.workspace }}/src/promptflow/promptflow `
          -t ${{ github.workspace }}/src/promptflow/tests/executor/unittests `
          -l eastus `
          -m "all" `
          -o "${{ github.workspace }}/test-results-executor-unit.xml"
    - name: Run Executor E2E Test
      shell: pwsh
      working-directory: ${{ github.workspace }}
      run: |
        pip install langchain langchain_community
        pip install numexpr
        python scripts/building/run_coverage_tests.py `
          -p ${{ github.workspace }}/src/promptflow/promptflow `
          -t ${{ github.workspace }}/src/promptflow/tests/executor/e2etests `
          -l eastus `
          -m "all" `
          -o "${{ github.workspace }}/test-results-executor-e2e.xml"
    - name: Upload pytest test results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }})
      if: ${{ always() }}
      uses: actions/upload-artifact@v3
      with:
        name: promptflow_executor_tests Test Results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }})
        path: ${{ github.workspace }}/*.xml


  publish-test-results:
    name: "Publish Tests Results"
    needs: [ promptflow_devkit_tests, promptflow_azure_tests, promptflow_executor_tests, promptflow_core_tests, promptflow_core_azureml_serving_tests ]
    runs-on: ubuntu-latest
    permissions:
      checks: write
      pull-requests: write
      contents: read
      issues: read
    if: always()

    steps:
      - name: Download Artifacts
        uses: actions/download-artifact@v3
        with:
          path: artifacts
      - name: Publish Test Results
        uses: EnricoMi/publish-unit-test-result-action@v2
        with:
          files: "artifacts/**/test-*.xml"