promptflow-release-testing-matrix #77
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: promptflow-release-testing-matrix | |
| on: | |
| workflow_call: | |
| workflow_dispatch: | |
| inputs: | |
| # can leave empty when trigger manually | |
| # GitHub Actions API for trigger does not return workflow run id | |
| # there we reference below Stack Overflow solution: | |
| # https://stackoverflow.com/a/69500478 | |
| # which adds an identifier in workflow run jobs and can be used for filter | |
| id: | |
| description: Identifier for the workflow run | |
| required: false | |
| type: string | |
| env: | |
| packageSetupType: promptflow_with_extra | |
| testWorkingDirectory: src/promptflow | |
| PYTHONPATH: ${{ github.workspace }}/src/promptflow | |
| IS_IN_CI_PIPELINE: "true" | |
| jobs: | |
| id: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: workflow run id - ${{ inputs.id }} | |
| run: | | |
| echo "workflow run id: ${{ inputs.id }}" | |
| all_tests: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest, macos-latest, windows-latest] | |
| testType: [sdk-cli, executor] | |
| pythonVersion: ['3.8', '3.9', '3.10', '3.11'] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v3 | |
| - name: Display and Set Environment Variables | |
| run: | |
| env | sort >> $GITHUB_OUTPUT | |
| shell: bash -el {0} | |
| - name: Python Env Setup - ${{ matrix.os }} - Python Version ${{ matrix.pythonVersion }} | |
| uses: "./.github/actions/step_create_python_environment" | |
| with: | |
| pythonVersion: ${{ matrix.pythonVersion }} | |
| - name: Build wheel | |
| uses: "./.github/actions/step_sdk_setup_win" | |
| with: | |
| setupType: promptflow_with_extra | |
| scriptPath: ${{ env.testWorkingDirectory }} | |
| - name: Azure Login | |
| uses: azure/login@v1 | |
| with: | |
| creds: ${{ secrets.AZURE_CREDENTIALS }} | |
| - name: Generate Configs | |
| uses: "./.github/actions/step_generate_configs" | |
| with: | |
| targetFolder: ${{ env.testWorkingDirectory }} | |
| - name: Get number of CPU cores | |
| uses: SimenB/github-actions-cpu-cores@v1 | |
| id: cpu-cores | |
| - name: Run SDK CLI Test | |
| if : ${{ matrix.testType == 'sdk-cli' }} | |
| shell: pwsh | |
| working-directory: ${{ env.testWorkingDirectory }} | |
| run: | | |
| gci env:* | sort-object name | |
| az account show | |
| python "../../scripts/building/run_coverage_tests.py" ` | |
| -p promptflow ` | |
| -t ${{ github.workspace }}/src/promptflow/tests/sdk_cli_azure_test ${{ github.workspace }}/src/promptflow/tests/sdk_cli_test ` | |
| -l eastus ` | |
| -m "unittest or e2etest" ` | |
| -n ${{ steps.cpu-cores.outputs.count }} | |
| mv ${{ github.workspace }}/${{ env.testWorkingDirectory }}/test-results.xml ${{ github.workspace }}/test-results-sdk-cli.xml | |
| - name: Run Executor Test | |
| if : ${{ matrix.testType == 'executor' }} | |
| shell: pwsh | |
| working-directory: ${{ github.workspace }} | |
| run: | | |
| gci env:* | sort-object name | |
| az account show | |
| pip install langchain | |
| pip install numexpr | |
| python scripts/building/run_coverage_tests.py ` | |
| -p ${{ github.workspace }}/src/promptflow/promptflow ` | |
| -t ${{ github.workspace }}/src/promptflow/tests/executor/e2etests ${{ github.workspace }}/src/promptflow/tests/executor/unittests ` | |
| -l eastus ` | |
| -m "all" ` | |
| -n ${{ steps.cpu-cores.outputs.count }} | |
| mv ./test-results.xml ./test-results-executor.xml | |
| - name: Upload pytest test results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }}) | |
| if: ${{ always() }} | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: Test Results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }}) | |
| path: ${{ github.workspace }}/*.xml | |
| publish-test-results: | |
| name: "Publish Tests Results" | |
| needs: | |
| - all_tests | |
| runs-on: ubuntu-latest | |
| permissions: write-all | |
| if: always() | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v3 | |
| - name: Publish Test Results | |
| uses: "./.github/actions/step_publish_test_results" | |
| with: | |
| testActionFileName: promptflow-release-testing-matrix.yml | |
| testResultTitle: Release Test Matrix | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| coverageThreshold: 0 |