[SDK/CLI] Write instance_results.jsonl path in run properties #10898
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: promptflow-sdk-cli-test | |
| on: | |
| schedule: | |
| - cron: "40 18 * * *" # Every day starting at 2:40 BJT | |
| pull_request: | |
| paths: | |
| - src/promptflow-core/** | |
| - src/promptflow-devkit/** | |
| - src/promptflow/** | |
| - src/promptflow-tracing/** | |
| - scripts/building/** | |
| - .github/workflows/promptflow-sdk-cli-test.yml | |
| - src/promptflow-recording/** | |
| workflow_dispatch: | |
| inputs: | |
| filepath: | |
| description: file or paths you want to trigger a test | |
| required: true | |
| default: "./tests/sdk_cli_test ./tests/sdk_pfs_test" | |
| type: string | |
| env: | |
| IS_IN_CI_PIPELINE: "true" | |
| RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording | |
| TRACING_DIRECTORY: ${{ github.workspace }}/src/promptflow-tracing | |
| CORE_DIRECTORY: ${{ github.workspace }}/src/promptflow-core | |
| WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-devkit | |
| PROMPTFLOW_DIRECTORY: ${{ github.workspace }}/src/promptflow | |
| TOOL_DIRECTORY: ${{ github.workspace }}/src/promptflow-tools | |
| jobs: | |
| sdk_cli_tests: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest] | |
| pythonVersion: ['3.8', '3.9', '3.10', '3.11'] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: set test mode | |
| run: | | |
| echo "PROMPT_FLOW_TEST_MODE=$(if [[ "${{ github.event_name }}" == "pull_request" ]]; then echo replay; else echo live; fi)" >> $GITHUB_ENV | |
| echo "FILE_PATHS=$(if [[ "${{ inputs.filepath }}" == "" ]]; then echo "./tests/sdk_cli_test ./tests/sdk_pfs_test"; else echo ${{ inputs.filepath }}; fi)" >> $GITHUB_ENV | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.pythonVersion }} | |
| - uses: snok/install-poetry@v1 | |
| - name: install test dependency group | |
| run: | | |
| set -xe | |
| poetry install -E pyarrow --with ci,test | |
| poetry run pip show promptflow-tracing | |
| poetry run pip show promptflow-core | |
| poetry run pip show promptflow-devkit | |
| poetry run pip show promptflow-tools | |
| working-directory: ${{ env.WORKING_DIRECTORY }} | |
| - name: Azure login (non pull_request workflow) | |
| if: github.event_name != 'pull_request' | |
| uses: azure/login@v1 | |
| with: | |
| creds: ${{ secrets.AZURE_CREDENTIALS }} | |
| - name: generate live test resources (non pull_request workflow) | |
| if: github.event_name != 'pull_request' | |
| uses: "./.github/actions/step_generate_configs" | |
| with: | |
| targetFolder: ${{ env.PROMPTFLOW_DIRECTORY }} | |
| - name: generate live test resources (pull_request workflow) | |
| if: github.event_name == 'pull_request' | |
| working-directory: ${{ env.PROMPTFLOW_DIRECTORY }} | |
| run: | | |
| cp ${{ github.workspace }}/src/promptflow/dev-connections.json.example ${{ github.workspace }}/src/promptflow/connections.json | |
| - name: run devkit tests | |
| run: | | |
| poetry run pytest ${{ env.FILE_PATHS }} --cov=promptflow --cov-config=pyproject.toml \ | |
| --cov-report=term --cov-report=html --cov-report=xml -n auto -m "(unittest or e2etest) and not csharp" \ | |
| --ignore-glob ./tests/sdk_cli_test/e2etests/test_executable.py | |
| working-directory: ${{ env.WORKING_DIRECTORY }} | |
| - name: Upload Test Results | |
| if: always() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: Test Results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }}) | |
| path: | | |
| ${{ env.WORKING_DIRECTORY }}/*.xml | |
| ${{ env.WORKING_DIRECTORY }}/htmlcov/ | |
| ${{ env.WORKING_DIRECTORY }}/tests/sdk_cli_test/count.json | |
| - run: poetry install -E executable | |
| working-directory: ${{ env.WORKING_DIRECTORY }} | |
| - name: run devkit executable tests | |
| run: | | |
| poetry run pytest -n auto -m "unittest or e2etest" ./tests/sdk_cli_test/e2etests/test_executable.py | |
| working-directory: ${{ env.WORKING_DIRECTORY }} | |
| publish-test-results-sdk-cli-test: | |
| needs: sdk_cli_tests | |
| runs-on: ubuntu-latest | |
| permissions: | |
| checks: write | |
| pull-requests: write | |
| contents: read | |
| issues: read | |
| if: always() | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| - name: Publish Test Results | |
| uses: "./.github/actions/step_publish_test_results" | |
| with: | |
| testActionFileName: promptflow-sdk-cli-test.yml | |
| testResultTitle: SDK CLI Test Result | |
| osVersion: ubuntu-latest | |
| pythonVersion: 3.9 | |
| coverageThreshold: 40 | |
| context: test/sdk_cli |