promptflow-sdk-cli-test.yml

name: promptflow-sdk-cli-test
on:
  schedule:
    - cron: "40 18 * * *" # Every day starting at 2:40 BJT
  pull_request_target:
    paths:
      - src/promptflow/**
      - scripts/**
      - '**promptflow-sdk-cli-test.yml'
  workflow_dispatch:
env:
  packageSetupType: promptflow_with_extra
  testWorkingDirectory: ${{ github.workspace }}/src/promptflow
  PYTHONPATH: ${{ github.workspace }}/src/promptflow
  IS_IN_CI_PIPELINE: "true"
jobs:
  authorize:
    environment:
      # forked prs from pull_request_target will be run in external environment, domain prs will be run in internal environment
      ${{ github.event_name == 'pull_request_target' &&
      github.event.pull_request.head.repo.full_name != github.repository &&
      'external' || 'internal' }}
    runs-on: ubuntu-latest
    steps:
      - run: true
  sdk_cli_tests:
    needs: authorize
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest]
    runs-on: ${{ matrix.os }}
    steps:
    - name: checkout
      uses: actions/checkout@v3
      with:
        ref: ${{ github.event.pull_request.head.sha || github.ref }}
    - name: Display and Set Environment Variables
      run: |
        if [ "ubuntu-latest" == "${{ matrix.os }}" ]; then
          export pyVersion="3.9";
        elif [ "macos-latest" == "${{ matrix.os }}" ]; then
          export pyVersion="3.10";
        else
          echo "Unsupported OS: ${{ matrix.os }}";
          exit 1;
        fi
        env | sort >> $GITHUB_OUTPUT
      id: display_env
      shell: bash -el {0}
    - name: Python Setup - ${{ matrix.os }} - Python Version ${{ steps.display_env.outputs.pyVersion }}
      uses: "./.github/actions/step_create_python_environment"
      with:
        pythonVersion: ${{ steps.display_env.outputs.pyVersion }}
    - name: Build wheel
      uses: "./.github/actions/step_sdk_setup"
      with:
        setupType: ${{ env.packageSetupType }}
        scriptPath: ${{ env.testWorkingDirectory }}
    - name: Azure Login
      uses: azure/login@v1
      with:
        creds: ${{ secrets.AZURE_CREDENTIALS }}
    - name: Generate Configs
      uses: "./.github/actions/step_generate_configs"
      with:
        targetFolder: ${{ env.testWorkingDirectory }}
    - name: Get number of CPU cores
      uses: SimenB/github-actions-cpu-cores@v1
      id: cpu-cores
    - name: Run Test
      shell: pwsh
      working-directory: ${{ env.testWorkingDirectory }}
      run: |
        gci env:* | sort-object name
        az account show
        python "../../scripts/building/run_coverage_tests.py" `
          -p promptflow `
          -t ${{ github.workspace }}/src/promptflow/tests/sdk_cli_azure_test ${{ github.workspace }}/src/promptflow/tests/sdk_cli_test `
          -l eastus `
          -m "unittest or e2etest" `
          -n ${{ steps.cpu-cores.outputs.count }} `
          --coverage-config ${{ github.workspace }}/src/promptflow/tests/sdk_cli_test/.coveragerc
    - name: Upload Test Results
      if: always()
      uses: actions/upload-artifact@v3
      with:
        name: Test Results (Python ${{ steps.display_env.outputs.pyVersion }}) (OS ${{ matrix.os }})
        path: |
          ${{ env.testWorkingDirectory }}/*.xml
          ${{ env.testWorkingDirectory }}/htmlcov/
  publish-test-results:
    name: "Publish Tests Results"
    needs: sdk_cli_tests
    runs-on: ubuntu-latest
    permissions: write-all
    if: always()

    steps:
    - name: checkout
      uses: actions/checkout@v3
      with:
        ref: ${{ github.event.pull_request.head.sha || github.ref }}
    - name: Publish Test Results
      uses: "./.github/actions/step_publish_test_results"
      with:
        testActionFileName: promptflow-sdk-cli-test.yml
        testResultTitle: SDK CLI Test Result
        osVersion: ubuntu-latest
        pythonVersion: 3.9
        coverageThreshold: 40
        token: ${{ secrets.GITHUB_TOKEN }}
        context: test/sdk_cli