Run CLI tests #846
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Run CLI tests | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '0 0 * * *' | |
| pull_request: | |
| types: [opened, synchronize, reopened, ready_for_review] | |
| paths: | |
| - .github/workflows/cli-test.yml | |
| - extensions/ql-vscode/src/codeql-cli/** | |
| - extensions/ql-vscode/src/language-support/** | |
| - extensions/ql-vscode/src/query-server/** | |
| - extensions/ql-vscode/supported_cli_versions.json | |
| - extensions/ql-vscode/src/variant-analysis/run-remote-query.ts | |
| jobs: | |
| find-nightly: | |
| name: Find Nightly Release | |
| runs-on: ubuntu-latest | |
| outputs: | |
| url: ${{ steps.get-url.outputs.nightly-url }} | |
| steps: | |
| - name: Get Nightly Release URL | |
| id: get-url | |
| env: | |
| GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}' | |
| shell: bash | |
| # This workflow step gets an unstable testing version of the CodeQL CLI. It should not be used outside of these tests. | |
| run: | | |
| LATEST=`gh api repos/dsp-testing/codeql-cli-nightlies/releases --jq '.[].tag_name' --method GET --raw-field 'per_page=1'` | |
| echo "nightly-url=https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/$LATEST" >> "$GITHUB_OUTPUT" | |
| set-matrix: | |
| name: Set Matrix for cli-test | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set the variables | |
| id: set-variables | |
| run: echo "cli-versions=$(cat ./extensions/ql-vscode/supported_cli_versions.json | jq -rc)" >> $GITHUB_OUTPUT | |
| outputs: | |
| cli-versions: ${{ steps.set-variables.outputs.cli-versions }} | |
| cli-test: | |
| name: CLI Test | |
| runs-on: ${{ matrix.os }} | |
| needs: [find-nightly, set-matrix] | |
| timeout-minutes: 30 | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest, windows-latest] | |
| version: ${{ fromJson(needs.set-matrix.outputs.cli-versions) }} | |
| fail-fast: false | |
| env: | |
| CLI_VERSION: ${{ matrix.version }} | |
| NIGHTLY_URL: ${{ needs.find-nightly.outputs.url }} | |
| TEST_CODEQL_PATH: '${{ github.workspace }}/codeql' | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version-file: extensions/ql-vscode/.nvmrc | |
| cache: 'npm' | |
| cache-dependency-path: extensions/ql-vscode/package-lock.json | |
| - name: Install dependencies | |
| working-directory: extensions/ql-vscode | |
| run: | | |
| npm ci | |
| shell: bash | |
| - name: Build | |
| working-directory: extensions/ql-vscode | |
| run: | | |
| npm run build | |
| shell: bash | |
| - name: Decide on ref of CodeQL repo | |
| id: choose-ref | |
| shell: bash | |
| run: | | |
| if [[ "${{ matrix.version }}" == "nightly" ]] | |
| then | |
| REF="codeql-cli/latest" | |
| else | |
| REF="codeql-cli/${{ matrix.version }}" | |
| fi | |
| echo "ref=$REF" >> "$GITHUB_OUTPUT" | |
| - name: Checkout QL | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: github/codeql | |
| ref: ${{ steps.choose-ref.outputs.ref }} | |
| path: codeql | |
| - name: Run CLI tests (Linux) | |
| working-directory: extensions/ql-vscode | |
| if: matrix.os == 'ubuntu-latest' | |
| run: | | |
| unset DBUS_SESSION_BUS_ADDRESS | |
| /usr/bin/xvfb-run npm run test:cli-integration | |
| - name: Run CLI tests (Windows) | |
| working-directory: extensions/ql-vscode | |
| if: matrix.os == 'windows-latest' | |
| run: | | |
| npm run test:cli-integration | |
| report-failure: | |
| name: Report failure on the default branch | |
| runs-on: ubuntu-latest | |
| needs: [cli-test] | |
| if: failure() && github.ref == 'refs/heads/main' | |
| permissions: | |
| contents: read | |
| issues: write | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| steps: | |
| - name: Create GitHub issue | |
| run: | | |
| # Set -eu so that we fail if the gh command fails. | |
| set -eu | |
| # Try to find an existing open issue if there is one | |
| ISSUE="$(gh issue list --repo "$GITHUB_REPOSITORY" --label "cli-test-failure" --state "open" --limit 1 --json number -q '.[0].number')" | |
| if [[ -n "$ISSUE" ]]; then | |
| echo "Found open issue number $ISSUE ($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/issues/$ISSUE)" | |
| else | |
| echo "Did not find an open tracking issue. Creating one." | |
| ISSUE_BODY="issue-body.md" | |
| printf "CLI tests have failed on the default branch.\n\n@github/code-scanning-secexp-reviewers" > "$ISSUE_BODY" | |
| ISSUE="$(gh issue create --repo "$GITHUB_REPOSITORY" --label "cli-test-failure" --title "CLI test failure" --body-file "$ISSUE_BODY")" | |
| # `gh issue create` returns the full issue URL, not just the number. | |
| echo "Created issue with URL $ISSUE" | |
| fi | |
| COMMENT_FILE="comment.md" | |
| RUN_URL=$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID | |
| printf 'CLI test [%s](%s) failed on ref `%s`' "$GITHUB_RUN_ID" "$RUN_URL" "$GITHUB_REF" > "$COMMENT_FILE" | |
| # `gh issue create` returns an issue URL, and `gh issue list | cut -f 1` returns an issue number. | |
| # Both are accepted here. | |
| gh issue comment "$ISSUE" --repo "$GITHUB_REPOSITORY" --body-file "$COMMENT_FILE" |