nightly.yaml

name: Nightly
on:
  schedule:
    # Times in UTC (PST+8), this translates to running Mon-Fri at midnight/1am (depending on DST)
    - cron: 0 8 * * 1-5
  # Only enable on pull requests for testing
  # pull_request:
  #   paths: [.github/workflows/nightly.yaml]
  workflow_dispatch: {}

permissions:
  actions: write
  checks: write
  contents: read
  deployments: read
  id-token: write
  issues: write
  discussions: read
  packages: read
  pages: read
  pull-requests: write
  repository-projects: read
  security-events: read
  statuses: write

jobs:
  # Run tests against all linters for snapshots and latest version as they exist on main
  # This job is used to diagnose plugin config health in advance of a release
  linter_tests_main:
    name: Plugin Tests Main
    # runs-on: [self-hosted, "${{ matrix.os }}"] TODO(Tyler): Set after Windows self-hosted are established.
    runs-on: ${{ matrix.os }}
    timeout-minutes: 120
    strategy:
      fail-fast: false
      matrix:
        linter-version: [Snapshots, Latest]
        os: [ubuntu-latest, macOS, windows-latest]
    steps:
      - name: Checkout
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

      - name: Cache tool downloads
        # ubuntu runner has persistent cache
        if: matrix.os == 'windows-latest'
        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
        with:
          path: /tmp/plugins_testing_download_cache
          # No need to key on trunk version unless we change how we store downloads.
          key: trunk-${{ runner.os }}

      - name: Delete cache
        # For now, avoid deleting cache on pull request changes to nightly. This improves PR experience.
        if: env.TRIGGER != 'pull_request'
        run: |
          if [ -d "/tmp/plugins_testing_download_cache" ]
          then
            tmp_dir=/tmp/${GITHUB_RUN_ID}-${GITHUB_RUN_NUMBER}-${GITHUB_RUN_ATTEMPT}
            mv "/tmp/plugins_testing_download_cache" ${tmp_dir}
            chmod -R u+w ${tmp_dir}
            rm -rf ${tmp_dir}
          fi
        shell: bash

      - name: Linter Tests
        uses: ./.github/actions/linter_tests
        with:
          linter-version: ${{ matrix.linter-version }}
          ref-type: main
          sourcery-token: ${{ secrets.TRUNK_SOURCERY_TOKEN }}
          trunk-staging-token: ${{ secrets.TRUNK_DEBUGGER_TOKEN }}
          trunk-prod-token: ${{ secrets.TRUNK_ORG_PROD_TOKEN }}

  # Run tests against all linters for snapshots and latest version as they exist in latest release
  # This job is used to update the list of validated versions
  linter_tests_release:
    name: Plugin Tests Release
    # runs-on: [self-hosted, "${{ matrix.os }}"] TODO(Tyler): Set after Windows self-hosted are established.
    runs-on: ${{ matrix.os }}
    timeout-minutes: 150
    strategy:
      fail-fast: false
      matrix:
        linter-version: [Snapshots, Latest]
        os: [ubuntu-latest, macOS, windows-latest]
        include:
          # Normalize the filenames as inputs for ease of parsing
          - os: ubuntu-latest
            results-file: ubuntu-latest
          - os: macOS
            results-file: macos-latest
          - os: windows-latest
            results-file: windows-latest
    outputs:
      plugin-version: ${{ steps.get-release.outputs.tag }}

    steps:
      - name: Retrieve git history
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          fetch-depth: 0

      # This assumes that any changes on main since the last release are backwards compatible
      # in terms of allowing existing linter tests to run.
      - name: Preserve test runner latest behavior
        shell: bash
        run: |
          cp -r tests tests.bak
          cp package.json package.json.bak
          cp package-lock.json package-lock.json.bak
          cp -r .github/actions .github/actions.bak
          cp -r .trunk .trunk.bak
          cp tsconfig.json tsconfig.json.bak
          cp jest.config.json jest.config.json.bak
          cp .gitattributes .gitattributes.bak
          # Include any newly generated snapshots that have been marked release-ready
          grep "// trunk-upgrade-validation:RELEASE" -r --include=*.shot -l | xargs -I {} cp {}{,.bak}
        continue-on-error: true

      - name: Get Latest Release
        id: get-release
        uses: WyriHaximus/github-action-get-previous-tag@04e8485ecb6487243907e330d522ff60f02283ce # v1.4.0
        with:
          # only use releases tagged v<semver>
          prefix: v

      - name: Checkout
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          ref: ${{ steps.get-release.outputs.tag }}
          clean: false

      - name: Overwrite test runners with latest behavior
        shell: bash
        run: |
          rm -rf tests
          mv tests.bak tests
          mv package.json.bak package.json
          mv package-lock.json.bak package-lock.json
          rm -rf .github/actions
          mv .github/actions.bak .github/actions
          rm -rf .trunk
          mv .trunk.bak .trunk
          mv tsconfig.json.bak tsconfig.json
          mv jest.config.json.bak jest.config.json
          mv .gitattributes.bak .gitattributes
          # Include any newly generated snapshots that have been marked release-ready, but don't replace if present
          grep "// trunk-upgrade-validation:RELEASE" -r --include=*.shot.bak -l | sed -e 's/.bak//' | xargs -I {} mv {}{.bak,}
        continue-on-error: true

      - name: Cache tool downloads
        # ubuntu, mac runners have persistent cache
        if: matrix.os == 'windows-latest'
        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
        with:
          path: /tmp/plugins_testing_download_cache
          # No need to key on trunk version unless we change how we store downloads.
          key: trunk-${{ runner.os }}

      - name: Delete cache
        # For now, avoid deleting cache on pull request changes to nightly. This improves PR experience.
        run: |
          if [ -d "${TMPDIR:-/tmp}/plugins_testing_download_cache" ]
          then
            tmp_dir=${TMPDIR:-/tmp}/${GITHUB_RUN_ID}-${GITHUB_RUN_NUMBER}-${GITHUB_RUN_ATTEMPT}
            mv "${TMPDIR:-/tmp}/plugins_testing_download_cache" ${tmp_dir}
            chmod -R u+w ${tmp_dir}
            rm -rf ${tmp_dir}
          fi
        shell: bash

      - name: Linter Tests ${{ matrix.os }}
        # Use overwritten dependency action, rather than released version
        uses: ./.github/actions/linter_tests
        with:
          linter-version: ${{ matrix.linter-version }}
          append-args: linters -- --json --outputFile=${{ matrix.results-file }}-res.json
          ref-type: release
          sourcery-token: ${{ secrets.TRUNK_SOURCERY_TOKEN }}
          trunk-staging-token: ${{ secrets.TRUNK_DEBUGGER_TOKEN }}
          trunk-prod-token: ${{ secrets.TRUNK_ORG_PROD_TOKEN }}

      - name: Upload Test Outputs for Upload Job
        # Only upload results from latest. Always run, except when cancelled.
        if: (failure() || success()) && matrix.linter-version == 'Latest'
        uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4
        with:
          name: ${{ matrix.results-file }}-test-results
          path: ${{ matrix.results-file }}-res.json

  upload_linter_tests:
    name: Upload Linter Test Results
    needs: linter_tests_release
    # Still run on test failure
    if: always()

    uses: ./.github/workflows/upload_results.reusable.yaml
    secrets:
      TRUNKBOT_SLACK_BOT_TOKEN: ${{ secrets.TRUNKBOT_SLACK_BOT_TOKEN }}
      TRUNK_STAGING_API_TOKEN: ${{ secrets.TRUNK_STAGING_API_TOKEN }}
      TRUNK_API_TOKEN: ${{ secrets.TRUNK_API_TOKEN }}
      TRUNK_OPEN_PR_APP_ID: ${{ secrets.TRUNK_OPEN_PR_APP_ID }}
      TRUNK_OPEN_PR_APP_PRIVATE_KEY: ${{ secrets.TRUNK_OPEN_PR_APP_PRIVATE_KEY }}
      TRUNK_SOURCERY_TOKEN: ${{ secrets.TRUNK_SOURCERY_TOKEN }}
      TRUNK_DEBUGGER_TOKEN: ${{ secrets.TRUNK_DEBUGGER_TOKEN }}
      TRUNK_ORG_PROD_TOKEN: ${{ secrets.TRUNK_ORG_PROD_TOKEN }}
    with:
      plugin-version: ${{ needs.linter_tests_release.outputs.plugin-version }}
      upload-validated-versions: true
      test-type: linter
      test-ref: latest release

  # Run tool tests only on main
  tool_tests_main:
    name: Tool Tests Main
    # runs-on: [self-hosted, "${{ matrix.os }}"] TODO(Tyler): Set after Windows self-hosted are established.
    runs-on: ${{ matrix.os }}
    timeout-minutes: 120
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macOS, windows-latest]
        include:
          # Normalize the filenames as inputs for ease of parsing
          - os: ubuntu-latest
            results-file: ubuntu-latest
          - os: macOS
            results-file: macos-latest
          - os: windows-latest
            results-file: windows-latest
    steps:
      - name: Checkout
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

      # TODO(lauri): For now this just runs on the hardcoded versions. We should configure this
      # akin to the linter_tests job.
      - name: Tool Tests ${{ matrix.os }}
        uses: ./.github/actions/tool_tests
        with:
          append-args: tools -- --json --outputFile=${{ matrix.results-file }}-res.json
          trunk-staging-token: ${{ secrets.TRUNK_DEBUGGER_TOKEN }}
          trunk-prod-token: ${{ secrets.TRUNK_ORG_PROD_TOKEN }}

      - name: Upload Test Outputs for Notification Job
        # Always run, except when cancelled.
        if: (failure() || success())
        uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4
        with:
          name: tools-${{ matrix.results-file }}-test-results
          path: ${{ matrix.results-file }}-res.json

  upload_tool_tests:
    name: Upload Tool Test Results
    needs: tool_tests_main
    # Still run on test failure
    if: always()
    uses: ./.github/workflows/upload_results.reusable.yaml
    secrets:
      TRUNKBOT_SLACK_BOT_TOKEN: ${{ secrets.TRUNKBOT_SLACK_BOT_TOKEN }}
      TRUNK_SOURCERY_TOKEN: ${{ secrets.TRUNK_SOURCERY_TOKEN }}
      TRUNK_DEBUGGER_TOKEN: ${{ secrets.TRUNK_DEBUGGER_TOKEN }}
      TRUNK_ORG_PROD_TOKEN: ${{ secrets.TRUNK_ORG_PROD_TOKEN }}
    with:
      plugin-version: main
      results-prefix: tools-
      upload-validated-versions: false
      test-type: tool
      test-ref: main

  # Run repo healthcheck tests
  repo_tests:
    name: Repo Tests
    uses: ./.github/workflows/repo_tests.reusable.yaml

  action_tests_main:
    name: Action Tests Main
    runs-on: [ubuntu-latest]
    timeout-minutes: 30
    steps:
      - name: Checkout
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

      - name: Action Tests
        uses: ./.github/actions/action_tests
        with:
          trunk-staging-token: ${{ secrets.TRUNK_DEBUGGER_TOKEN }}
          trunk-prod-token: ${{ secrets.TRUNK_ORG_PROD_TOKEN }}