Auto generate snapshots from nightly (#596)

Attach additional information about the type of testing failure that
occurred (i.e. `passed`, `unknown`, `task_failure`, or
`assertion_failure`). If the latter occurred on all platforms (other
than skipped), we can likely conclude that either:
1. The linter released a new version with a differing output
2. The linter's output relies on a live service, and all of its
snapshots will need to be changed

The second case is out of scope for now, so for case 1 we will
proactively generate a snapshot, mark for release, and create a PR. If
this PR passes tests and a brief sanity check, then we will be back to
business as usual.

Notes:
- Because it is marked for release, in most cases we won't have to worry
about this being spammy for other edge cases. If issues do occur though,
we can always add an ignore filter to the jest rerun invocation.
- The failure notification will include _`(rerunning)`_ if it triggers a
rerun test.
- If something goes wrong or a snapshot cannot be generated on the
rerun, the job will fail, and it can be triaged through other nightly
email notification triage.
- If we ever add any additional complexity to the `parse/index.ts`
script, we should probably write a test for it.

Successful test
[run](https://github.com/trunk-io/plugins/actions/runs/7380972339/job/20079163879)
creating [PR](https://github.com/trunk-io/plugins/pull/604/files)

.github/workflows/nightly.yaml

@@ -201,6 +201,10 @@ jobs:
       TRUNKBOT_SLACK_BOT_TOKEN: ${{ secrets.TRUNKBOT_SLACK_BOT_TOKEN }}
       TRUNK_STAGING_API_TOKEN: ${{ secrets.TRUNK_STAGING_API_TOKEN }}
       TRUNK_API_TOKEN: ${{ secrets.TRUNK_API_TOKEN }}
+      TRUNK_OPEN_PR_APP_ID: ${{ secrets.TRUNK_OPEN_PR_APP_ID }}
+      TRUNK_OPEN_PR_APP_PRIVATE_KEY: ${{ secrets.TRUNK_OPEN_PR_APP_PRIVATE_KEY }}
+      TRUNK_SOURCERY_TOKEN: ${{ secrets.TRUNK_SOURCERY_TOKEN }}
+      TRUNK_DEBUGGER_TOKEN: ${{ secrets.TRUNK_DEBUGGER_TOKEN }}
     with:
       plugin-version: ${{ needs.linter_tests_release.outputs.plugin-version }}
       upload-validated-versions: true
@@ -253,6 +257,8 @@ jobs:
     uses: ./.github/workflows/upload_results.reusable.yaml
     secrets:
       TRUNKBOT_SLACK_BOT_TOKEN: ${{ secrets.TRUNKBOT_SLACK_BOT_TOKEN }}
+      TRUNK_SOURCERY_TOKEN: ${{ secrets.TRUNK_SOURCERY_TOKEN }}
+      TRUNK_DEBUGGER_TOKEN: ${{ secrets.TRUNK_DEBUGGER_TOKEN }}
     with:
       plugin-version: main
       results-prefix: tools-

.github/workflows/upload_results.reusable.yaml

@@ -1,5 +1,6 @@
 # Parse test results and
-# Send upload notifications
+# Send upload notifications and
+# Generate new snapshots via PR
 name: Parse Test Results
 on:
   workflow_call:
@@ -29,6 +30,15 @@ on:
         required: false
       TRUNK_API_TOKEN:
         required: false
+      TRUNK_OPEN_PR_APP_ID:
+        required: false
+      TRUNK_OPEN_PR_APP_PRIVATE_KEY:
+        required: false
+      TRUNK_SOURCERY_TOKEN:
+        required: false
+      TRUNK_DEBUGGER_TOKEN:
+        required: false
+
 permissions: read-all
 
 jobs:
@@ -38,6 +48,8 @@ jobs:
     timeout-minutes: 10
     env:
       SLACK_CHANNEL_ID: plugins-notifications
+    outputs:
+      reruns: ${{ steps.parse.outputs.reruns }}
     steps:
       - name: Checkout
         uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
@@ -123,6 +135,7 @@ jobs:
           npm run parse
           echo "failures=$([[ -f failures.json ]] && echo "true" || echo "false")" >> "$GITHUB_OUTPUT"
           echo "failures-payload=$(cat failures.json)" >> "$GITHUB_OUTPUT"
+          echo "reruns=$(cat reruns.txt)" >> "$GITHUB_OUTPUT"
         env:
           PLUGIN_VERSION: ${{ inputs.plugin-version }}
           # Used to format Slack notification for failures
@@ -211,3 +224,63 @@ jobs:
             }
         env:
           SLACK_BOT_TOKEN: ${{ secrets.TRUNKBOT_SLACK_BOT_TOKEN }}
+  generate_snapshots_pr:
+    name: Generate Snapshots PR
+    runs-on: ubuntu-x64
+    timeout-minutes: 30
+    needs: upload_test_results
+    if: needs.upload_test_results.outputs.reruns != ''
+    steps:
+      - name: Checkout
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+
+      - name: Setup Node
+        uses: actions/setup-node@1a4442cacd436585916779262731d5b162bc6ec7 # v3.8.2
+        with:
+          node-version: 18
+
+      - name: Install Dependencies
+        shell: bash
+        run: npm ci
+
+      - name: Add npm bin to path
+        shell: bash
+        run: echo "$PWD/node_modules/.bin" >> "$GITHUB_PATH"
+
+      - name: Linter Tests Rerun Linux
+        uses: ./.github/actions/linter_tests
+        with:
+          linter-version: Latest
+          append-args: ${{ needs.upload_test_results.outputs.reruns }} -- -u
+          sourcery-token: ${{ secrets.TRUNK_SOURCERY_TOKEN }}
+          trunk-token: ${{ secrets.TRUNK_DEBUGGER_TOKEN }}
+        env:
+          PLUGINS_TEST_UPDATE_SNAPSHOTS: "true"
+
+      - name: Mark generated snapshots as ready for release
+        shell: bash
+        run: |
+          git ls-files --others --exclude-standard | grep ".shot" | xargs sed -i '2i // trunk-upgrade-validation:RELEASE'
+
+      - name: Create App Token for TrunkBuild App (Internal)
+        uses: tibdex/github-app-token@v1
+        id: generate-token
+        with:
+          app_id: ${{ secrets.TRUNK_OPEN_PR_APP_ID }}
+          private_key: ${{ secrets.TRUNK_OPEN_PR_APP_PRIVATE_KEY }}
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v5
+        with:
+          title: Auto-add missing snapshots
+          body:
+            Create new snapshots from https://github.com/trunk-io/plugins/actions/runs/${{
+            github.run_id }}
+          base: main
+          branch: trunk-io/auto-update-snapshots
+          labels: trunk
+          add-paths: "**/*.shot"
+          commit-message: Auto add snapshots
+          delete-branch: true
+          token: ${{ steps.generate-token.outputs.token }}
+          reviewers: TylerJang27

tests/index.ts

@@ -2,7 +2,7 @@ import caller from "caller";
 import * as fs from "fs";
 import * as path from "path";
 import { SetupSettings, TestTarget, TrunkLintDriver, TrunkToolDriver } from "tests/driver";
-import { FileIssue, LandingState } from "tests/types";
+import { FailureMode, FileIssue, LandingState } from "tests/types";
 
 import specific_snapshot = require("jest-specific-snapshot");
 import Debug from "debug";
@@ -61,6 +61,21 @@ const registerVersion = (testType: string, linterVersion?: string) => {
   );
 };
 
+/**
+ * Adds the predictive failure mode to the test reporter. This is used to proactively generate snapshots
+ * For linters and add metadata about the type of failure to notifications.
+ * @param failureMode The type of suspected failure mode based on landing state properties.
+ */
+const registerFailureMode = (failureMode: FailureMode) => {
+  // @ts-expect-error: `_buffer` is `private`, see `tests/reporter/reporters.ts` for rationale
+  // trunk-ignore(eslint): Manual patch is required here for most reliable implementation
+  console._buffer?.push({
+    message: failureMode,
+    origin: expect.getState().currentTestName,
+    type: "suspected-failure-mode",
+  });
+};
+
 const baseDebug = Debug("Tests");
 
 const CUSTOM_SNAPSHOT_PREFIX = "CUSTOM";
@@ -306,8 +321,6 @@ export const toolTest = ({
   });
 };
 
-// TODO(Tyler): Add additional assertion options to the custom checks, including checking failures, etc.
-// TODO(Tyler): Add additional options to the custom checks, including OS and CI-specific runs.
 /**
  * Test that running a linter filtered by `linterName` with any custom `args` produces the desired output
  * json. Optionally specify additional file paths to snapshot.
@@ -363,6 +376,9 @@ export const customLinterCheckTest = ({
           const debug = baseDebug.extend(driver.debugNamespace);
 
           const testRunResult = await driver.runCheck({ args, linter: linterName });
+          if (!testRunResult.success || testRunResult.landingState?.taskFailures?.length) {
+            registerFailureMode("task_failure");
+          }
           expect(testRunResult).toMatchObject({
             success: true,
           });
@@ -390,6 +406,8 @@ export const customLinterCheckTest = ({
             driver.enabledVersion ?? "no version",
             primarySnapshotPath,
           );
+
+          registerFailureMode("assertion_failure");
           expect(testRunResult.landingState).toMatchSpecificSnapshot(
             primarySnapshotPath,
             landingStateWrapper(testRunResult.landingState, primarySnapshotPath),
@@ -478,6 +496,9 @@ export const customLinterFmtTest = ({
           const debug = baseDebug.extend(driver.debugNamespace);
 
           const testRunResult = await driver.runFmt({ args, linter: linterName });
+          if (!testRunResult.success || testRunResult.landingState?.taskFailures?.length) {
+            registerFailureMode("task_failure");
+          }
           expect(testRunResult).toMatchObject({
             success: true,
             landingState: {
@@ -487,6 +508,7 @@ export const customLinterFmtTest = ({
 
           // Step 4: Verify that any specified files match their expected snapshots for that linter version.
           const snapshotDir = path.resolve(dirname, TEST_DATA);
+          registerFailureMode("assertion_failure");
           pathsToSnapshot.forEach((pathToSnapshot) => {
             const normalizedName = `${testName}.${pathToSnapshot
               .replaceAll("/", ".")
@@ -576,6 +598,9 @@ export const fuzzyLinterCheckTest = ({
           const debug = baseDebug.extend(driver.debugNamespace);
 
           const testRunResult = await driver.runCheck({ args, linter: linterName });
+          if (!testRunResult.success || testRunResult.landingState?.taskFailures?.length) {
+            registerFailureMode("task_failure");
+          }
           expect(testRunResult).toMatchObject({
             success: true,
           });
@@ -605,6 +630,7 @@ export const fuzzyLinterCheckTest = ({
             driver.enabledVersion ?? "no version",
             primarySnapshotPath,
           );
+          registerFailureMode("assertion_failure");
           expect(strippedLandingState).toMatchSpecificSnapshot(
             primarySnapshotPath,
             landingStateWrapper(strippedLandingState, primarySnapshotPath),
@@ -670,6 +696,9 @@ export const linterCheckTest = ({
           conditionalTest(skipTestIf(linterVersion), prefix, async () => {
             const debug = baseDebug.extend(driver.debugNamespace);
             const testRunResult = await driver.runCheckUnit(inputPath, linterName);
+            if (!testRunResult.success || testRunResult.landingState?.taskFailures?.length) {
+              registerFailureMode("task_failure");
+            }
             expect(testRunResult).toMatchObject({
               success: true,
             });
@@ -698,6 +727,7 @@ export const linterCheckTest = ({
               driver.enabledVersion ?? "no version",
               snapshotPath,
             );
+            registerFailureMode("assertion_failure");
             expect(testRunResult.landingState).toMatchSpecificSnapshot(
               snapshotPath,
               landingStateWrapper(testRunResult.landingState, snapshotPath),
@@ -758,6 +788,9 @@ export const linterFmtTest = ({
             const debug = baseDebug.extend(driver.debugNamespace);
             const testRunResult = await driver.runFmtUnit(inputPath, linterName);
 
+            if (!testRunResult.success || testRunResult.landingState?.taskFailures?.length) {
+              registerFailureMode("task_failure");
+            }
             expect(testRunResult).toMatchObject({
               success: true,
               landingState: {
@@ -782,6 +815,7 @@ export const linterFmtTest = ({
               driver.enabledVersion ?? "no version",
               snapshotPath,
             );
+            registerFailureMode("assertion_failure");
             expect(fs.readFileSync(testRunResult.targetPath!, "utf-8")).toMatchSpecificSnapshot(
               snapshotPath,
             );