test(fuzz): differential fuzzing against upstream Baileys #43
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,144 @@ | ||
| name: Nightly Fuzz | ||
|
|
||
| # Two modes, on purpose. | ||
| # | ||
| # Every pull request already runs the fuzz suite through `npm test`, with a fixed | ||
| # seed and small per-target budgets. That run is deterministic: it cannot fail | ||
| # because of an unlucky draw, which is the only way a fuzz suite survives contact | ||
| # with a CI system people have to trust. | ||
| # | ||
| # This job is where the searching happens. It varies the seed per run, raises the | ||
| # budgets, and enforces the parts of the known-divergence registry that would be | ||
| # hostile on a pull request — entries past their review date, and entries that no | ||
| # longer excuse anything. | ||
|
|
||
| on: | ||
| schedule: | ||
| # 03:17 UTC, off the hour so it does not queue behind everything else. | ||
| - cron: '17 3 * * *' | ||
| workflow_dispatch: | ||
| inputs: | ||
| seed: | ||
| description: 'Fuzz seed (defaults to the run id)' | ||
| required: false | ||
| type: string | ||
| mode: | ||
| description: 'smoke or deep' | ||
| required: false | ||
| default: 'deep' | ||
| type: string | ||
|
|
||
| permissions: | ||
| contents: read | ||
| issues: write | ||
|
|
||
| jobs: | ||
| fuzz: | ||
| runs-on: ubuntu-latest | ||
| timeout-minutes: 45 | ||
|
|
||
| steps: | ||
| - name: Checkout code | ||
| uses: actions/checkout@v4 | ||
|
|
||
| - name: Setup Node.js | ||
| uses: actions/setup-node@v4 | ||
| with: | ||
| node-version: '24' | ||
| cache: 'npm' | ||
|
|
||
| - name: Install dependencies | ||
| run: npm ci | ||
|
|
||
| - name: Run the fuzz suite | ||
| id: fuzz | ||
| env: | ||
| # A fresh seed per run is the point: the fixed-seed smoke run on every | ||
| # PR has already searched its own corner exhaustively. | ||
| FUZZ_SEED: ${{ inputs.seed || github.run_id }} | ||
| FUZZ_MODE: ${{ inputs.mode || 'deep' }} | ||
| FUZZ_TIME_BUDGET_MS: '180000' | ||
| FUZZ_REPORT_DIR: fuzz-reports | ||
| FUZZ_STRICT_ALLOWLIST: '1' | ||
| # --expose-gc turns on the WASM handle-leak probe, which skips without it. | ||
| run: node --expose-gc --test "./src/__fuzz__/**/*.test.ts" | ||
| continue-on-error: true | ||
|
jlucaso1 marked this conversation as resolved.
cubic-dev-ai[bot] marked this conversation as resolved.
|
||
|
|
||
| - name: Summarise | ||
| id: report | ||
| if: always() | ||
| run: | | ||
| { | ||
| node scripts/fuzz/report.ts fuzz-reports --markdown --fail-on-stale | ||
| } > fuzz-summary.md 2>&1 && echo "clean=true" >> "$GITHUB_OUTPUT" || echo "clean=false" >> "$GITHUB_OUTPUT" | ||
| cat fuzz-summary.md >> "$GITHUB_STEP_SUMMARY" | ||
|
coderabbitai[bot] marked this conversation as resolved.
|
||
|
|
||
| - name: Upload reports | ||
| if: always() | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: fuzz-reports-${{ github.run_id }} | ||
| path: | | ||
| fuzz-reports/ | ||
| fuzz-summary.md | ||
| retention-days: 30 | ||
|
|
||
| - name: Open an issue for the findings | ||
| if: always() && steps.report.outputs.clean == 'false' | ||
|
There was a problem hiding this comment.
When a generated check or generator crashes without producing a Useful? React with 👍 / 👎. |
||
| uses: actions/github-script@v7 | ||
| with: | ||
| script: | | ||
| const fs = require('node:fs') | ||
| const summary = fs.readFileSync('fuzz-summary.md', 'utf8') | ||
| const seed = process.env.FUZZ_SEED | ||
| const title = `Nightly fuzz: findings on seed ${seed}` | ||
|
|
||
| // One open issue per topic, updated rather than duplicated: a fuzzer | ||
| // that opens a fresh issue every night trains people to close them | ||
| // unread. | ||
| const existing = await github.rest.issues.listForRepo({ | ||
| owner: context.repo.owner, | ||
| repo: context.repo.repo, | ||
| state: 'open', | ||
| labels: 'fuzz' | ||
| }) | ||
|
There was a problem hiding this comment.
When the repository has any unrelated open issue—or even a pull request—carrying the generic Useful? React with 👍 / 👎. |
||
|
|
||
| const body = [ | ||
| summary, | ||
| '', | ||
| `Run: ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`, | ||
| `Reproduce locally: \`FUZZ_SEED=${seed} FUZZ_MODE=deep npm run fuzz\``, | ||
|
There was a problem hiding this comment.
When the nightly fails in the GC-only leak probe, this command runs Useful? React with 👍 / 👎. |
||
| '', | ||
| '---', | ||
| '_Generated by [Claude Code](https://claude.ai/code)_' | ||
| ].join('\n') | ||
|
|
||
| if (existing.data.length > 0) { | ||
| await github.rest.issues.createComment({ | ||
| owner: context.repo.owner, | ||
| repo: context.repo.repo, | ||
| issue_number: existing.data[0].number, | ||
| body | ||
| }) | ||
| } else { | ||
| await github.rest.issues.create({ | ||
| owner: context.repo.owner, | ||
| repo: context.repo.repo, | ||
| title, | ||
| body, | ||
| labels: ['fuzz'] | ||
| }) | ||
| } | ||
| env: | ||
| FUZZ_SEED: ${{ inputs.seed || github.run_id }} | ||
|
|
||
| # The fuzz and summarise steps deliberately do not fail on the spot, so the | ||
| # artifacts get uploaded and the issue gets filed first. Without a final gate | ||
| # the job would then finish green holding findings, which is the one outcome | ||
| # that would make the whole nightly pointless. | ||
| - name: Fail the job when the run was not clean | ||
| if: always() && (steps.fuzz.outcome != 'success' || steps.report.outputs.clean != 'true') | ||
| run: | | ||
| echo "fuzz outcome: ${{ steps.fuzz.outcome }}" | ||
| echo "report clean: ${{ steps.report.outputs.clean }}" | ||
| exit 1 | ||
Uh oh!
There was an error while loading. Please reload this page.