fix(changelog): harden release evidence handling

Author: nibzard

scripts/changelog/README.md

@@ -2,19 +2,25 @@
 
 ## Replay a historical week locally
 
-Set `OPENAI_API_KEY` and a GitHub token, then run:
+Set a GitHub token and, when the window contains eligible facts, `OPENAI_API_KEY`, then run:
 
 ```bash
 GITHUB_TOKEN="$(gh auth token)" bun run preview-changelog \
   --number 35 \
   --since 2026-07-17T15:58:38.000Z \
-  --until 2026-07-24T13:35:35.621Z
+  --until 2026-07-24T13:35:35.621Z \
+  --application-release-base-sha d2f91834047984bf37e09caba27231fb07a4109a \
+  --application-release-head-sha 4cc44305bae19df7135edda12a602841ad90ffe7
 ```
 
-Preview mode resolves the application release branch at both historical cutoffs, uses the same
-collection, filtering, and model paths as automation, and writes the draft and its review evidence
-to a new temporary directory. It does not update changelog content, metadata, generator state, or
-GitHub Actions output.
+Preview mode requires the exact application release commits at both historical cutoffs because
+commit timestamps cannot reconstruct when the release branch moved. It uses the same collection,
+filtering, and model paths as automation and writes the draft and its review evidence to a new
+temporary directory. It does not update changelog content, metadata, generator state, or GitHub
+Actions output.
+
+For later replays, use `applicationReleaseSha` from the generator state at the previous and target
+published changelog revisions.
 
 ## Auditing filtered evidence
 

scripts/changelog/source.ts

@@ -315,6 +315,12 @@ function allFilesMatch(group: ChangeGroup, predicate: (filename: string) => bool
   return group.changedFiles.every((file) => predicate(file.filename));
 }
 
+function fileWasAdded(group: ChangeGroup, filename: string): boolean {
+  return group.commits.some((commit) =>
+    commit.changedFiles.some((file) => file.filename === filename && file.status === 'added'),
+  );
+}
+
 function isCommonlyIgnoredFile(filename: string): boolean {
   return COMMON_IGNORED_FILE_PATTERNS.some((pattern) => pattern.test(filename));
 }
@@ -422,7 +428,7 @@ function classifyGroup(group: ChangeGroup): EligibilityReason {
     const materialFiles = readerFacingFiles.filter(
       (file) => !/(^|\/)meta\.json$/.test(file.filename),
     );
-    if (materialFiles.some((file) => file.status === 'added')) {
+    if (materialFiles.some((file) => fileWasAdded(group, file.filename))) {
       return 'eligible';
     }
 
@@ -442,7 +448,7 @@ function classifyGroup(group: ChangeGroup): EligibilityReason {
       /^(examples|recipes)\//.test(file.filename),
     );
     const addsRecipe =
-      recipeFiles.some((file) => file.status === 'added') ||
+      recipeFiles.some((file) => fileWasAdded(group, file.filename)) ||
       (/\b(add|added|new|introduc(e|ed|es|ing))\b/.test(text) &&
         group.changedFiles.some((file) => file.filename === 'registry.yaml'));
 
@@ -456,7 +462,8 @@ function classifyGroup(group: ChangeGroup): EligibilityReason {
       ) ||
       group.changedFiles.some(
         (file) =>
-          file.status === 'added' && /(^|\/)benchmarks?\//.test(file.filename.toLowerCase()),
+          fileWasAdded(group, file.filename) &&
+          /(^|\/)benchmarks?\//.test(file.filename.toLowerCase()),
       );
 
     if (addsBenchmark) {

scripts/generate-changelog-draft.ts

@@ -133,6 +133,8 @@ export interface CliOptions {
   number?: number;
   since?: string;
   until?: string;
+  applicationReleaseBaseSha?: string;
+  applicationReleaseHeadSha?: string;
 }
 
 interface ChangelogState {
@@ -194,15 +196,44 @@ export function parseArgs(argv: string[]): CliOptions {
       continue;
     }
 
+    if (value === '--application-release-base-sha' || value === '--application-release-head-sha') {
+      const sha = requireOptionValue(argv, index, value);
+      if (!/^[0-9a-f]{40}$/i.test(sha)) {
+        throw new Error(`Invalid SHA for ${value}: ${sha}`);
+      }
+
+      if (value === '--application-release-base-sha') {
+        options.applicationReleaseBaseSha = sha;
+      } else {
+        options.applicationReleaseHeadSha = sha;
+      }
+      index += 1;
+      continue;
+    }
+
     throw new Error(`Unknown argument: ${value}`);
   }
 
   if (options.preview) {
-    if (options.number === undefined || !options.since || !options.until) {
-      throw new Error('Preview mode requires --number, --since, and --until.');
+    if (
+      options.number === undefined ||
+      !options.since ||
+      !options.until ||
+      !options.applicationReleaseBaseSha ||
+      !options.applicationReleaseHeadSha
+    ) {
+      throw new Error(
+        'Preview mode requires --number, --since, --until, --application-release-base-sha, and --application-release-head-sha.',
+      );
     }
-  } else if (options.number !== undefined) {
-    throw new Error('--number can only be used with --preview.');
+  } else if (
+    options.number !== undefined ||
+    options.applicationReleaseBaseSha ||
+    options.applicationReleaseHeadSha
+  ) {
+    throw new Error(
+      '--number, --application-release-base-sha, and --application-release-head-sha can only be used with --preview.',
+    );
   }
 
   return options;
@@ -554,10 +585,18 @@ async function fetchSubmoduleShaAtRef(
 }
 
 class NonAheadCompareError extends Error {
-  constructor(repoConfig: ChangelogRepository, baseSha: string, headSha: string, status: string) {
+  readonly status: GitHubCompareResponse['status'];
+
+  constructor(
+    repoConfig: ChangelogRepository,
+    baseSha: string,
+    headSha: string,
+    status: GitHubCompareResponse['status'],
+  ) {
     super(
       `Expected an ahead range for ${repoConfig.owner}/${repoConfig.repo}, received ${status} for ${baseSha}...${headSha}.`,
     );
+    this.status = status;
   }
 }
 
@@ -570,7 +609,7 @@ async function fetchComparedCommits(
   const commits: GitHubListCommit[] = [];
   let totalCommits: number | null = null;
 
-  for (let page = 1; page <= 3; page += 1) {
+  for (let page = 1; ; page += 1) {
     const url = new URL(
       `https://api.github.com/repos/${repoConfig.owner}/${repoConfig.repo}/compare/${baseSha}...${headSha}`,
     );
@@ -665,9 +704,9 @@ export async function expandReleasedBrowserCommits(
           token,
         );
       } catch (error) {
-        // A rolled-back or rewritten submodule pointer must not block the whole draft; the
-        // application release comparison still fails loudly on the same condition.
-        if (error instanceof NonAheadCompareError) {
+        // A pure rollback contains no newly released browser commits. Diverged ranges may contain
+        // head-only commits, so they must fail instead of silently dropping release evidence.
+        if (error instanceof NonAheadCompareError && error.status === 'behind') {
           console.warn(
             `Skipping submodule expansion for ${hostCommit.owner}/${hostCommit.repo}@${hostCommit.shortSha}: ${error.message}`,
           );
@@ -1330,8 +1369,8 @@ async function writePreviewDraftArtifacts(input: {
   ]);
 }
 
-async function main() {
-  const options = parseArgs(process.argv.slice(2));
+export async function main(argv = process.argv.slice(2)) {
+  const options = parseArgs(argv);
   const githubToken = getGithubToken();
   let nextNumber: number;
   let windowSelection: WindowSelection;
@@ -1344,18 +1383,8 @@ async function main() {
       since: options.since as string,
       until: options.until as string,
     });
-    [applicationReleaseBaseSha, applicationReleaseSha] = await Promise.all([
-      fetchBranchHeadAtOrBefore(
-        CHANGELOG_APPLICATION_REPOSITORY,
-        githubToken,
-        windowSelection.since,
-      ),
-      fetchBranchHeadAtOrBefore(
-        CHANGELOG_APPLICATION_REPOSITORY,
-        githubToken,
-        windowSelection.until,
-      ),
-    ]);
+    applicationReleaseBaseSha = options.applicationReleaseBaseSha as string;
+    applicationReleaseSha = options.applicationReleaseHeadSha as string;
   } else {
     const state = await readChangelogState();
     windowSelection = await selectWindow(options, state);
@@ -1444,16 +1473,6 @@ async function main() {
     );
   }
 
-  if (eligible.length === 0) {
-    console.log(
-      'The source window contains no public changelog facts. Treating it as a quiet week.',
-    );
-    if (!options.preview) {
-      await appendGithubOutput('has_changes', 'false');
-    }
-    return;
-  }
-
   const previewWorkspace = options.preview
     ? await createPreviewWorkspace({
         number: nextNumber,
@@ -1468,6 +1487,16 @@ async function main() {
     console.log(`Prepared isolated preview evidence at ${previewWorkspace.directory}`);
   }
 
+  if (eligible.length === 0) {
+    console.log(
+      'The source window contains no public changelog facts. Treating it as a quiet week.',
+    );
+    if (!options.preview) {
+      await appendGithubOutput('has_changes', 'false');
+    }
+    return;
+  }
+
   const openAiToken = getOpenAiToken();
   const draft = await requestDraftFromModel(eligible, nextNumber, windowSelection, openAiToken);
   if (!draft.sections.some((section) => section.entries.length > 0)) {