From 573764c77666b37f720a0101026c709fea615ebf Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Sat, 29 Aug 2026 20:45:22 -0400 Subject: [PATCH 1/2] fix(cli): persist what the server answered for a captured page MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A page that renders is not a page that succeeded. An error page has a title, a palette, typefaces and a DOM, so every extractor downstream reads it happily and produces a design system belonging to whoever wrote the error page rather than to the site's owner. The status was already read, once, to feed `detectBlockedPage`, and then dropped. That helper cannot stand in for it: it decides whether the rendered document LOOKS like a protection wall, over a minimal-DOM heuristic that a rich error page passes. "Was this blocked?" and "what did the server answer?" are two questions, and widening the first to carry the second would leave a heuristic owning a fact. So the response status is persisted plainly, as its own record, and every consumer decides for itself what a non-success response means for its product. Written before the blocked-page check runs, so the record's absence means "navigation never produced a response" — a third state distinct from a status of 404 and from a status of null. --- packages/cli/src/capture/index.ts | 10 ++++- .../cli/src/capture/responseRecord.test.ts | 44 +++++++++++++++++++ packages/cli/src/capture/responseRecord.ts | 32 ++++++++++++++ packages/cli/src/capture/types.ts | 5 +++ 4 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 packages/cli/src/capture/responseRecord.test.ts create mode 100644 packages/cli/src/capture/responseRecord.ts diff --git a/packages/cli/src/capture/index.ts b/packages/cli/src/capture/index.ts index cdcbfd1e9c..fc3ad8b831 100644 --- a/packages/cli/src/capture/index.ts +++ b/packages/cli/src/capture/index.ts @@ -42,6 +42,7 @@ import { import type { VisionCaptionOutcome } from "./contentExtractor.js"; import { loadEnvFile, generateProjectScaffold } from "./scaffolding.js"; import { detectBlockedPage } from "./pageBlockDetection.js"; +import { writeResponseRecord } from "./responseRecord.js"; import { navigateForCapture } from "./navigateForCapture.js"; import { captureProtocolTimeoutMs, @@ -295,8 +296,14 @@ export async function captureWebsite( progress("warn", message); } + // Persisted before the blocked-page check, so a capture that reaches navigation always leaves + // a record of what the server said. That makes the file's ABSENCE mean "capture never got a + // response", which is a third state distinct from a status of 404 and from a status of null. + const httpStatus = navigationResponse?.status() ?? null; + writeResponseRecord(join(outputDir, "extracted"), { status: httpStatus }); + const blockedReason = detectBlockedPage({ - httpStatus: navigationResponse?.status() ?? null, + httpStatus, ...(contentCheckTimedOut ? { title: "", @@ -874,6 +881,7 @@ export async function captureWebsite( ok: true, projectDir: outputDir, url, + httpStatus, title: tokens.title, extracted, screenshots, diff --git a/packages/cli/src/capture/responseRecord.test.ts b/packages/cli/src/capture/responseRecord.test.ts new file mode 100644 index 0000000000..d865f17ca0 --- /dev/null +++ b/packages/cli/src/capture/responseRecord.test.ts @@ -0,0 +1,44 @@ +import { mkdtempSync, mkdirSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { + RESPONSE_RECORD_FILENAME, + writeResponseRecord, + type CaptureResponseRecord, +} from "./responseRecord.js"; + +describe("writeResponseRecord", () => { + let extractedDir: string; + let projectDir: string; + + beforeEach(() => { + projectDir = mkdtempSync(join(tmpdir(), "hf-response-record-")); + extractedDir = join(projectDir, "extracted"); + mkdirSync(extractedDir); + }); + + afterEach(() => { + rmSync(projectDir, { recursive: true, force: true }); + }); + + const readBack = (): CaptureResponseRecord => + JSON.parse(readFileSync(join(extractedDir, RESPONSE_RECORD_FILENAME), "utf-8")); + + it("records the status the server answered", () => { + writeResponseRecord(extractedDir, { status: 404 }); + + expect(readBack()).toEqual({ status: 404 }); + }); + + it("keeps a missing status distinguishable from a status of zero and from success", () => { + writeResponseRecord(extractedDir, { status: null }); + const record = readBack(); + + // The three states a consumer must be able to separate: the server answered, the server + // answered nothing, and — never — a falsy stand-in that reads as either. + expect(record.status).toBeNull(); + expect(record.status).not.toBe(0); + expect("status" in record).toBe(true); + }); +}); diff --git a/packages/cli/src/capture/responseRecord.ts b/packages/cli/src/capture/responseRecord.ts new file mode 100644 index 0000000000..53c1066dbe --- /dev/null +++ b/packages/cli/src/capture/responseRecord.ts @@ -0,0 +1,32 @@ +import { writeFileSync } from "node:fs"; +import { join } from "node:path"; + +/** + * What the server answered for the captured page, written beside the extraction. + * + * A capture that renders is not the same fact as a capture that succeeded: an error page has a + * title, colors, typefaces and a DOM, so every extractor downstream reads it happily and produces + * a design system belonging to whoever wrote the error page. `detectBlockedPage` cannot answer + * this — it decides whether the page LOOKS like a protection wall, which is a heuristic over the + * rendered document, and a rich 404 passes it. So the status is persisted as its own plain fact + * and consumers decide for themselves what a non-success response means for their product. + */ +export const RESPONSE_RECORD_FILENAME = "response.json"; + +export interface CaptureResponseRecord { + /** + * The final response's status after redirects, or null when navigation produced no response at + * all. Null is NOT "fine": it means we never learned what the server said, which is a different + * fact from a 200 and from a 404, and a consumer must be able to tell the three apart. + */ + status: number | null; +} + +/** Writes the record into an already-created `extracted/` directory. */ +export function writeResponseRecord(extractedDir: string, record: CaptureResponseRecord): void { + writeFileSync( + join(extractedDir, RESPONSE_RECORD_FILENAME), + JSON.stringify(record, null, 2), + "utf-8", + ); +} diff --git a/packages/cli/src/capture/types.ts b/packages/cli/src/capture/types.ts index f64b5f318d..bf88224daf 100644 --- a/packages/cli/src/capture/types.ts +++ b/packages/cli/src/capture/types.ts @@ -69,6 +69,11 @@ export interface CaptureResult { projectDir: string; /** Source URL */ url: string; + /** + * What the server answered for `url`, after redirects; null when navigation produced no + * response. Also persisted to `extracted/response.json` for out-of-process consumers. + */ + httpStatus: number | null; /** Page title */ title: string; /** Extracted HTML data */ From c3f279d299e299170882fc0a40bda07379caf216 Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Sat, 29 Aug 2026 21:22:17 -0400 Subject: [PATCH 2/2] fix(cli): report the response status in the capture command's JSON output The status reached `CaptureResult` and was then dropped at the CLI boundary, which is the same read-once-and-discard that made the error page harvestable in the first place. `--json` is the documented programmatic surface, and an agent reading `ok: true` off a capture of a 404 has no way to see it there. --- packages/cli/src/commands/capture.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/cli/src/commands/capture.ts b/packages/cli/src/commands/capture.ts index 7d4d3a2ed4..5948d1a83b 100644 --- a/packages/cli/src/commands/capture.ts +++ b/packages/cli/src/commands/capture.ts @@ -204,6 +204,9 @@ export default defineCommand({ ok: result.ok, projectDir: result.projectDir, url: result.url, + // Reported beside `ok`, because they answer different questions: a capture of an + // error page is `ok: true` with a status the caller has to see to know it. + httpStatus: result.httpStatus, title: result.title, screenshots: result.screenshots.length, assets: result.assets.length,