diff --git a/packages/cli/src/commands/render.test.ts b/packages/cli/src/commands/render.test.ts index 48f0ce7c20..e3a1fdabed 100644 --- a/packages/cli/src/commands/render.test.ts +++ b/packages/cli/src/commands/render.test.ts @@ -123,6 +123,20 @@ vi.mock("../telemetry/config.js", () => ({ configState.cache = { ...configState.disk }; return { ...configState.disk }; }), + recordRecentRender: vi.fn((id: string, ok: boolean) => { + // Mirrors the real ring update (readConfigFresh → append, cap 5 → write) + // against the mock's disk state, so a render's recent-renders write is + // modeled like every other config mutation here. Fixed timestamp keeps it + // deterministic (tests never assert on `at`). + const disk = configState.disk as Record; + const ring = [ + ...((disk.recentRenders as unknown[]) ?? []), + { id, at: "2026-01-01T00:00:00Z", ok }, + ]; + const next = { ...disk, recentRenders: ring.slice(-5) }; + configState.disk = next; + configState.cache = { ...next }; + }), writeConfig: vi.fn((config: Record) => { configState.writeConfigCalls.push({ ...config }); if (configState.failWrites > 0) { diff --git a/packages/engine/src/index.ts b/packages/engine/src/index.ts index 62ed54ab22..5996ee7a29 100644 --- a/packages/engine/src/index.ts +++ b/packages/engine/src/index.ts @@ -209,6 +209,8 @@ export type { export { calculateOptimalWorkers, computeWorkerSizing, + selectVerifySampleIndicesForTask, + verifyDiskDrawElementSamples, distributeFrames, distributeFramesInterleaved, executeParallelCapture, @@ -273,6 +275,10 @@ export { export { trackChildProcess, killTrackedProcesses } from "./utils/processTracker.js"; +// drawElement self-verify comparison — shared by the streaming drain +// (producer) and the parallel disk-path verify (parallelCoordinator). +export { psnrDb, resolveDeVerifyMinDb } from "./utils/psnr.js"; + export { decodePng, decodePngToRgb48le, diff --git a/packages/engine/src/services/parallelCoordinator.test.ts b/packages/engine/src/services/parallelCoordinator.test.ts index 31f7b59410..43d4386d76 100644 --- a/packages/engine/src/services/parallelCoordinator.test.ts +++ b/packages/engine/src/services/parallelCoordinator.test.ts @@ -6,6 +6,7 @@ import { expectedFramesForTask, flagSilentWorkerExits, formatWorkerFailure, + selectVerifySampleIndicesForTask, selectWorkerDiagnostics, shouldDisableBrowserPoolForParallelWorker, shouldVerifyWorkerGpu, @@ -349,6 +350,35 @@ describe("flagSilentWorkerExits", () => { }); }); +describe("selectVerifySampleIndicesForTask", () => { + it("keeps only samples inside the task's contiguous range, sorted", () => { + // 2-worker split of 3032 frames: worker 1 owns [1516, 3032). + expect( + selectVerifySampleIndicesForTask([2274, 758, 1516, 3031, 3032], { + startFrame: 1516, + endFrame: 3032, + }), + ).toEqual([1516, 2274, 3031]); + }); + + it("respects the stride lattice for interleaved tasks", () => { + // Worker 1 of a 3-way interleave over [1, 30): captures 1, 4, 7, ... + expect( + selectVerifySampleIndicesForTask([1, 2, 4, 6, 7, 28, 29], { + startFrame: 1, + endFrame: 30, + frameStride: 3, + }), + ).toEqual([1, 4, 7, 28]); + }); + + it("returns empty when no samples fall in the range", () => { + expect( + selectVerifySampleIndicesForTask([0, 10, 20], { startFrame: 100, endFrame: 200 }), + ).toEqual([]); + }); +}); + describe("shouldVerifyWorkerGpu", () => { const softwareConfig: Partial = { browserGpuMode: "software" }; diff --git a/packages/engine/src/services/parallelCoordinator.ts b/packages/engine/src/services/parallelCoordinator.ts index 934055a70d..3ee1bb637f 100644 --- a/packages/engine/src/services/parallelCoordinator.ts +++ b/packages/engine/src/services/parallelCoordinator.ts @@ -7,7 +7,7 @@ import { cpus, freemem } from "os"; import { existsSync, mkdirSync, readdirSync } from "fs"; -import { copyFile, rename } from "fs/promises"; +import { copyFile, readFile, rename } from "fs/promises"; import { join } from "path"; import { getHeapStatistics } from "v8"; @@ -19,11 +19,13 @@ import { captureFrameToBufferPipelined, captureFrameToBuffer, getCapturePerfSummary, + DrawElementVerificationError, type CaptureSession, type CaptureOptions, type CapturePerfSummary, type BeforeCaptureHook, } from "./frameCapture.js"; +import { psnrDb, resolveDeVerifyMinDb } from "../utils/psnr.js"; import { DEFAULT_CONFIG, type EngineConfig } from "../config.js"; import { assertSwiftShader } from "../utils/assertSwiftShader.js"; import { readWebGlVendorInfoFromCanvas } from "../utils/readWebGlVendorInfoFromCanvas.js"; @@ -551,6 +553,126 @@ async function captureFrameRange( return framesCaptured; } +/** + * The armed self-verify sample indices this task actually captured: inside + * `[startFrame, endFrame)` and on the task's stride lattice. Mirrors the + * capture loop in `captureFrameRange` (`i += stride` from `startFrame`). + */ +export function selectVerifySampleIndicesForTask( + sampleIndices: Iterable, + task: Pick, +): number[] { + const stride = task.frameStride ?? 1; + const selected: number[] = []; + for (const idx of sampleIndices) { + if (idx < task.startFrame || idx >= task.endFrame) continue; + if ((idx - task.startFrame) % stride !== 0) continue; + selected.push(idx); + } + return selected.sort((a, b) => a - b); +} + +/** + * Disk-path drawElement self-verification (PRINFRA-352). Parallel DISK + * workers arm the same pre-injection ground-truth samples as the streaming + * path (`resolveParallelDeVerifySamples` even raises the density for + * multi-worker capture) — but only the streaming drain ever CHECKED them, + * so an explicit `--experimental-fast-capture --workers N` render shipped + * unverified drawElement frames. On a 16GB host, two concurrent + * hardware-GPU Chrome instances hit the documented compositor-tile-eviction + * damage class (frames displaced into vertical strips for one worker's + * whole range — reads as "corruption from the exact worker boundary"). + * + * After a worker's range completes, re-read its captured files for the + * sampled indices and PSNR-compare against the session's ground truth. + * A breach throws `DrawElementVerificationError`, which the orchestrator's + * existing pinned-fallback retry converts into a screenshot re-render — + * the same recovery the streaming drain gets. + */ +/** HF_DE_PAR_DEBUG=1 gated per-worker trace line (message built lazily). */ +function logParDebug(message: () => string): void { + if (process.env.HF_DE_PAR_DEBUG === "1") console.log(message()); +} + +/** + * Throw the verification error for a sample below the PSNR floor; log the + * pass otherwise. Split from the sampling loop for the complexity gate. + */ +function assertDiskSampleAboveFloor( + db: number, + verifyMinDb: number, + idx: number, + workerId: number, +): void { + if (db < verifyMinDb) { + // Message keeps the contiguous "drawElement self-verify" phrase — + // captureFailure's VERIFICATION_ERROR_PATTERNS classifies on it. + throw new DrawElementVerificationError( + `drawElement self-verify failed at frame ${idx} (disk path, worker ${workerId}): ` + + `${db.toFixed(1)}dB < ${verifyMinDb}dB vs pre-injection screenshot`, + { kind: "psnr", frameIndex: idx, failedDb: db, verifyThresholdDb: verifyMinDb }, + ); + } + console.log( + `[Parallel] drawElement disk self-verify passed (worker ${workerId}, frame ${idx}, ` + + `${db === Infinity ? "inf" : db.toFixed(1)}dB)`, + ); +} + +/** + * Compare one captured frame file against its ground truth. Returns the + * PSNR, or null on infrastructure failure (missing file already surfaces + * via the frame completeness check; ffmpeg spawn/tmpdir here) — a skipped + * sample is not damage evidence and must not fail the capture. + */ +async function psnrForDiskSample( + framePath: string, + truth: Buffer, + workerId: number, + idx: number, +): Promise { + try { + return await psnrDb(await readFile(framePath), truth); + } catch (err) { + console.warn( + `[Parallel] drawElement disk self-verify sample skipped (worker ${workerId}, ` + + `frame ${idx}): ${err instanceof Error ? err.message : String(err)}`, + ); + return null; + } +} + +// Branches are the gate conditions themselves (mode/armed/streaming guards + +// per-sample skip/breach) — already decomposed into psnrForDiskSample + +// assertDiskSampleAboveFloor; further splitting obscures the check. +// fallow-ignore-next-line complexity +export async function verifyDiskDrawElementSamples( + session: CaptureSession, + task: WorkerTask, + streaming: boolean, +): Promise { + // Streaming capture verifies every sampled frame in the drain guard already. + if (streaming || session.captureMode !== "drawelement") return; + const truths = session.deVerifyFrames; + if (!truths || truths.size === 0) return; + const verifyMinDb = resolveDeVerifyMinDb(); + const ext = session.options.format === "png" ? "png" : "jpg"; + const offset = task.outputFrameOffset ?? 0; + for (const idx of selectVerifySampleIndicesForTask(truths.keys(), task)) { + const truth = truths.get(idx); + if (!truth) continue; + const framePath = join(task.outputDir, `frame_${String(idx - offset).padStart(6, "0")}.${ext}`); + const db = await psnrForDiskSample(framePath, truth, task.workerId, idx); + if (db === null) continue; + assertDiskSampleAboveFloor(db, verifyMinDb, idx, task.workerId); + } +} + +// Inherited worker-lifecycle shape (session create → verify GPU → init → +// capture → self-verify → perf, with a classifying catch + closing finally); +// flagged only because the disk self-verify call shifted its line range into +// the changed-code audit. Not restructured by this PR. +// fallow-ignore-next-line complexity async function executeWorkerTask( task: WorkerTask, serverUrl: string, @@ -590,19 +712,16 @@ async function executeWorkerTask( createBeforeCaptureHook(), workerConfig, ); - if (process.env.HF_DE_PAR_DEBUG === "1") { - console.log(`[par:w${task.workerId}] session created`); - } + logParDebug(() => `[par:w${task.workerId}] session created`); // Worker-0-only SwiftShader assertion — see `shouldVerifyWorkerGpu` and #955. if (shouldVerifyWorkerGpu(task.workerId, workerConfig)) { await assertSwiftShader(session.page, readWebGlVendorInfoFromCanvas); } await initializeSession(session); - if (process.env.HF_DE_PAR_DEBUG === "1") { - console.log( - `[par:w${task.workerId}] init done (mode=${session.captureMode} workerEncode=${session.workerEncodeEnabled === true})`, - ); - } + logParDebug( + () => + `[par:w${task.workerId}] init done (mode=${session?.captureMode} workerEncode=${session?.workerEncodeEnabled === true})`, + ); framesCaptured = await captureFrameRange( session, task, @@ -612,6 +731,8 @@ async function executeWorkerTask( onFrameBuffer, ); + await verifyDiskDrawElementSamples(session, task, Boolean(onFrameBuffer)); + perf = getCapturePerfSummary(session); return { workerId: task.workerId, diff --git a/packages/engine/src/utils/psnr.ts b/packages/engine/src/utils/psnr.ts new file mode 100644 index 0000000000..675594ec47 --- /dev/null +++ b/packages/engine/src/utils/psnr.ts @@ -0,0 +1,45 @@ +import { execFile } from "node:child_process"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { getFfmpegBinary } from "./ffmpegBinaries.js"; + +/** + * PSNR (average, dB) between two same-dimension encoded images via ffmpeg. + * Infinity means bit-identical pixels. Single source of truth for every + * drawElement self-verify comparison (streaming drain + parallel disk path). + */ +export async function psnrDb(a: Buffer, b: Buffer): Promise { + // promisify(execFile) lazily, not at module load: this module is in the + // engine's parallel-capture import chain, and a top-level call to a builtin + // crashes any downstream test that partially mocks node:child_process + // without an execFile export (vitest surfaces it as a load-time error). + const execFileP = promisify(execFile); + const dir = await mkdtemp(join(tmpdir(), "hf-de-verify-")); + try { + const pa = join(dir, "a.jpg"); + const pb = join(dir, "b.jpg"); + await Promise.all([writeFile(pa, a), writeFile(pb, b)]); + const { stderr } = await execFileP( + getFfmpegBinary(), + ["-hide_banner", "-i", pa, "-i", pb, "-lavfi", "psnr", "-f", "null", "-"], + { maxBuffer: 4 * 1024 * 1024 }, + ); + const m = /average:(inf|[\d.]+)/.exec(stderr); + if (!m) throw new Error(`psnr parse failed: ${stderr.slice(-300)}`); + return m[1] === "inf" ? Infinity : Number(m[1]); + } finally { + await rm(dir, { recursive: true, force: true }).catch(() => {}); + } +} + +/** + * The drawElement self-verify PSNR floor (dB). HF_DE_VERIFY_MIN_DB overrides, + * clamped to [10, 60]; out-of-range or unset falls back to 32 — the threshold + * every prior eval used to separate real compositor damage from encoder noise. + */ +export function resolveDeVerifyMinDb(): number { + const raw = Number(process.env.HF_DE_VERIFY_MIN_DB ?? "32"); + return Number.isFinite(raw) && raw >= 10 && raw <= 60 ? raw : 32; +} diff --git a/packages/producer/src/services/render/capturePlan.test.ts b/packages/producer/src/services/render/capturePlan.test.ts index a8b8c0ac0c..8e8352de42 100644 --- a/packages/producer/src/services/render/capturePlan.test.ts +++ b/packages/producer/src/services/render/capturePlan.test.ts @@ -140,6 +140,28 @@ describe("CapturePlan", () => { }); }); + it("forces screenshot on a disk-plan drawElement verify failure (PRINFRA-352 recovery)", () => { + // Parallel disk workers under the explicit fast-capture opt-in verify their + // own captured samples; a breach must re-render the DISK plan on the + // screenshot baseline (not throw, and not stay on drawElement). + const disk = createCapturePlan({ + workerCount: 2, + forceScreenshot: false, + useStreamingEncode: false, + useLayeredComposite: false, + usePageSideCompositing: false, + hasHdrContent: false, + needsAlpha: false, + }); + const next = replanAfterFailure(disk, { kind: "draw_element_verification" }); + expect(next).toMatchObject({ + kind: "sdr_disk", + forceScreenshot: true, + forceParallelStream: false, + workerCount: 2, + }); + }); + it("rejects a streaming transition from a non-streaming plan", () => { const disk = createCapturePlan({ workerCount: 2, diff --git a/packages/producer/src/services/render/capturePlan.ts b/packages/producer/src/services/render/capturePlan.ts index d704f4f952..6e04857124 100644 --- a/packages/producer/src/services/render/capturePlan.ts +++ b/packages/producer/src/services/render/capturePlan.ts @@ -124,6 +124,18 @@ function revertedRouting(routing: CaptureRouting): CaptureRouting { /** Pure, exhaustive capture fallback transition. The input plan is never mutated. */ export function replanAfterFailure(plan: CapturePlan, failure: CapturePlanFailure): CapturePlan { + // Disk-path drawElement self-verification (parallel disk workers under the + // explicit fast-capture opt-in) can also trip — the retry stays on the disk + // path but forces the screenshot baseline. + if (plan.kind === "sdr_disk" && failure.kind === "draw_element_verification") { + return createCapturePlan({ + ...plan, + forceScreenshot: true, + useStreamingEncode: false, + useLayeredComposite: false, + forceParallelStream: false, + }); + } if (plan.kind !== "sdr_streaming") { throw new Error(`Cannot apply ${failure.kind} to ${plan.kind} capture plan`); } diff --git a/packages/producer/src/services/render/stages/captureStage.ts b/packages/producer/src/services/render/stages/captureStage.ts index 808b00bffe..005eae259f 100644 --- a/packages/producer/src/services/render/stages/captureStage.ts +++ b/packages/producer/src/services/render/stages/captureStage.ts @@ -44,6 +44,7 @@ import { type EngineConfig, captureFrame, captureFrameToBufferPipelined, + verifyDiskDrawElementSamples, writeCapturedFrame, closeCaptureSession, completeDeferredDrawElementInit, @@ -331,6 +332,24 @@ export async function runCaptureStage(input: CaptureStageInput): Promise { - const dir = await mkdtemp(join(tmpdir(), "hf-de-verify-")); - try { - const pa = join(dir, "a.jpg"); - const pb = join(dir, "b.jpg"); - await Promise.all([writeFile(pa, a), writeFile(pb, b)]); - const { stderr } = await execFileP( - getFfmpegBinary(), - ["-hide_banner", "-i", pa, "-i", pb, "-lavfi", "psnr", "-f", "null", "-"], - { maxBuffer: 4 * 1024 * 1024 }, - ); - const m = /average:(inf|[\d.]+)/.exec(stderr); - if (!m) throw new Error(`psnr parse failed: ${stderr.slice(-300)}`); - return m[1] === "inf" ? Infinity : Number(m[1]); - } finally { - await rm(dir, { recursive: true, force: true }).catch(() => {}); - } -} +// psnrDb moved to @hyperframes/engine (utils/psnr.ts) so the parallel +// disk-path verify (parallelCoordinator) and this drain guard share one +// comparison implementation. // ── drawElement drain-time safety checks (ungated-release safety net) ── // Shared by the sequential worker-encode loop and the interleaved parallel @@ -277,14 +258,14 @@ function createDrainFrameGuard(args: { // check stops meaning anything); above ~60dB natural DE-vs-screenshot // encoder differences (~45dB+) would force a screenshot fallback on every // verified render. Out-of-range or malformed values fall back to 32. - const verifyMinDbRaw = Number(process.env.HF_DE_VERIFY_MIN_DB ?? "32"); - const verifyMinDb = - Number.isFinite(verifyMinDbRaw) && verifyMinDbRaw >= 10 && verifyMinDbRaw <= 60 - ? verifyMinDbRaw - : 32; - if (process.env.HF_DE_VERIFY_MIN_DB !== undefined && verifyMinDb !== verifyMinDbRaw) { - log.warn("[Render] HF_DE_VERIFY_MIN_DB out of range [10,60]; using 32", { - raw: process.env.HF_DE_VERIFY_MIN_DB, + // Single-sourced clamp (psnr.ts) so the disk and streaming verify paths can + // never apply different PSNR floors to the same composition. The warn stays + // here because only this path has a logger in scope. + const verifyMinDb = resolveDeVerifyMinDb(); + const rawEnv = process.env.HF_DE_VERIFY_MIN_DB; + if (rawEnv !== undefined && Number(rawEnv) !== verifyMinDb) { + log.warn(`[Render] HF_DE_VERIFY_MIN_DB out of range [10,60]; using ${verifyMinDb}`, { + raw: rawEnv, }); } const sizes: number[] = []; diff --git a/packages/producer/src/services/renderOrchestrator.test.ts b/packages/producer/src/services/renderOrchestrator.test.ts index e900b68f21..deec16be0d 100644 --- a/packages/producer/src/services/renderOrchestrator.test.ts +++ b/packages/producer/src/services/renderOrchestrator.test.ts @@ -17,6 +17,7 @@ vi.mock("@hyperframes/engine", async (importOriginal) => { import { buildMissingFrameRetryBatches, captureAttemptMadeProgress, + closeOrphanedProbeForRetry, describeMemoryExhaustion, executeDiskCaptureWithAdaptiveRetry, collectVideoMetadataHints, @@ -2130,3 +2131,61 @@ describe("shouldStreamParallelCapture (non-DE parallel streaming router)", () => expect(shouldStreamParallelCapture({ ...eligible, layeredOrEffectRoute: true })).toBe(false); }); }); + +describe("closeOrphanedProbeForRetry (probe cleanup before verify-triggered retry)", () => { + // Enough of a CaptureSession stand-in to exercise the closer path — the + // helper never inspects the object; it just hands it to the injected closer. + const stubSession = { browserConsoleBuffer: [] } as unknown as Parameters< + typeof closeOrphanedProbeForRetry + >[0]; + + it("hands the still-owned probe to the closer before the caller clears it", async () => { + const closer = vi.fn(async () => {}); + const log = { warn: vi.fn() }; + + await closeOrphanedProbeForRetry(stubSession, closer, log, "streaming"); + + expect(closer).toHaveBeenCalledTimes(1); + expect(closer).toHaveBeenCalledWith(stubSession); + expect(log.warn).not.toHaveBeenCalled(); + }); + + it("swallows a close failure with a warn so the caller's retry can proceed", async () => { + const closer = vi.fn(async () => { + throw new Error("chrome zombie"); + }); + const log = { warn: vi.fn() }; + + await expect( + closeOrphanedProbeForRetry(stubSession, closer, log, "disk verify"), + ).resolves.toBeUndefined(); + + expect(closer).toHaveBeenCalledTimes(1); + expect(log.warn).toHaveBeenCalledTimes(1); + const [message, meta] = log.warn.mock.calls[0]; + expect(message).toContain("disk verify"); + expect((meta as { error: string }).error).toBe("chrome zombie"); + }); + + it("preserves the retry context in the warn message so the audit trail names which retry path leaked", async () => { + const closer = vi.fn(async () => { + throw new Error("session already closed"); + }); + const log = { warn: vi.fn() }; + + await closeOrphanedProbeForRetry(stubSession, closer, log, "streaming"); + + expect(log.warn.mock.calls[0][0]).toContain("streaming"); + expect(log.warn.mock.calls[0][0]).not.toContain("disk verify"); + }); + + it("stringifies non-Error rejections so the log entry still names the cause", async () => { + const closer = vi.fn(async () => Promise.reject("string-only rejection")); + const log = { warn: vi.fn() }; + + await closeOrphanedProbeForRetry(stubSession, closer, log, "streaming"); + + expect(log.warn).toHaveBeenCalledTimes(1); + expect((log.warn.mock.calls[0][1] as { error: string }).error).toBe("string-only rejection"); + }); +}); diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.ts index db47fb4c18..c9a0c81292 100644 --- a/packages/producer/src/services/renderOrchestrator.ts +++ b/packages/producer/src/services/renderOrchestrator.ts @@ -109,6 +109,7 @@ import { createCapturePlan, replanAfterFailure, type CapturePlan, + type SdrDiskCapturePlan, type CaptureRouting, } from "./render/capturePlan.js"; import { normalizeErrorMessage } from "../utils/errorMessage.js"; @@ -1017,6 +1018,20 @@ export async function executeDiskCaptureWithAdaptiveRetry(options: { if (failure.kind === "cancelled") { throw error; } + // A drawElement self-verify breach (a parallel disk worker's sampled + // frame diverged from its pre-injection ground truth) is a CORRECTNESS + // failure, not a missing-frame one: the damaged frames are written + // complete to disk, so the presence/size-only findMissingFrameRanges + // below would count them present and wrongly return success — shipping + // the exact compositor damage this verify exists to catch. Rethrow so + // the orchestrator's disk-stage screenshot retry fires (mirrors the + // `cancelled` guard; a worker-halving retry here would only re-run + // drawElement and re-damage). Structural detection walks the aggregated + // CaptureFailure → worker CaptureFailure → DrawElementVerificationError + // cause chain. + if (isDrawElementVerificationError(error)) { + throw error; + } const remaining = findMissingFrameRanges( options.totalFrames, options.framesDir, @@ -1453,6 +1468,31 @@ export function shouldRetryViaPinnedFallback(args: { return args.deWorkerInversion === "inverted" || args.deParallelRouter === "routed"; } +/** + * When a self-verify (or pinned-fallback) retry is triggered mid-capture, the + * caller may still hold a live probe session that the failed stage was passed + * but did not (or could not) close in its own `finally` before it threw. Left + * behind, that session's Chrome process orphans until the containing render + * exits — precisely when we are recovering from GPU/memory pressure and can + * least afford an unaccounted Chrome. Close it before the caller clears its + * reference; swallow any close error with a warn so the retry itself is never + * derailed by a shutdown hiccup. + */ +export async function closeOrphanedProbeForRetry( + probe: CaptureSession, + closer: (session: CaptureSession) => Promise, + log: Pick, + retryContext: string, +): Promise { + try { + await closer(probe); + } catch (closeErr) { + log.warn(`[Render] probe close before ${retryContext} retry failed; continuing with retry`, { + error: closeErr instanceof Error ? closeErr.message : String(closeErr), + }); + } +} + /** * Parallel-streaming router for NON-drawElement capture (screenshot on * macOS/Windows/forced-screenshot, BeginFrame on Linux): should this @@ -1559,6 +1599,29 @@ export function extractStandaloneEntryFromIndex( return document.toString(); } +/** + * Telemetry fields a drawElement self-verify failure contributes to the + * fallback record. Shared by the streaming and parallel-disk verify catches + * so the `verifyDetails` → `de_fallback_*` mapping lives in one place — a new + * field is added once, not once per capture path. `kind` is read structurally + * off the error (never from message text), so a reworded/translated/ + * cross-module-serialized error can't flip "blank" into "psnr". + */ +function deVerifyFallbackTelemetry(err: unknown): { + reason: "psnr" | "blank"; + failedDb?: number; + frameIndex?: number; + thresholdDb?: number; +} { + const details = getDrawElementVerificationDetails(err); + return { + reason: details?.kind ?? "psnr", + failedDb: roundDb(details?.failedDb), + frameIndex: details?.frameIndex, + thresholdDb: roundDb(details?.verifyThresholdDb), + }; +} + /** * Render a `RenderJob` end-to-end: compile → probe → extract videos → * audio → capture → encode → assemble. The function body is a thin @@ -3011,20 +3074,14 @@ async function executeRenderPipeline(input: { throw err; const isMemoryExhaustion = !isVerifyError && isMemoryExhaustionError(err); deSelfVerifyFallback = isVerifyError; - // `kind` is a structural field on the error (DrawElementVerificationDetails), - // never derived from message text — a reworded message, a translated - // string, or a cross-module/serialized error must never be able to - // flip "blank" into "psnr" or vice versa (review finding). - const verifyDetails = isVerifyError ? getDrawElementVerificationDetails(err) : undefined; - deFallbackReason = isVerifyError - ? (verifyDetails?.kind ?? "psnr") - : isMemoryExhaustion - ? "oom" - : "capture_error"; if (isVerifyError) { - deFallbackFailedDb = roundDb(verifyDetails?.failedDb); - deFallbackFrameIndex = verifyDetails?.frameIndex; - deFallbackThresholdDb = roundDb(verifyDetails?.verifyThresholdDb); + const t = deVerifyFallbackTelemetry(err); + deFallbackReason = t.reason; + deFallbackFailedDb = t.failedDb; + deFallbackFrameIndex = t.frameIndex; + deFallbackThresholdDb = t.thresholdDb; + } else { + deFallbackReason = isMemoryExhaustion ? "oom" : "capture_error"; } log.warn( isVerifyError @@ -3058,7 +3115,16 @@ async function executeRenderPipeline(input: { deWorkerInversion, deParallelRouter, }); - probeSession = null; + // Streaming stage aims to close the probe in its own finally; if it + // threw before doing so, the Chrome process would orphan through the + // pinned-fallback retry. Close defensively before we release the + // reference — see closeOrphanedProbeForRetry. + if (probeSession) { + lastBrowserConsole = probeSession.browserConsoleBuffer; + const orphaned = probeSession; + probeSession = null; + await closeOrphanedProbeForRetry(orphaned, closeCaptureSession, log, "streaming"); + } if (failedRouting === "worker_inversion") { // The inversion bet on drawElement and lost — re-render on the // pre-inversion parallel screenshot path instead of single-worker @@ -3144,34 +3210,102 @@ async function executeRenderPipeline(input: { if (capturePlan.kind !== "sdr_disk") { throw new Error(`Disk capture requires sdr_disk plan; got ${capturePlan.kind}`); } - const diskPlan = capturePlan; // ── Disk-based capture (original flow) ──────────────────────────── resetCaptureAttemptProgress(job); const captureFrameStart = Date.now(); - const captureRes = await observeRenderStage( - observability, - "capture_disk", - captureStageObservationData({ needsAlpha: diskPlan.needsAlpha }), - () => - runCaptureStage({ - fileServer: activeFileServer, - workDir, - framesDir, - job, - totalFrames, - cfg, - plan: diskPlan, - log, - probeSession, - captureAttempts, - dedupPerfs, - buildCaptureOptions, - createRenderVideoFrameInjector, - abortSignal: executionSignal, - assertNotAborted, - onProgress, - }), - ); + const invokeDiskCapture = (diskPlan: SdrDiskCapturePlan) => + observeRenderStage( + observability, + "capture_disk", + captureStageObservationData({ needsAlpha: diskPlan.needsAlpha }), + () => + runCaptureStage({ + fileServer: activeFileServer, + workDir, + framesDir, + job, + totalFrames, + cfg, + plan: diskPlan, + log, + probeSession, + captureAttempts, + dedupPerfs, + buildCaptureOptions, + createRenderVideoFrameInjector, + abortSignal: executionSignal, + assertNotAborted, + onProgress, + }), + ); + let captureRes; + try { + captureRes = await invokeDiskCapture(capturePlan); + } catch (err) { + // Disk-path drawElement self-verification tripped (a parallel disk + // worker's sampled frame diverged from its pre-injection ground + // truth — reachable only under the explicit fast-capture opt-in). + // Same recovery contract as the streaming drain: re-render on the + // screenshot baseline. Anything else keeps its existing semantics. + if ( + !isDrawElementVerificationError(err) || + err instanceof RenderCancelledError || + executionSignal?.aborted === true + ) { + throw err; + } + deSelfVerifyFallback = true; + const t = deVerifyFallbackTelemetry(err); + deFallbackReason = t.reason; + deFallbackFailedDb = t.failedDb; + deFallbackFrameIndex = t.frameIndex; + deFallbackThresholdDb = t.thresholdDb; + log.warn( + "[Render] drawElement self-verification failed on the parallel disk path; " + + "re-rendering via screenshot", + { error: err instanceof Error ? err.message : String(err) }, + ); + observability.checkpoint( + "capture_disk", + "drawElement self-verify failed; retrying with forceScreenshot", + ); + // The failed attempt's frames are untrusted BUT satisfy the + // completeness check — wipe them so the retry re-captures everything + // instead of silently keeping damaged files. + rmSync(framesDir, { recursive: true, force: true }); + mkdirSync(framesDir, { recursive: true }); + resetCaptureAttemptProgress(job); + dedupPerfs.length = 0; + cfg.useDrawElement = false; + // Same shape as the streaming retry above: `runCaptureStage` was + // passed the probe and threw before it could close it, so we must + // release the Chrome process ourselves before starting the + // screenshot-baseline retry — otherwise it orphans until render + // exit. See closeOrphanedProbeForRetry. + if (probeSession) { + lastBrowserConsole = probeSession.browserConsoleBuffer; + const orphaned = probeSession; + probeSession = null; + await closeOrphanedProbeForRetry(orphaned, closeCaptureSession, log, "disk verify"); + } + capturePlan = replanAfterFailure(capturePlan, { kind: "draw_element_verification" }); + syncCapturePlan(); + updateCaptureObservability({ + forceScreenshot: capturePlan.forceScreenshot, + deSelfVerifyFallback, + deFallbackReason, + deFallbackFailedDb, + deFallbackFrameIndex, + deFallbackThresholdDb, + }); + if (capturePlan.kind !== "sdr_disk") { + throw new Error(`Disk verify retry requires sdr_disk plan; got ${capturePlan.kind}`); + } + captureRes = await invokeDiskCapture(capturePlan); + // The first attempt's error marked the phase failed; the retry + // recovered it — don't brand the render as failed in telemetry. + observability.clearFailure("capture_disk"); + } const captureFrameMs = Date.now() - captureFrameStart; workerCount = captureRes.workerCount; updateCaptureObservability({ workerCount });