feat(producer): refuse distributed-unsupported formats (webm + HDR mp4) (#815)

## What

Phase 3 of the distributed rendering plan: §11 PR 3.5 format banlist. Extends `plan()` to refuse two v1-unsupported formats up front with a typed non-retryable `FormatNotSupportedInDistributedError` (`code === "FORMAT_NOT_SUPPORTED_IN_DISTRIBUTED"`).

## Why

Both webm and HDR mp4 are documented as deferred to v1.5 (§7.2 + §12), but until this PR the only signal at the runtime layer is the in-process pipeline silently producing wrong output (chunk concat-copy doesn't round-trip VP9; HDR signaling gets stripped at the chunk boundary). Failing fast at `plan()` time keeps adopters from spending fan-out compute on a render that can't succeed and gives them a typed error code their workflow adapter can route on.

## How

- New exports in `services/distributed/plan.ts`:
  - `FORMAT_NOT_SUPPORTED_IN_DISTRIBUTED` — non-retryable error code matching §11's wording.
  - `FormatNotSupportedInDistributedError` — typed error class with `code`, `format`, and `reason` fields. Message names the rejected format and tells adopters to fall back to the in-process renderer (`executeRenderJob`) which has full format support.
  - `rejectUnsupportedDistributedFormat(config)` — pure helper exported separately so adapters can run the same gate at their input layer (Step Functions input validation, Temporal workflow start) before the activity even runs.
- `plan()` calls `rejectUnsupportedDistributedFormat(config)` as the first line of the function — BEFORE `mkdirSync(planDir)` so a banned input never produces a partial planDir.
- Replaced the previous ad-hoc `if (hdrMode === "force-hdr") throw new Error(...)` with the typed error class.

### What did NOT change

`executeRenderJob`, the in-process orchestrator, the `hyperframes render` CLI, producer HTTP routes — all unchanged. The in-process renderer continues to accept webm + HDR (its existing functionality).

## Test plan

- [x] Unit tests added — `packages/producer/src/services/distributed/planFormatBanlist.test.ts`. 5 cases:
  - `rejectUnsupportedDistributedFormat` accepts the v1-supported formats (mp4, mov, png-sequence) with both `auto` and `force-sdr` hdrMode.
  - Rejects webm — error has `code === FORMAT_NOT_SUPPORTED_IN_DISTRIBUTED`, `format === "webm"`, message mentions in-process renderer.
  - Rejects HDR mp4 (`hdrMode === "force-hdr"`) — error has `format === "mp4-hdr"`, message mentions HDR.
  - End-to-end via `plan()`: webm throws with no planDir leaking to disk.
  - End-to-end via `plan()`: HDR mp4 throws with no planDir leaking to disk.
- [x] `bun test packages/producer/src/services/distributed/` — 30 pass.
- [x] `bun run --filter @hyperframes/producer typecheck` — clean.
- [x] `bunx oxlint` + `bunx oxfmt --check` — clean on changed files.
- [ ] Producer Docker regression harness — pending CI. `executeRenderJob` is unchanged; PSNR baselines should hold.

This is PR 5 of a 6-PR Phase 3 stack:

- 3.1 — `services/distributed/plan.ts` (#808)
- 3.2 — `services/distributed/renderChunk.ts` (#809)
- 3.3 — `services/distributed/assemble.ts` (#813)
- 3.4 — `planDir` size cap (`PLAN_TOO_LARGE`) (#814)
- **3.5 (this PR)** — distributed format banlist (webm + HDR mp4)
- 3.6 — public exports + `@hyperframes/producer/distributed` subpath

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Author: jrusso1020

packages/producer/src/services/distributed/plan.ts

@@ -171,13 +171,83 @@ export class PlanTooLargeError extends Error {
   }
 }
 
+/**
+ * Non-retryable error code raised when `plan()` is asked for an output
+ * format that distributed mode doesn't support (webm, HDR mp4). The same
+ * config would fail on every retry, so the failure must not auto-retry.
+ */
+export const FORMAT_NOT_SUPPORTED_IN_DISTRIBUTED = "FORMAT_NOT_SUPPORTED_IN_DISTRIBUTED";
+
+/**
+ * Typed error raised by `plan()` for outputs that distributed mode
+ * refuses to ship.
+ *
+ *   - webm — VP9 + matroska concat-copy is fragile across libvpx-vp9
+ *     builds, and the chunked pipeline can't guarantee bit-identical
+ *     concat output across worker versions.
+ *   - mp4 + HDR (PQ / HLG) — chunked HDR pre-extract + HDR signaling
+ *     re-apply on the assembled file is not implemented yet.
+ *
+ * The in-process renderer (`executeRenderJob`) handles both natively.
+ */
+export class FormatNotSupportedInDistributedError extends Error {
+  readonly code: typeof FORMAT_NOT_SUPPORTED_IN_DISTRIBUTED = FORMAT_NOT_SUPPORTED_IN_DISTRIBUTED;
+  readonly format: string;
+  readonly reason: string;
+  constructor(format: string, reason: string) {
+    super(
+      `[plan] format ${JSON.stringify(format)} is not supported in distributed mode: ${reason}. ` +
+        `Render with the in-process renderer (\`executeRenderJob\`) — it has full format ` +
+        `support — or pick a distributed-supported format: mp4 SDR, mov ProRes 4444, or ` +
+        `png-sequence.`,
+    );
+    this.name = "FormatNotSupportedInDistributedError";
+    this.format = format;
+    this.reason = reason;
+  }
+}
+
 function formatBytes(bytes: number): string {
   if (bytes < 1024) return `${bytes} B`;
   if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KiB`;
   if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(1)} MiB`;
   return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GiB`;
 }
 
+/**
+ * Reject formats the distributed pipeline cannot ship (webm + HDR mp4).
+ * Throws {@link FormatNotSupportedInDistributedError} with a message
+ * naming the rejected format. Runs at the very top of `plan()` so a
+ * banned input never produces a partial planDir.
+ *
+ * Exported so adapters can call the same gate at their own input layer
+ * (Step Functions input validation, Temporal workflow start) before the
+ * activity even runs — the resulting non-retryable error then matches
+ * what `plan()` would have thrown.
+ */
+export function rejectUnsupportedDistributedFormat(
+  config: Pick<DistributedRenderConfig, "format" | "hdrMode">,
+): void {
+  // The TypeScript type for `DistributedRenderConfig.format` already
+  // excludes webm, but a JS caller (or a caller that built the config
+  // dynamically from JSON) can still pass it. Belt-and-suspenders runtime
+  // check at the gate.
+  if ((config.format as string) === "webm") {
+    throw new FormatNotSupportedInDistributedError(
+      "webm",
+      "VP9 + matroska concat-copy is fragile across libvpx-vp9 builds, so chunked output " +
+        "can't be guaranteed byte-identical across workers",
+    );
+  }
+  if ((config.hdrMode as string) === "force-hdr") {
+    throw new FormatNotSupportedInDistributedError(
+      "mp4-hdr",
+      "HDR (PQ / HLG) requires per-source HDR pre-extract + HDR signaling re-apply on the " +
+        "assembled file; neither is implemented for the distributed pipeline",
+    );
+  }
+}
+
 /**
  * Walk `<planDir>/` depth-first and sum all regular file sizes. Symlinks
  * are not traversed — they shouldn't appear inside a planDir to begin with
@@ -373,9 +443,11 @@ export async function plan(
   config: DistributedRenderConfig,
   planDir: string,
 ): Promise<PlanResult> {
-  // ── Plan-time validation ──
-  // Rejections here surface as typed `PlanValidationError`s with non-retryable
-  // codes so workflow adapters don't waste retry budget on banned configs.
+  // Plan-time validation. Rejections here surface as typed errors with
+  // non-retryable codes so workflow adapters don't waste retry budget on
+  // banned configs. Runs BEFORE any directory creation so a banned input
+  // never produces a partial planDir.
+  rejectUnsupportedDistributedFormat(config);
   validateNoGpuEncode({
     useGpu: false,
     browserGpuMode: "software",

packages/producer/src/services/distributed/planFormatBanlist.test.ts

@@ -0,0 +1,143 @@
+/**
+ * Unit tests for the distributed format banlist.
+ *
+ * Two formats `plan()` refuses up front:
+ *   - webm — VP9 + matroska concat-copy is fragile across libvpx-vp9 builds.
+ *   - mp4 + HDR (`hdrMode === "force-hdr"`) — chunked HDR pre-extract +
+ *     HDR signaling re-apply on the assembled file is not implemented.
+ *
+ * The banlist must trip BEFORE any other work runs (file server, browser,
+ * ffprobe) — otherwise a banned config can leak a partial planDir on disk.
+ * Each case asserts `existsSync(planDir)` is `false` after the throw to
+ * pin the early-exit contract.
+ */
+
+import { afterAll, beforeAll, describe, expect, it } from "bun:test";
+import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import {
+  FORMAT_NOT_SUPPORTED_IN_DISTRIBUTED,
+  FormatNotSupportedInDistributedError,
+  plan,
+  rejectUnsupportedDistributedFormat,
+  type DistributedRenderConfig,
+} from "./plan.js";
+
+const FIXTURE_HTML = `<!doctype html>
+<html><body>
+  <div data-composition-id="root" data-width="320" data-height="240" data-duration="1">hi</div>
+</body></html>`;
+
+let runRoot: string;
+let projectDir: string;
+
+beforeAll(() => {
+  runRoot = mkdtempSync(join(tmpdir(), "hf-plan-format-ban-"));
+  projectDir = join(runRoot, "project");
+  mkdirSync(projectDir, { recursive: true });
+  writeFileSync(join(projectDir, "index.html"), FIXTURE_HTML, "utf-8");
+});
+
+afterAll(() => {
+  rmSync(runRoot, { recursive: true, force: true });
+});
+
+describe("rejectUnsupportedDistributedFormat (pure)", () => {
+  it("accepts the v1-supported formats (mp4 / mov / png-sequence)", () => {
+    expect(() => rejectUnsupportedDistributedFormat({ format: "mp4" })).not.toThrow();
+    expect(() => rejectUnsupportedDistributedFormat({ format: "mov" })).not.toThrow();
+    expect(() => rejectUnsupportedDistributedFormat({ format: "png-sequence" })).not.toThrow();
+    expect(() =>
+      rejectUnsupportedDistributedFormat({ format: "mp4", hdrMode: "auto" }),
+    ).not.toThrow();
+    expect(() =>
+      rejectUnsupportedDistributedFormat({ format: "mp4", hdrMode: "force-sdr" }),
+    ).not.toThrow();
+  });
+
+  it("rejects webm with FORMAT_NOT_SUPPORTED_IN_DISTRIBUTED", () => {
+    let caught: unknown;
+    try {
+      // Cast forces the runtime check even though the type narrows webm out.
+      rejectUnsupportedDistributedFormat({
+        format: "webm" as DistributedRenderConfig["format"],
+      });
+    } catch (err) {
+      caught = err;
+    }
+    expect(caught).toBeInstanceOf(FormatNotSupportedInDistributedError);
+    expect((caught as FormatNotSupportedInDistributedError).code).toBe(
+      FORMAT_NOT_SUPPORTED_IN_DISTRIBUTED,
+    );
+    expect((caught as FormatNotSupportedInDistributedError).format).toBe("webm");
+    expect((caught as Error).message).toMatch(/webm/);
+    expect((caught as Error).message).toMatch(/in-process|executeRenderJob/);
+  });
+
+  it('rejects HDR mp4 (`hdrMode === "force-hdr"`)', () => {
+    let caught: unknown;
+    try {
+      rejectUnsupportedDistributedFormat({
+        format: "mp4",
+        hdrMode: "force-hdr" as DistributedRenderConfig["hdrMode"],
+      });
+    } catch (err) {
+      caught = err;
+    }
+    expect(caught).toBeInstanceOf(FormatNotSupportedInDistributedError);
+    expect((caught as FormatNotSupportedInDistributedError).code).toBe(
+      FORMAT_NOT_SUPPORTED_IN_DISTRIBUTED,
+    );
+    expect((caught as FormatNotSupportedInDistributedError).format).toBe("mp4-hdr");
+    expect((caught as Error).message).toMatch(/HDR/);
+  });
+});
+
+describe("plan() banlist (end-to-end)", () => {
+  it("throws on webm and does not create the planDir", async () => {
+    const planDir = join(runRoot, "plandir-webm-bans");
+    // Don't pre-create planDir — plan() shouldn't create it on the throw path.
+    let caught: unknown;
+    try {
+      await plan(
+        projectDir,
+        {
+          format: "webm" as DistributedRenderConfig["format"],
+          fps: 30,
+          width: 320,
+          height: 240,
+        },
+        planDir,
+      );
+    } catch (err) {
+      caught = err;
+    }
+    expect(caught).toBeInstanceOf(FormatNotSupportedInDistributedError);
+    expect((caught as FormatNotSupportedInDistributedError).format).toBe("webm");
+    expect(existsSync(planDir)).toBe(false);
+  });
+
+  it("throws on HDR mp4 and does not create the planDir", async () => {
+    const planDir = join(runRoot, "plandir-hdr-bans");
+    let caught: unknown;
+    try {
+      await plan(
+        projectDir,
+        {
+          format: "mp4",
+          fps: 30,
+          width: 320,
+          height: 240,
+          hdrMode: "force-hdr" as DistributedRenderConfig["hdrMode"],
+        },
+        planDir,
+      );
+    } catch (err) {
+      caught = err;
+    }
+    expect(caught).toBeInstanceOf(FormatNotSupportedInDistributedError);
+    expect((caught as FormatNotSupportedInDistributedError).format).toBe("mp4-hdr");
+    expect(existsSync(planDir)).toBe(false);
+  });
+});