fix(producer): mix audio into a container that can record encoder delay

packages/aws-lambda/src/events.ts

@@ -117,7 +117,7 @@ interface AssembleEventBase {
   Action: "assemble";
   /** S3 URIs of every chunk, ordered by chunk index. Length must equal `chunkCount`. */
   ChunkS3Uris: string[];
-  /** S3 URI of the planDir's `audio.aac` if the composition has audio; `null` otherwise. */
+  /** S3 URI of the planDir's audio artifact if the composition has audio; `null` otherwise. */
   AudioS3Uri: string | null;
   /** Final output S3 URI (`s3://bucket/key.mp4`). */
   OutputS3Uri: string;

packages/aws-lambda/src/handler.test.ts

@@ -563,7 +563,7 @@ describe("handler dispatch", () => {
     );
     const renderChunkMock = mock(
       async (planDir: string, _chunkIndex: number, outputPath: string): Promise<ChunkResult> => {
-        expect(existsSync(join(planDir, "audio.aac"))).toBe(false);
+        expect(existsSync(join(planDir, "audio.m4a"))).toBe(false);
         writeFileSync(outputPath, "V2-CHUNK");
         return {
           outputPath,
@@ -818,7 +818,7 @@ function makeMinimalV1PlanDir(dir: string, withAudio: boolean): void {
     JSON.stringify([{ index: 0, startFrame: 0, endFrame: 30 }]),
   );
   writeFileSync(join(dir, "meta", "encoder.json"), "{}");
-  if (withAudio) writeFileSync(join(dir, "audio.aac"), "AAC");
+  if (withAudio) writeFileSync(join(dir, "audio.m4a"), "AAC");
   planJson.planHash = recomputePlanHashFromPlanDir(dir);
   writeFileSync(join(dir, "plan.json"), JSON.stringify(planJson));
 }

packages/aws-lambda/src/handler.ts

@@ -24,6 +24,7 @@ import {
   listPlanV2ArtifactsForTarget,
   materializePlanV2Target,
   plan,
+  PLAN_AUDIO_RELATIVE_PATH,
   planV2WithPublisher,
   type PlanResult,
   type PlanV2Artifact,
@@ -318,9 +319,11 @@ async function handlePlan(event: PlanEvent, deps?: HandlerDeps): Promise<PlanLam
     const planTar = join(work, "plan.tar.gz");
     await tarDirectory(planDir, planTar);
     const planTarUri = `${trimTrailingSlash(event.PlanOutputS3Prefix)}/plan.tar.gz`;
-    const audioPath = join(planDir, "audio.aac");
+    const audioPath = join(planDir, PLAN_AUDIO_RELATIVE_PATH);
     const hasAudio = existsSync(audioPath) && statSync(audioPath).size > 0;
-    const audioUri = hasAudio ? `${trimTrailingSlash(event.PlanOutputS3Prefix)}/audio.aac` : null;
+    const audioUri = hasAudio
+      ? `${trimTrailingSlash(event.PlanOutputS3Prefix)}/${PLAN_AUDIO_RELATIVE_PATH}`
+      : null;
     // Plan and audio are independent S3 PUTs; run them in parallel so
     // the response returns as soon as the slower of the two completes.
     await Promise.all([
@@ -390,7 +393,7 @@ async function handlePlanV2(
       Width: manifest.width,
       Height: manifest.height,
       Format: manifest.format,
-      HasAudio: manifest.artifacts.some((artifact) => artifact.path === "audio.aac"),
+      HasAudio: manifest.artifacts.some((artifact) => artifact.path === PLAN_AUDIO_RELATIVE_PATH),
       AudioS3Uri: null,
       FfmpegVersion: manifest.ffmpegVersion,
       ProducerVersion: manifest.producerVersion,
@@ -568,7 +571,7 @@ async function handleAssemble(
 
     let audioPath: string | null = null;
     if (event.AudioS3Uri) {
-      audioPath = join(planDir, "audio.aac");
+      audioPath = join(planDir, PLAN_AUDIO_RELATIVE_PATH);
       await downloadS3ObjectToFile(s3, event.AudioS3Uri, audioPath);
     }
 
@@ -616,7 +619,9 @@ async function handleAssembleV2(
     const planDir = await downloadAndMaterializePlanV2(s3, event, { role: "assembler" }, work);
     // `downloadAndMaterializePlanV2` materializes atomically. Audio is
     // assembler-only and lives at the familiar v1-compatible location.
-    const audioPath = existsSync(join(planDir, "audio.aac")) ? join(planDir, "audio.aac") : null;
+    const audioPath = existsSync(join(planDir, PLAN_AUDIO_RELATIVE_PATH))
+      ? join(planDir, PLAN_AUDIO_RELATIVE_PATH)
+      : null;
     const chunkPaths = await downloadChunkObjects(s3, event.ChunkS3Uris, work, event.Format);
     const finalOutput =
       event.Format === "png-sequence"

packages/engine/src/index.ts

@@ -213,7 +213,11 @@ export {
 
 export { createVideoFrameInjector } from "./services/videoFrameInjector.js";
 
-export { parseAudioElements, processCompositionAudio } from "./services/audioMixer.js";
+export {
+  MIXED_AUDIO_FILENAME,
+  parseAudioElements,
+  processCompositionAudio,
+} from "./services/audioMixer.js";
 export { cloneCaptureWarning, cloneCaptureWarnings } from "./services/captureWarning.js";
 export type {
   AudioElement,

packages/engine/src/services/audioMixer.level.test.ts

@@ -4,7 +4,7 @@ import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { afterEach, describe, expect, it } from "vitest";
 import { getFfmpegBinary } from "../utils/ffmpegBinaries.js";
-import { processCompositionAudio } from "./audioMixer.js";
+import { MIXED_AUDIO_FILENAME, processCompositionAudio } from "./audioMixer.js";
 
 const HAS_FFMPEG = spawnSync(getFfmpegBinary(), ["-version"], { encoding: "utf-8" }).status === 0;
 const tempDirs: string[] = [];
@@ -22,6 +22,39 @@ function meanVolumeDb(path: string): number {
   return Number(match[1]);
 }
 
+/** Seconds until the first sample loud enough to be signal rather than codec noise. */
+function firstAudibleSeconds(path: string): number {
+  const sampleRate = 48_000;
+  const result = spawnSync(
+    getFfmpegBinary(),
+    [
+      "-nostdin",
+      "-v",
+      "error",
+      "-i",
+      path,
+      "-map",
+      "0:a",
+      "-ac",
+      "1",
+      "-ar",
+      String(sampleRate),
+      "-f",
+      "s16le",
+      "-",
+    ],
+    { maxBuffer: 1 << 28 },
+  );
+  if (result.status !== 0) {
+    throw new Error(`Could not decode ${path}: ${result.stderr?.toString()}`);
+  }
+  const pcm = result.stdout;
+  for (let i = 0; i < pcm.length / 2; i += 1) {
+    if (Math.abs(pcm.readInt16LE(i * 2)) > 512) return i / sampleRate;
+  }
+  throw new Error(`No audible sample found in ${path}`);
+}
+
 describe.skipIf(!HAS_FFMPEG)("processCompositionAudio levels", () => {
   afterEach(() => {
     for (const dir of tempDirs.splice(0)) rmSync(dir, { recursive: true, force: true });
@@ -75,4 +108,57 @@ describe.skipIf(!HAS_FFMPEG)("processCompositionAudio levels", () => {
     expect(result.success).toBe(true);
     expect(meanVolumeDb(outputPath) - meanVolumeDb(sourcePath)).toBeGreaterThan(-0.3);
   });
+
+  it("places a delayed track on its authored start, not one AAC frame later", async () => {
+    // The mix is AAC-encoded, and AAC encoders emit ~1024 priming samples. A
+    // raw ADTS container has nowhere to record that delay, so it decodes as
+    // real leading silence and drags the whole track 21.33 ms late against a
+    // frame-accurate video. MIXED_AUDIO_FILENAME picks a container that stores
+    // the delay as an edit list instead; this asserts the artifact we actually
+    // ship lands on time.
+    const projectDir = mkdtempSync(join(tmpdir(), "hf-onset-"));
+    const workDir = mkdtempSync(join(tmpdir(), "hf-onset-work-"));
+    tempDirs.push(projectDir, workDir);
+    const sourcePath = join(projectDir, "tone.wav");
+    const outputPath = join(projectDir, MIXED_AUDIO_FILENAME);
+    const setup = spawnSync(
+      getFfmpegBinary(),
+      [
+        "-nostdin",
+        "-v",
+        "error",
+        "-f",
+        "lavfi",
+        "-i",
+        "sine=frequency=1000:duration=1:sample_rate=48000",
+        "-c:a",
+        "pcm_s16le",
+        sourcePath,
+      ],
+      { encoding: "utf-8" },
+    );
+    expect(setup.status, setup.stderr).toBe(0);
+
+    const result = await processCompositionAudio(
+      [
+        {
+          id: "tone",
+          src: "tone.wav",
+          start: 2,
+          end: 3,
+          mediaStart: 0,
+          layer: 0,
+          volume: 1,
+          type: "audio",
+        },
+      ],
+      projectDir,
+      workDir,
+      outputPath,
+      4,
+    );
+
+    expect(result.success).toBe(true);
+    expect(firstAudibleSeconds(outputPath)).toBeCloseTo(2, 2);
+  });
 });

packages/engine/src/services/audioMixer.ts

@@ -32,6 +32,21 @@ import { applyVolumeEnvelopeToWav } from "./audioVolumeEnvelope.js";
 
 export type { AudioElement, MixResult } from "./audioMixer.types.js";
 
+/**
+ * Filename every caller must use for the mixed-audio artifact.
+ *
+ * The extension is load-bearing, not cosmetic: FFmpeg picks the muxer from it,
+ * and the mix is AAC-encoded. A raw ADTS `.aac` stream has nowhere to record
+ * the encoder's priming delay, so those leading samples decode as real silence
+ * and shift the whole track ~1024 samples (21.33 ms at 48 kHz) late against a
+ * frame-accurate video track. An MP4-family container carries the delay as an
+ * edit list, which every decoder then strips, so the mix lands on its authored
+ * start. Keep the choice here rather than at each call site: the same file is
+ * muxed into the video, shipped in a distributed plan, and handed to users as
+ * the PNG-sequence sidecar, and all three have to agree.
+ */
+export const MIXED_AUDIO_FILENAME = "audio.m4a";
+
 function clampVolume(volume: number): number {
   if (!Number.isFinite(volume)) return 1;
   return Math.max(0, Math.min(1, volume));

packages/gcp-cloud-run/src/server.test.ts

@@ -87,7 +87,7 @@ function makeMinimalV1PlanDir(dir: string, withAudio: boolean): void {
     JSON.stringify([{ index: 0, startFrame: 0, endFrame: 30 }]),
   );
   writeFileSync(join(dir, "meta", "encoder.json"), "{}");
-  if (withAudio) writeFileSync(join(dir, "audio.aac"), "AAC");
+  if (withAudio) writeFileSync(join(dir, "audio.m4a"), "AAC");
   planJson.planHash = recomputePlanHashFromPlanDir(dir);
   writeFileSync(join(dir, "plan.json"), JSON.stringify(planJson));
 }
@@ -273,7 +273,7 @@ describe("dispatch", () => {
       chunkIndex: number,
       outputBase: string,
     ): Promise<ChunkResult> => {
-      expect(existsSync(join(planDir, "audio.aac"))).toBe(false);
+      expect(existsSync(join(planDir, "audio.m4a"))).toBe(false);
       writeFileSync(outputBase, `chunk-${chunkIndex}`);
       return {
         outputPath: outputBase,

packages/gcp-cloud-run/src/server.ts

@@ -32,6 +32,7 @@ import {
   listPlanV2ArtifactsForTarget,
   materializePlanV2Target,
   plan,
+  PLAN_AUDIO_RELATIVE_PATH,
   planV2WithPublisher,
   type PlanResult,
   type PlanV2Artifact,
@@ -322,7 +323,7 @@ async function handlePlan(event: PlanEvent, deps?: HandlerDeps): Promise<PlanRes
 
     // Upload the planDir as a single tarball. The workflow cannot pass a
     // directory-shaped artifact between steps; we serialize and rely on the
-    // consumer (renderChunk / assemble) to untar. `audio.aac` lives inside
+    // consumer (renderChunk / assemble) to untar. The audio artifact lives inside
     // planDir, so it already rides along in this tarball — every consumer
     // (including assemble) gets it from the untar. We deliberately do NOT
     // upload a separate audio object: it would duplicate the bytes on every
@@ -331,7 +332,7 @@ async function handlePlan(event: PlanEvent, deps?: HandlerDeps): Promise<PlanRes
     const planTar = join(work, "plan.tar.gz");
     await tarDirectory(planDir, planTar);
     const planTarUri = `${trimTrailingSlash(event.PlanOutputGcsPrefix)}/plan.tar.gz`;
-    const audioPath = join(planDir, "audio.aac");
+    const audioPath = join(planDir, PLAN_AUDIO_RELATIVE_PATH);
     const hasAudio = existsSync(audioPath) && statSync(audioPath).size > 0;
     await uploadFileToGcs(storage, planTar, planTarUri, "application/gzip");
 
@@ -397,7 +398,7 @@ async function handlePlanV2(
       Width: manifest.width,
       Height: manifest.height,
       Format: manifest.format,
-      HasAudio: manifest.artifacts.some((artifact) => artifact.path === "audio.aac"),
+      HasAudio: manifest.artifacts.some((artifact) => artifact.path === PLAN_AUDIO_RELATIVE_PATH),
       AudioGcsUri: null,
       FfmpegVersion: manifest.ffmpegVersion,
       ProducerVersion: manifest.producerVersion,
@@ -567,7 +568,7 @@ async function handleAssemble(
     // only for backward compatibility with an older Plan that uploaded it
     // standalone.
     let audioPath: string | null = null;
-    const planAudio = join(planDir, "audio.aac");
+    const planAudio = join(planDir, PLAN_AUDIO_RELATIVE_PATH);
     if (existsSync(planAudio) && statSync(planAudio).size > 0) {
       audioPath = planAudio;
     } else if (event.AudioGcsUri) {
@@ -619,7 +620,9 @@ async function handleAssembleV2(
   const work = mkdtempSync(join(deps?.tmpRoot ?? tmpdir(), "hf-cr-assemble-v2-"));
   try {
     const planDir = await downloadAndMaterializePlanV2(storage, event, { role: "assembler" }, work);
-    const audioPath = existsSync(join(planDir, "audio.aac")) ? join(planDir, "audio.aac") : null;
+    const audioPath = existsSync(join(planDir, PLAN_AUDIO_RELATIVE_PATH))
+      ? join(planDir, PLAN_AUDIO_RELATIVE_PATH)
+      : null;
     const chunkPaths = await downloadChunkObjects(storage, event.ChunkGcsUris, work, event.Format);
     const finalOutput =
       event.Format === "png-sequence"

packages/producer/src/distributed.ts

@@ -149,6 +149,13 @@ export {
 // CLI / adopter SDKs can derive runtime allowlists from one source.
 export { PlanVideosMetadataError, type DistributedFormat } from "./services/distributed/shared.js";
 
+// ── Plan artifact names ─────────────────────────────────────────────────────
+// The cloud adapters locate and publish the plan's audio artifact by name. Its
+// extension selects the container, so they must read it from here rather than
+// restate it: a literal that drifts from the writer's is a silently missing
+// audio track, not a loud failure.
+export { PLAN_AUDIO_RELATIVE_PATH } from "./services/distributed/shared.js";
+
 // ── Plan-time shared types from `freezePlan` ───────────────────────────────
 // Re-exported so adopters that deserialize a planDir's `meta/encoder.json`
 // or `meta/chunks.json` see the same shapes the producer wrote them as.

packages/producer/src/regression-harness-distributed.ts

@@ -35,7 +35,7 @@ import { existsSync, mkdirSync } from "node:fs";
 import { join } from "node:path";
 import type { Fps } from "@hyperframes/core";
 import { assemble, plan, renderChunk } from "./distributed.js";
-import type { DistributedFormat } from "./services/distributed/shared.js";
+import { PLAN_AUDIO_RELATIVE_PATH, type DistributedFormat } from "./services/distributed/shared.js";
 
 /**
  * Three-mode contract that backs `--mode=<value>` on the regression
@@ -199,9 +199,9 @@ export async function runDistributedSimulatedRender(
     chunkPaths.push(chunkPath);
   }
 
-  // Step C: assemble. `audio.aac` only exists when the composition has
+  // Step C: assemble. The audio artifact only exists when the composition has
   // audio — pass null otherwise so `assemble()` doesn't try to mux silence.
-  const audioPath = join(planDir, "audio.aac");
+  const audioPath = join(planDir, PLAN_AUDIO_RELATIVE_PATH);
   const audioForAssemble = existsSync(audioPath) ? audioPath : null;
   await assemble(planDir, chunkPaths, audioForAssemble, input.renderedOutputPath);
 }

packages/producer/src/services/distributed/assemble.test.ts

@@ -37,7 +37,7 @@ afterAll(() => {
 /**
  * Build a synthetic planDir whose `meta/chunks.json` declares N chunks of
  * `framesPerChunk` frames each. Does NOT materialize compiled/, video-frames/,
- * audio.aac — assemble only reads `plan.json` + `meta/chunks.json`, and we
+ * audio.m4a — assemble only reads `plan.json` + `meta/chunks.json`, and we
  * pass chunk paths explicitly. Keeping the dir lean speeds up the test
  * loop.
  */
@@ -287,7 +287,7 @@ describe("assemble()", () => {
   );
 
   it(
-    "muxes audio with frame-count-derived duration when audio.aac is present",
+    "muxes audio with frame-count-derived duration when audio.m4a is present",
     async () => {
       if (!hasFfmpeg) return;
 
@@ -301,7 +301,7 @@ describe("assemble()", () => {
 
       const chunkAPath = join(planDir, "chunk-0.mp4");
       const chunkBPath = join(planDir, "chunk-1.mp4");
-      const audioPath = join(planDir, "audio.aac");
+      const audioPath = join(planDir, "audio.m4a");
       makeMp4Chunk(chunkAPath, 6);
       makeMp4Chunk(chunkBPath, 6);
       // Audio is half a second longer than the video — `padOrTrimAudioToVideoFrameCount`
@@ -346,7 +346,7 @@ describe("assemble()", () => {
 
       const chunkAPath = join(planDir, "chunk-0.mp4");
       const chunkBPath = join(planDir, "chunk-1.mp4");
-      const audioPath = join(planDir, "audio.aac");
+      const audioPath = join(planDir, "audio.m4a");
       makeMp4Chunk(chunkAPath, 6);
       makeMp4Chunk(chunkBPath, 6);
       // Audio is shorter than the video, forcing the distributed pad branch.