fix: preserve audio automation plateaus

Author: miguel-heygen

packages/core/src/runtime/mediaVolumeEnvelope.test.ts

@@ -1,6 +1,74 @@
 /** @vitest-environment jsdom */
 import { describe, expect, it } from "vitest";
-import { probeAndCacheElementVolume } from "./mediaVolumeEnvelope";
+import { probeAndCacheElementVolume, probeElementVolumeKeyframes } from "./mediaVolumeEnvelope";
+
+describe("probeElementVolumeKeyframes", () => {
+  it("retains the last plateau sample before a short volume change", () => {
+    const audio = document.createElement("audio");
+    audio.dataset.start = "0";
+    audio.dataset.duration = "2";
+    audio.dataset.volume = "0.8";
+
+    const keyframes = probeElementVolumeKeyframes(
+      audio,
+      (time) => {
+        audio.volume = time < 1.05 ? 0.8 : 0.2;
+      },
+      2,
+      10,
+    );
+
+    expect(keyframes).toContainEqual({ time: 1, volume: 0.8 });
+    expect(keyframes).toContainEqual({ time: 1.1, volume: 0.2 });
+  });
+
+  it("samples a short transition at a clip end between frame intervals", () => {
+    const audio = document.createElement("audio");
+    audio.dataset.start = "0";
+    audio.dataset.duration = "1.05";
+    audio.dataset.volume = "0.7";
+
+    const keyframes = probeElementVolumeKeyframes(
+      audio,
+      (time) => {
+        audio.volume = time < 1.02 ? 0.7 : 0.1;
+      },
+      1.05,
+      10,
+    );
+
+    expect(keyframes).toEqual([
+      { time: 0, volume: 0.7 },
+      { time: 1, volume: 0.7 },
+      { time: 1.05, volume: 0.1 },
+    ]);
+  });
+
+  it("preserves every sampled point of a continuous ramp", () => {
+    const audio = document.createElement("audio");
+    audio.dataset.start = "0";
+    audio.dataset.duration = "0.5";
+    audio.dataset.volume = "0";
+
+    const keyframes = probeElementVolumeKeyframes(
+      audio,
+      (time) => {
+        audio.volume = time * 2;
+      },
+      0.5,
+      10,
+    );
+
+    expect(keyframes).toEqual([
+      { time: 0, volume: 0 },
+      { time: 0.1, volume: 0.2 },
+      { time: 0.2, volume: 0.4 },
+      { time: 0.3, volume: 0.6 },
+      { time: 0.4, volume: 0.8 },
+      { time: 0.5, volume: 1 },
+    ]);
+  });
+});
 
 describe("probeAndCacheElementVolume", () => {
   it("does not seek or cache when live timeline probing is disabled", () => {

packages/core/src/runtime/mediaVolumeEnvelope.ts

@@ -61,6 +61,9 @@ export function interpolateVolumeGain(envelope: VolumeKeyframe[], t: number): nu
   if (envelope.length === 0) return 1;
 
   let segment = 0;
+  // The PCM baker intentionally inlines this lookup with a monotonic cursor
+  // because calling this preview-oriented helper per sample would be O(N×M).
+  // fallow-ignore-next-line code-duplication
   while (segment < envelope.length - 2 && t >= envelope[segment + 1]!.time) {
     segment += 1;
   }
@@ -72,7 +75,49 @@ export function interpolateVolumeGain(envelope: VolumeKeyframe[], t: number): nu
   return a.volume + (b.volume - a.volume) * progress;
 }
 
-// fallow-ignore-next-line complexity
+function recordVolumeSample(
+  keyframes: VolumeKeyframe[],
+  previousSample: VolumeKeyframe | undefined,
+  sample: VolumeKeyframe,
+  isFinalSample: boolean,
+): void {
+  const last = keyframes.at(-1);
+  if (!last || Math.abs(last.volume - sample.volume) > 0.0001) {
+    // Change-only compression must retain the preceding real sample so a
+    // flat run stays flat instead of being interpolated into the next value.
+    // During a continuous ramp, that sample is already the last keyframe.
+    if (last && previousSample && previousSample.time > last.time) {
+      keyframes.push(previousSample);
+    }
+    keyframes.push(sample);
+  } else if (isFinalSample && sample.time > last.time) {
+    keyframes.push(sample);
+  }
+}
+
+function parseFiniteDatasetNumber(value: string | undefined): number | undefined {
+  const parsed = Number.parseFloat(value ?? "");
+  return Number.isFinite(parsed) ? parsed : undefined;
+}
+
+function resolveVolumeProbeWindow(
+  el: HTMLAudioElement | HTMLVideoElement,
+  compositionDuration: number,
+): { start: number; end: number; staticVolume: number } {
+  const start = parseFiniteDatasetNumber(el.dataset.start) ?? 0;
+  const endAttr = parseFiniteDatasetNumber(el.dataset.end);
+  const durAttr = parseFiniteDatasetNumber(el.dataset.duration);
+  let end = compositionDuration;
+  if (endAttr !== undefined && endAttr > start) {
+    end = endAttr;
+  } else if (durAttr !== undefined && durAttr > 0) {
+    end = start + durAttr;
+  }
+  const staticAttr = parseFiniteDatasetNumber(el.dataset.volume) ?? 1;
+  const staticVolume = Math.max(0, Math.min(1, staticAttr));
+  return { start, end, staticVolume };
+}
+
 /**
  * Probe a single media element's volume automation by seeking a GSAP timeline
  * through the element's active window.
@@ -89,18 +134,7 @@ export function probeElementVolumeKeyframes(
   compositionDuration: number,
   sampleFps: number,
 ): VolumeKeyframe[] | null {
-  const start = Number.parseFloat(el.dataset.start ?? "0") || 0;
-  const endAttr = Number.parseFloat(el.dataset.end ?? "");
-  const durAttr = Number.parseFloat(el.dataset.duration ?? "");
-  const end =
-    Number.isFinite(endAttr) && endAttr > start
-      ? endAttr
-      : Number.isFinite(durAttr) && durAttr > 0
-        ? start + durAttr
-        : compositionDuration;
-
-  const staticAttr = Number.parseFloat(el.dataset.volume ?? "");
-  const staticVolume = Number.isFinite(staticAttr) ? Math.max(0, Math.min(1, staticAttr)) : 1;
+  const { start, end, staticVolume } = resolveVolumeProbeWindow(el, compositionDuration);
 
   // Reset to data-volume so GSAP captures the correct FROM value.
   el.volume = staticVolume;
@@ -110,15 +144,19 @@ export function probeElementVolumeKeyframes(
   const sampleEnd = Math.min(compositionDuration, end);
 
   const keyframes: VolumeKeyframe[] = [];
-  for (let t = sampleStart; t <= sampleEnd + 1e-6; t += step) {
+  let previousSample: VolumeKeyframe | undefined;
+  for (let t = sampleStart; t <= sampleEnd + 1e-6; t = Math.min(sampleEnd, t + step)) {
     const bounded = Math.min(sampleEnd, t);
     seekTimeline(bounded);
     const raw = Number(el.volume);
-    if (!Number.isFinite(raw)) continue;
-    const volume = Math.max(0, Math.min(1, raw));
-    const last = keyframes.at(-1);
-    if (!last || Math.abs(last.volume - volume) > 0.0001 || bounded === sampleEnd) {
-      keyframes.push({ time: Number(bounded.toFixed(6)), volume: Number(volume.toFixed(6)) });
+    if (Number.isFinite(raw)) {
+      const volume = Math.max(0, Math.min(1, raw));
+      const sample = {
+        time: Number(bounded.toFixed(6)),
+        volume: Number(volume.toFixed(6)),
+      };
+      recordVolumeSample(keyframes, previousSample, sample, bounded === sampleEnd);
+      previousSample = sample;
     }
     if (bounded === sampleEnd) break;
   }

packages/producer/src/services/htmlCompiler.test.ts

@@ -4,6 +4,7 @@ import { mkdtempSync, writeFileSync, mkdirSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { parseHTML } from "linkedom";
+import { interpolateVolumeGain } from "@hyperframes/core/media-volume-envelope";
 import { defaultLogger } from "../logger.js";
 import {
   collectExternalAssets,
@@ -1763,6 +1764,58 @@ h1 { font-size: 2rem; }`;
 });
 
 describe("discoverAudioVolumeAutomationFromTimeline", () => {
+  it("emits plateau boundaries around a sampled volume change", async () => {
+    class TestAudioElement {
+      id = "music";
+      dataset = { start: "0", duration: "3", volume: "0.8" };
+      volume = 0.8;
+    }
+    class TestVideoElement {}
+
+    const audio = new TestAudioElement();
+    const previousWindow = globalThis.window;
+    const previousDocument = globalThis.document;
+    const previousAudioElement = globalThis.HTMLAudioElement;
+    const previousVideoElement = globalThis.HTMLVideoElement;
+
+    globalThis.window = {
+      __timelines: {
+        root: {
+          totalTime: (time: number) => {
+            audio.volume = time < 1.05 ? 0.8 : 0.2;
+          },
+        },
+      },
+    } as any;
+    globalThis.document = {
+      querySelector: (selector: string) =>
+        selector === "[data-composition-id]"
+          ? { getAttribute: (name: string) => (name === "data-composition-id" ? "root" : null) }
+          : null,
+      getElementById: (id: string) => (id === "music" ? audio : null),
+    } as any;
+    globalThis.HTMLAudioElement = TestAudioElement as any;
+    globalThis.HTMLVideoElement = TestVideoElement as any;
+
+    try {
+      const page = {
+        evaluate: async (fn: (arg: unknown) => unknown, arg: unknown) => fn(arg),
+      } as any;
+
+      const [automation] = await discoverAudioVolumeAutomationFromTimeline(page, ["music"], 3, 10);
+
+      expect(automation?.keyframes).toContainEqual({ time: 1, volume: 0.8 });
+      expect(automation?.keyframes).toContainEqual({ time: 1.1, volume: 0.2 });
+      expect(interpolateVolumeGain(automation?.keyframes ?? [], 0.5)).toBeCloseTo(0.8, 6);
+      expect(interpolateVolumeGain(automation?.keyframes ?? [], 1.5)).toBeCloseTo(0.2, 6);
+    } finally {
+      globalThis.window = previousWindow;
+      globalThis.document = previousDocument;
+      globalThis.HTMLAudioElement = previousAudioElement;
+      globalThis.HTMLVideoElement = previousVideoElement;
+    }
+  });
+
   it("prefers runtime duration over stale data-end while sampling video-derived audio", async () => {
     class TestAudioElement {}
     class TestVideoElement {

packages/producer/src/services/htmlCompiler.ts

@@ -2147,19 +2147,33 @@ export async function discoverAudioVolumeAutomationFromTimeline(
         }
 
         const keyframes: { time: number; volume: number }[] = [];
-        for (let t = sampleStart; t <= sampleEnd + 0.000001; t += step) {
+        let previousSample: { time: number; volume: number } | undefined;
+        for (let t = sampleStart; t <= sampleEnd + 0.000001; t = Math.min(sampleEnd, t + step)) {
           const boundedTime = Math.min(sampleEnd, t);
           seekTl(boundedTime);
           const rawVolume = Number(el.volume);
-          if (!Number.isFinite(rawVolume)) continue;
+          if (!Number.isFinite(rawVolume)) {
+            if (boundedTime === sampleEnd) break;
+            continue;
+          }
           const volume = Math.max(0, Math.min(1, rawVolume));
+          const sample = {
+            time: Number(boundedTime.toFixed(6)),
+            volume: Number(volume.toFixed(6)),
+          };
           const last = keyframes.at(-1);
-          if (!last || Math.abs(last.volume - volume) > 0.0001 || boundedTime === sampleEnd) {
-            keyframes.push({
-              time: Number(boundedTime.toFixed(6)),
-              volume: Number(volume.toFixed(6)),
-            });
+          if (!last || Math.abs(last.volume - volume) > 0.0001) {
+            // Retain the preceding real sample when compression omitted a flat
+            // run. Continuous ramps already have that sample as their last
+            // keyframe, so their interpolation remains unchanged.
+            if (last && previousSample && previousSample.time > last.time) {
+              keyframes.push(previousSample);
+            }
+            keyframes.push(sample);
+          } else if (boundedTime === sampleEnd && sample.time > last.time) {
+            keyframes.push(sample);
           }
+          previousSample = sample;
           if (boundedTime === sampleEnd) break;
         }