fix(core): key the preview volume envelope to the clip, not the timeline (#3198)

A GSAP volume fade on an audio clip that starts after t=0 left the preview
silent for the clip's whole length while the encoded render was correct.

`mediaVolumeEnvelope` is meant to keep preview and render on one envelope, and
its contract is "normalise, then read with track-relative seconds". The preview
skipped both halves. `probeElementVolumeKeyframes` stamps each keyframe with the
TIMELINE seek time it sampled at, and `normaliseEnvelope` — the function that
rebases those onto the track — had exactly one caller, the renderer's PCM baker.
The preview handed the raw keyframes to `interpolateVolumeGain` along with
`relTime`, so for a clip at t=2 every lookup fell two seconds before the first
keyframe and clamped to its volume: 0 for a fade-in.

Rebase once, where the cache is filled, so the cached envelope has a single
documented time base. Read it with elapsed-time-in-clip rather than `relTime`,
which is a position inside the media SOURCE — it carries `mediaStart` and the
playback rate, and only coincides with the envelope's time base for an
untrimmed clip playing at 1x from zero. That second half also fixes a latent
sibling: a trimmed clip read the wrong envelope point even when it started at 0.

Author: miguel-heygen

packages/core/src/runtime/media.ts

@@ -273,7 +273,13 @@ export function syncRuntimeMedia(params: {
       if (clip.volumeKeyframes && clip.volumeKeyframes.length > 0) {
         // Keyframes probed from the GSAP timeline — same source as the renderer.
         // Use the interpolated envelope value directly; no need to track GSAP changes.
-        authorVolume = clampVolume(interpolateVolumeGain(clip.volumeKeyframes, relTime));
+        // Index by elapsed time on the TIMELINE since the clip began, which is what
+        // a normalised envelope is keyed by (and what the renderer's PCM baker uses).
+        // `relTime` is a position inside the media SOURCE — it carries `mediaStart`
+        // and the playback rate — so it only coincides with the envelope's time base
+        // for an untrimmed clip playing at 1x from t=0.
+        const elapsedInClip = params.timeSeconds - clip.start;
+        authorVolume = clampVolume(interpolateVolumeGain(clip.volumeKeyframes, elapsedInClip));
       } else if (previousRuntimeVolume === undefined) {
         // First tick this clip is active. The transport has already seeked GSAP
         // to the current time (seekTimelineAndAdapters runs before syncRuntimeMedia),

packages/core/src/runtime/mediaVolumeEnvelope.test.ts

@@ -1,6 +1,10 @@
 /** @vitest-environment jsdom */
 import { describe, expect, it } from "vitest";
-import { probeAndCacheElementVolume, probeElementVolumeKeyframes } from "./mediaVolumeEnvelope";
+import {
+  interpolateVolumeGain,
+  probeAndCacheElementVolume,
+  probeElementVolumeKeyframes,
+} from "./mediaVolumeEnvelope";
 
 describe("probeElementVolumeKeyframes", () => {
   it("retains the last plateau sample before a short volume change", () => {
@@ -143,4 +147,40 @@ describe("probeAndCacheElementVolume", () => {
       expect.arrayContaining([expect.objectContaining({ volume: 0 })]),
     );
   });
+
+  it("caches a track-relative envelope for a clip that starts after t=0", () => {
+    // The probe stamps timeline seek times. A clip starting at 2s therefore
+    // yields keyframes at 2.0+, and reading them with track-relative time landed
+    // before the first keyframe and clamped to its volume — 0 for a fade-in, so
+    // the preview stayed silent for the whole clip while the render was correct.
+    const audio = document.createElement("audio");
+    audio.dataset.start = "2";
+    audio.dataset.duration = "1";
+    audio.dataset.volume = "1";
+    document.body.append(audio);
+
+    const timeline = {
+      totalTime(next?: number) {
+        if (next !== undefined) {
+          // 0.05s linear fade-in at the clip's start (timeline t=2).
+          audio.volume = Math.max(0, Math.min(1, (next - 2) / 0.05));
+        }
+        return 0;
+      },
+    };
+    const cache = new WeakMap<HTMLMediaElement, { time: number; volume: number }[]>();
+
+    probeAndCacheElementVolume(audio, timeline, 3, cache);
+
+    const envelope = cache.get(audio);
+    if (!envelope) throw new Error("Expected a cached envelope");
+    expect(envelope[0]).toEqual({ time: 0, volume: 0 });
+    expect(envelope.at(-1)?.time).toBeCloseTo(1, 5);
+
+    // Silent at the clip's start, full once the fade is done, and it stays there.
+    expect(interpolateVolumeGain(envelope, 0)).toBeCloseTo(0, 5);
+    expect(interpolateVolumeGain(envelope, 0.05)).toBeCloseTo(1, 5);
+    expect(interpolateVolumeGain(envelope, 0.5)).toBeCloseTo(1, 5);
+    expect(interpolateVolumeGain(envelope, 1)).toBeCloseTo(1, 5);
+  });
 });

packages/core/src/runtime/mediaVolumeEnvelope.ts

@@ -179,8 +179,15 @@ export interface VolumeProbeOptions {
 }
 
 /**
- * Probe a media element and, if volume automation is detected, store the
- * keyframes in `cache`. Safe to call with a null timeline — returns early.
+ * Probe a media element and, if volume automation is detected, store a
+ * NORMALISED envelope in `cache`. Safe to call with a null timeline — returns
+ * early.
+ *
+ * `probeElementVolumeKeyframes` stamps each keyframe with the timeline seek
+ * time it was sampled at. Everything downstream of this cache — like the
+ * renderer's PCM baker — indexes an envelope by track-relative seconds, so the
+ * rebase belongs here, at the one point that fills the cache, rather than at
+ * each read.
  */
 export function probeAndCacheElementVolume(
   mediaEl: HTMLMediaElement,
@@ -217,6 +224,8 @@ export function probeAndCacheElementVolume(
   const keyframes = probeElementVolumeKeyframes(mediaEl, seekFn, compositionDuration, 60);
   if (Number.isFinite(originalTime)) seekFn(originalTime);
   if (keyframes) {
-    cache.set(mediaEl, keyframes);
+    const { start, staticVolume } = resolveVolumeProbeWindow(mediaEl, compositionDuration);
+    const envelope = normaliseEnvelope(keyframes, start, staticVolume);
+    if (envelope.length > 0) cache.set(mediaEl, envelope);
   }
 }