perf(core): pre-decode audio at mount, upcoming-clip readiness, stall telemetry

Cut-boundary playback smoothness for segment-heavy compositions (an editor
emitting one media element per cut):

- init: decode-only warm pass over audio[data-start] as soon as the WebAudio
  transport exists (and periodically for late-loaded sub-compositions), so
  first play after a document (re)load doesn't run full-file decodeAudioData
  lazily while sound falls back to the buffering HTMLMedia element.
- init: staged preload — when a composition carries >6 timed media elements,
  far-future segments (start >10s) begin preload=metadata; the sync layer
  upgrades them as the playhead approaches. Stops the mount-time fetch
  stampede that leaves EARLY segments buffering late.
- media: two readiness stages for upcoming clips while playing: pre-seek to
  the clip's media offset ~3s ahead (buffer the RIGHT region), and a muted
  video pre-roll ~350ms ahead so activation is a visibility flip instead of
  a cold seek+play (decoder reset ~150ms freeze).
- init: bounded runtime_media_stall diagnostics (waiting/stalled while
  playing) + freeze the transport clock when a buffering ACTIVE video has no
  audio mastering the clock (video-only compositions), mirroring the
  existing audio-buffering freeze.

Author: I-bangash

packages/core/src/runtime/init.test.ts

@@ -2683,4 +2683,61 @@ describe("initSandboxRuntimeModular", () => {
       }).not.toThrow();
     });
   });
+
+  describe("audio buffer pre-decode at mount", () => {
+    it("warm-decodes every timed audio source before any play()", async () => {
+      const decodeAudioData = vi.fn(async () => ({}) as AudioBuffer);
+      class MockAudioContext {
+        currentTime = 0;
+        state = "running";
+        destination = {};
+        decodeAudioData = decodeAudioData;
+        createGain() {
+          return { gain: { value: 1 }, connect: vi.fn(), disconnect: vi.fn() };
+        }
+        resume = vi.fn();
+        close = vi.fn();
+      }
+      vi.stubGlobal("AudioContext", MockAudioContext);
+      const fetchMock = vi.fn(async () => ({
+        ok: true,
+        arrayBuffer: async () => new ArrayBuffer(8),
+      }));
+      vi.stubGlobal("fetch", fetchMock);
+
+      try {
+        const root = document.createElement("div");
+        root.setAttribute("data-composition-id", "main");
+        root.setAttribute("data-root", "true");
+        root.setAttribute("data-start", "0");
+        root.setAttribute("data-duration", "10");
+        root.setAttribute("data-width", "1920");
+        root.setAttribute("data-height", "1080");
+        const audioA = document.createElement("audio");
+        audioA.setAttribute("data-start", "0");
+        audioA.setAttribute("data-duration", "5");
+        audioA.setAttribute("src", "https://media.example/a.mp3");
+        const audioB = document.createElement("audio");
+        audioB.setAttribute("data-start", "5");
+        audioB.setAttribute("data-duration", "5");
+        audioB.setAttribute("src", "https://media.example/b.mp3");
+        root.append(audioA, audioB);
+        document.body.appendChild(root);
+        window.__timelines = { main: createMockTimeline(10) };
+
+        initSandboxRuntimeModular();
+
+        // No play() has happened — the decode must be driven by mount alone.
+        await vi.waitFor(() => {
+          expect(fetchMock).toHaveBeenCalledTimes(2);
+          expect(decodeAudioData).toHaveBeenCalledTimes(2);
+        });
+        const urls = fetchMock.mock.calls.map((call) => String(call[0]));
+        expect(urls.some((u) => u.endsWith("a.mp3"))).toBe(true);
+        expect(urls.some((u) => u.endsWith("b.mp3"))).toBe(true);
+      } finally {
+        vi.unstubAllGlobals();
+      }
+    });
+  });
 });

packages/core/src/runtime/init.ts

@@ -174,6 +174,9 @@ export function initSandboxRuntimeModular(): void {
   let webAudioReady = false;
   void webAudio.init().then((ok) => {
     webAudioReady = ok;
+    // Warm the decoded-buffer cache as soon as the transport exists (decode
+    // works on a suspended AudioContext, so no user gesture is needed).
+    if (ok) predecodeAudioClipBuffers();
   });
   // `_auto` is a Studio-internal keyframe marker (an auto-tracked endpoint the
   // parser reads back), NOT an animatable property. Register it as a no-op GSAP
@@ -1808,19 +1811,69 @@ export function initSandboxRuntimeModular(): void {
     }
   };
 
+  // Boundary-readiness telemetry: surface playback stalls (`waiting`/`stalled`
+  // on a timed media element while the transport plays) so cut-boundary tuning
+  // can target the right layer. Bounded per document to avoid diagnostic spam.
+  let mediaStallDiagnosticsPosted = 0;
+  const MAX_MEDIA_STALL_DIAGNOSTICS = 20;
+  const onMediaStallEvent = (event: Event) => {
+    const el = event.currentTarget;
+    if (!(el instanceof HTMLMediaElement)) return;
+    if (!state.isPlaying || state.tornDown) return;
+    if (mediaStallDiagnosticsPosted >= MAX_MEDIA_STALL_DIAGNOSTICS) return;
+    mediaStallDiagnosticsPosted += 1;
+    let bufferedEnd: number | null = null;
+    try {
+      bufferedEnd = el.buffered.length > 0 ? el.buffered.end(el.buffered.length - 1) : null;
+    } catch {
+      // buffered ranges unavailable
+    }
+    postRuntimeMessage({
+      source: "hf-preview",
+      type: "diagnostic",
+      code: "runtime_media_stall",
+      details: {
+        event: event.type,
+        tagName: el.tagName.toLowerCase(),
+        currentSrc: el.currentSrc || null,
+        readyState: el.readyState,
+        networkState: el.networkState,
+        mediaTime: el.currentTime,
+        bufferedEnd,
+        compositionTime: state.currentTime,
+        clipStart: Number.parseFloat(el.dataset.start ?? "") || null,
+      },
+    });
+  };
+
   const unbindMediaMetadataListeners = () => {
     for (const mediaEl of metadataBoundMedia) {
       mediaEl.removeEventListener("loadedmetadata", scheduleMetadataDurationHydration);
       mediaEl.removeEventListener("durationchange", scheduleMetadataDurationHydration);
       mediaEl.removeEventListener("loadedmetadata", onMediaLoadedMetadataForProxy);
       mediaEl.removeEventListener("error", onMediaErrorForProxy);
+      mediaEl.removeEventListener("waiting", onMediaStallEvent);
+      mediaEl.removeEventListener("stalled", onMediaStallEvent);
     }
     metadataBoundMedia.clear();
   };
 
   const bindMediaMetadataListeners = () => {
     if (state.tornDown) return;
     const mediaEls = Array.from(document.querySelectorAll("video, audio")) as HTMLMediaElement[];
+    // Segment-heavy compositions (editors emit one media element per cut) must
+    // not full-preload EVERYTHING at mount: dozens of parallel fetches contend
+    // for the connection pool and the EARLY segments buffer late. Above the
+    // threshold, far-future timed segments start metadata-only; the sync layer
+    // upgrades them to full preload as the playhead approaches (media.ts
+    // readiness stages set preload="auto" ~3s ahead).
+    const EAGER_PRELOAD_MAX_TIMED_MEDIA = 6;
+    const STAGED_PRELOAD_HORIZON_SECONDS = 10;
+    const timedMediaCount = mediaEls.reduce(
+      (count, el) => count + (el.hasAttribute("data-start") ? 1 : 0),
+      0,
+    );
+    const stagePreload = timedMediaCount > EAGER_PRELOAD_MAX_TIMED_MEDIA;
     for (const mediaEl of mediaEls) {
       if (metadataBoundMedia.has(mediaEl)) continue;
       metadataBoundMedia.add(mediaEl);
@@ -1835,6 +1888,8 @@ export function initSandboxRuntimeModular(): void {
       // for <audio> — all guarded inside mediaProxy.ts itself.
       mediaEl.addEventListener("loadedmetadata", onMediaLoadedMetadataForProxy);
       mediaEl.addEventListener("error", onMediaErrorForProxy);
+      mediaEl.addEventListener("waiting", onMediaStallEvent);
+      mediaEl.addEventListener("stalled", onMediaStallEvent);
 
       // Proactive proxy-fallback trigger: consult the codec map and swap
       // BEFORE the eager load() below, so a known-hostile asset never even
@@ -1845,8 +1900,12 @@ export function initSandboxRuntimeModular(): void {
       // Eagerly preload media data so audio/video is buffered before the user
       // clicks play. Without this, the first play() call fires on un-fetched
       // media, producing silence or choppy audio until the browser caches it.
-      if (mediaEl.preload !== "auto") {
-        mediaEl.preload = "auto";
+      // Exception: staged preload for far-future segments (see above).
+      const startAttr = Number.parseFloat(mediaEl.dataset.start ?? "");
+      const farFuture = Number.isFinite(startAttr) && startAttr > STAGED_PRELOAD_HORIZON_SECONDS;
+      const targetPreload = stagePreload && farFuture ? "metadata" : "auto";
+      if (mediaEl.preload !== targetPreload) {
+        mediaEl.preload = targetPreload;
       }
       if (mediaEl.readyState < HTMLMediaElement.HAVE_FUTURE_DATA) {
         mediaEl.load();
@@ -1861,6 +1920,29 @@ export function initSandboxRuntimeModular(): void {
     }
   };
 
+  // Decode-only warm pass over every timed audio source so WebAudio buffers are
+  // ready BEFORE the first play() instead of being decoded lazily inside it.
+  // The lazy path costs a full-file fetch+decodeAudioData per source at the
+  // moment the user hits play — on a freshly (re)loaded document (every editor
+  // edit reloads the preview iframe) that gap plays through the HTMLMedia
+  // fallback, heard as an audio dropout. Pre-decoding overlaps that work with
+  // document load. `decodeAudioElement` dedupes via its buffer cache and
+  // failure blacklist, so repeated calls per element are cheap map hits.
+  // Skipped during render capture: the producer mixes audio with ffmpeg from
+  // source files and never plays through WebAudio.
+  const predecodeAudioClipBuffers = () => {
+    if (state.tornDown || !webAudioReady) return;
+    if (state.nativeMediaSyncDisabled || state.webAudioMediaDisabled) return;
+    if ((window as Window & { __HF_RENDER_CAPTURE_MODE?: boolean }).__HF_RENDER_CAPTURE_MODE) {
+      return;
+    }
+    const audioEls = document.querySelectorAll("audio[data-start]");
+    for (const el of audioEls) {
+      if (!(el instanceof HTMLMediaElement) || !el.isConnected) continue;
+      void webAudio.decodeAudioElement(el);
+    }
+  };
+
   const probeAndCacheVolumeKeyframes = (mediaEl: HTMLMediaElement) => {
     if (volumeKeyframeCache.has(mediaEl)) return;
     probeAndCacheElementVolume(
@@ -2882,6 +2964,9 @@ export function initSandboxRuntimeModular(): void {
       }
       if (transportTickCount % 30 === 0) {
         bindMediaMetadataListeners();
+        // Also warm decode for audio elements discovered after mount (loaded
+        // sub-compositions) or bound before the async WebAudio init resolved.
+        predecodeAudioClipBuffers();
       }
 
       // Sync clock duration with the resolved timeline each tick (catches async
@@ -2933,6 +3018,31 @@ export function initSandboxRuntimeModular(): void {
               break;
             }
           }
+          if (!foundActive) {
+            // No audio is mastering the clock (video-only composition, or the
+            // audio hasn't reached its window). A buffering ACTIVE video should
+            // freeze time the same way buffering audio does above — otherwise
+            // the monotonic clock runs ahead while frames are stuck, and the
+            // drift correction later hard-seeks (decoder reset) to catch up.
+            const videoEls = document.querySelectorAll("video[data-start]");
+            for (const rawEl of videoEls) {
+              if (!(rawEl instanceof HTMLMediaElement) || !rawEl.isConnected) continue;
+              const start = Number.parseFloat(rawEl.dataset.start ?? "");
+              if (!Number.isFinite(start)) continue;
+              const durAttr = Number.parseFloat(rawEl.dataset.duration ?? "");
+              const end = Number.isFinite(durAttr) && durAttr > 0 ? start + durAttr : Infinity;
+              if (state.currentTime < start || state.currentTime >= end) continue;
+              if (
+                !rawEl.paused &&
+                !rawEl.error &&
+                rawEl.readyState < HTMLMediaElement.HAVE_FUTURE_DATA
+              ) {
+                clock.attachAudioSource({ currentTimeSeconds: state.currentTime });
+                foundActive = true;
+              }
+              break;
+            }
+          }
           if (!foundActive && clock.hasAudioSource()) {
             clock.detachAudioSource();
           }

packages/core/src/runtime/media.test.ts

@@ -1045,4 +1045,74 @@ describe("syncRuntimeMedia", () => {
     });
     expect(clip.el.muted).toBe(true);
   });
+
+  describe("upcoming-clip readiness (boundary smoothness)", () => {
+    function readyClip(overrides?: Partial<RuntimeMediaClip>): RuntimeMediaClip {
+      const clip = createMockClip(overrides);
+      Object.defineProperty(clip.el, "readyState", { value: 4, writable: true });
+      return clip;
+    }
+
+    function syncAt(
+      clip: RuntimeMediaClip,
+      timeSeconds: number,
+      extra?: { playing?: boolean; outputMuted?: boolean },
+    ): void {
+      syncRuntimeMedia({
+        clips: [clip],
+        timeSeconds,
+        playing: extra?.playing ?? true,
+        playbackRate: 1,
+        outputMuted: extra?.outputMuted,
+      });
+    }
+
+    it("pre-seeks an upcoming clip to its media offset within the lookahead window", () => {
+      const clip = readyClip({ start: 7, end: 12, mediaStart: 30 });
+      clip.el.preload = "metadata";
+      syncAt(clip, 5);
+      expect(clip.el.currentTime).toBe(30);
+      expect(clip.el.preload).toBe("auto");
+      expect(clip.el.play).not.toHaveBeenCalled();
+    });
+
+    it("leaves clips beyond the lookahead window untouched", () => {
+      const clip = readyClip({ start: 20, end: 30, mediaStart: 30 });
+      clip.el.currentTime = 3;
+      syncAt(clip, 5);
+      expect(clip.el.currentTime).toBe(3);
+      expect(clip.el.play).not.toHaveBeenCalled();
+    });
+
+    it("pre-rolls a mutable upcoming video just before its boundary — playing, muted, not paused", () => {
+      const clip = readyClip({ start: 5.2, end: 12, mediaStart: 30 });
+      syncAt(clip, 5, { outputMuted: true });
+      expect(clip.el.play).toHaveBeenCalled();
+      expect(clip.el.muted).toBe(true);
+      // Rolls in from mediaStart - lead so playback crosses the boundary at
+      // exactly mediaStart — activation then sees ~zero drift (no cold seek).
+      expect(clip.el.currentTime).toBeCloseTo(29.8, 5);
+      expect(clip.el.pause).not.toHaveBeenCalled();
+    });
+
+    it("never pre-rolls a video that would be audible — falls back to pre-seek", () => {
+      const clip = readyClip({ start: 5.2, end: 12, mediaStart: 30 });
+      syncAt(clip, 5);
+      expect(clip.el.play).not.toHaveBeenCalled();
+      expect(clip.el.currentTime).toBe(30);
+    });
+
+    it("skips the pre-roll when the roll-in would cross media time 0", () => {
+      const clip = readyClip({ start: 5.2, end: 12, mediaStart: 0.05 });
+      syncAt(clip, 5, { outputMuted: true });
+      expect(clip.el.play).not.toHaveBeenCalled();
+    });
+
+    it("paused transport leaves upcoming clips untouched", () => {
+      const clip = readyClip({ start: 6, end: 12, mediaStart: 30 });
+      syncAt(clip, 5, { playing: false });
+      expect(clip.el.currentTime).toBe(0);
+      expect(clip.el.play).not.toHaveBeenCalled();
+    });
+  });
 });