feat: make creator media edits render-safe (#3322)

* feat: make creator media edits render-safe

* fix: align media playback timing

* docs: add creator editing recipes

* docs: expand creator editing guidance

* fix: unify media source offsets

* fix: scale natural media duration

* fix: preserve natural media zero spans

* fix: align compiled natural media timing

* test: classify compiler media test as integration

* fix: drop inactive media windows

* fix: unify literal timing parsing

* fix: keep browser media parsing serializable

* fix: keep page timing readers strict

* fix: close remaining preview timing gaps

* fix(core): preserve Studio voice pitch at playback speed

* chore: keep creator contract source-neutral

Author: miguel-heygen

packages/core/src/compiler/timingCompiler.test.ts

@@ -19,6 +19,23 @@ it("source contains no raw NUL bytes", () => {
 });
 
 describe("compileTimingAttrs", () => {
+  it.each(["", "   ", "0s", "0abc", "0px", "-1s", "Infinity", "NaN"])(
+    "does not partially parse invalid literal data-duration=%j",
+    (duration) => {
+      const html = `<video id="v1" src="a.mp4" data-start="2" data-duration="${duration}">`;
+      const { html: compiled } = compileTimingAttrs(html);
+
+      expect(compiled).not.toContain("data-end=");
+    },
+  );
+
+  it("uses Number semantics for hexadecimal literal timing", () => {
+    const { html: compiled } = compileTimingAttrs(
+      '<video id="v1" src="a.mp4" data-start="2" data-duration="0x10">',
+    );
+    expect(compiled).toContain('data-end="18"');
+  });
+
   it("adds data-end when data-start and data-duration are present on a video", () => {
     const html = '<video id="v1" src="a.mp4" data-start="2" data-duration="5">';
     const { html: compiled, unresolved } = compileTimingAttrs(html);

packages/core/src/compiler/timingCompiler.ts

@@ -15,6 +15,12 @@
  * and call injectDurations() to complete the compilation.
  */
 
+import {
+  parseStrictFiniteTimingNumber,
+  readElementPlaybackRate,
+  readMediaStart,
+} from "../runtime/playbackRate.js";
+
 // ── Types ────────────────────────────────────────────────────────────────
 
 export interface UnresolvedElement {
@@ -25,6 +31,7 @@ export interface UnresolvedElement {
   end?: number;
   duration?: number;
   mediaStart: number;
+  playbackRate: number;
   compositionSrc?: string;
 }
 
@@ -40,6 +47,7 @@ export interface ResolvedMediaElement {
   start: number;
   duration: number;
   mediaStart: number;
+  playbackRate: number;
   loop: boolean;
 }
 
@@ -92,6 +100,11 @@ function injectAttr(tag: string, attr: string, value: string): string {
   return tag.replace(/>$/, ` ${attr}="${value}">`);
 }
 
+function setAttr(tag: string, attr: string, value: string): string {
+  if (!hasAttr(tag, attr)) return injectAttr(tag, attr, value);
+  return tag.replace(new RegExp(`(${attr}=["'])[^"']*(["'])`), `$1${value}$2`);
+}
+
 // Real media/timing elements never live inside comments, <script>, or <style>.
 // The tag regexes below aren't comment-aware, so a comment that merely mentions
 // `<video>`/`<audio>` gets rewritten as if it were a real element (issue #1938).
@@ -137,14 +150,16 @@ function compileTag(
     startStr = "0";
   }
   const start = parseFloat(startStr);
-  const mediaStartStr = getAttr(result, "data-media-start");
-  const mediaStart = mediaStartStr ? parseFloat(mediaStartStr) : 0;
+  const attrReader = { getAttribute: (name: string) => getAttr(result, name) };
+  const mediaStart = readMediaStart(attrReader);
+  const playbackRate = readElementPlaybackRate(attrReader);
 
   // 1. Compute data-end from data-start + data-duration
   if (!hasAttr(result, "data-end")) {
     const durationStr = getAttr(result, "data-duration");
-    if (durationStr !== null) {
-      const end = start + parseFloat(durationStr);
+    const duration = parseStrictFiniteTimingNumber(durationStr);
+    if (duration != null) {
+      const end = start + duration;
       result = injectAttr(result, "data-end", String(end));
     } else if (id) {
       // No data-duration: mark as unresolved so caller can provide it
@@ -154,6 +169,7 @@ function compileTag(
         src: getAttr(result, "src") ?? undefined,
         start,
         mediaStart,
+        playbackRate,
       };
     }
   }
@@ -215,6 +231,7 @@ export function compileTimingAttrs(html: string): CompilationResult {
         tagName: "div",
         start: startStr ? parseFloat(startStr) : 0,
         mediaStart: 0,
+        playbackRate: 1,
         compositionSrc: compositionSrc ?? undefined,
       });
     }
@@ -241,8 +258,8 @@ export function injectDurations(html: string, resolutions: ResolvedDuration[]):
       let result = tag;
 
       // Add data-duration if missing
-      if (!hasAttr(result, "data-duration")) {
-        result = injectAttr(result, "data-duration", String(duration));
+      if (parseStrictFiniteTimingNumber(getAttr(result, "data-duration")) == null) {
+        result = setAttr(result, "data-duration", String(duration));
       }
 
       // Add data-end if missing
@@ -279,20 +296,21 @@ export function extractResolvedMedia(html: string): ResolvedMediaElement[] {
     const durationStr = getAttr(tag, "data-duration");
     if (!id || durationStr === null) continue;
 
-    const duration = parseFloat(durationStr);
-    if (!Number.isFinite(duration) || duration <= 0) continue;
+    const duration = parseStrictFiniteTimingNumber(durationStr);
+    if (duration == null || duration <= 0) continue;
 
     const isVideo = /^<video/i.test(tag);
     const startStr = getAttr(tag, "data-start");
-    const mediaStartStr = getAttr(tag, "data-media-start");
+    const attrReader = { getAttribute: (name: string) => getAttr(tag, name) };
 
     resolved.push({
       id,
       tagName: isVideo ? "video" : "audio",
       src: getAttr(tag, "src") ?? undefined,
       start: startStr !== null ? parseFloat(startStr) : 0,
       duration,
-      mediaStart: mediaStartStr ? parseFloat(mediaStartStr) : 0,
+      mediaStart: readMediaStart(attrReader),
+      playbackRate: readElementPlaybackRate(attrReader),
       loop: hasAttr(tag, "loop"),
     });
   }

packages/core/src/index.ts

@@ -298,6 +298,14 @@ export {
 // publishConfig entry points at a file the pack doesn't contain
 // (verify:packed-manifests catches exactly that).
 export { createRuntimeStartTimeResolver } from "./runtime/startResolver.js";
+export {
+  normalizePlaybackRate,
+  parseStrictFiniteTimingNumber,
+  readElementPlaybackRate,
+  readMediaStart,
+  resolveNaturalMediaTimelineDuration,
+  resolveNaturalMediaTimelineDurationFromValues,
+} from "./runtime/playbackRate.js";
 
 // Variable validation (CLI / tooling-side)
 export {

packages/core/src/runtime/audioFx.test.ts

@@ -1,7 +1,7 @@
 // @vitest-environment happy-dom
 import { describe, expect, it } from "vitest";
 import { attachElementFxChain } from "./audioFx.js";
-import { defaultAudioFxParams, HF_AUDIO_FX } from "../audioFx.js";
+import { defaultAudioFxParams, HF_AUDIO_FX, HF_AUDIO_FX_ATTR } from "../audioFx.js";
 
 /**
  * The DSP is proven in a real browser by the engine's render tests. What needs
@@ -107,7 +107,9 @@ describe("attachElementFxChain", () => {
     const dst = new Node();
     const handle = attachElementFxChain(
       ctx(),
-      { getAttribute: () => "{not json" },
+      {
+        getAttribute: (name: string) => (name === HF_AUDIO_FX_ATTR ? "{not json" : null),
+      },
       src as never,
       dst as never,
     );

packages/core/src/runtime/clipTree.test.ts

@@ -27,6 +27,17 @@ describe("createClipTree", () => {
     rootDuration: 10,
   };
 
+  it.each(["", "   ", "0s", "0abc", "0px", "-1s", "Infinity", "NaN"])(
+    "keeps a clip with invalid literal data-duration=%j on the fallback window",
+    (duration) => {
+      document.body.innerHTML = `
+        <div data-composition-id="root" data-duration="10" data-start="0" id="root">
+          <div data-start="0" data-duration="${duration}" id="clip"></div>
+        </div>`;
+      expect(createClipTree(params).roots.map((node) => node.id)).toContain("clip");
+    },
+  );
+
   // Regression: id-less children (root index.html uses data-hf-id, not id) must
   // get their data-hf-id as the node id — not a synthetic `__clip-N` — so the
   // tree aligns with __clipManifest (which also keys on data-hf-id) and inline
@@ -47,4 +58,18 @@ describe("createClipTree", () => {
     expect(child!.id).not.toMatch(/^__clip-/);
     expect(child!.parentId).toBe("scene");
   });
+
+  it.each([10, 11])(
+    "does not replace a known zero media span with root duration (start=%s)",
+    (start) => {
+      document.body.innerHTML = `
+      <div data-composition-id="root" data-duration="100" data-start="0" id="root">
+        <video id="at-eof" data-start="0" data-media-start="${start}"></video>
+      </div>`;
+      const video = document.querySelector("video")!;
+      Object.defineProperty(video, "duration", { value: 10, configurable: true });
+
+      expect(createClipTree({ ...params, rootDuration: 100 }).roots).toEqual([]);
+    },
+  );
 });

packages/core/src/runtime/clipTree.ts

@@ -11,6 +11,7 @@
  */
 
 import type { RuntimeTimelineLike } from "./types";
+import { parseStrictFiniteTimingNumber, resolveNaturalMediaTimelineDuration } from "./playbackRate";
 
 export interface ClipNode {
   readonly id: string;
@@ -51,9 +52,7 @@ interface StartResolverLike {
 }
 
 function parseNum(value: string | null): number | null {
-  if (value == null) return null;
-  const n = Number(value);
-  return Number.isFinite(n) ? n : null;
+  return parseStrictFiniteTimingNumber(value);
 }
 
 function durationFromTimeline(
@@ -68,11 +67,7 @@ function durationFromTimeline(
 
 function durationFromMedia(el: Element): number | null {
   if (!(el instanceof HTMLMediaElement) || !Number.isFinite(el.duration)) return null;
-  const mediaStart =
-    parseNum(el.getAttribute("data-playback-start")) ??
-    parseNum(el.getAttribute("data-media-start")) ??
-    0;
-  return el.duration > mediaStart ? el.duration - mediaStart : null;
+  return resolveNaturalMediaTimelineDuration(el, el.duration);
 }
 
 // Used only to filter out zero-duration (decorative) elements at build time.

packages/core/src/runtime/init.test.ts

@@ -110,6 +110,30 @@ describe("initSandboxRuntimeModular", () => {
     window.cancelAnimationFrame = (() => {}) as typeof window.cancelAnimationFrame;
   });
 
+  it.each([
+    ["2x", 5],
+    ["0x2", 10],
+  ])("derives a native-parsed natural media window for rate %s", (rate, expected) => {
+    document.body.innerHTML = `<div data-composition-id="main" data-root="true"><video data-start="0" data-playback-rate="${rate}"></video></div>`;
+    const video = document.querySelector("video")!;
+    Object.defineProperty(video, "duration", { value: 10, configurable: true });
+    window.__timelines = {};
+    initSandboxRuntimeModular();
+    expect(window.__player?.getDuration()).toBe(expected);
+  });
+
+  it.each([10, 11])(
+    "preserves a known zero natural media window at source EOF (start=%s)",
+    (start) => {
+      document.body.innerHTML = `<div data-composition-id="main" data-root="true"><video data-start="0" data-media-start="${start}"></video></div>`;
+      const video = document.querySelector("video")!;
+      Object.defineProperty(video, "duration", { value: 10, configurable: true });
+      window.__timelines = {};
+      initSandboxRuntimeModular();
+      expect(window.__player?.getDuration()).toBe(0);
+    },
+  );
+
   afterEach(() => {
     window.__hfRuntimeTeardown?.();
     document.body.innerHTML = "";