fix(engine,producer,lint): resolve <source> children for media extract and localize

Parent src-only scans skipped multi-format <video>/<audio> markup, so those
elements were never extracted, downloaded, or mixed and rendered blank/silent.
Lint now accepts a child <source src> as a resolvable media src.

Author: valeriangalliat

packages/engine/src/services/audioMixer.test.ts

@@ -80,6 +80,28 @@ describe("parseAudioElements strict literal timing", () => {
   );
 });
 
+describe("parseAudioElements — <source> children", () => {
+  it("discovers audio and audible-video tracks that use <source> children", () => {
+    const tracks = parseAudioElements(`
+      <audio id="bgm" data-start="2" data-end="7">
+        <source src="https://cdn.example.com/bgm.mp3" type="audio/mpeg">
+        <source src="_remote_media/bgm.ogg" type="audio/ogg">
+      </audio>
+      <video id="rec" data-has-audio="true" data-start="4" data-end="10">
+        <source src="https://cdn.example.com/rec.mp4" type="video/mp4">
+        <source src="_remote_media/rec.webm" type="video/webm">
+      </video>
+    `);
+    expect(tracks).toEqual([
+      expect.objectContaining({ id: "bgm", src: "_remote_media/bgm.ogg", type: "audio" }),
+      expect.objectContaining({
+        id: "rec-audio",
+        src: "_remote_media/rec.webm",
+        type: "video",
+      }),
+    ]);
+  });
+});
 describe("processCompositionAudio", () => {
   const tempDirs: string[] = [];
 

packages/engine/src/services/audioMixer.ts

@@ -20,7 +20,7 @@ import {
 import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
 import { formatFfmpegError, runFfmpeg, type RunFfmpegResult } from "../utils/runFfmpeg.js";
 import { unwrapTemplate } from "../utils/htmlTemplate.js";
-import { resolveProjectRelativeSrc } from "./videoFrameExtractor.js";
+import { resolveMediaElementSrc, resolveProjectRelativeSrc } from "./videoFrameExtractor.js";
 import { resolveReferencedStart, type RefResolverEl } from "./referenceResolver.js";
 import { isKnownInactiveTimelineWindow } from "./mediaTimelineWindow.js";
 import type {
@@ -512,7 +512,12 @@ export function parseAudioElements(html: string): AudioElement[] {
   // <audio> and <video data-has-audio> tracks differ only in the emitted id
 
   // and `type`; everything else (timing, layer, volume) is read identically.
-  const build = (el: RefResolverEl, id: string, type: AudioElement["type"]): AudioElement => {
+  const build = (
+    el: RefResolverEl,
+    id: string,
+    src: string,
+    type: AudioElement["type"],
+  ): AudioElement => {
     const playbackRateAttr = el.getAttribute("data-playback-rate");
     const layerAttr = el.getAttribute("data-layer");
     const volumeAttr = el.getAttribute("data-volume");
@@ -524,7 +529,7 @@ export function parseAudioElements(html: string): AudioElement[] {
     const group = groupId ? groupsById.get(groupId) : undefined;
     return {
       id,
-      src: el.getAttribute("src") as string,
+      src,
       start: resolveStart(el),
       end: parseEnd(el.getAttribute("data-end")),
       mediaStart: readMediaStart(el),
@@ -553,20 +558,22 @@ export function parseAudioElements(html: string): AudioElement[] {
   const trackId = (el: RefResolverEl): string | null =>
     el.getAttribute(MEDIA_RENDER_ID_ATTR) || el.getAttribute("id");
 
-  for (const el of document.querySelectorAll("audio[id][src]")) {
+  for (const el of document.querySelectorAll("audio[id]")) {
     const id = trackId(el);
+    const src = resolveMediaElementSrc(el);
     // `memberGroupHidden` is the group's own mute: a hidden BUS drops every
     // member from the mix, the same way `isHidden` drops one track.
-    if (!id || !el.getAttribute("src") || isHidden(el) || memberGroupHidden(el)) continue;
+    if (!id || !src || isHidden(el) || memberGroupHidden(el)) continue;
     if (isKnownInactiveTimelineWindow(el, resolveStart(el))) continue;
-    elements.push(build(el, id, "audio"));
+    elements.push(build(el, id, src, "audio"));
   }
 
-  for (const el of document.querySelectorAll('video[id][src][data-has-audio="true"]')) {
+  for (const el of document.querySelectorAll('video[id][data-has-audio="true"]')) {
     const id = trackId(el);
-    if (!id || !el.getAttribute("src") || isHidden(el)) continue;
+    const src = resolveMediaElementSrc(el);
+    if (!id || !src || isHidden(el)) continue;
     if (isKnownInactiveTimelineWindow(el, resolveStart(el))) continue;
-    elements.push(build(el, `${id}-audio`, "video"));
+    elements.push(build(el, `${id}-audio`, src, "video"));
   }
 
   return elements;

packages/engine/src/services/videoFrameExtractor.test.ts

@@ -860,6 +860,17 @@ describe("parseVideoElements", () => {
       expect(Number.isNaN(v.start)).toBe(false);
     }
   });
+
+  it("discovers <video> elements that use <source> children", () => {
+    const videos = parseVideoElements(
+      '<video id="rec" data-start="1" data-duration="4">' +
+        '<source src="https://cdn.example.com/rec.mp4" type="video/mp4">' +
+        '<source src="_remote_media/rec.webm" type="video/webm">' +
+        "</video>",
+    );
+    expect(videos).toHaveLength(1);
+    expect(videos[0]).toMatchObject({ id: "rec", src: "_remote_media/rec.webm", start: 1, end: 5 });
+  });
 });
 
 describe("FrameLookupTable", () => {

packages/engine/src/services/videoFrameExtractor.ts

@@ -527,16 +527,39 @@ export interface ExtractionResult {
   phaseBreakdown: ExtractionPhaseBreakdown;
 }
 
+/** Minimal structural shape for resolving parent/`<source>` media `src`. */
+interface MediaSrcEl {
+  getAttribute(name: string): string | null;
+  querySelectorAll(selectors: string): Iterable<{ getAttribute(name: string): string | null }>;
+}
+
+/**
+ * Parent `src`, else a `<source src>`. Prefer local paths over http(s) so a
+ * localized sibling wins when another `<source>` failed to download.
+ */
+export function resolveMediaElementSrc(el: MediaSrcEl): string | null {
+  const direct = el.getAttribute("src");
+  if (direct) return direct;
+  let remote: string | null = null;
+  for (const source of el.querySelectorAll("source")) {
+    const src = source.getAttribute("src");
+    if (!src) continue;
+    if (!/^https?:\/\//i.test(src)) return src;
+    remote ??= src;
+  }
+  return remote;
+}
+
 export function parseVideoElements(html: string): VideoElement[] {
   const videos: VideoElement[] = [];
   const { document } = parseHTML(unwrapTemplate(html));
   const startCache = new Map<RefResolverEl, number>();
   const visiting = new Set<RefResolverEl>();
 
-  const videoEls = document.querySelectorAll("video[src]");
+  const videoEls = document.querySelectorAll("video");
   let autoIdCounter = 0;
   for (const el of videoEls) {
-    const src = el.getAttribute("src");
+    const src = resolveMediaElementSrc(el);
     if (!src) continue;
     // Generate a stable ID for videos without one — the producer needs IDs
     // to track extracted frames and composite them during encoding.

packages/lint/src/rules/media.test.ts

@@ -217,6 +217,37 @@ describe("media rules", () => {
     expect(finding?.severity).toBe("error");
   });
 
+  it("accepts <source src> children in place of parent src", async () => {
+    const html = `
+<html><body>
+  <div id="root" data-composition-id="c1" data-width="1920" data-height="1080">
+    <video id="rec" data-start="0" data-duration="4" muted playsinline>
+      <source src="clip.mp4" type="video/mp4">
+      <source src="clip.webm" type="video/webm">
+    </video>
+  </div>
+  <script>window.__timelines = window.__timelines || {}; window.__timelines["c1"] = gsap.timeline({ paused: true });</script>
+</body></html>`;
+    const result = await lintHyperframeHtml(html);
+    expect(result.findings.some((f) => f.code === "media_missing_src")).toBe(false);
+  });
+
+  it("reports error for <source>-only media with no data-start", async () => {
+    const html = `
+<html><body>
+  <div id="root" data-composition-id="c1" data-width="1920" data-height="1080">
+    <video id="rec" muted playsinline>
+      <source src="clip.mp4" type="video/mp4">
+    </video>
+  </div>
+  <script>window.__timelines = window.__timelines || {}; window.__timelines["c1"] = gsap.timeline({ paused: true });</script>
+</body></html>`;
+    const result = await lintHyperframeHtml(html);
+    const finding = result.findings.find((f) => f.code === "media_missing_data_start");
+    expect(finding).toBeDefined();
+    expect(finding?.elementId).toBe("rec");
+  });
+
   it("reports error for media with src but no data-start", async () => {
     const html = `
 <html><body>

packages/lint/src/rules/media.ts

@@ -1,4 +1,4 @@
-import type { LintContext, HyperframeLintFinding } from "../context";
+import type { LintContext, HyperframeLintFinding, OpenTag } from "../context";
 import { readAttr, readDecodedAttr, stripJsComments, truncateSnippet, isMediaTag } from "../utils";
 import { validateColorGradingContract } from "@hyperframes/parsers/color-grading-contract";
 
@@ -41,6 +41,20 @@ function hasAttrName(tagSource: string, attr: string): boolean {
   return new RegExp(`(?:^|\\s)${escaped}(?:\\s*=|\\s|/?>)`, "i").test(attrs);
 }
 
+/** Parent `src`, else a descendant `<source src>` (matches engine resolveMediaElementSrc). */
+function mediaHasResolvableSrc(tag: OpenTag, tags: readonly OpenTag[]): boolean {
+  if (readAttr(tag.raw, "src")) return true;
+  const end = tag.closeIndex ?? tag.endIndex;
+  if (end == null) return false;
+  return tags.some(
+    (child) =>
+      child.name === "source" &&
+      child.index > tag.index &&
+      child.index < end &&
+      Boolean(readAttr(child.raw, "src")),
+  );
+}
+
 function classNamesFromAttr(classAttr: string | null): string[] {
   if (!classAttr) return [];
   return classAttr.split(/\s+/).filter(Boolean);
@@ -499,7 +513,7 @@ export const mediaRules: Array<(ctx: LintContext) => HyperframeLintFinding[]> =
       if (tag.name !== "video" && tag.name !== "audio") continue;
       const hasDataStart = readAttr(tag.raw, "data-start");
       const hasId = readAttr(tag.raw, "id");
-      const hasSrc = readAttr(tag.raw, "src");
+      const hasSrc = mediaHasResolvableSrc(tag, tags);
       if (hasSrc && !hasDataStart) {
         findings.push({
           code: "media_missing_data_start",
@@ -538,9 +552,9 @@ export const mediaRules: Array<(ctx: LintContext) => HyperframeLintFinding[]> =
           findings.push({
             code: "media_missing_src",
             severity: "error",
-            message: `<${tag.name} id="${hasId}"> has data-start but no src attribute. The renderer cannot load this media.`,
+            message: `<${tag.name} id="${hasId}"> has data-start but no src (on the element or a <source> child). The renderer cannot load this media.`,
             elementId: hasId,
-            fixHint: `Add a src attribute to the <${tag.name}> element directly. If using <source> children, the renderer still requires src on the parent element.`,
+            fixHint: `Add src on the <${tag.name}> element, or a <source src="..."> child.`,
             snippet: truncateSnippet(tag.raw),
           });
         }

packages/producer/src/services/htmlCompiler.test.ts

@@ -1545,6 +1545,25 @@ describe("localizeRemoteMediaSources", () => {
     expect(remoteMediaAssets.size).toBe(0);
   });
 
+  it("localizes remote <source> children of a <video>", async () => {
+    const orig = globalThis.fetch;
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    (globalThis as any).fetch = async () => validTestMediaResponse();
+    try {
+      const dl = mkdtempSync(join(tmpdir(), "hf-dl-src-"));
+      const html = `<video id="rec" data-start="0" data-end="5" muted>
+<source src="https://src-ok.example.com/rec.mp4" type="video/mp4">
+<source src="https://src-ok.example.com/rec.webm" type="video/webm">
+</video>`;
+      const { html: result, remoteMediaAssets } = await localizeRemoteMediaSources(html, dl);
+      expect(result).not.toContain("https://src-ok.example.com/");
+      expect(remoteMediaAssets.size).toBe(2);
+    } finally {
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      (globalThis as any).fetch = orig;
+    }
+  });
+
   it("rewrites src in both double-quoted and single-quoted attributes", async () => {
     const orig = globalThis.fetch;
     // eslint-disable-next-line @typescript-eslint/no-explicit-any

packages/producer/src/services/htmlCompiler.ts

@@ -1280,6 +1280,8 @@ const REMOTE_MEDIA_SUBDIR = "_remote_media";
 // have `>` inside quoted attribute values (data-title etc.).
 const REMOTE_MEDIA_TAG_RE =
   /<(?:video|audio)\b[^>]*?\bsrc\s*=\s*["'](https?:\/\/[^"']+)["'][^>]*>/gi;
+// <source src> on media elements (picture uses srcset, not src).
+const REMOTE_SOURCE_TAG_RE = /<source\b[^>]*?\bsrc\s*=\s*["'](https?:\/\/[^"']+)["'][^>]*>/gi;
 // Match <img> tags (including agent-pipeline-emitted variants where `src` is
 // not the first attribute). Producer-side localisation is the primary fix for
 // the remote-<img> flicker; frameCapture's `pollImagesReady`/`decodeAllImages`
@@ -1350,10 +1352,11 @@ async function downloadAndRewriteUrls(
 }
 
 /**
- * Download any remote `src` URLs on `<video>` and `<audio>` elements into a
- * local subdirectory of `downloadDir`, rewrite the HTML src attributes to
- * relative paths, and return the updated HTML along with a map of
- * `{ relativePath → absoluteLocalPath }` for callers to add to `externalAssets`.
+ * Download any remote `src` URLs on `<video>` / `<audio>` elements and their
+ * `<source>` children into a local subdirectory of `downloadDir`, rewrite the
+ * HTML src attributes to relative paths, and return the updated HTML along with
+ * a map of `{ relativePath → absoluteLocalPath }` for callers to add to
+ * `externalAssets`.
  *
  * Skips URLs that fail to download (warns and preserves the original URL so
  * the browser can still attempt the remote fetch as a fallback).
@@ -1369,12 +1372,13 @@ export async function localizeRemoteMediaSources(
   html: string,
   downloadDir: string,
 ): Promise<{ html: string; remoteMediaAssets: Map<string, string> }> {
-  // Collect unique HTTP URLs from <video>/<audio> src attributes.
   const urlSet = new Set<string>();
-  const re = new RegExp(REMOTE_MEDIA_TAG_RE.source, REMOTE_MEDIA_TAG_RE.flags);
-  let m: RegExpExecArray | null;
-  while ((m = re.exec(html)) !== null) {
-    if (m[1]) urlSet.add(m[1]);
+  for (const tagRe of [REMOTE_MEDIA_TAG_RE, REMOTE_SOURCE_TAG_RE]) {
+    const re = new RegExp(tagRe.source, tagRe.flags);
+    let m: RegExpExecArray | null;
+    while ((m = re.exec(html)) !== null) {
+      if (m[1]) urlSet.add(m[1]);
+    }
   }
   return downloadAndRewriteUrls(
     urlSet,