Skip to content

Commit 0c9d234

Browse files
Merge pull request #3481 from heygen-com/fix/web-audio-cross-origin-silence-v2
fix(core): prevent cross-origin Web Audio capture from silencing audio
2 parents a7e8674 + f7fc001 commit 0c9d234

12 files changed

Lines changed: 1136 additions & 40 deletions

File tree

‎packages/cli/src/utils/checkBrowser.test.ts‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -436,6 +436,53 @@ it("surfaces the runtime's media-proxy-unavailable console.info line as its own
436436
);
437437
});
438438

439+
it("surfaces the runtime's web-audio-bypass console.info line as its own info finding", async () => {
440+
// The whole complaint in #3458 is that nothing was reported. The runtime
441+
// emits this from media DISCOVERY, not from playback scheduling, precisely
442+
// because `check` seeks and never calls play() — a diagnostic raised from
443+
// the transport would never reach this scraper.
444+
vi.spyOn(Date, "now").mockReturnValue(100);
445+
mountCanvasFixture();
446+
const page = fakePage();
447+
const bypassMessage = fakeConsoleMessage(
448+
"info",
449+
'[hyperframes] runtime_web_audio_bypass: "https://cdn.example.com/track.mp3" ' +
450+
"(cross_origin_no_cors): Web Audio capture withheld; the track plays through native " +
451+
"HTMLMediaElement output. Native playback cannot reproduce: fx-chain — proxy or download " +
452+
"the asset to a same-origin URL to keep it.",
453+
);
454+
const authorInfo = fakeConsoleMessage("info", "debug runtime_web_audio_bypass lookalike");
455+
page.on = vi.fn(
456+
(event: string, handler: (message: ReturnType<typeof fakeConsoleMessage>) => void) => {
457+
if (event === "console") {
458+
handler(bypassMessage);
459+
handler(authorInfo);
460+
}
461+
},
462+
);
463+
installSessionMock(page);
464+
465+
const result = await runBrowserCheck(
466+
PROJECT,
467+
{ ...DEFAULT_CHECK_OPTIONS, samples: 1, contrast: false },
468+
{ kind: "none" },
469+
runAuditGrid,
470+
);
471+
472+
expect(result.runtimeFindings).toContainEqual(
473+
expect.objectContaining({
474+
code: "web_audio_bypass",
475+
severity: "info",
476+
message: bypassMessage.text(),
477+
}),
478+
);
479+
// Prefix-anchored, so a composition author's own console.info that merely
480+
// mentions the code is not promoted into a finding.
481+
expect(result.runtimeFindings.some((finding) => finding.message === authorInfo.text())).toBe(
482+
false,
483+
);
484+
});
485+
439486
it("elevates and deduplicates WebGPU validation warnings while preserving ordinary warnings", async () => {
440487
vi.spyOn(Date, "now").mockReturnValue(100);
441488
mountCanvasFixture();

‎packages/cli/src/utils/checkBrowser.ts‎

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -264,6 +264,11 @@ export async function captureFindingCrops(
264264
// `console.info` from a composition author's own script must not.
265265
const MEDIA_PROXY_MARKER_PREFIX = "[hyperframes] runtime_media_proxy_";
266266
const MEDIA_PROXY_UNAVAILABLE_MARKER = "[hyperframes] runtime_media_proxy_unavailable";
267+
// `reportWebAudioMediaRoute` (packages/core/src/runtime/webAudioRoute.ts) uses
268+
// the same code-in-the-console-line contract. It is emitted from the media
269+
// DISCOVERY phase rather than from playback scheduling, precisely so this
270+
// scraper can see it — `check` seeks, it never plays.
271+
const WEB_AUDIO_BYPASS_MARKER = "[hyperframes] runtime_web_audio_bypass";
267272
const WEBGPU_RUNTIME_FAILURE =
268273
/\b(?:GPUValidationError|GPUOutOfMemoryError|GPUInternalError)\b|WebGPU uncaptured error|(?:destroyed\b.*\b(?:GPU )?(?:resource|buffer|texture)\b.*\bsubmit)|(?:(?:GPU )?(?:resource|buffer|texture)\b.*\bdestroyed\b.*\bsubmit)/i;
269274

@@ -290,6 +295,20 @@ function pushRuntimeDraft(drafts: RuntimeDraft[], draft: RuntimeDraft): void {
290295
drafts.push({ ...draft, count: 1 });
291296
}
292297

298+
/**
299+
* The finding code for a runtime-emitted `console.info` line, or null for the
300+
* ordinary info logging a composition author's own script produces. Matching is
301+
* prefix-anchored on the stable diagnostic codes the runtime deliberately embeds
302+
* in the text, so a line that merely mentions one is not promoted.
303+
*/
304+
function runtimeInfoFindingCode(text: string): string | null {
305+
if (text.startsWith(WEB_AUDIO_BYPASS_MARKER)) return "web_audio_bypass";
306+
if (!text.startsWith(MEDIA_PROXY_MARKER_PREFIX)) return null;
307+
return text.includes(MEDIA_PROXY_UNAVAILABLE_MARKER)
308+
? "media_proxy_unavailable"
309+
: "media_proxy_fallback";
310+
}
311+
293312
function wireRuntimeListeners(page: Page, drafts: RuntimeDraft[], currentTime: () => number): void {
294313
page.on("console", (message) => {
295314
const type = message.type();
@@ -315,12 +334,12 @@ function wireRuntimeListeners(page: Page, drafts: RuntimeDraft[], currentTime: (
315334
url: location.url,
316335
line: location.lineNumber,
317336
});
318-
} else if (type === "info" && text.startsWith(MEDIA_PROXY_MARKER_PREFIX)) {
337+
} else if (type === "info") {
338+
const code = runtimeInfoFindingCode(text);
339+
if (!code) return;
319340
const location = message.location();
320341
pushRuntimeDraft(drafts, {
321-
code: text.includes(MEDIA_PROXY_UNAVAILABLE_MARKER)
322-
? "media_proxy_unavailable"
323-
: "media_proxy_fallback",
342+
code,
324343
severity: "info",
325344
message: text,
326345
time: currentTime(),

‎packages/core/package-subpaths.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -236,6 +236,12 @@
236236
"types": "./dist/runtime/stackingContext.d.ts",
237237
"environments": ["browser", "bun", "node"]
238238
},
239+
"./runtime/web-audio-route": {
240+
"source": "./src/runtime/webAudioRoute.ts",
241+
"runtime": "./dist/runtime/webAudioRoute.js",
242+
"types": "./dist/runtime/webAudioRoute.d.ts",
243+
"environments": ["browser", "bun", "node"]
244+
},
239245
"./compiler/html-document": {
240246
"source": "./src/compiler/htmlDocument.ts",
241247
"runtime": "./dist/compiler/htmlDocument.js",

‎packages/core/package.json‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -245,6 +245,12 @@
245245
"import": "./src/runtime/stackingContext.ts",
246246
"types": "./src/runtime/stackingContext.ts"
247247
},
248+
"./runtime/web-audio-route": {
249+
"bun": "./src/runtime/webAudioRoute.ts",
250+
"node": "./dist/runtime/webAudioRoute.js",
251+
"import": "./src/runtime/webAudioRoute.ts",
252+
"types": "./src/runtime/webAudioRoute.ts"
253+
},
248254
"./compiler/html-document": {
249255
"bun": "./src/compiler/htmlDocument.ts",
250256
"node": "./dist/compiler/htmlDocument.js",
@@ -539,6 +545,10 @@
539545
"import": "./dist/runtime/stackingContext.js",
540546
"types": "./dist/runtime/stackingContext.d.ts"
541547
},
548+
"./runtime/web-audio-route": {
549+
"import": "./dist/runtime/webAudioRoute.js",
550+
"types": "./dist/runtime/webAudioRoute.d.ts"
551+
},
542552
"./compiler/html-document": {
543553
"import": "./dist/compiler/htmlDocument.js",
544554
"types": "./dist/compiler/htmlDocument.d.ts"

‎packages/core/src/runtime/init.test.ts‎

Lines changed: 216 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3053,4 +3053,220 @@ describe("initSandboxRuntimeModular", () => {
30533053
}).not.toThrow();
30543054
});
30553055
});
3056+
3057+
// #3458: cross-origin media with no CORS opt-in. `createMediaElementSource`
3058+
// returns a node that outputs silence per the Web Audio spec rather than
3059+
// throwing, so the composition played through with visuals animating and no
3060+
// sound, and nothing was logged.
3061+
describe("cross-origin audio without a CORS opt-in", () => {
3062+
// `WebAudioTransport.init()` does `new AudioContext()`, which jsdom does not
3063+
// provide — without a stub it returns false, `webAudioReady` stays false,
3064+
// and `scheduleWebAudioForActiveClips` is never reached at all, so every
3065+
// assertion below would pass for the wrong reason.
3066+
class MockAudioContext {
3067+
currentTime = 0;
3068+
state = "running";
3069+
destination = {};
3070+
resume() {
3071+
return Promise.resolve();
3072+
}
3073+
createGain() {
3074+
return { gain: { value: 1 }, connect() {}, disconnect() {} };
3075+
}
3076+
}
3077+
const originalAudioContext = (globalThis as Record<string, unknown>).AudioContext;
3078+
3079+
beforeEach(() => {
3080+
(globalThis as Record<string, unknown>).AudioContext = MockAudioContext;
3081+
});
3082+
3083+
afterEach(() => {
3084+
(globalThis as Record<string, unknown>).AudioContext = originalAudioContext;
3085+
});
3086+
3087+
/** `webAudio.init()` resolves on a microtask, so `webAudioReady` is still
3088+
* false on the tick `initSandboxRuntimeModular()` returns. */
3089+
async function startPlayback() {
3090+
initSandboxRuntimeModular();
3091+
await Promise.resolve();
3092+
window.__player?.play();
3093+
await Promise.resolve();
3094+
await Promise.resolve();
3095+
}
3096+
3097+
function mountAudio(src: string, attrs: Record<string, string> = {}) {
3098+
const root = document.createElement("div");
3099+
root.setAttribute("data-composition-id", "main");
3100+
root.setAttribute("data-root", "true");
3101+
root.setAttribute("data-start", "0");
3102+
root.setAttribute("data-duration", "10");
3103+
root.setAttribute("data-width", "1920");
3104+
root.setAttribute("data-height", "1080");
3105+
document.body.appendChild(root);
3106+
3107+
const audio = document.createElement("audio");
3108+
audio.setAttribute("data-start", "0");
3109+
audio.setAttribute("data-duration", "10");
3110+
audio.setAttribute("src", src);
3111+
for (const [name, value] of Object.entries(attrs)) audio.setAttribute(name, value);
3112+
audio.load = () => {};
3113+
audio.play = vi.fn(() => Promise.resolve());
3114+
root.appendChild(audio);
3115+
3116+
window.__timelines = { main: createMockTimeline(10) };
3117+
return audio;
3118+
}
3119+
3120+
it("withholds Web Audio capture but still tries decode, which keeps the FX graph", async () => {
3121+
// Decode is the BEST outcome here, not a consolation: a CDN that sends
3122+
// `Access-Control-Allow-Origin` while the author simply never wrote the
3123+
// `crossorigin` attribute decodes fine, and that route keeps every
3124+
// effect and automation lane the media-element route would have had.
3125+
const audio = mountAudio("https://cdn.example.com/track.mp3");
3126+
vi.spyOn(console, "info").mockImplementation(() => {});
3127+
const captureSpy = vi.spyOn(WebAudioTransport.prototype, "scheduleMediaElementPlayback");
3128+
const decodeSpy = vi
3129+
.spyOn(WebAudioTransport.prototype, "decodeAudioElement")
3130+
.mockResolvedValue(null);
3131+
3132+
await startPlayback();
3133+
3134+
expect(captureSpy).not.toHaveBeenCalled();
3135+
expect(decodeSpy).toHaveBeenCalledWith(audio);
3136+
});
3137+
3138+
it("leaves the element audible on native output when decode also fails", async () => {
3139+
const audio = mountAudio("https://cdn.example.com/track.mp3");
3140+
vi.spyOn(console, "info").mockImplementation(() => {});
3141+
vi.spyOn(WebAudioTransport.prototype, "decodeAudioElement").mockResolvedValue(null);
3142+
3143+
await startPlayback();
3144+
3145+
// The three things that add up to "the user hears it".
3146+
expect(audio.muted).toBe(false);
3147+
expect(audio.volume).toBeGreaterThan(0);
3148+
expect(audio.play).toHaveBeenCalled();
3149+
expect(window.__player?.isPlaying()).toBe(true);
3150+
});
3151+
3152+
it("does not fail closed into silence for an FX track it deliberately withheld", async () => {
3153+
// The pre-existing non-unit-rate rule mutes a processed track rather than
3154+
// let it lose its graph. On this route capture was withheld ON PURPOSE
3155+
// and native output IS the fix, so muting would hand back the exact
3156+
// silence being fixed — now with the runtime's blessing.
3157+
const audio = mountAudio("https://cdn.example.com/track.mp3", {
3158+
"data-fx-chain": "[]",
3159+
"data-playback-rate": "2",
3160+
});
3161+
vi.spyOn(console, "info").mockImplementation(() => {});
3162+
vi.spyOn(WebAudioTransport.prototype, "decodeAudioElement").mockResolvedValue(null);
3163+
3164+
await startPlayback();
3165+
3166+
expect(audio.muted).toBe(false);
3167+
});
3168+
3169+
it("reports the bypass at media discovery, without anyone calling play()", () => {
3170+
// `hyperframes check` seeks, it never plays. A diagnostic raised only
3171+
// from the schedule path would be invisible to the one gate whose job is
3172+
// to surface this.
3173+
mountAudio("https://cdn.example.com/track.mp3", { "data-fx-chain": "[]" });
3174+
const info = vi.spyOn(console, "info").mockImplementation(() => {});
3175+
3176+
initSandboxRuntimeModular();
3177+
3178+
const line = info.mock.calls.find(([first]) =>
3179+
String(first).includes("runtime_web_audio_bypass"),
3180+
);
3181+
expect(line).toBeDefined();
3182+
// Names what native playback cannot carry, so the author knows the track
3183+
// is audible but no longer processed.
3184+
expect(String(line?.[0])).toContain("fx-chain");
3185+
});
3186+
3187+
it("says nothing about a cross-origin <video>, which never routes through Web Audio", () => {
3188+
const root = document.createElement("div");
3189+
root.setAttribute("data-composition-id", "main");
3190+
root.setAttribute("data-root", "true");
3191+
root.setAttribute("data-width", "1920");
3192+
root.setAttribute("data-height", "1080");
3193+
document.body.appendChild(root);
3194+
const video = document.createElement("video");
3195+
video.setAttribute("data-start", "0");
3196+
video.setAttribute("src", "https://cdn.example.com/clip.mp4");
3197+
video.load = () => {};
3198+
root.appendChild(video);
3199+
window.__timelines = { main: createMockTimeline(10) };
3200+
const info = vi.spyOn(console, "info").mockImplementation(() => {});
3201+
3202+
initSandboxRuntimeModular();
3203+
3204+
expect(
3205+
info.mock.calls.some(([first]) => String(first).includes("runtime_web_audio_bypass")),
3206+
).toBe(false);
3207+
});
3208+
3209+
it("still routes same-origin audio through Web Audio", async () => {
3210+
const audio = mountAudio("/assets/vo.mp3");
3211+
const captureSpy = vi
3212+
.spyOn(WebAudioTransport.prototype, "scheduleMediaElementPlayback")
3213+
.mockResolvedValue(null);
3214+
3215+
await startPlayback();
3216+
3217+
expect(captureSpy).toHaveBeenCalledTimes(1);
3218+
expect(captureSpy.mock.calls[0]?.[0]).toBe(audio);
3219+
});
3220+
3221+
// The fail-closed rule and the bypass diagnostic answer different
3222+
// questions, so they deliberately test different attributes. The
3223+
// diagnostic lists everything native output cannot carry; the rule below
3224+
// only decides whether losing the FX graph is worse than silence.
3225+
describe("the non-unit-rate fail-closed rule keeps its original scope", () => {
3226+
function playWithFailedCapture() {
3227+
vi.spyOn(WebAudioTransport.prototype, "scheduleMediaElementPlayback").mockResolvedValue(
3228+
null,
3229+
);
3230+
vi.spyOn(WebAudioTransport.prototype, "decodeAudioElement").mockResolvedValue(null);
3231+
return startPlayback();
3232+
}
3233+
3234+
it("still mutes an fx-chain track whose capture failed at a non-unit rate", async () => {
3235+
const audio = mountAudio("/assets/vo.mp3", {
3236+
"data-fx-chain": "[]",
3237+
"data-playback-rate": "2",
3238+
});
3239+
3240+
await playWithFailedCapture();
3241+
3242+
expect(audio.muted).toBe(true);
3243+
});
3244+
3245+
it("leaves a grouped track audible, as it was before #3458", async () => {
3246+
// Group membership is reported as unexpressible on the bypass route,
3247+
// but it was never part of the fail-closed pair. Folding it in here
3248+
// would silence a same-origin grouped clip at a non-unit rate that
3249+
// plays today — a behaviour change #3458 does not call for.
3250+
const audio = mountAudio("/assets/vo.mp3", {
3251+
"data-audio-group": "vo",
3252+
"data-playback-rate": "2",
3253+
});
3254+
3255+
await playWithFailedCapture();
3256+
3257+
expect(audio.muted).toBe(false);
3258+
});
3259+
3260+
it("leaves an above-unity data-volume track audible", async () => {
3261+
const audio = mountAudio("/assets/vo.mp3", {
3262+
"data-volume": "2",
3263+
"data-playback-rate": "2",
3264+
});
3265+
3266+
await playWithFailedCapture();
3267+
3268+
expect(audio.muted).toBe(false);
3269+
});
3270+
});
3271+
});
30563272
});

0 commit comments

Comments
 (0)