Skip to content

Commit a2094eb

Browse files
committed
feat(runtime): render media treatments deterministically
1 parent 6703b7c commit a2094eb

11 files changed

Lines changed: 3087 additions & 299 deletions

File tree

‎packages/core/src/runtime/colorGrading.test.ts‎

Lines changed: 736 additions & 37 deletions
Large diffs are not rendered by default.

‎packages/core/src/runtime/colorGrading.ts‎

Lines changed: 1893 additions & 239 deletions
Large diffs are not rendered by default.

‎packages/core/src/runtime/init.test.ts‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1320,6 +1320,46 @@ describe("initSandboxRuntimeModular", () => {
13201320
expect(video.style.visibility).toBe("hidden");
13211321
});
13221322

1323+
it("allocates color grading only for the active timed media", () => {
1324+
const getContextSpy = vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue(null);
1325+
const root = document.createElement("div");
1326+
root.setAttribute("data-composition-id", "main");
1327+
root.setAttribute("data-root", "true");
1328+
root.setAttribute("data-start", "0");
1329+
root.setAttribute("data-duration", "4");
1330+
root.setAttribute("data-width", "1920");
1331+
root.setAttribute("data-height", "1080");
1332+
document.body.appendChild(root);
1333+
1334+
for (const [id, start] of [
1335+
["first", "0"],
1336+
["second", "2"],
1337+
]) {
1338+
const video = document.createElement("video");
1339+
video.id = id;
1340+
video.setAttribute("data-start", start);
1341+
video.setAttribute("data-duration", "2");
1342+
video.setAttribute("data-color-grading", '{"adjust":{"exposure":0.1}}');
1343+
Object.defineProperty(video, "paused", { value: true, configurable: true });
1344+
Object.defineProperty(video, "readyState", { value: 0, configurable: true });
1345+
video.load = () => {};
1346+
root.appendChild(video);
1347+
}
1348+
1349+
window.__timelines = { main: createMockTimeline(4) };
1350+
initSandboxRuntimeModular();
1351+
1352+
expect(getContextSpy).toHaveBeenCalledTimes(1);
1353+
expect(document.getElementById("first")?.style.visibility).toBe("visible");
1354+
expect(document.getElementById("second")?.style.visibility).toBe("hidden");
1355+
1356+
window.__player?.seek(3);
1357+
1358+
expect(getContextSpy).toHaveBeenCalledTimes(2);
1359+
expect(document.getElementById("first")?.style.visibility).toBe("hidden");
1360+
expect(document.getElementById("second")?.style.visibility).toBe("visible");
1361+
});
1362+
13231363
it("plays scheduled child timelines without a captured root timeline when audio has failed", () => {
13241364
const raf = createManualRaf();
13251365
vi.spyOn(performance, "now").mockImplementation(() => raf.now());
@@ -2122,6 +2162,36 @@ describe("initSandboxRuntimeModular", () => {
21222162
expect(seekTimes.length).toBeGreaterThan(beforeResume);
21232163
});
21242164

2165+
it("redraws animated grading from the transport clock only during playback", () => {
2166+
const raf = createManualRaf();
2167+
vi.spyOn(performance, "now").mockImplementation(() => raf.now());
2168+
window.requestAnimationFrame = raf.requestAnimationFrame as typeof window.requestAnimationFrame;
2169+
window.cancelAnimationFrame = raf.cancelAnimationFrame as typeof window.cancelAnimationFrame;
2170+
2171+
document.body.innerHTML = `
2172+
<div data-composition-id="root" data-duration="5" data-width="1920" data-height="1080"></div>
2173+
`;
2174+
window.__timelines = { root: createMockTimeline(5) };
2175+
initSandboxRuntimeModular();
2176+
2177+
const runtime = (
2178+
window as Window & { __hf?: { colorGrading?: { redrawAnimated: () => number } } }
2179+
).__hf?.colorGrading;
2180+
if (!runtime) throw new Error("Expected color grading runtime");
2181+
const redrawAnimated = vi.spyOn(runtime, "redrawAnimated");
2182+
2183+
raf.step(16);
2184+
expect(redrawAnimated).not.toHaveBeenCalled();
2185+
2186+
window.__player?.play();
2187+
raf.step(16);
2188+
expect(redrawAnimated).toHaveBeenCalledTimes(1);
2189+
2190+
window.__player?.pause();
2191+
raf.step(16);
2192+
expect(redrawAnimated).toHaveBeenCalledTimes(1);
2193+
});
2194+
21252195
it("keeps a usable bound timeline when the registry entry is replaced", () => {
21262196
const raf = createManualRaf();
21272197
vi.spyOn(performance, "now").mockImplementation(() => raf.now());

‎packages/core/src/runtime/init.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2174,6 +2174,7 @@ export function initSandboxRuntimeModular(): void {
21742174
});
21752175
picker.installPickerApi();
21762176

2177+
syncTimedElementVisibility(state.currentTime);
21772178
const colorGrading = createColorGradingRuntime();
21782179
colorGradingRuntime = colorGrading;
21792180
registerRuntimeCleanup(() => {
@@ -2839,6 +2840,9 @@ export function initSandboxRuntimeModular(): void {
28392840
if (clock.isPlaying() || !hasActiveStudioManualEditGesture()) {
28402841
seekTimelineAndAdapters(t);
28412842
}
2843+
if (clock.isPlaying()) {
2844+
colorGrading.redrawAnimated();
2845+
}
28422846

28432847
// Looping is handled at the player layer (<hyperframes-player>),
28442848
// not the runtime. The clock pauses at duration; GSAP's repeat:-1

‎packages/engine/src/services/frameCapture.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2356,6 +2356,14 @@ async function prepareFrameForCapture(
23562356
if (session.onBeforeCapture) {
23572357
await session.onBeforeCapture(page, quantizedTime);
23582358
}
2359+
await page.evaluate(async () => {
2360+
const runtime = (
2361+
window as Window & {
2362+
__hf?: { colorGrading?: { waitForActiveLuts?: () => Promise<number> } };
2363+
}
2364+
).__hf?.colorGrading;
2365+
await runtime?.waitForActiveLuts?.();
2366+
});
23592367
const beforeCaptureMs = Date.now() - beforeCaptureStart;
23602368

23612369
// Page-side compositing three-phase protocol:

‎packages/engine/src/services/screenshotService.test.ts‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -181,9 +181,12 @@ describe("injectVideoFramesBatch replacement layout", () => {
181181
'<html><body><div id="root"><video id="clip" style="position:absolute;inset:0;width:100%;height:100%;object-fit:cover"></video></div></body></html>',
182182
);
183183

184+
const events: string[] = [];
184185
Object.defineProperty(window.HTMLImageElement.prototype, "decode", {
185186
configurable: true,
186-
value: () => Promise.resolve(),
187+
value: async () => {
188+
events.push("decode");
189+
},
187190
});
188191

189192
const video = document.getElementById("clip") as HTMLVideoElement;
@@ -232,6 +235,10 @@ describe("injectVideoFramesBatch replacement layout", () => {
232235
const previousDocument = globals.document;
233236
globals.window = window;
234237
globals.document = document;
238+
const redraw = vi.fn(() => events.push("redraw"));
239+
(window as unknown as { __hf: { colorGrading: { redraw: () => void } } }).__hf = {
240+
colorGrading: { redraw },
241+
};
235242
try {
236243
const page = {
237244
evaluate: async (
@@ -266,6 +273,8 @@ describe("injectVideoFramesBatch replacement layout", () => {
266273
expect(img?.style.right).toBe("auto");
267274
expect(img?.style.bottom).toBe("auto");
268275
expect(img?.style.inset).toBe("auto");
276+
expect(redraw).toHaveBeenCalledOnce();
277+
expect(events).toEqual(["decode", "redraw"]);
269278
});
270279
});
271280

@@ -569,6 +578,10 @@ describe("video-frame injection respects ancestor visibility", () => {
569578
seededImg.classList.add("__render_frame__");
570579
seededImg.style.opacity = "0";
571580
setup.video.parentNode?.insertBefore(seededImg, setup.video.nextSibling);
581+
const setSourceVisibility = vi.fn();
582+
(setup.window as unknown as { __hf: unknown }).__hf = {
583+
colorGrading: { setSourceVisibility },
584+
};
572585

573586
try {
574587
await syncVideoFrameVisibility(passthroughPage(), ["pip"]);
@@ -578,6 +591,7 @@ describe("video-frame injection respects ancestor visibility", () => {
578591

579592
expect(seededImg.style.opacity).toBe("1");
580593
expect(seededImg.style.visibility).toBe("visible");
594+
expect(setSourceVisibility).toHaveBeenCalledWith(setup.video, true);
581595
});
582596

583597
it("syncVideoFrameVisibility shows the replacement <img> when a plain [data-start] host is visibility:hidden", async () => {
@@ -643,6 +657,10 @@ describe("video-frame injection respects ancestor visibility", () => {
643657
seededImg.style.visibility = "visible";
644658
setup.video.parentNode?.insertBefore(seededImg, setup.video.nextSibling);
645659
const setPropertySpy = vi.spyOn(seededImg.style, "setProperty");
660+
const setSourceVisibility = vi.fn();
661+
(setup.window as unknown as { __hf: unknown }).__hf = {
662+
colorGrading: { setSourceVisibility },
663+
};
646664

647665
try {
648666
await syncVideoFrameVisibility(passthroughPage(), ["pip"]);
@@ -652,6 +670,7 @@ describe("video-frame injection respects ancestor visibility", () => {
652670

653671
expect(seededImg.style.visibility).toBe("hidden");
654672
expect(setPropertySpy).toHaveBeenCalledWith("visibility", "hidden", "important");
673+
expect(setSourceVisibility).toHaveBeenCalledWith(setup.video, false);
655674
});
656675

657676
it("applyDomLayerMask does not revive hidden idless timed descendants of a shown layer", async () => {

‎packages/engine/src/services/screenshotService.ts‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -789,6 +789,11 @@ export async function injectVideoFramesBatch(
789789
if (pendingDecodes.length > 0) {
790790
await Promise.all(pendingDecodes);
791791
}
792+
if (injectedIds.length > 0) {
793+
const redraw = (window as Window & { __hf?: { colorGrading?: { redraw?: () => void } } })
794+
.__hf?.colorGrading?.redraw;
795+
redraw?.();
796+
}
792797
return injectedIds;
793798
},
794799
updates,
@@ -825,14 +830,22 @@ export async function syncVideoFrameVisibility(
825830
return false;
826831
};
827832
const active = new Set(ids);
833+
const setColorGradingVisibility = (
834+
window as Window & {
835+
__hf?: {
836+
colorGrading?: { setSourceVisibility?: (target: Element, visible: boolean) => boolean };
837+
};
838+
}
839+
).__hf?.colorGrading?.setSourceVisibility;
828840
const videos = Array.from(
829841
document.querySelectorAll("video[data-start]"),
830842
) as HTMLVideoElement[];
831843
for (const video of videos) {
832844
const img = video.nextElementSibling as HTMLElement | null;
833845
const hasImg = img && img.classList.contains("__render_frame__");
834846
const ancestorHidden = isVisualAncestorHidden(video);
835-
if (active.has(video.id) && !ancestorHidden) {
847+
const visible = active.has(video.id) && !ancestorHidden;
848+
if (visible) {
836849
// Active video: show injected <img>, hide native <video>.
837850
// Do NOT clobber inline opacity here — GSAP-controlled opacity must
838851
// survive until injectVideoFramesBatch reads it via getComputedStyle.
@@ -859,6 +872,7 @@ export async function syncVideoFrameVisibility(
859872
img.style.setProperty("visibility", "hidden", "important");
860873
}
861874
}
875+
setColorGradingVisibility?.(video, visible);
862876
}
863877
},
864878
activeVideoIds,

‎packages/engine/src/services/videoFrameInjector.ts‎

Lines changed: 0 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -148,25 +148,6 @@ function createFrameSourceCache(
148148

149149
export const __testing = { createFrameSourceCache };
150150

151-
async function redrawRuntimeColorGrading(page: Page): Promise<void> {
152-
await page.evaluate(() => {
153-
const hf = (
154-
window as Window & {
155-
__hf?: {
156-
colorGrading?: { redraw?: () => unknown };
157-
};
158-
}
159-
).__hf;
160-
const redraw = hf?.colorGrading?.redraw;
161-
if (typeof redraw !== "function") return;
162-
try {
163-
redraw();
164-
} catch {
165-
// Optional page-side shader layer.
166-
}
167-
});
168-
}
169-
170151
/**
171152
* Creates a BeforeCaptureHook that injects pre-extracted video frames
172153
* into the page, replacing native <video> elements with frame images.
@@ -248,7 +229,6 @@ export function createVideoFrameInjector(
248229
}, time);
249230
}
250231
}
251-
await redrawRuntimeColorGrading(page);
252232
};
253233
}
254234

‎packages/lint/src/rules/media.test.ts‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,114 @@ describe("media rules", () => {
8181
expect(finding).toBeUndefined();
8282
});
8383

84+
it("reports grading controls placed outside their schema sections", async () => {
85+
const html = `
86+
<html><body>
87+
<div id="root" data-composition-id="c1" data-width="1920" data-height="1080">
88+
<video id="v1" data-start="0" data-duration="5" src="clip.mp4" muted data-color-grading='{"preset":"skin-soft","intensity":0.58,"highlights":-0.06,"temperature":0.02}'></video>
89+
</div>
90+
<script>window.__timelines = {};</script>
91+
</body></html>`;
92+
const result = await lintHyperframeHtml(html);
93+
const finding = result.findings.find((f) => f.code === "color_grading_invalid_structure");
94+
expect(finding?.severity).toBe("error");
95+
expect(finding?.message).toContain("highlights");
96+
expect(finding?.fixHint).toContain('"adjust"');
97+
});
98+
99+
it("accepts grading controls inside their schema sections", async () => {
100+
const html = `
101+
<html><body>
102+
<div id="root" data-composition-id="c1" data-width="1920" data-height="1080">
103+
<video id="v1" data-start="0" data-duration="5" src="clip.mp4" muted data-color-grading='{"preset":"skin-soft","intensity":0.58,"adjust":{"highlights":-0.06,"temperature":0.02},"details":{"vignette":0.03},"effects":{"blur":0.1,"chromaBleed":0.2,"tapeDamage":0.3,"tapeTracking":0.4,"tapeNoise":0.5,"tapeSpeed":0.6,"filmArtifacts":0.4,"halftone":0.5,"halftoneSize":0.6,"twoInkPrint":0.7,"twoInkPrintSize":0.8,"ascii":0.9,"asciiSize":0.4,"asciiInvert":1,"dither":0.8,"ditherSize":0.3,"bloom":0.5,"bloomRadius":8,"asciiStyle":4,"asciiColor":1,"asciiRotation":1,"monoScreen":0.5,"monoScreenSize":0.4,"monoScreenAngle":0.3,"monoScreenSpread":0.2,"monoScreenShape":3,"monoScreenInvert":1,"scanlines":0.3,"scanlineCount":0.4,"scanlineSoftness":0.5,"chromaticAberration":0.2,"chromaticAngle":0.6,"crtCurvature":0.25,"digitalGlitch":0.4,"digitalGlitchColorSplit":0.45,"digitalGlitchLineTear":0.5,"digitalGlitchPixelate":0.55,"digitalGlitchBlockAmount":0.6,"digitalGlitchBlockDisplacement":0.7,"digitalGlitchBlockOpacity":0.2,"digitalGlitchSpeed":0.7,"engraving":1,"engravingSpacing":0.4117647,"engravingMinThickness":0.2,"engravingMaxThickness":0.4571429,"engravingAngle":0.25,"engravingContrast":0.4666667,"engravingSharpness":0.59,"engravingWave":0.2,"engravingWaveFrequency":0.2222222},"palette":["#ff6b66","#080717","#d9339f","#3c185f"],"lut":null}'></video>
104+
</div>
105+
<script>window.__timelines = {};</script>
106+
</body></html>`;
107+
const result = await lintHyperframeHtml(html);
108+
expect(result.findings.find((f) => f.code.startsWith("color_grading_"))).toBeUndefined();
109+
});
110+
111+
it("accepts crosshatch controls in the effects section", async () => {
112+
const html = `
113+
<html><body>
114+
<div id="root" data-composition-id="c1" data-width="1920" data-height="1080">
115+
<video data-start="0" data-duration="5" src="clip.mp4" muted data-color-grading='{"effects":{"crosshatch":1,"crosshatchSpacing":0.28,"crosshatchThickness":0.25,"crosshatchAngle":0.25,"crosshatchContrast":0.3333333,"crosshatchEdges":0.5,"crosshatchLineWeight":0,"crosshatchWave":0.33,"crosshatchWaveFrequency":0.2222222}}'></video>
116+
</div>
117+
<script>window.__timelines = {};</script>
118+
</body></html>`;
119+
const result = await lintHyperframeHtml(html);
120+
expect(result.findings.find((f) => f.code.startsWith("color_grading_"))).toBeUndefined();
121+
});
122+
123+
it("accepts Kuwahara controls in the effects section", async () => {
124+
const html = `
125+
<html><body>
126+
<div id="root" data-composition-id="c1" data-width="1920" data-height="1080">
127+
<video data-start="0" data-duration="5" src="clip.mp4" muted data-color-grading='{"effects":{"kuwahara":1,"kuwaharaRadius":0.142857,"kuwaharaSharpness":0.3125,"kuwaharaSaturation":0.5}}'></video>
128+
</div>
129+
<script>window.__timelines = {};</script>
130+
</body></html>`;
131+
const result = await lintHyperframeHtml(html);
132+
expect(result.findings.find((f) => f.code.startsWith("color_grading_"))).toBeUndefined();
133+
});
134+
135+
it("reports malformed or out-of-range color grading palettes", async () => {
136+
const html = `
137+
<html><body>
138+
<div id="root" data-composition-id="c1" data-width="1920" data-height="1080">
139+
<video id="v1" data-start="0" data-duration="5" src="clip.mp4" muted data-color-grading='{"effects":{"dither":1},"palette":["#000000","red"]}'></video>
140+
</div>
141+
<script>window.__timelines = {};</script>
142+
</body></html>`;
143+
const result = await lintHyperframeHtml(html);
144+
const finding = result.findings.find((f) => f.code === "color_grading_invalid_structure");
145+
expect(finding?.severity).toBe("error");
146+
expect(finding?.fixHint).toContain("2 to 6");
147+
expect(finding?.fixHint).toContain("#RRGGBB");
148+
});
149+
150+
it("reports malformed grading JSON", async () => {
151+
const html = `
152+
<html><body>
153+
<div id="root" data-composition-id="c1" data-width="1920" data-height="1080">
154+
<video id="v1" data-start="0" data-duration="5" src="clip.mp4" muted data-color-grading='{"preset":"skin-soft"'></video>
155+
</div>
156+
<script>window.__timelines = {};</script>
157+
</body></html>`;
158+
const result = await lintHyperframeHtml(html);
159+
expect(result.findings.find((f) => f.code === "color_grading_invalid_json")?.severity).toBe(
160+
"error",
161+
);
162+
});
163+
164+
it("reports invalid string values for structured grading sections", async () => {
165+
const html = `
166+
<html><body>
167+
<div id="root" data-composition-id="c1" data-width="1920" data-height="1080">
168+
<video id="v1" data-start="0" data-duration="5" src="clip.mp4" muted data-color-grading='{"adjust":"cinematic"}'></video>
169+
</div>
170+
<script>window.__timelines = {};</script>
171+
</body></html>`;
172+
const result = await lintHyperframeHtml(html);
173+
expect(
174+
result.findings.find((f) => f.code === "color_grading_invalid_structure")?.severity,
175+
).toBe("error");
176+
});
177+
178+
it("reports color grading on non-media elements", async () => {
179+
const html = `
180+
<html><body>
181+
<div id="root" data-composition-id="c1" data-width="1920" data-height="1080">
182+
<div id="background" data-color-grading='{"preset":"skin-soft"}'></div>
183+
</div>
184+
<script>window.__timelines = {};</script>
185+
</body></html>`;
186+
const result = await lintHyperframeHtml(html);
187+
expect(result.findings.find((f) => f.code === "color_grading_non_media")?.severity).toBe(
188+
"error",
189+
);
190+
});
191+
84192
it("reports warning for media with preload=none", async () => {
85193
const html = `
86194
<html><body>

0 commit comments

Comments
 (0)