feat(studio): give an agent eyes with studio_frame

Renders the composition to a PNG at a given time and returns the URL. This is
what turns the tool set from a remote control into a loop: author a change,
capture the instant it affects, look, adjust. No agent can judge motion from
source, because "what does this look like at 2.4 seconds" is not a question a
file answers.

Reuses Studio's existing capture endpoint via `buildFrameCaptureUrl` rather
than inventing a second one.

Two things this does not fake:

It reports the time the playhead LANDED on, not the time requested. The player
clamps, so those differ at the ends, and attaching the wrong time to a frame is
how an agent draws a confident wrong conclusion about motion.

It waits before capturing, by default 150ms. The frame is rendered from the
file on disk, and the render cache is cleared by a file watcher with a 40ms
write-stability threshold, so a capture that beats the watcher renders the
PRE-edit composition. That exact staleness was a real bug here once. An agent
reading a stale frame as "my edit failed" would thrash, so the wait is on by
default, `settleMs` makes it tunable, and the tool description names the
failure rather than leaving it to be rediscovered.

It probes with HEAD before returning, so a URL that 404s comes back as a
failure with a hint instead of as a link the agent cannot render.

Author: miguel-heygen

packages/studio/src/webmcp/StudioAgentTools.tsx

@@ -63,8 +63,29 @@ export function StudioAgentTools() {
           isPlaying: player.isPlaying,
         };
       },
+      getProjectId: () => projectId,
+      getCompositionPath: () => activeCompPath,
+      // HEAD, not GET: the tool only needs to know the frame renders. Pulling
+      // the PNG here would download it once for nothing, since the agent
+      // fetches the URL itself.
+      probeFrame: async (url) => {
+        try {
+          const response = await fetch(url, { method: "HEAD" });
+          return { ok: response.ok, status: response.status };
+        } catch {
+          return { ok: false, status: 0 };
+        }
+      },
+      wait: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
     }),
-    [getSnapshot, previewIframeRef, buildDomSelectionFromTarget, applyDomSelection],
+    [
+      getSnapshot,
+      previewIframeRef,
+      buildDomSelectionFromTarget,
+      applyDomSelection,
+      projectId,
+      activeCompPath,
+    ],
   );
 
   useStudioAgentTools(deps);

packages/studio/src/webmcp/tools/frameTools.test.ts

@@ -0,0 +1,146 @@
+// @vitest-environment jsdom
+import { describe, expect, it, vi } from "vitest";
+import { studioFrame, type FrameToolDeps, type StudioFrameResult } from "./frameTools";
+import type { ToolFailure, ToolResult } from "../toolResult";
+
+function frameDeps(overrides: Partial<FrameToolDeps> = {}): FrameToolDeps {
+  return {
+    getProjectId: () => "demo",
+    getCompositionPath: () => "index.html",
+    readPlayhead: () => ({ currentTime: 2.4, duration: 10, isPlaying: false }),
+    requestSeek: () => undefined,
+    probeFrame: async () => ({ ok: true, status: 200 }),
+    wait: async () => undefined,
+    ...overrides,
+  };
+}
+
+function expectOk<T>(result: ToolResult<T>): { ok: true } & T {
+  if (!result.ok) throw new Error(`expected ok, got ${JSON.stringify(result)}`);
+  return result;
+}
+
+function expectFailure(result: ToolResult<unknown>): ToolFailure {
+  if (result.ok) throw new Error(`expected failure, got ${JSON.stringify(result)}`);
+  return result;
+}
+
+describe("studioFrame", () => {
+  it("returns a URL for the composition at the playhead", async () => {
+    const result = await studioFrame(frameDeps());
+
+    const ok = expectOk<StudioFrameResult>(result);
+    expect(ok.time).toBe(2.4);
+    expect(ok.compositionPath).toBe("index.html");
+    expect(ok.url).toContain("/thumbnail/");
+    expect(ok.url).toContain("t=2.400");
+    expect(ok.url).toContain("format=png");
+  });
+
+  it("seeks first when given a time", async () => {
+    const requestSeek = vi.fn();
+
+    await studioFrame(frameDeps({ requestSeek }), { time: 5 });
+
+    expect(requestSeek).toHaveBeenCalledWith(5);
+  });
+
+  it("captures where the playhead LANDED, not what was asked for", async () => {
+    // The player clamps. Reporting the request would attach the wrong time to
+    // the frame, and an agent judging motion would draw the wrong conclusion.
+    const result = await studioFrame(
+      frameDeps({ readPlayhead: () => ({ currentTime: 10, duration: 10, isPlaying: false }) }),
+      { time: 999 },
+    );
+
+    const ok = expectOk<StudioFrameResult>(result);
+    expect(ok.time).toBe(10);
+    expect(ok.url).toContain("t=10.000");
+  });
+
+  it("waits before capturing, so a just-made edit is in the frame", async () => {
+    // The render cache is cleared by a file watcher with a write-stability
+    // threshold. Capturing faster than that renders the PRE-edit composition.
+    const wait = vi.fn(async () => undefined);
+    const order: string[] = [];
+
+    await studioFrame(
+      frameDeps({
+        wait: async (ms) => {
+          order.push(`wait:${ms}`);
+          await wait();
+        },
+        probeFrame: async () => {
+          order.push("probe");
+          return { ok: true, status: 200 };
+        },
+      }),
+    );
+
+    expect(order).toEqual(["wait:150", "probe"]);
+  });
+
+  it("honours a caller-supplied settle time and reports it", async () => {
+    const result = await studioFrame(frameDeps(), { settleMs: 800 });
+
+    expect(expectOk<StudioFrameResult>(result).settledMs).toBe(800);
+  });
+
+  it("clamps an absurd settle time rather than hanging", async () => {
+    const result = await studioFrame(frameDeps(), { settleMs: 10 * 60 * 1000 });
+
+    expect(expectOk<StudioFrameResult>(result).settledMs).toBe(5000);
+  });
+
+  it("falls back to the default for a nonsense settle time", async () => {
+    for (const settleMs of [-1, Number.NaN]) {
+      const result = await studioFrame(frameDeps(), { settleMs });
+      expect(expectOk<StudioFrameResult>(result).settledMs).toBe(150);
+    }
+  });
+
+  it("skips the wait entirely when asked for zero", async () => {
+    const wait = vi.fn(async () => undefined);
+
+    await studioFrame(frameDeps({ wait }), { settleMs: 0 });
+
+    expect(wait).not.toHaveBeenCalled();
+  });
+
+  it("reports a renderer failure instead of handing back a dead URL", async () => {
+    const result = expectFailure(
+      await studioFrame(frameDeps({ probeFrame: async () => ({ ok: false, status: 500 }) })),
+    );
+
+    expect(result.kind).toBe("failed");
+    expect(result.reason).toContain("500");
+    expect(result.hint).toBeDefined();
+  });
+
+  it("fails when no project is open, before touching the renderer", async () => {
+    const probeFrame = vi.fn();
+
+    const result = expectFailure(
+      await studioFrame(frameDeps({ getProjectId: () => null, probeFrame })),
+    );
+
+    expect(result.kind).toBe("blocked");
+    expect(probeFrame).not.toHaveBeenCalled();
+  });
+
+  it("rejects a negative or non-finite time without seeking", async () => {
+    const requestSeek = vi.fn();
+
+    for (const time of [-1, Number.NaN, Number.POSITIVE_INFINITY]) {
+      const result = expectFailure(await studioFrame(frameDeps({ requestSeek }), { time }));
+      expect(result.kind).toBe("invalid");
+    }
+    expect(requestSeek).not.toHaveBeenCalled();
+  });
+
+  it("captures the master composition when no path is active", async () => {
+    const result = await studioFrame(frameDeps({ getCompositionPath: () => null }));
+
+    expect(expectOk<StudioFrameResult>(result).compositionPath).toBe("index.html");
+  });
+});

packages/studio/src/webmcp/tools/frameTools.ts

@@ -0,0 +1,133 @@
+/**
+ * `studio_frame`: the eyes.
+ *
+ * Without this the tool set is a remote control. With it an agent can author a
+ * change, look at the instant it affects, judge it, and adjust. That loop is the
+ * one thing source alone cannot support, because "what does this look like at
+ * 2.4 seconds" is not a question a file can answer.
+ *
+ * Reuses Studio's existing capture endpoint (`utils/frameCapture`) rather than
+ * inventing a second one. The server renders the composition with Puppeteer, so
+ * the frame reflects the file on disk, not the live preview DOM.
+ */
+
+import { buildFrameCaptureUrl } from "../../utils/frameCapture";
+import { toolFailure, toolOk, type ToolResult } from "../toolResult";
+
+export interface FrameToolDeps {
+  getProjectId: () => string | null;
+  getCompositionPath: () => string | null;
+  readPlayhead: () => { currentTime: number; duration: number; isPlaying: boolean };
+  requestSeek: (time: number) => void;
+  /** Confirms the URL renders. Injected so tests need no network. */
+  probeFrame: (url: string) => Promise<{ ok: boolean; status: number }>;
+  wait: (ms: number) => Promise<void>;
+}
+
+export interface StudioFrameResult {
+  /** Fetch this to see the frame. A PNG of the composition at `time`. */
+  url: string;
+  time: number;
+  compositionPath: string;
+  /** How long the tool waited for a pending write to settle before capturing. */
+  settledMs: number;
+}
+
+export interface StudioFrameInput {
+  /** Seconds. Omit to capture wherever the playhead already is. */
+  time?: number;
+  /**
+   * Milliseconds to wait before capturing, so a just-written edit is visible.
+   * See the staleness note in the description.
+   */
+  settleMs?: number;
+}
+
+/**
+ * Long enough to cover the project watcher's 40ms write-stability threshold
+ * plus filesystem latency, short enough not to be felt. This is the mitigation
+ * for a real, previously-fixed bug: the preview signature is invalidated by a
+ * file watcher, and a capture that beats the watcher renders the PRE-edit
+ * composition. An agent reading that as "my edit failed" would thrash.
+ */
+const DEFAULT_SETTLE_MS = 150;
+const MAX_SETTLE_MS = 5_000;
+
+export async function studioFrame(
+  deps: FrameToolDeps,
+  input: StudioFrameInput = {},
+): Promise<ToolResult<StudioFrameResult>> {
+  const projectId = deps.getProjectId();
+  if (!projectId) {
+    return toolFailure("blocked", "no project is open");
+  }
+
+  if (input.time !== undefined) {
+    if (typeof input.time !== "number" || !Number.isFinite(input.time) || input.time < 0) {
+      return toolFailure("invalid", "time must be a non-negative, finite number of seconds");
+    }
+    deps.requestSeek(input.time);
+  }
+
+  const settledMs = clampSettle(input.settleMs);
+  if (settledMs > 0) await deps.wait(settledMs);
+
+  // Capture whatever the playhead now reads, rather than what was requested:
+  // the player clamps, so those can differ and the frame belongs to the former.
+  const { currentTime } = deps.readPlayhead();
+  const compositionPath = deps.getCompositionPath();
+  const url = buildFrameCaptureUrl({ projectId, compositionPath, currentTime });
+
+  const probe = await deps.probeFrame(url);
+  if (!probe.ok) {
+    return toolFailure(
+      "failed",
+      `the renderer returned ${probe.status} for this frame`,
+      "The composition may not build. Try `hyperframes check`.",
+    );
+  }
+
+  return toolOk<StudioFrameResult>({
+    url,
+    time: currentTime,
+    compositionPath: compositionPath ?? "index.html",
+    settledMs,
+  });
+}
+
+function clampSettle(requested: number | undefined): number {
+  if (requested === undefined) return DEFAULT_SETTLE_MS;
+  if (typeof requested !== "number" || !Number.isFinite(requested) || requested < 0) {
+    return DEFAULT_SETTLE_MS;
+  }
+  return Math.min(requested, MAX_SETTLE_MS);
+}
+
+export const STUDIO_FRAME_INPUT_SCHEMA = {
+  type: "object",
+  properties: {
+    time: {
+      type: "number",
+      minimum: 0,
+      description: "Seconds. Omit to capture wherever the playhead already is.",
+    },
+    settleMs: {
+      type: "integer",
+      minimum: 0,
+      maximum: MAX_SETTLE_MS,
+      description: `Wait this long before capturing so a just-made edit is included. Default ${DEFAULT_SETTLE_MS}.`,
+    },
+  },
+  additionalProperties: false,
+} as const;
+
+export const STUDIO_FRAME_DESCRIPTION = [
+  "Render the composition to a PNG at a given time and return its URL, so you can",
+  "SEE the result instead of inferring it from source. Use this to judge a change:",
+  "edit, capture the instant it affects, look, adjust.",
+  "The frame is rendered from the file on disk, not the live preview.",
+  "A capture taken immediately after an edit can therefore predate that edit, because",
+  "the render cache is cleared by a file watcher. The tool waits briefly to cover that;",
+  "raise `settleMs` if a frame still looks stale, rather than concluding the edit failed.",
+  "Returns `ok: true` with `url` and the `time` actually captured, or `ok: false`.",
+].join(" ");

packages/studio/src/webmcp/useStudioAgentTools.test.tsx

@@ -36,6 +36,10 @@ function deps(overrides: Partial<StudioAgentToolsDeps> = {}): StudioAgentToolsDe
     applySelection: () => undefined,
     requestSeek: () => undefined,
     readPlayhead: () => ({ currentTime: 0, duration: 10, isPlaying: false }),
+    getProjectId: () => "demo",
+    getCompositionPath: () => "index.html",
+    probeFrame: async () => ({ ok: true, status: 200 }),
+    wait: async () => undefined,
     ...overrides,
   };
 }
@@ -99,6 +103,7 @@ describe("useStudioAgentTools", () => {
       "studio_look",
       "studio_select",
       "studio_seek",
+      "studio_frame",
     ]);
   });
 
@@ -112,14 +117,14 @@ describe("useStudioAgentTools", () => {
     await act(async () => {
       harness = mountTools(deps({ getSnapshot: () => snapshot() }));
     });
-    expect(registerTool).toHaveBeenCalledTimes(3);
+    expect(registerTool).toHaveBeenCalledTimes(4);
 
     await act(async () => {
       harness?.rerenderWith(deps({ getSnapshot: () => snapshot({ currentTime: 5 }) }));
       harness?.rerenderWith(deps({ getSnapshot: () => snapshot({ currentTime: 6 }) }));
     });
 
-    expect(registerTool).toHaveBeenCalledTimes(3);
+    expect(registerTool).toHaveBeenCalledTimes(4);
   });
 
   it("executes against the LATEST deps, not the ones present at registration", async () => {
@@ -192,7 +197,7 @@ describe("useStudioAgentTools", () => {
       mountTools(deps({ getSnapshot: () => snapshot() }));
     });
 
-    expect(registerTool).toHaveBeenCalledTimes(3);
+    expect(registerTool).toHaveBeenCalledTimes(4);
   });
 
   it("reports a tool that throws as an internal failure instead of rejecting", async () => {

packages/studio/src/webmcp/useStudioAgentTools.ts

@@ -24,10 +24,18 @@ import {
   type StudioSeekResult,
   type StudioSelectResult,
 } from "./tools/selectionTools";
+import {
+  studioFrame,
+  STUDIO_FRAME_DESCRIPTION,
+  STUDIO_FRAME_INPUT_SCHEMA,
+  type FrameToolDeps,
+  type StudioFrameInput,
+  type StudioFrameResult,
+} from "./tools/frameTools";
 
 const log = makeStudioDebugLogger("webmcp");
 
-export interface StudioAgentToolsDeps extends SelectionToolDeps {
+export interface StudioAgentToolsDeps extends SelectionToolDeps, FrameToolDeps {
   /** Read Studio's current state. Called per tool invocation, never cached. */
   getSnapshot: () => StudioLookSnapshot;
 }
@@ -79,6 +87,15 @@ function buildStudioTools(depsRef: { readonly current: StudioAgentToolsDeps }):
           studioSeek(depsRef.current, readNumberInput(input, "time")),
         ),
     },
+    {
+      name: "studio_frame",
+      title: "See the composition",
+      description: STUDIO_FRAME_DESCRIPTION,
+      inputSchema: STUDIO_FRAME_INPUT_SCHEMA,
+      annotations: { readOnlyHint: true, untrustedContentHint: true },
+      execute: (input): Promise<ToolResult<StudioFrameResult>> =>
+        runToolBody("studio_frame", () => studioFrame(depsRef.current, input as StudioFrameInput)),
+    },
   ];
 }