feat(studio): let an agent drive Studio's selection and playhead

Adds `studio_select` and `studio_seek`, so an agent and the human are looking
at the same element and the same instant. Selecting reveals the inspector,
exactly as a click does, which is what makes the agent's move visible.

Selection is shared state, not a per-call argument, and that is forced rather
than chosen. Most of Studio's edit handlers read the ambient React selection,
and `applyDomSelection` only schedules a state update, so selecting and
committing inside ONE call would write to whatever was selected before. Two
tool calls are separated by a render, so the contract is select first, then
act. That is also how a human works: click, then type.

`studio_seek` uses `requestSeek`, not `setCurrentTime`. The latter only moves
the timeline's displayed number and leaves the composition where it was.

Two things the tools refuse to fake:

Seek does not clamp. `seek()` already clamps against the adapter's duration,
which can differ from the store's, and clamping again would give that
invariant two owners that can disagree. The tool reports where the playhead
actually landed instead, read back afterwards.

`requestSeek` is fire-and-forget, so it cannot report that no adapter was
mounted to receive it. The tool compares the playhead before and after and
fails rather than claiming a seek that never happened.

Select separates three failures that a single message would have merged: the
preview is not mounted yet (wait), no element matches the handle (re-read),
and the element cannot be selected (try a neighbour). The agent's next move
differs for each, so collapsing them would cost it a round trip or a retry
loop.

Author: miguel-heygen

packages/studio/src/webmcp/StudioAgentTools.tsx

@@ -1,8 +1,8 @@
-import { useCallback } from "react";
-import { useDomEditSelectionContext } from "../contexts/DomEditContext";
+import { useCallback, useMemo } from "react";
+import { useDomEditActionsContext, useDomEditSelectionContext } from "../contexts/DomEditContext";
 import { useStudioShellContext } from "../contexts/StudioContext";
 import { usePlayerStore } from "../player";
-import { useStudioAgentTools } from "./useStudioAgentTools";
+import { useStudioAgentTools, type StudioAgentToolsDeps } from "./useStudioAgentTools";
 import type { StudioLookSnapshot } from "./tools/lookTools";
 
 /**
@@ -12,14 +12,15 @@ import type { StudioLookSnapshot } from "./tools/lookTools";
  * contexts are only readable below `DomEditProvider`, which `App` renders, and
  * `App.tsx` sits three lines under the 600-line cap.
  *
- * The player store is read IMPERATIVELY through `getState()` inside the
- * snapshot callback rather than subscribed to. Subscribing to `currentTime`
- * would re-render this component on every animation frame during playback for
- * a value nothing here displays.
+ * The player store is read IMPERATIVELY through `getState()` rather than
+ * subscribed to. Subscribing to `currentTime` would re-render this component on
+ * every animation frame during playback for a value nothing here displays.
  */
 export function StudioAgentTools() {
   const { projectId, activeCompPath, editHistory } = useStudioShellContext();
   const { domEditSelection, selectedGsapAnimations } = useDomEditSelectionContext();
+  const { previewIframeRef, buildDomSelectionFromTarget, applyDomSelection } =
+    useDomEditActionsContext();
 
   const getSnapshot = useCallback((): StudioLookSnapshot => {
     const player = usePlayerStore.getState();
@@ -41,6 +42,25 @@ export function StudioAgentTools() {
     };
   }, [projectId, activeCompPath, domEditSelection, selectedGsapAnimations, editHistory]);
 
-  useStudioAgentTools({ getSnapshot });
+  const deps = useMemo<StudioAgentToolsDeps>(
+    () => ({
+      getSnapshot,
+      getPreviewDocument: () => previewIframeRef.current?.contentDocument ?? null,
+      buildSelection: (element) => buildDomSelectionFromTarget(element),
+      applySelection: (selection) => applyDomSelection(selection, { revealPanel: true }),
+      requestSeek: (time) => usePlayerStore.getState().requestSeek(time),
+      readPlayhead: () => {
+        const player = usePlayerStore.getState();
+        return {
+          currentTime: player.currentTime,
+          duration: player.duration,
+          isPlaying: player.isPlaying,
+        };
+      },
+    }),
+    [getSnapshot, previewIframeRef, buildDomSelectionFromTarget, applyDomSelection],
+  );
+
+  useStudioAgentTools(deps);
   return null;
 }

packages/studio/src/webmcp/toolResult.ts

@@ -36,7 +36,7 @@ export function toolOk<T extends object>(value: T): { ok: true } & T {
   return { ok: true, ...value };
 }
 
-function toolFailure(kind: ToolFailureKind, reason: string, hint?: string): ToolFailure {
+export function toolFailure(kind: ToolFailureKind, reason: string, hint?: string): ToolFailure {
   return hint ? { ok: false, kind, reason, hint } : { ok: false, kind, reason };
 }
 

packages/studio/src/webmcp/tools/selectionTools.test.ts

@@ -0,0 +1,214 @@
+// @vitest-environment jsdom
+import { describe, expect, it, vi } from "vitest";
+import type { DomEditSelection } from "../../components/editor/domEditingTypes";
+import {
+  studioSeek,
+  studioSelect,
+  type SelectionToolDeps,
+  type StudioSeekResult,
+  type StudioSelectResult,
+} from "./selectionTools";
+import type { ToolFailure, ToolResult } from "../toolResult";
+
+function previewDoc(html: string): Document {
+  const iframe = document.createElement("iframe");
+  document.body.append(iframe);
+  const doc = iframe.contentDocument;
+  if (!doc) throw new Error("expected iframe document");
+  doc.body.innerHTML = html;
+  return doc;
+}
+
+function selectionFor(element: HTMLElement): DomEditSelection {
+  return {
+    id: element.id || undefined,
+    hfId: element.getAttribute("data-hf-id") ?? undefined,
+    element,
+    label: "Headline",
+    tagName: element.tagName.toLowerCase(),
+    sourceFile: "index.html",
+    compositionPath: "index.html",
+    isCompositionHost: false,
+    isInsideLockedComposition: false,
+    boundingBox: { x: 40, y: 12, width: 880, height: 96 },
+    textContent: element.textContent,
+    dataAttributes: {},
+    inlineStyles: {},
+    computedStyles: {},
+    textFields: [],
+    capabilities: {
+      canSelect: true,
+      canEditStyles: true,
+      canCrop: true,
+      canMove: true,
+      canResize: true,
+      canApplyManualOffset: true,
+      canApplyManualSize: true,
+      canApplyManualRotation: true,
+    },
+  };
+}
+
+function selectionDeps(overrides: Partial<SelectionToolDeps> = {}): SelectionToolDeps {
+  return {
+    getPreviewDocument: () => null,
+    buildSelection: async (element) => selectionFor(element),
+    applySelection: () => undefined,
+    requestSeek: () => undefined,
+    readPlayhead: () => ({ currentTime: 0, duration: 10, isPlaying: false }),
+    ...overrides,
+  };
+}
+
+function expectFailure(result: ToolResult<unknown>): ToolFailure {
+  if (result.ok) throw new Error(`expected failure, got ${JSON.stringify(result)}`);
+  return result;
+}
+
+function expectOk<T>(result: ToolResult<T>): { ok: true } & T {
+  if (!result.ok) throw new Error(`expected ok, got ${JSON.stringify(result)}`);
+  return result;
+}
+
+describe("studioSelect", () => {
+  it("applies the selection a click would produce and reports it back", async () => {
+    const doc = previewDoc('<h1 id="headline" data-hf-id="abc">Ship it</h1>');
+    const applySelection = vi.fn();
+
+    const result = await studioSelect(
+      selectionDeps({ getPreviewDocument: () => doc, applySelection }),
+      "hf:abc",
+    );
+
+    const ok = expectOk<StudioSelectResult>(result);
+    expect(ok.handle).toBe("hf:abc");
+    expect(ok.label).toBe("Headline");
+    expect(ok.box.width).toBe(880);
+    // Reveals the inspector, which is what makes the human see what the agent did.
+    expect(applySelection).toHaveBeenCalledTimes(1);
+  });
+
+  it("distinguishes a preview that is not mounted from a handle that does not match", async () => {
+    const notMounted = expectFailure(await studioSelect(selectionDeps(), "dom:headline"));
+    expect(notMounted.kind).toBe("blocked");
+    expect(notMounted.reason).toMatch(/not mounted/);
+
+    const doc = previewDoc('<h1 id="headline">Ship it</h1>');
+    const noMatch = expectFailure(
+      await studioSelect(selectionDeps({ getPreviewDocument: () => doc }), "dom:missing"),
+    );
+    expect(noMatch.kind).toBe("invalid");
+    expect(noMatch.reason).toMatch(/no element matches/);
+    // The two must not be the same message: waiting and re-reading are different fixes.
+    expect(noMatch.reason).not.toBe(notMounted.reason);
+  });
+
+  it("reports an element Studio cannot build a selection for, as a third case", async () => {
+    const doc = previewDoc('<h1 id="headline">Ship it</h1>');
+
+    const result = expectFailure(
+      await studioSelect(
+        selectionDeps({ getPreviewDocument: () => doc, buildSelection: async () => null }),
+        "dom:headline",
+      ),
+    );
+
+    expect(result.kind).toBe("blocked");
+    expect(result.reason).toMatch(/cannot select/);
+  });
+
+  it("rejects a missing handle without touching the preview", async () => {
+    const getPreviewDocument = vi.fn(() => null);
+
+    const result = expectFailure(await studioSelect(selectionDeps({ getPreviewDocument }), "  "));
+
+    expect(result.kind).toBe("invalid");
+    expect(getPreviewDocument).not.toHaveBeenCalled();
+  });
+
+  it("leaves the existing selection alone when it fails", async () => {
+    const doc = previewDoc('<h1 id="headline">Ship it</h1>');
+    const applySelection = vi.fn();
+
+    await studioSelect(
+      selectionDeps({ getPreviewDocument: () => doc, applySelection }),
+      "dom:missing",
+    );
+
+    expect(applySelection).not.toHaveBeenCalled();
+  });
+});
+
+describe("studioSeek", () => {
+  it("reports where the playhead landed, not what was requested", () => {
+    // The player clamps against the ADAPTER's duration, which the wrapper
+    // deliberately does not second-guess.
+    let currentTime = 0;
+    const result = studioSeek(
+      selectionDeps({
+        requestSeek: () => {
+          currentTime = 10;
+        },
+        readPlayhead: () => ({ currentTime, duration: 10, isPlaying: false }),
+      }),
+      999,
+    );
+
+    const ok = expectOk<StudioSeekResult>(result);
+    expect(ok.playhead).toBe(10);
+    expect(ok.moved).toBe(true);
+  });
+
+  it("reports that playback stopped", () => {
+    let isPlaying = true;
+    let currentTime = 0;
+    const result = studioSeek(
+      selectionDeps({
+        requestSeek: () => {
+          currentTime = 2;
+          isPlaying = false;
+        },
+        readPlayhead: () => ({ currentTime, duration: 10, isPlaying }),
+      }),
+      2,
+    );
+
+    expect(expectOk<StudioSeekResult>(result).isPlaying).toBe(false);
+  });
+
+  it("fails rather than claiming a seek the player never received", () => {
+    // `requestSeek` is fire-and-forget: with no adapter mounted it silently does
+    // nothing, and reporting ok would be a lie the agent builds on.
+    const result = expectFailure(
+      studioSeek(
+        selectionDeps({ readPlayhead: () => ({ currentTime: 0, duration: 10, isPlaying: false }) }),
+        5,
+      ),
+    );
+
+    expect(result.kind).toBe("blocked");
+    expect(result.reason).toMatch(/did not move/);
+  });
+
+  it("succeeds when asked to seek to where the playhead already is", () => {
+    const result = studioSeek(
+      selectionDeps({ readPlayhead: () => ({ currentTime: 3, duration: 10, isPlaying: false }) }),
+      3,
+    );
+
+    // Nothing moved, but nothing failed either, and `moved` says which.
+    const ok = expectOk<StudioSeekResult>(result);
+    expect(ok.moved).toBe(false);
+    expect(ok.playhead).toBe(3);
+  });
+
+  it("rejects a non-finite time without calling the player", () => {
+    const requestSeek = vi.fn();
+
+    for (const time of [Number.NaN, Number.POSITIVE_INFINITY]) {
+      const result = expectFailure(studioSeek(selectionDeps({ requestSeek }), time));
+      expect(result.kind).toBe("invalid");
+    }
+    expect(requestSeek).not.toHaveBeenCalled();
+  });
+});