feat(vom): make canvas-rendered content observable | 支持在 observe 中发现并截图 canvas 视觉区域

apps/extension/src/session-manager/__tests__/ref-store.test.ts

@@ -13,6 +13,8 @@ describe("RefStore", () => {
       backendNodeId: 42,
       tabId: 7,
       generation: 0,
+      kind: "dom",
+      capabilities: ["interact", "screenshot"],
     });
     expect(s.size()).toBe(1);
     expect(s.isEmpty()).toBe(false);
@@ -48,4 +50,31 @@ describe("RefStore", () => {
       cdpSessionId: "child-session",
     });
   });
+
+  it("preserves explicit capabilities for visual surface refs", () => {
+    const s = new RefStore();
+    s.set("e1", 42, {
+      tabId: 7,
+      kind: "surface",
+      capabilities: ["screenshot"],
+    });
+
+    expect(s.resolveEntry("e1")).toMatchObject({
+      kind: "surface",
+      capabilities: ["screenshot"],
+    });
+  });
+
+  it("never grants ordinary interaction to a surface through defaults or caller input", () => {
+    const s = new RefStore();
+    s.set("e1", 42, { tabId: 7, kind: "surface" });
+    s.set("e2", 43, {
+      tabId: 7,
+      kind: "surface",
+      capabilities: ["interact", "screenshot"],
+    });
+
+    expect(s.resolveEntry("e1")?.capabilities).toEqual(["screenshot"]);
+    expect(s.resolveEntry("e2")?.capabilities).toEqual(["screenshot"]);
+  });
 });

apps/extension/src/session-manager/ref-store.ts

@@ -1,3 +1,5 @@
+import type { Rect } from "@browser-skill/vom";
+
 /**
  * Per-session map from `@e<N>` snapshot refs to a CDP node address.
  *
@@ -12,12 +14,17 @@
  * inside the `SessionContext`.
  */
 export type BackendNodeId = number;
+export type RefCapability = "interact" | "screenshot";
+export type RefTargetKind = "dom" | "surface";
 
 export interface RefEntry {
   backendNodeId: BackendNodeId;
   tabId: number | null;
   frameId?: string;
   cdpSessionId?: string;
+  visibleRect?: Rect;
+  kind: RefTargetKind;
+  capabilities: RefCapability[];
   generation: number;
 }
 
@@ -28,8 +35,19 @@ export type RefInput =
       tabId: number;
       frameId?: string;
       cdpSessionId?: string;
+      visibleRect?: Rect;
+      kind?: RefTargetKind;
+      capabilities?: RefCapability[];
     };
 
+function capabilitiesFor(
+  kind: RefTargetKind,
+  capabilities: RefCapability[] | undefined,
+): RefCapability[] {
+  if (kind === "surface") return ["screenshot"];
+  return capabilities ?? ["interact", "screenshot"];
+}
+
 export class RefStore {
   private readonly map = new Map<string, RefEntry>();
   private generation = 0;
@@ -70,13 +88,20 @@ export class RefStore {
       tabId?: number;
       frameId?: string;
       cdpSessionId?: string;
+      visibleRect?: Rect;
+      kind?: RefTargetKind;
+      capabilities?: RefCapability[];
     } = {},
   ): void {
+    const kind = opts.kind ?? "dom";
     this.map.set(normaliseRef(ref), {
       backendNodeId: id,
       tabId: opts.tabId ?? null,
       ...(opts.frameId ? { frameId: opts.frameId } : {}),
       ...(opts.cdpSessionId ? { cdpSessionId: opts.cdpSessionId } : {}),
+      ...(opts.visibleRect ? { visibleRect: opts.visibleRect } : {}),
+      kind,
+      capabilities: capabilitiesFor(kind, opts.capabilities),
       generation: this.generation,
     });
   }
@@ -94,14 +119,20 @@ export class RefStore {
       return {
         backendNodeId: input,
         tabId: null,
+        kind: "dom",
+        capabilities: ["interact", "screenshot"],
         generation: this.generation,
       };
     }
+    const kind = input.kind ?? "dom";
     return {
       backendNodeId: input.backendNodeId,
       tabId: input.tabId,
       ...(input.frameId ? { frameId: input.frameId } : {}),
       ...(input.cdpSessionId ? { cdpSessionId: input.cdpSessionId } : {}),
+      ...(input.visibleRect ? { visibleRect: input.visibleRect } : {}),
+      kind,
+      capabilities: capabilitiesFor(kind, input.capabilities),
       generation: this.generation,
     };
   }

apps/extension/src/tools/__tests__/interaction.test.ts

@@ -123,6 +123,29 @@ describe("handleClick", () => {
     expect(fake.cdp.send).not.toHaveBeenCalled();
   });
 
+  it("rejects screenshot-only visual surface refs before issuing CDP calls", async () => {
+    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
+    const ctx = await sm.start("aa11");
+    ctx.refStore.set("e3", 1234, {
+      tabId: 4,
+      kind: "surface",
+      capabilities: ["screenshot"],
+    });
+    const fake = makeFakeCdp({});
+
+    const res = await handleClick(
+      sm,
+      { session_id: "aa11", ref: "@e3" },
+      { cdp: fake.cdp, tabsApi: fake.tabsApi },
+    );
+
+    expect(res).toMatchObject({
+      code: "permission_denied",
+      data: { reason: "ref_capability_denied", required_capability: "interact" },
+    });
+    expect(fake.cdp.send).not.toHaveBeenCalled();
+  });
+
   it("clicks by ref, computes the quad centre, dispatches three mouse events", async () => {
     const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
     const ctx = await sm.start("aa11");

apps/extension/src/tools/__tests__/observation.test.ts

@@ -300,6 +300,76 @@ describe("handleScreenshot", () => {
     expect(clip).toMatchObject({ x: 10, y: 20, width: 100, height: 40 });
   });
 
+  it("captures a bounded visible crop for a visual surface without scrolling it", async () => {
+    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
+    const ctx = await sm.start("aa11");
+    ctx.refStore.set("e5", 999, {
+      tabId: 7,
+      kind: "surface",
+      capabilities: ["screenshot"],
+    });
+    const { cdp, sent } = makeFakeCdp({
+      "Page.getLayoutMetrics": () => ({
+        cssLayoutViewport: { clientWidth: 4096, clientHeight: 4096 },
+      }),
+      "DOM.getContentQuads": () => ({ quads: [[0, 0, 4096, 0, 4096, 4096, 0, 4096]] }),
+      "Page.captureScreenshot": () => ({ data: TINY_PNG }),
+    });
+
+    const res = await handleScreenshot(
+      sm,
+      { session_id: "aa11", ref: "@e5", tab_id: 7 },
+      makeScreenshotDeps({
+        cdp,
+        get: vi.fn(async () => ({ id: 7, windowId: 100, active: false }) as chrome.tabs.Tab),
+        query: vi.fn(),
+        captureVisibleTab: vi.fn(),
+      }),
+    );
+
+    if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
+    expect(sent.some((call) => call.method === "DOM.scrollIntoViewIfNeeded")).toBe(false);
+    const clip = sent.find((call) => call.method === "Page.captureScreenshot")?.params as {
+      clip?: { width: number; height: number; scale: number };
+    };
+    expect(clip.clip).toMatchObject({ width: 4096, height: 4096 });
+    expect(clip.clip?.scale).toBeCloseTo(Math.sqrt(4_000_000 / (4096 * 4096)));
+  });
+
+  it("crops a visual surface screenshot to its observation-time visible region", async () => {
+    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
+    const ctx = await sm.start("aa11");
+    ctx.refStore.set("e5", 999, {
+      tabId: 7,
+      kind: "surface",
+      visibleRect: { x: 25, y: 30, w: 50, h: 40 },
+    });
+    const { cdp, sent } = makeFakeCdp({
+      "Page.getLayoutMetrics": () => ({
+        cssLayoutViewport: { clientWidth: 800, clientHeight: 600 },
+      }),
+      "DOM.getContentQuads": () => ({ quads: [[0, 0, 200, 0, 200, 100, 0, 100]] }),
+      "Page.captureScreenshot": () => ({ data: TINY_PNG }),
+    });
+
+    const res = await handleScreenshot(
+      sm,
+      { session_id: "aa11", ref: "@e5", tab_id: 7 },
+      makeScreenshotDeps({
+        cdp,
+        get: vi.fn(async () => ({ id: 7, windowId: 100, active: false }) as chrome.tabs.Tab),
+        query: vi.fn(),
+        captureVisibleTab: vi.fn(),
+      }),
+    );
+
+    if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
+    const clip = sent.find((call) => call.method === "Page.captureScreenshot")?.params as {
+      clip?: { x: number; y: number; width: number; height: number };
+    };
+    expect(clip.clip).toMatchObject({ x: 25, y: 30, width: 50, height: 40 });
+  });
+
   it("returns not_found for unknown ref", async () => {
     const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
     await sm.start("aa11");
@@ -2693,6 +2763,85 @@ describe("handleSnapshot", () => {
     );
   });
 
+  it("discovers rendered canvases only in observe and stores a screenshot-only ref", async () => {
+    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
+    const ctx = await sm.start("aa11");
+    const root: CdpAxNode = {
+      nodeId: "1",
+      role: { type: "role", value: "RootWebArea" },
+      name: { type: "computedString", value: "Canvas app" },
+      backendDOMNodeId: 100,
+    };
+    const strings = ["body", "canvas", "static", "auto"];
+    const i = (value: string) => strings.indexOf(value);
+    const send = vi.fn(async (_tabId: number, method: string) => {
+      if (method === "Accessibility.enable" || method === "DOMSnapshot.enable") return {};
+      if (method === "Accessibility.getFullAXTree") return { nodes: [root] };
+      if (method === "Page.getLayoutMetrics") {
+        return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
+      }
+      if (method === "DOMSnapshot.captureSnapshot") {
+        return {
+          strings,
+          documents: [
+            {
+              nodes: {
+                parentIndex: [-1, 0],
+                nodeName: [i("body"), i("canvas")],
+                backendNodeId: [100, 200],
+                attributes: [[], []],
+              },
+              layout: {
+                nodeIndex: [0, 1],
+                styles: [
+                  [i("static"), i("auto"), i("auto")],
+                  [i("static"), i("auto"), i("auto")],
+                ],
+                bounds: [
+                  [0, 0, 1000, 800],
+                  [20, 30, 800, 500],
+                ],
+                paintOrders: [0, 1],
+              },
+            },
+          ],
+        };
+      }
+      throw new Error(`unexpected CDP method ${method}`);
+    });
+    const deps = {
+      cdp: {
+        send: send as unknown as <T = unknown>(
+          tabId: number,
+          method: string,
+          params?: object,
+        ) => Promise<T>,
+        trackSessionTab: vi.fn(),
+      },
+      tabsApi: {
+        get: vi.fn(
+          async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
+        ),
+        query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
+      },
+      conditionalSurfaceProbe: false,
+    };
+
+    const observed = await handleObserve(sm, { session_id: "aa11" }, deps);
+    if ("code" in observed) throw new Error(`unexpected error: ${JSON.stringify(observed)}`);
+    expect(observed.text).toContain('@e1 surface "canvas visual surface"');
+    expect(ctx.refStore.resolveEntry("e1")).toMatchObject({
+      backendNodeId: 200,
+      kind: "surface",
+      capabilities: ["screenshot"],
+    });
+
+    const snapshot = await handleSnapshot(sm, { session_id: "aa11" }, deps);
+    if ("code" in snapshot) throw new Error(`unexpected error: ${JSON.stringify(snapshot)}`);
+    expect(snapshot.text).not.toContain("visual surface");
+    expect(snapshot.ref_count).toBe(0);
+  });
+
   it("allows passive snapshots of explicit user-window tabs", async () => {
     const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
     await sm.start("aa11");

apps/extension/src/tools/__tests__/snapshot-ref.test.ts

@@ -24,10 +24,14 @@ describe("lookupSnapshotRef", () => {
     expect(lookupSnapshotRef(ctx, "@e3", 4)).toEqual({
       backendNodeId: 1234,
       refKey: "e3",
+      kind: "dom",
+      capabilities: ["interact", "screenshot"],
     });
     expect(lookupSnapshotRef(ctx, "e3", 4)).toEqual({
       backendNodeId: 1234,
       refKey: "e3",
+      kind: "dom",
+      capabilities: ["interact", "screenshot"],
     });
   });
 
@@ -53,13 +57,17 @@ describe("resolveSnapshotRef", () => {
       tabId: 4,
       frameId: "child-frame",
       cdpSessionId: "child-session",
+      kind: "dom",
+      capabilities: ["interact", "screenshot"],
     });
 
     const expected = {
       backendNodeId: 1234,
       refKey: "e3",
       frameId: "child-frame",
       cdpSessionId: "child-session",
+      kind: "dom" as const,
+      capabilities: ["interact", "screenshot"] as const,
     };
     expect(lookupSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
     expect(resolveSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
@@ -105,6 +113,28 @@ describe("resolveSnapshotRef", () => {
     expect(resolveSnapshotRef(ctx, "@e3", 4)).toEqual({
       backendNodeId: 1234,
       refKey: "e3",
+      kind: "dom",
+      capabilities: ["interact", "screenshot"],
+    });
+  });
+
+  it("rejects an operation outside the ref's declared capabilities", async () => {
+    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
+    const ctx = await sm.start("aa11");
+    ctx.refStore.set("e3", 1234, {
+      tabId: 4,
+      kind: "surface",
+      capabilities: ["screenshot"],
+    });
+
+    expect(resolveSnapshotRef(ctx, "@e3", 4, "interact")).toMatchObject({
+      code: "permission_denied",
+      message: "ref @e3 does not support interact",
+      data: {
+        reason: "ref_capability_denied",
+        required_capability: "interact",
+        kind: "surface",
+      },
     });
   });
 });

apps/extension/src/tools/interaction.ts

@@ -159,7 +159,7 @@ async function resolveBackendNode(
     };
   }
   if (hasRef) {
-    const resolved = resolveSnapshotRef(ctx, params.ref as string, target.tabId);
+    const resolved = resolveSnapshotRef(ctx, params.ref as string, target.tabId, "interact");
     if (isRpcError(resolved)) return resolved;
     return {
       backendNodeId: resolved.backendNodeId,