diff --git a/apps/extension/src/browser-driver/__tests__/chromium-cdp.test.ts b/apps/extension/src/browser-driver/__tests__/chromium-cdp.test.ts index 06f80289..ed26feeb 100644 --- a/apps/extension/src/browser-driver/__tests__/chromium-cdp.test.ts +++ b/apps/extension/src/browser-driver/__tests__/chromium-cdp.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from "vitest"; -import { type CdpDebuggerApi, ChromiumCdp } from "../chromium-cdp"; +import { type CdpDebuggee, type CdpDebuggerApi, ChromiumCdp } from "../chromium-cdp"; function fakeChromeEvent() { const listeners = new Set<(...args: TArgs) => void>(); @@ -14,7 +14,7 @@ function fakeChromeEvent() { } function fakeApi() { - const onEvent = fakeChromeEvent<[chrome.debugger.Debuggee, string, unknown]>(); + const onEvent = fakeChromeEvent<[CdpDebuggee, string, unknown]>(); const onDetach = fakeChromeEvent<[chrome.debugger.Debuggee, string]>(); const api: CdpDebuggerApi = { attach: vi.fn(async () => {}), @@ -29,6 +29,135 @@ function fakeApi() { } describe("ChromiumCdp", () => { + it("discovers multiple iframe targets and recursively routes nested OOPIF commands", async () => { + const { api, onEvent } = fakeApi(); + (api.sendCommand as ReturnType).mockImplementation( + async ( + target: chrome.debugger.Debuggee & { sessionId?: string }, + method: string, + params?: { frameId?: string }, + ) => { + if (method === "DOM.getFrameOwner") { + return { backendNodeId: params?.frameId === "nested" ? 300 : 200 }; + } + if (method !== "Page.getFrameTree") return {}; + if (target.sessionId === "right-session") { + return { + frameTree: { + frame: { id: "right", url: "https://right.test" }, + childFrames: [ + { frame: { id: "nested", parentId: "right", url: "https://nested.test" } }, + ], + }, + }; + } + if (target.sessionId === "nested-session") { + return { frameTree: { frame: { id: "nested", url: "https://nested.test" } } }; + } + return { + frameTree: { + frame: { id: "main", url: "https://app.test" }, + childFrames: [ + { frame: { id: "left", parentId: "main", url: "https://left.test" } }, + { + frame: { id: "right", parentId: "main", url: "https://right.test" }, + childFrames: [ + { frame: { id: "nested", parentId: "right", url: "https://nested.test" } }, + ], + }, + ], + }, + }; + }, + ); + const cdp = new ChromiumCdp(api); + await cdp.ensureAttached(4); + onEvent.fire({ tabId: 4 }, "Target.attachedToTarget", { sessionId: "right-session" }); + onEvent.fire({ tabId: 4 } as chrome.debugger.Debuggee, "Target.attachedToTarget", { + sessionId: "nested-session", + }); + + const graph = await cdp.getFrameGraph(4); + + expect(graph.frames).toHaveLength(4); + expect(graph.frames.find((frame) => frame.frameId === "left")?.target).toEqual({ tabId: 4 }); + expect(graph.frames.find((frame) => frame.frameId === "right")?.target.sessionId).toBe( + "right-session", + ); + expect(graph.frames.find((frame) => frame.frameId === "nested")?.target.sessionId).toBe( + "nested-session", + ); + expect(graph.frames.find((frame) => frame.frameId === "right")?.ownerBackendNodeId).toBe(200); + expect(graph.frames.find((frame) => frame.frameId === "nested")?.ownerBackendNodeId).toBe(300); + expect(api.sendCommand).toHaveBeenCalledWith( + { tabId: 4, sessionId: "right-session" }, + "DOM.getFrameOwner", + { frameId: "nested" }, + ); + expect(api.sendCommand).toHaveBeenCalledWith( + { tabId: 4, sessionId: "nested-session" }, + "Target.setAutoAttach", + expect.objectContaining({ flatten: true }), + ); + }); + + it("waits for recursively attached iframe targets without a depth limit", async () => { + const { api, onEvent } = fakeApi(); + const depth = 6; + const scheduled = new Set(); + const nestedTree = (index: number): Record => ({ + frame: { + id: index === 0 ? "main" : `frame-${index}`, + ...(index > 0 ? { parentId: index === 1 ? "main" : `frame-${index - 1}` } : {}), + }, + ...(index < depth ? { childFrames: [nestedTree(index + 1)] } : {}), + }); + (api.sendCommand as ReturnType).mockImplementation( + async (target: CdpDebuggee, method: string, params?: { frameId?: string }) => { + if (method === "Target.setAutoAttach") { + const parentIndex = target.sessionId + ? Number(target.sessionId.replace("session-", "")) + : 0; + const nextIndex = parentIndex + 1; + const key = `${target.sessionId ?? "root"}:${nextIndex}`; + if (nextIndex <= depth && !scheduled.has(key)) { + scheduled.add(key); + setTimeout(() => { + onEvent.fire( + { tabId: 4, ...(target.sessionId ? { sessionId: target.sessionId } : {}) }, + "Target.attachedToTarget", + { + sessionId: `session-${nextIndex}`, + targetInfo: { type: "iframe" }, + }, + ); + }, 5); + } + return {}; + } + if (method === "Page.getFrameTree") { + if (!target.sessionId) return { frameTree: nestedTree(0) }; + const index = Number(target.sessionId.replace("session-", "")); + return { frameTree: nestedTree(index) }; + } + if (method === "DOM.getFrameOwner") { + return { backendNodeId: Number(params?.frameId?.replace("frame-", "")) + 100 }; + } + return {}; + }, + ); + const cdp = new ChromiumCdp(api); + + const graph = await cdp.getFrameGraph(4); + + expect(graph.frames).toHaveLength(depth + 1); + for (let index = 1; index <= depth; index += 1) { + expect( + graph.frames.find((frame) => frame.frameId === `frame-${index}`)?.target.sessionId, + ).toBe(`session-${index}`); + } + }); + it("coalesces concurrent attach calls for the same tab", async () => { const { api } = fakeApi(); let releaseAttach!: () => void; diff --git a/apps/extension/src/browser-driver/__tests__/frame-graph.test.ts b/apps/extension/src/browser-driver/__tests__/frame-graph.test.ts new file mode 100644 index 00000000..b9d4b408 --- /dev/null +++ b/apps/extension/src/browser-driver/__tests__/frame-graph.test.ts @@ -0,0 +1,136 @@ +import { describe, expect, it } from "vitest"; +import { buildFrameGraph, type CdpFrameTreeNode, type CdpFrameTreeSource } from "../frame-graph"; + +describe("buildFrameGraph", () => { + it("keeps sibling frames distinct and routes nested OOPIFs to their child sessions", () => { + const graph = buildFrameGraph([ + { + target: { tabId: 4 }, + tree: { + frame: { id: "main", url: "https://app.test" }, + childFrames: [ + { frame: { id: "left", parentId: "main", url: "https://left.test" } }, + { + frame: { id: "right", parentId: "main", url: "https://right.test" }, + childFrames: [ + { frame: { id: "nested", parentId: "right", url: "https://nested.test" } }, + ], + }, + ], + }, + }, + { + target: { tabId: 4, sessionId: "right-session" }, + tree: { + frame: { id: "right", url: "https://right.test" }, + childFrames: [{ frame: { id: "nested", parentId: "right", url: "https://nested.test" } }], + }, + }, + { + target: { tabId: 4, sessionId: "nested-session" }, + tree: { frame: { id: "nested", url: "https://nested.test" } }, + }, + ]); + + expect(graph?.frames).toHaveLength(4); + expect(graph?.frames.find((frame) => frame.frameId === "left")?.target).toEqual({ tabId: 4 }); + expect(graph?.frames.find((frame) => frame.frameId === "right")).toMatchObject({ + parentFrameId: "main", + target: { tabId: 4, sessionId: "right-session" }, + }); + expect(graph?.frames.find((frame) => frame.frameId === "nested")).toMatchObject({ + parentFrameId: "right", + target: { tabId: 4, sessionId: "nested-session" }, + }); + }); + + it("propagates each target boundary to its same-process descendants", () => { + const sources: CdpFrameTreeSource[] = [ + { + target: { tabId: 4 }, + tree: { + frame: { id: "main" }, + childFrames: [ + { + frame: { id: "oopif-a", parentId: "main" }, + childFrames: [ + { + frame: { id: "same-process-b", parentId: "oopif-a" }, + childFrames: [ + { + frame: { id: "oopif-c", parentId: "same-process-b" }, + childFrames: [{ frame: { id: "same-process-d", parentId: "oopif-c" } }], + }, + ], + }, + ], + }, + ], + }, + }, + { + target: { tabId: 4, sessionId: "session-a" }, + tree: { + frame: { id: "oopif-a" }, + childFrames: [ + { + frame: { id: "same-process-b", parentId: "oopif-a" }, + childFrames: [{ frame: { id: "oopif-c", parentId: "same-process-b" } }], + }, + ], + }, + }, + { + target: { tabId: 4, sessionId: "session-c" }, + tree: { + frame: { id: "oopif-c" }, + childFrames: [{ frame: { id: "same-process-d", parentId: "oopif-c" } }], + }, + }, + ]; + + const graph = buildFrameGraph(sources); + + expect(graph?.frames.map((frame) => [frame.frameId, frame.target.sessionId])).toEqual([ + ["main", undefined], + ["oopif-a", "session-a"], + ["same-process-b", "session-a"], + ["oopif-c", "session-c"], + ["same-process-d", "session-c"], + ]); + expect(graph?.frames.find((frame) => frame.frameId === "oopif-a")?.parentFrameId).toBe("main"); + expect(graph?.frames.find((frame) => frame.frameId === "oopif-c")?.parentFrameId).toBe( + "same-process-b", + ); + + const reordered = buildFrameGraph([sources[2], sources[1], sources[0]]); + expect(reordered).toEqual(graph); + }); + + it("walks deeply nested frame trees without consuming the JavaScript call stack", () => { + const depth = 5_000; + const root: CdpFrameTreeNode = { frame: { id: "frame-0" } }; + let parent = root; + for (let index = 1; index <= depth; index += 1) { + const child: CdpFrameTreeNode = { + frame: { id: `frame-${index}`, parentId: `frame-${index - 1}` }, + }; + parent.childFrames = [child]; + parent = child; + } + + const graph = buildFrameGraph([{ target: { tabId: 4 }, tree: root }]); + + expect(graph?.frames).toHaveLength(depth + 1); + expect(graph?.frames.at(-1)?.frameId).toBe(`frame-${depth}`); + }); + + it("does not loop when an in-memory frame tree contains an object cycle", () => { + const root: CdpFrameTreeNode = { frame: { id: "main" } }; + root.childFrames = [root]; + + expect(buildFrameGraph([{ target: { tabId: 4 }, tree: root }])?.frames).toEqual([ + { frameId: "main", target: { tabId: 4 } }, + ]); + }); +}); diff --git a/apps/extension/src/browser-driver/chromium-cdp.ts b/apps/extension/src/browser-driver/chromium-cdp.ts index 52283b16..2a24a259 100644 --- a/apps/extension/src/browser-driver/chromium-cdp.ts +++ b/apps/extension/src/browser-driver/chromium-cdp.ts @@ -31,6 +31,15 @@ import type { NetworkEntryKind, NetworkResult, } from "@/transport/types"; +import { + buildFrameGraph, + type CdpFrameGraph, + type CdpFrameTreeNode, + type CdpFrameTreeSource, + type CdpTarget, +} from "./frame-graph"; + +export type CdpDebuggee = chrome.debugger.Debuggee & { sessionId?: string }; /** * Minimal slice of `chrome.debugger` the rest of the extension @@ -38,21 +47,15 @@ import type { * fake without monkey-patching the real `chrome` global. */ export interface CdpDebuggerApi { - attach(target: chrome.debugger.Debuggee, requiredVersion: string): Promise; - detach(target: chrome.debugger.Debuggee): Promise; - sendCommand( - target: chrome.debugger.Debuggee, - method: string, - commandParams?: object, - ): Promise; + attach(target: CdpDebuggee, requiredVersion: string): Promise; + detach(target: CdpDebuggee): Promise; + sendCommand(target: CdpDebuggee, method: string, commandParams?: object): Promise; /** * Fires for every CDP event (`Page.lifecycleEvent`, `DOM.documentUpdated`, * …). The first callback argument is the source debuggee; the second * is the CDP method name; the third is the payload. */ - onEvent: chrome.events.Event< - (source: chrome.debugger.Debuggee, method: string, params: unknown) => void - >; + onEvent: chrome.events.Event<(source: CdpDebuggee, method: string, params: unknown) => void>; /** * Fires when Chrome unilaterally detaches us — most commonly because * the tab navigated to a chrome:// URL or the user clicked @@ -110,6 +113,8 @@ const MAX_CONSOLE_STACK_FRAMES = 20; const MAX_NETWORK_BUFFER = 200; const MAX_NETWORK_FIELD_LENGTH = 4096; const MAX_NETWORK_REQUEST_META = 1024; +const FRAME_DISCOVERY_TIMEOUT_MS = 1000; +const FRAME_DISCOVERY_QUIET_MS = 20; interface ParsedDialogOpening { type: JavaScriptDialogType; @@ -131,6 +136,27 @@ interface NetworkRequestMeta { truncated: boolean; } +interface FrameDiscoveryState { + sessions: Set; + pending: Set>; + generation: number; +} + +async function settleBeforeDeadline(promises: Promise[], deadline: number): Promise { + if (promises.length === 0) return true; + const remaining = deadline - Date.now(); + if (remaining <= 0) return false; + let timer: ReturnType | undefined; + const timedOut = await Promise.race([ + Promise.allSettled(promises).then(() => false), + new Promise((resolve) => { + timer = setTimeout(() => resolve(true), remaining); + }), + ]); + if (timer) clearTimeout(timer); + return !timedOut; +} + /** * Wrapper around `chrome.debugger` that owns the "attach once per * tabId" cache and exposes typed `send()`. @@ -149,10 +175,12 @@ export class ChromiumCdp { private readonly networkSequences = new Map(); private readonly networkDomainsEnabledTabs = new Set(); private readonly networkRequestMeta = new Map>(); + private readonly frameDiscovery = new Map(); private detachSubscription: { dispose(): void } | null = null; private dialogSubscription: { dispose(): void } | null = null; private consoleSubscription: { dispose(): void } | null = null; private networkSubscription: { dispose(): void } | null = null; + private frameTargetSubscription: { dispose(): void } | null = null; constructor(api: CdpDebuggerApi = chromeDebuggerApi) { this.api = api; @@ -160,6 +188,7 @@ export class ChromiumCdp { this.bindDialogHandler(); this.bindConsoleHandler(); this.bindNetworkHandler(); + this.bindFrameTargetHandler(); } /** Attach to `tabId` if we haven't already in this driver. */ @@ -177,6 +206,9 @@ export class ChromiumCdp { await this.enableConsoleDomains(tabId); await this.enableNetworkDomainBestEffort(tabId); this.attachedTabs.add(tabId); + await this.enableFrameDiscovery({ tabId }).catch((err) => { + console.debug("[bsk cdp] frame discovery unavailable", { tabId, err }); + }); } catch (err) { // A CDP domain enable failed after the raw attach succeeded // (e.g. `Page.enable` rejects because the tab just navigated to @@ -222,6 +254,73 @@ export class ChromiumCdp { } } + async sendToTarget(target: CdpTarget, method: string, params?: object): Promise { + if (!this.attachedTabs.has(target.tabId)) { + await this.ensureAttached(target.tabId); + } + try { + return (await this.api.sendCommand(target, method, params ?? {})) as T; + } catch (err) { + throw normalizeError(err); + } + } + + async getFrameGraph(tabId: number): Promise { + await this.ensureAttached(tabId); + await this.enableFrameDiscovery({ tabId }).catch(() => {}); + await this.drainFrameAttachTasks(tabId); + + const sources: CdpFrameTreeSource[] = []; + const root = await this.sendToTarget<{ frameTree?: CdpFrameTreeNode }>( + { tabId }, + "Page.getFrameTree", + {}, + ); + if (root.frameTree) sources.push({ target: { tabId }, tree: root.frameTree }); + + const sessions = [...(this.frameDiscovery.get(tabId)?.sessions ?? [])]; + const childTrees = await Promise.all( + sessions.map(async (sessionId): Promise => { + const target = { tabId, sessionId }; + try { + const reply = await this.sendToTarget<{ frameTree?: CdpFrameTreeNode }>( + target, + "Page.getFrameTree", + {}, + ); + return reply.frameTree ? { target, tree: reply.frameTree } : null; + } catch { + return null; + } + }), + ); + for (const source of childTrees) { + if (source) sources.push(source); + } + + const graph = buildFrameGraph(sources); + if (!graph) throw new Error("Page.getFrameTree returned no root frame"); + const frameById = new Map(graph.frames.map((frame) => [frame.frameId, frame])); + await Promise.all( + graph.frames.map(async (frame) => { + if (!frame.parentFrameId) return; + const parent = frameById.get(frame.parentFrameId); + if (!parent) return; + try { + const owner = await this.sendToTarget<{ backendNodeId?: number }>( + parent.target, + "DOM.getFrameOwner", + { frameId: frame.frameId }, + ); + if (owner.backendNodeId !== undefined) frame.ownerBackendNodeId = owner.backendNodeId; + } catch { + // The frame may have navigated between tree capture and owner lookup. + } + }), + ); + return graph; + } + /** Return a cursor marking the current dialog sequence for `tabId`. */ dialogCursor(tabId: number): DialogCursor { return this.dialogSequences.get(tabId) ?? 0; @@ -332,6 +431,7 @@ export class ChromiumCdp { this.clearDialogState(tabId); this.clearConsoleState(tabId); this.clearNetworkState(tabId); + this.clearFrameState(tabId); try { await this.api.detach({ tabId }); } catch (err) { @@ -354,7 +454,7 @@ export class ChromiumCdp { } /** Subscribe to all CDP events. Returned disposable removes the listener. */ - onEvent(handler: (source: chrome.debugger.Debuggee, method: string, params: unknown) => void): { + onEvent(handler: (source: CdpDebuggee, method: string, params: unknown) => void): { dispose(): void; } { this.api.onEvent.addListener(handler); @@ -378,6 +478,7 @@ export class ChromiumCdp { this.networkSequences.clear(); this.networkDomainsEnabledTabs.clear(); this.networkRequestMeta.clear(); + this.frameDiscovery.clear(); await Promise.all( tabs.map(async (tabId) => { try { @@ -428,9 +529,104 @@ export class ChromiumCdp { } } + private async enableFrameDiscovery(target: CdpTarget): Promise { + await this.api.sendCommand(target, "Target.setAutoAttach", { + autoAttach: true, + waitForDebuggerOnStart: false, + flatten: true, + filter: [{ type: "iframe", exclude: false }], + }); + } + + private bindFrameTargetHandler(): void { + if (this.frameTargetSubscription) return; + const listener = (source: CdpDebuggee, method: string, params: unknown) => { + const tabId = source.tabId; + if (typeof tabId !== "number") return; + const raw = (params ?? {}) as Record; + const sessionId = typeof raw.sessionId === "string" ? raw.sessionId : undefined; + if (!sessionId) return; + + if (method === "Target.detachedFromTarget") { + const state = this.frameDiscovery.get(tabId); + if (state?.sessions.delete(sessionId)) state.generation += 1; + return; + } + if (method !== "Target.attachedToTarget") return; + const targetInfo = raw.targetInfo as { type?: string } | undefined; + if (targetInfo?.type && targetInfo.type !== "iframe") return; + + const state = this.frameDiscoveryState(tabId); + if (state.sessions.has(sessionId)) return; + state.sessions.add(sessionId); + state.generation += 1; + + const task = this.initializeFrameTarget({ tabId, sessionId }); + state.pending.add(task); + void task.finally(() => { + state.pending.delete(task); + }); + }; + this.api.onEvent.addListener(listener); + this.frameTargetSubscription = { + dispose: () => this.api.onEvent.removeListener(listener), + }; + } + + private async initializeFrameTarget(target: CdpTarget): Promise { + try { + await this.enableFrameDiscovery(target); + } catch (err) { + const state = this.frameDiscovery.get(target.tabId); + if (state?.sessions.delete(target.sessionId as string)) state.generation += 1; + console.debug("[bsk cdp] child frame target initialization failed", { target, err }); + } + } + + private async drainFrameAttachTasks(tabId: number): Promise { + const deadline = Date.now() + FRAME_DISCOVERY_TIMEOUT_MS; + while (Date.now() < deadline) { + const state = this.frameDiscovery.get(tabId); + const generation = state?.generation ?? 0; + if (state?.pending.size && !(await settleBeforeDeadline([...state.pending], deadline))) break; + + // Target.setAutoAttach may enqueue the next attachedToTarget event after + // its command promise settles. Wait for a short quiet window, then finish + // only if neither the state object nor its generation changed. + const quietTime = Math.min(FRAME_DISCOVERY_QUIET_MS, deadline - Date.now()); + if (quietTime <= 0) break; + await new Promise((resolve) => setTimeout(resolve, quietTime)); + const current = this.frameDiscovery.get(tabId); + if ( + current === state && + (current?.generation ?? 0) === generation && + !current?.pending.size + ) { + return; + } + } + console.debug("[bsk cdp] frame discovery did not reach quiescence before timeout", { tabId }); + } + + private frameDiscoveryState(tabId: number): FrameDiscoveryState { + const existing = this.frameDiscovery.get(tabId); + if (existing) return existing; + const created: FrameDiscoveryState = { + sessions: new Set(), + pending: new Set(), + generation: 0, + }; + this.frameDiscovery.set(tabId, created); + return created; + } + + private clearFrameState(tabId: number): void { + this.frameDiscovery.delete(tabId); + } + private bindDialogHandler(): void { if (this.dialogSubscription) return; - const listener = (source: chrome.debugger.Debuggee, method: string, params: unknown) => { + const listener = (source: CdpDebuggee, method: string, params: unknown) => { if (method !== "Page.javascriptDialogOpening") return; const tabId = source.tabId; if (typeof tabId !== "number") return; @@ -614,6 +810,7 @@ export class ChromiumCdp { this.clearDialogState(source.tabId); this.clearConsoleState(source.tabId); this.clearNetworkState(source.tabId); + this.clearFrameState(source.tabId); } }; this.api.onDetach.addListener(listener); @@ -632,6 +829,8 @@ export class ChromiumCdp { this.consoleSubscription = null; this.networkSubscription?.dispose(); this.networkSubscription = null; + this.frameTargetSubscription?.dispose(); + this.frameTargetSubscription = null; } /** Detach tabs only when no other live session has claimed them. */ diff --git a/apps/extension/src/browser-driver/frame-graph.ts b/apps/extension/src/browser-driver/frame-graph.ts new file mode 100644 index 00000000..1dc2ead3 --- /dev/null +++ b/apps/extension/src/browser-driver/frame-graph.ts @@ -0,0 +1,107 @@ +export interface CdpTarget { + tabId: number; + sessionId?: string; +} + +export interface CdpFrame { + frameId: string; + parentFrameId?: string; + ownerBackendNodeId?: number; + url?: string; + target: CdpTarget; +} + +export interface CdpFrameGraph { + rootFrameId: string; + frames: CdpFrame[]; +} + +export interface CdpFrameTreeNode { + frame: { + id: string; + parentId?: string; + url?: string; + }; + childFrames?: CdpFrameTreeNode[]; +} + +export interface CdpFrameTreeSource { + target: CdpTarget; + tree: CdpFrameTreeNode; +} + +export function cdpTargetKey(target: CdpTarget): string { + return `${target.tabId}:${target.sessionId ?? "root"}`; +} + +function mergeFrame( + frames: Map, + order: string[], + node: CdpFrameTreeNode, + target: CdpTarget, +): void { + const existing = frames.get(node.frame.id); + if (!existing) order.push(node.frame.id); + const parentFrameId = node.frame.parentId || existing?.parentFrameId; + const url = node.frame.url || existing?.url; + frames.set(node.frame.id, { + frameId: node.frame.id, + ...(parentFrameId ? { parentFrameId } : {}), + ...(url ? { url } : {}), + target, + }); +} + +function walkFrameTree( + source: CdpFrameTreeSource, + targetByRootFrameId: ReadonlyMap, + frames: Map, + order: string[], +): void { + const stack: Array<{ node: CdpFrameTreeNode; inheritedTarget: CdpTarget }> = [ + { node: source.tree, inheritedTarget: source.target }, + ]; + const expanded = new WeakSet(); + + while (stack.length > 0) { + const current = stack.pop(); + if (!current || expanded.has(current.node)) continue; + expanded.add(current.node); + + const target = targetByRootFrameId.get(current.node.frame.id) ?? current.inheritedTarget; + mergeFrame(frames, order, current.node, target); + + const children = current.node.childFrames ?? []; + for (let index = children.length - 1; index >= 0; index -= 1) { + stack.push({ node: children[index], inheritedTarget: target }); + } + } +} + +export function buildFrameGraph(sources: CdpFrameTreeSource[]): CdpFrameGraph | null { + const rootSource = sources.find((source) => source.target.sessionId === undefined) ?? sources[0]; + if (!rootSource) return null; + + const frames = new Map(); + const order: string[] = []; + const targetByRootFrameId = new Map( + sources.map((source) => [source.tree.frame.id, source.target] as const), + ); + // The top-level source is authoritative if malformed input reports the same + // frame as both a root target and a child target. + targetByRootFrameId.set(rootSource.tree.frame.id, rootSource.target); + + walkFrameTree(rootSource, targetByRootFrameId, frames, order); + for (const source of sources) { + if (source === rootSource) continue; + walkFrameTree(source, targetByRootFrameId, frames, order); + } + + return { + rootFrameId: rootSource.tree.frame.id, + frames: order.flatMap((frameId) => { + const frame = frames.get(frameId); + return frame ? [frame] : []; + }), + }; +} diff --git a/apps/extension/src/content/HelpRequestOverlay.tsx b/apps/extension/src/content/HelpRequestOverlay.tsx index 087df654..0f39bc67 100644 --- a/apps/extension/src/content/HelpRequestOverlay.tsx +++ b/apps/extension/src/content/HelpRequestOverlay.tsx @@ -8,6 +8,7 @@ import { useRef, useState, } from "react"; +import type { HelpHighlightRect } from "@/lib/help-bridge"; import logoUrl from "../../assets/logo.png"; export interface HelpRequestData { @@ -19,6 +20,10 @@ export interface HelpRequestData { displayMode?: "full" | "compact"; /** CSS selectors to scroll to + flash-highlight. */ selectors: string[]; + /** Pre-resolved top-viewport rectangles, used for cross-frame targets. */ + rects?: HelpHighlightRect[]; + /** Re-resolve cross-frame rectangles after scroll, resize, or layout changes. */ + refreshRects?: () => Promise; onContinue: (note: string) => void; onCancel: () => void; } @@ -49,6 +54,7 @@ const PANEL_WIDTH = 420; const FALLBACK_PANEL_H = 180; const FALLBACK_PANEL_H_COLLAPSED = 44; const EMPTY_SELECTORS: string[] = []; +const EMPTY_RECTS: HelpHighlightRect[] = []; function clamp(value: number, min: number, max: number): number { return Math.max(min, Math.min(max, value)); @@ -179,6 +185,8 @@ export function HelpRequestOverlay({ request }: Props) { const { t } = useTranslation("extension"); const isCompact = request?.displayMode === "compact"; const effectiveSelectors = isCompact ? EMPTY_SELECTORS : (request?.selectors ?? EMPTY_SELECTORS); + const [liveRects, setLiveRects] = useState(null); + const effectiveRects = isCompact ? EMPTY_RECTS : (liveRects ?? request?.rects ?? EMPTY_RECTS); const [note, setNote] = useState(""); const [collapsed, setCollapsed] = useState(false); const [boxes, setBoxes] = useState([]); @@ -201,6 +209,7 @@ export function HelpRequestOverlay({ request }: Props) { // Reset the note and collapse state whenever a new request appears. useEffect(() => { setNote(""); + setLiveRects(null); setCollapsed(false); placementRef.current = null; setDragPos(null); @@ -209,6 +218,32 @@ export function HelpRequestOverlay({ request }: Props) { dragStartRef.current = null; }, [request?.id]); + useEffect(() => { + if (!request || isCompact || !request.refreshRects) return; + let disposed = false; + let resolving = false; + const refresh = async () => { + if (resolving) return; + resolving = true; + try { + const rects = await request.refreshRects?.(); + if (!disposed && rects) setLiveRects(rects); + } finally { + resolving = false; + } + }; + const onViewportChange = () => void refresh(); + window.addEventListener("scroll", onViewportChange, true); + window.addEventListener("resize", onViewportChange); + const interval = window.setInterval(onViewportChange, 500); + return () => { + disposed = true; + window.removeEventListener("scroll", onViewportChange, true); + window.removeEventListener("resize", onViewportChange); + window.clearInterval(interval); + }; + }, [request, isCompact]); + // Scroll the first matched target into view once per distinct request id. useLayoutEffect(() => { if (!request) return; @@ -231,7 +266,7 @@ export function HelpRequestOverlay({ request }: Props) { return; } const update = () => { - const nextBoxes = measure(effectiveSelectors); + const nextBoxes = [...measure(effectiveSelectors), ...effectiveRects]; setBoxes((prev) => (boxesEqual(prev, nextBoxes) ? prev : nextBoxes)); }; update(); @@ -243,7 +278,7 @@ export function HelpRequestOverlay({ request }: Props) { window.removeEventListener("resize", update); window.clearInterval(interval); }; - }, [request, effectiveSelectors]); + }, [request, effectiveSelectors, effectiveRects]); const onHeaderPointerDown = useCallback( (e: ReactPointerEvent) => { @@ -305,7 +340,7 @@ export function HelpRequestOverlay({ request }: Props) { return; } if (userMovedRef.current) return; - const measured = boxes.length > 0 ? boxes : measure(effectiveSelectors); + const measured = boxes.length > 0 ? boxes : [...measure(effectiveSelectors), ...effectiveRects]; const anchor = unionBox(measured); const banner = bannerRef.current; if (!anchor || !banner) { @@ -329,7 +364,7 @@ export function HelpRequestOverlay({ request }: Props) { } const nextPos = placePanel(anchor, panelW, panelH, placementRef.current.placement); setPanelPos((prev) => (panelPosEqual(prev, nextPos) ? prev : nextPos)); - }, [request, boxes, collapsed, isCompact, effectiveSelectors]); + }, [request, boxes, collapsed, isCompact, effectiveSelectors, effectiveRects]); if (!request) return null; diff --git a/apps/extension/src/content/__tests__/HelpRequestOverlay.test.tsx b/apps/extension/src/content/__tests__/HelpRequestOverlay.test.tsx index e1ca276c..d2f03df6 100644 --- a/apps/extension/src/content/__tests__/HelpRequestOverlay.test.tsx +++ b/apps/extension/src/content/__tests__/HelpRequestOverlay.test.tsx @@ -1,6 +1,6 @@ import { i18n } from "@browser-skill/i18n"; import { I18nextProvider } from "@browser-skill/i18n/react"; -import { cleanup, fireEvent, render, screen } from "@testing-library/react"; +import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react"; import { createElement } from "react"; import { afterEach, describe, expect, it, vi } from "vitest"; import { type HelpRequestData, HelpRequestOverlay } from "../HelpRequestOverlay"; @@ -38,6 +38,36 @@ describe("HelpRequestOverlay", () => { expect(screen.getByText("Please complete the captcha")).toBeTruthy(); }); + it("renders explicit viewport rectangles for cross-frame targets", () => { + const { container } = renderOverlay( + baseRequest({ rects: [{ top: 30, left: 40, width: 120, height: 50 }] }), + ); + const highlight = container.querySelector("[data-slot='help-highlight']"); + expect(highlight?.style.top).toBe("30px"); + expect(highlight?.style.left).toBe("40px"); + expect(highlight?.style.width).toBe("120px"); + expect(highlight?.style.height).toBe("50px"); + }); + + it("re-resolves cross-frame rectangles after the viewport changes", async () => { + const refreshRects = vi.fn(async () => [{ top: 80, left: 90, width: 140, height: 60 }]); + const { container } = renderOverlay( + baseRequest({ + rects: [{ top: 30, left: 40, width: 120, height: 50 }], + refreshRects, + }), + ); + + window.dispatchEvent(new Event("scroll")); + + await waitFor(() => { + const highlight = container.querySelector("[data-slot='help-highlight']"); + expect(highlight?.style.top).toBe("80px"); + expect(highlight?.style.left).toBe("90px"); + }); + expect(refreshRects).toHaveBeenCalledTimes(1); + }); + it("keeps the inactive render stable", () => { const { container, rerender } = renderOverlay(null); diff --git a/apps/extension/src/content/__tests__/help-request.test.ts b/apps/extension/src/content/__tests__/help-request.test.ts new file mode 100644 index 00000000..e1da24ef --- /dev/null +++ b/apps/extension/src/content/__tests__/help-request.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it, vi } from "vitest"; +import { createHelpRequestData } from "../help-request"; + +describe("createHelpRequestData", () => { + it("gives live delivery and recovery the same rectangle refresh mechanism", async () => { + const finish = vi.fn(); + const query = vi.fn(async () => ({ + active: true, + request: { + requestId: "help-1", + prompt: "complete the challenge", + selectors: [], + rects: [{ top: 80, left: 90, width: 140, height: 60 }], + timeoutMs: 1_000, + }, + })); + const request = createHelpRequestData( + { + requestId: "help-1", + prompt: "complete the challenge", + selectors: [], + rects: [{ top: 10, left: 20, width: 30, height: 40 }], + timeoutMs: 1_000, + }, + { finish, query }, + ); + + await expect(request.refreshRects?.()).resolves.toEqual([ + { top: 80, left: 90, width: 140, height: 60 }, + ]); + request.onContinue(" done "); + request.onCancel(); + + expect(query).toHaveBeenCalledOnce(); + expect(finish).toHaveBeenNthCalledWith(1, "help-1", "continued", " done "); + expect(finish).toHaveBeenNthCalledWith(2, "help-1", "cancelled"); + }); + + it("ignores rectangle replies for a replaced request", async () => { + const request = createHelpRequestData( + { + requestId: "help-1", + prompt: "complete the challenge", + selectors: [], + timeoutMs: 1_000, + }, + { + finish: vi.fn(), + query: vi.fn(async () => ({ + active: true, + request: { + requestId: "help-2", + prompt: "new challenge", + selectors: [], + rects: [{ top: 1, left: 2, width: 3, height: 4 }], + timeoutMs: 1_000, + }, + })), + }, + ); + + await expect(request.refreshRects?.()).resolves.toBeUndefined(); + }); +}); diff --git a/apps/extension/src/content/help-request.ts b/apps/extension/src/content/help-request.ts new file mode 100644 index 00000000..f5fa9f70 --- /dev/null +++ b/apps/extension/src/content/help-request.ts @@ -0,0 +1,31 @@ +import type { HelpQueryResponse, HelpRequestMessage } from "@/lib/help-bridge"; +import type { HelpRequestData } from "./HelpRequestOverlay"; + +export interface HelpRequestDataDeps { + finish(requestId: string, outcome: "continued" | "cancelled", note?: string): void; + query(): Promise; +} + +/** Build the content-side representation shared by live delivery and recovery. */ +export function createHelpRequestData( + message: Omit, + deps: HelpRequestDataDeps, +): HelpRequestData { + return { + id: message.requestId, + prompt: message.prompt, + ...(message.title ? { title: message.title } : {}), + ...(message.displayMode ? { displayMode: message.displayMode } : {}), + selectors: message.selectors, + rects: message.rects, + refreshRects: async () => { + const response = await deps.query(); + return response?.active && response.request?.requestId === message.requestId + ? response.request.rects + : undefined; + }, + onContinue: (note: string) => + deps.finish(message.requestId, "continued", note.trim() ? note : undefined), + onCancel: () => deps.finish(message.requestId, "cancelled"), + }; +} diff --git a/apps/extension/src/entrypoints/content.ts b/apps/extension/src/entrypoints/content.ts index 52536ad8..7552a098 100644 --- a/apps/extension/src/entrypoints/content.ts +++ b/apps/extension/src/entrypoints/content.ts @@ -7,6 +7,7 @@ import { BorrowConfirmationOverlay } from "@/content/BorrowConfirmationOverlay"; import { ControlOverlay } from "@/content/ControlOverlay"; import { createCaptureSuppressController } from "@/content/capture-suppress"; import { HelpRequestOverlay } from "@/content/HelpRequestOverlay"; +import { createHelpRequestData } from "@/content/help-request"; import overlayCss from "@/content/overlay.css?inline"; import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller"; import { RecordOverlay } from "@/content/RecordOverlay"; @@ -344,16 +345,7 @@ export default defineContentScript({ if (isHelpRequestMessage(message)) { const helpMsg = message as HelpRequestMessage; - const previousHelp = overlays.setAgentHelpRequest({ - id: helpMsg.requestId, - prompt: helpMsg.prompt, - ...(helpMsg.title ? { title: helpMsg.title } : {}), - ...(helpMsg.displayMode ? { displayMode: helpMsg.displayMode } : {}), - selectors: helpMsg.selectors, - onContinue: (note: string) => - void sendHelpFinish(helpMsg.requestId, "continued", note.trim() ? note : undefined), - onCancel: () => void sendHelpFinish(helpMsg.requestId, "cancelled"), - }); + const previousHelp = mountHelpRequest(helpMsg); if (previousHelp && previousHelp.id !== helpMsg.requestId) { void sendHelpFinish(previousHelp.id, "cancelled"); } @@ -406,17 +398,18 @@ export default defineContentScript({ }); } - function mountHelpRequest(helpMsg: Omit): void { - overlays.setAgentHelpRequest({ - id: helpMsg.requestId, - prompt: helpMsg.prompt, - ...(helpMsg.title ? { title: helpMsg.title } : {}), - ...(helpMsg.displayMode ? { displayMode: helpMsg.displayMode } : {}), - selectors: helpMsg.selectors, - onContinue: (note: string) => - void sendHelpFinish(helpMsg.requestId, "continued", note.trim() ? note : undefined), - onCancel: () => void sendHelpFinish(helpMsg.requestId, "cancelled"), - }); + function mountHelpRequest(helpMsg: Omit) { + return overlays.setAgentHelpRequest( + createHelpRequestData(helpMsg, { + finish: (requestId, outcome, note) => { + void sendHelpFinish(requestId, outcome, note); + }, + query: () => + chrome.runtime.sendMessage({ type: HELP_QUERY }) as Promise< + HelpQueryResponse | undefined + >, + }), + ); } async function queryActiveHelpWithRetry(): Promise { diff --git a/apps/extension/src/lib/__tests__/help-bridge.test.ts b/apps/extension/src/lib/__tests__/help-bridge.test.ts index 50b4b81f..db524457 100644 --- a/apps/extension/src/lib/__tests__/help-bridge.test.ts +++ b/apps/extension/src/lib/__tests__/help-bridge.test.ts @@ -32,6 +32,7 @@ describe("help-bridge", () => { prompt: "log in", displayMode: "compact", selectors: ["#login"], + rects: [{ top: 10, left: 20, width: 100, height: 40 }], timeoutMs: 1000, }), ).toBe(true); diff --git a/apps/extension/src/lib/help-bridge.ts b/apps/extension/src/lib/help-bridge.ts index b73b45c2..46557502 100644 --- a/apps/extension/src/lib/help-bridge.ts +++ b/apps/extension/src/lib/help-bridge.ts @@ -16,6 +16,13 @@ export const HELP_ACK = "bsk-help-ack"; export const HELP_QUERY = "bsk-help-query"; export const HELP_FINISH = "bsk-help-finish"; +export interface HelpHighlightRect { + top: number; + left: number; + width: number; + height: number; +} + export interface HelpRequestMessage { type: typeof HELP_REQUEST; requestId: string; @@ -26,6 +33,8 @@ export interface HelpRequestMessage { displayMode?: "full" | "compact"; /** CSS selectors to scroll to + flash-highlight (may be empty). */ selectors: string[]; + /** Top-viewport rectangles for targets that cannot be represented by a root selector. */ + rects?: HelpHighlightRect[]; timeoutMs: number; } @@ -74,6 +83,16 @@ export function isHelpRequestMessage(msg: unknown): msg is HelpRequestMessage { (m.displayMode === undefined || m.displayMode === "full" || m.displayMode === "compact") && Array.isArray(m.selectors) && m.selectors.every((selector) => typeof selector === "string") && + (m.rects === undefined || + (Array.isArray(m.rects) && + m.rects.every( + (rect) => + typeof rect === "object" && + rect !== null && + ["top", "left", "width", "height"].every((key) => + Number.isFinite((rect as Record)[key]), + ), + ))) && typeof m.timeoutMs === "number" ); } diff --git a/apps/extension/src/session-manager/__tests__/ref-store.test.ts b/apps/extension/src/session-manager/__tests__/ref-store.test.ts index 8a544470..1a8d003c 100644 --- a/apps/extension/src/session-manager/__tests__/ref-store.test.ts +++ b/apps/extension/src/session-manager/__tests__/ref-store.test.ts @@ -33,4 +33,19 @@ describe("RefStore", () => { expect(s.resolve("@e11")).toBe(11); expect(s.size()).toBe(2); }); + + it("preserves the child CDP session as part of ref identity", () => { + const s = new RefStore(); + s.set("e1", 42, { + tabId: 7, + frameId: "child-frame", + cdpSessionId: "child-session", + }); + expect(s.resolveEntry("e1")).toMatchObject({ + backendNodeId: 42, + tabId: 7, + frameId: "child-frame", + cdpSessionId: "child-session", + }); + }); }); diff --git a/apps/extension/src/session-manager/ref-store.ts b/apps/extension/src/session-manager/ref-store.ts index 2098aafa..568392fc 100644 --- a/apps/extension/src/session-manager/ref-store.ts +++ b/apps/extension/src/session-manager/ref-store.ts @@ -1,9 +1,10 @@ /** - * Per-session map from `@e` snapshot refs to CDP `backendNodeId`. + * Per-session map from `@e` snapshot refs to a CDP node address. * * Each fresh `tool.snapshot` resets the store: M6 will call - * `replace(...)` with the new ref → backendNodeId pairs. Tools like - * `tool.click` consume the store via `resolve("@e1")`. + * `replace(...)` with the new ref → node address pairs. A node address + * includes the owning flat CDP session/frame when the element lives in + * an OOPIF; tools resolve live geometry from that identity at call time. * * Refs are session-scoped (§7): looking up a ref in the wrong session * returns `null`, never silently leaks. Storing values in different @@ -15,10 +16,19 @@ export type BackendNodeId = number; export interface RefEntry { backendNodeId: BackendNodeId; tabId: number | null; + frameId?: string; + cdpSessionId?: string; generation: number; } -type RefInput = BackendNodeId | { backendNodeId: BackendNodeId; tabId: number }; +export type RefInput = + | BackendNodeId + | { + backendNodeId: BackendNodeId; + tabId: number; + frameId?: string; + cdpSessionId?: string; + }; export class RefStore { private readonly map = new Map(); @@ -44,7 +54,7 @@ export class RefStore { } /** - * Replace the entire store with a new ref → backendNodeId mapping. + * Replace the entire store with a new ref → CDP node identity mapping. * Used after every fresh `tool.snapshot`. */ replace(entries: Iterable): void { @@ -53,10 +63,20 @@ export class RefStore { for (const [ref, input] of entries) this.map.set(normaliseRef(ref), this.entry(input)); } - set(ref: string, id: BackendNodeId, opts: { tabId?: number } = {}): void { + set( + ref: string, + id: BackendNodeId, + opts: { + tabId?: number; + frameId?: string; + cdpSessionId?: string; + } = {}, + ): void { this.map.set(normaliseRef(ref), { backendNodeId: id, tabId: opts.tabId ?? null, + ...(opts.frameId ? { frameId: opts.frameId } : {}), + ...(opts.cdpSessionId ? { cdpSessionId: opts.cdpSessionId } : {}), generation: this.generation, }); } @@ -80,6 +100,8 @@ export class RefStore { return { backendNodeId: input.backendNodeId, tabId: input.tabId, + ...(input.frameId ? { frameId: input.frameId } : {}), + ...(input.cdpSessionId ? { cdpSessionId: input.cdpSessionId } : {}), generation: this.generation, }; } diff --git a/apps/extension/src/tools/__tests__/element-geometry.test.ts b/apps/extension/src/tools/__tests__/element-geometry.test.ts index 9b454fad..b5fcfbe9 100644 --- a/apps/extension/src/tools/__tests__/element-geometry.test.ts +++ b/apps/extension/src/tools/__tests__/element-geometry.test.ts @@ -1,24 +1,54 @@ import { describe, expect, it } from "vitest"; -import { nodeBoundingRect, quadBoundingRect } from "../element-geometry"; +import { nodeContentRegion } from "../element-geometry"; import type { CdpRunner } from "../shared"; -describe("quadBoundingRect", () => { - it("computes axis-aligned bounds from an 8-double quad", () => { - expect(quadBoundingRect([10, 20, 110, 20, 110, 60, 10, 60])).toEqual({ - x: 10, - y: 20, - width: 100, - height: 40, - }); +describe("nodeContentRegion", () => { + it("rejects invalid content quads before using the box model fallback", async () => { + const send = async (_tabId: number, method: string) => { + if (method === "DOM.getContentQuads") return { quads: [[0, 0, 0, 0, 0, 0, 0, 0]] }; + if (method === "DOM.getBoxModel") { + return { model: { content: [5, 10, 25, 10, 25, 30, 5, 30] } }; + } + throw new Error(`unexpected CDP call ${method}`); + }; + + await expect(nodeContentRegion({ send: send as CdpRunner["send"] }, 7, 555)).resolves.toEqual([ + [ + { x: 5, y: 10 }, + { x: 25, y: 10 }, + { x: 25, y: 30 }, + { x: 5, y: 30 }, + ], + ]); }); - it("returns null for degenerate quads", () => { - expect(quadBoundingRect([0, 0, 0, 0, 0, 0, 0, 0])).toBeNull(); - expect(quadBoundingRect([1, 2, 3])).toBeNull(); + it("preserves every visible content fragment", async () => { + const send = async () => ({ + quads: [ + [10, 20, 60, 20, 60, 40, 10, 40], + [15, 50, 90, 50, 90, 80, 15, 80], + ], + }); + const cdp: CdpRunner = { + send: send as CdpRunner["send"], + }; + + await expect(nodeContentRegion(cdp, 7, 555)).resolves.toEqual([ + [ + { x: 10, y: 20 }, + { x: 60, y: 20 }, + { x: 60, y: 40 }, + { x: 10, y: 40 }, + ], + [ + { x: 15, y: 50 }, + { x: 90, y: 50 }, + { x: 90, y: 80 }, + { x: 15, y: 80 }, + ], + ]); }); -}); -describe("nodeBoundingRect", () => { it("falls back to visible descendant bounds for zero-size containers", async () => { const calls: Array<{ method: string; params?: object }> = []; const send = async (_tabId: number, method: string, params?: object) => { @@ -38,12 +68,14 @@ describe("nodeBoundingRect", () => { }; const cdp = { send: send as CdpRunner["send"] }; - await expect(nodeBoundingRect(cdp, 7, 555)).resolves.toEqual({ - x: 242, - y: 468, - width: 769, - height: 180, - }); + await expect(nodeContentRegion(cdp, 7, 555)).resolves.toEqual([ + [ + { x: 242, y: 468 }, + { x: 1011, y: 468 }, + { x: 1011, y: 648 }, + { x: 242, y: 648 }, + ], + ]); expect(calls.map((c) => c.method)).toEqual([ "DOM.getContentQuads", "DOM.getBoxModel", diff --git a/apps/extension/src/tools/__tests__/frame-geometry.test.ts b/apps/extension/src/tools/__tests__/frame-geometry.test.ts new file mode 100644 index 00000000..c7f82b4d --- /dev/null +++ b/apps/extension/src/tools/__tests__/frame-geometry.test.ts @@ -0,0 +1,286 @@ +import { describe, expect, it, vi } from "vitest"; +import { resolveFrameProjection, resolveNodeGeometry } from "../frame-geometry"; +import { + clipPolygon, + polygonArea, + polygonBounds, + projectAndClipRegion, + projectUnitPoint, + type Quad, + rectPolygon, + regionBounds, +} from "../geometry"; +import type { CdpRunner } from "../shared"; + +describe("frame geometry projection", () => { + it("keeps region bounds separate from polygon area", () => { + const region = [ + rectPolygon({ x: 0, y: 0, w: 10, h: 10 }), + rectPolygon({ x: 20, y: 5, w: 5, h: 5 }), + ]; + expect(regionBounds(region)).toEqual({ x: 0, y: 0, width: 25, height: 10 }); + expect(polygonArea(region[0])).toBe(100); + }); + + it("maps through the iframe content box instead of its border box", () => { + const contentQuad: Quad = [ + { x: 204, y: 306 }, + { x: 604, y: 306 }, + { x: 604, y: 506 }, + { x: 204, y: 506 }, + ]; + const projected = projectAndClipRegion( + [rectPolygon({ x: 10, y: 20, w: 100, h: 40 })], + [{ sourceViewport: { width: 200, height: 100 }, destinationQuad: contentQuad }], + { width: 1000, height: 800 }, + ); + expect(polygonBounds(projected.flat())).toEqual({ x: 224, y: 346, width: 200, height: 80 }); + }); + + it("uses a projective mapping for perspective-transformed iframe quads", () => { + const quad: Quad = [ + { x: 0, y: 0 }, + { x: 200, y: 20 }, + { x: 180, y: 140 }, + { x: 20, y: 100 }, + ]; + expect(projectUnitPoint({ x: 0, y: 0 }, quad)).toEqual(quad[0]); + expect(projectUnitPoint({ x: 1, y: 0 }, quad)).toEqual(quad[1]); + expect(projectUnitPoint({ x: 1, y: 1 }, quad)).toEqual(quad[2]); + expect(projectUnitPoint({ x: 0, y: 1 }, quad)).toEqual(quad[3]); + }); + + it("clips against the iframe polygon rather than its bounding box", () => { + const frameQuad: Quad = [ + { x: 50, y: 0 }, + { x: 100, y: 50 }, + { x: 50, y: 100 }, + { x: 0, y: 50 }, + ]; + expect(clipPolygon(rectPolygon({ x: 0, y: 0, w: 10, h: 10 }), frameQuad)).toEqual([]); + }); + + it("does not reapply same-target frame transforms around an OOPIF", async () => { + const send = vi.fn(async (_tabId, method, params) => { + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } }; + } + if (method === "DOM.getBoxModel") { + const backendNodeId = (params as { backendNodeId?: number }).backendNodeId; + return backendNodeId === 20 + ? { model: { content: [110, 70, 310, 70, 310, 170, 110, 170] } } + : { model: { content: [100, 50, 400, 50, 400, 250, 100, 250] } }; + } + throw new Error(`unexpected root command ${method}`); + }); + const cdp: CdpRunner = { + send: send as CdpRunner["send"], + sendToTarget: vi.fn(async (_target, method) => { + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } }; + } + throw new Error(`unexpected child command ${method}`); + }) as CdpRunner["sendToTarget"], + }; + const projection = await resolveFrameProjection( + cdp, + { + rootFrameId: "main", + frames: [ + { frameId: "main", target: { tabId: 4 } }, + { + frameId: "same-process-parent", + parentFrameId: "main", + ownerBackendNodeId: 10, + target: { tabId: 4 }, + }, + { + frameId: "oopif", + parentFrameId: "same-process-parent", + ownerBackendNodeId: 20, + target: { tabId: 4, sessionId: "oopif-session" }, + }, + ], + }, + "oopif", + ); + + expect(projection?.edges).toEqual([ + { + sourceViewport: { width: 200, height: 100 }, + destinationQuad: [ + { x: 110, y: 70 }, + { x: 310, y: 70 }, + { x: 310, y: 170 }, + { x: 110, y: 170 }, + ], + destinationClips: [ + [ + { x: 100, y: 50 }, + { x: 400, y: 50 }, + { x: 400, y: 250 }, + { x: 100, y: 250 }, + ], + ], + }, + ]); + }); + + it("clips live OOPIF geometry at every frame boundary", async () => { + const cdp: CdpRunner = { + send: vi.fn(async (_tabId, method) => { + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } }; + } + if (method === "DOM.getBoxModel") { + return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } }; + } + throw new Error(`unexpected root command ${method}`); + }) as CdpRunner["send"], + sendToTarget: vi.fn(async (_target, method) => { + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } }; + } + if (method === "DOM.getContentQuads") { + return { quads: [[-50, -20, 250, -20, 250, 120, -50, 120]] }; + } + throw new Error(`unexpected child command ${method}`); + }) as CdpRunner["sendToTarget"], + getFrameGraph: vi.fn(async () => ({ + rootFrameId: "main", + frames: [ + { frameId: "main", target: { tabId: 4 } }, + { + frameId: "child", + parentFrameId: "main", + ownerBackendNodeId: 10, + target: { tabId: 4, sessionId: "child-session" }, + }, + ], + })), + }; + + const geometry = await resolveNodeGeometry(cdp, 4, { + target: { tabId: 4, sessionId: "child-session" }, + frameId: "child", + backendNodeId: 101, + }); + + expect(geometry).toMatchObject({ + topBounds: { x: 100, y: 100, width: 200, height: 100 }, + actionPoint: { x: 200, y: 150 }, + }); + }); + + it("reuses one frame graph while scrolling and projecting live geometry", async () => { + const getFrameGraph = vi.fn(async () => ({ + rootFrameId: "main", + frames: [ + { frameId: "main", target: { tabId: 4 } }, + { + frameId: "child", + parentFrameId: "main", + ownerBackendNodeId: 10, + target: { tabId: 4, sessionId: "child-session" }, + }, + ], + })); + const cdp: CdpRunner = { + send: vi.fn(async (_tabId, method) => { + if (method === "DOM.scrollIntoViewIfNeeded") return {}; + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } }; + } + if (method === "DOM.getBoxModel") { + return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } }; + } + throw new Error(`unexpected root command ${method}`); + }) as CdpRunner["send"], + sendToTarget: vi.fn(async (_target, method) => { + if (method === "DOM.scrollIntoViewIfNeeded") return {}; + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } }; + } + if (method === "DOM.getContentQuads") { + return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] }; + } + throw new Error(`unexpected child command ${method}`); + }) as CdpRunner["sendToTarget"], + getFrameGraph, + }; + + await expect( + resolveNodeGeometry( + cdp, + 4, + { + target: { tabId: 4, sessionId: "child-session" }, + frameId: "child", + backendNodeId: 101, + }, + { scrollIntoView: true }, + ), + ).resolves.not.toMatchObject({ code: expect.any(String) }); + expect(getFrameGraph).toHaveBeenCalledOnce(); + }); + + it("clips live geometry to same-target iframe ancestors", async () => { + const cdp: CdpRunner = { + send: vi.fn(async (_tabId, method) => { + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } }; + } + if (method === "DOM.getBoxModel") { + return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } }; + } + if (method === "DOM.getContentQuads") { + return { quads: [[50, 50, 350, 50, 350, 250, 50, 250]] }; + } + throw new Error(`unexpected ${method}`); + }) as CdpRunner["send"], + getFrameGraph: vi.fn(async () => ({ + rootFrameId: "main", + frames: [ + { frameId: "main", target: { tabId: 4 } }, + { + frameId: "child", + parentFrameId: "main", + ownerBackendNodeId: 10, + target: { tabId: 4 }, + }, + ], + })), + }; + + const geometry = await resolveNodeGeometry(cdp, 4, { + target: { tabId: 4 }, + frameId: "child", + backendNodeId: 101, + }); + + expect(geometry).toMatchObject({ + topBounds: { x: 100, y: 100, width: 200, height: 100 }, + actionPoint: { x: 200, y: 150 }, + }); + }); + + it("fails closed when an OOPIF frame graph is unavailable", async () => { + const cdp: CdpRunner = { + send: vi.fn(), + sendToTarget: vi.fn(async (_target, method) => { + if (method === "DOM.getContentQuads") { + return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] }; + } + throw new Error(`unexpected ${method}`); + }) as CdpRunner["sendToTarget"], + }; + + await expect( + resolveNodeGeometry(cdp, 4, { + target: { tabId: 4, sessionId: "child-session" }, + frameId: "child", + backendNodeId: 101, + }), + ).resolves.toMatchObject({ code: "cdp_failed" }); + }); +}); diff --git a/apps/extension/src/tools/__tests__/human-loop.test.ts b/apps/extension/src/tools/__tests__/human-loop.test.ts index 960758bc..a500735f 100644 --- a/apps/extension/src/tools/__tests__/human-loop.test.ts +++ b/apps/extension/src/tools/__tests__/human-loop.test.ts @@ -1,5 +1,6 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { SessionManager } from "@/session-manager/manager"; +import { RefStore } from "@/session-manager/ref-store"; import type { RequestHelpParams } from "@/transport/types"; import { handleRequestHelp, type RequestHelpDeps, resetHelpLifecycleForTests } from "../human-loop"; @@ -50,11 +51,10 @@ function installHelpLifecycleChrome() { } function fakeManager(sessionId: string, agentWindowId: number, tabId: number) { + const refStore = new RefStore(); const mgr = { get: (id: string) => - id === sessionId - ? { sessionId, agentWindowId, refStore: { resolve: () => null }, borrowedTabs: new Map() } - : null, + id === sessionId ? { sessionId, agentWindowId, refStore, borrowedTabs: new Map() } : null, findByWindowId: (wid: number) => (wid === agentWindowId ? { sessionId } : null), } as unknown as SessionManager; return mgr; @@ -339,26 +339,23 @@ describe("handleRequestHelp", () => { ac.abort(); await expect(pending).resolves.toMatchObject({ code: "cancelled" }); - expect(cdpSend).toHaveBeenCalledWith( - 5, - "DOM.querySelectorAll", - expect.objectContaining({ selector: "[data-bsk-help]" }), - ); - expect(cdpSend).toHaveBeenCalledWith( - 6, - "DOM.querySelectorAll", - expect.objectContaining({ selector: "[data-bsk-help]" }), + expect(cdpSend).not.toHaveBeenCalledWith( + expect.anything(), + "DOM.removeAttribute", + expect.anything(), ); }); - it("tags ref targets via CDP and reports them matched", async () => { + it("resolves ref targets to explicit viewport rectangles without mutating the page", async () => { + const refStore = new RefStore(); + refStore.set("e1", 42, { tabId: 5 }); const mgr = { get: (id: string) => id === "abcd" ? { sessionId: "abcd", agentWindowId: 99, - refStore: { resolve: () => 42 }, + refStore, borrowedTabs: new Map(), } : null, @@ -366,7 +363,17 @@ describe("handleRequestHelp", () => { } as unknown as SessionManager; const deps = baseDeps({ cdp: { - send: vi.fn(async () => ({ object: { objectId: "obj-1" } })), + send: vi.fn(async (_tabId, method) => { + if (method === "DOM.getDocument") return { root: { nodeId: 1 } }; + if (method === "DOM.scrollIntoViewIfNeeded") return {}; + if (method === "DOM.getContentQuads") { + return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }; + } + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } }; + } + throw new Error(`unexpected ${method}`); + }), } as unknown as RequestHelpDeps["cdp"], }); const res = await handleRequestHelp( @@ -375,12 +382,95 @@ describe("handleRequestHelp", () => { deps, ); const sentMsg = (deps.sendToTab as ReturnType).mock.calls[0][1]; - expect(sentMsg.selectors).toContain('[data-bsk-help="0"]'); + expect(sentMsg.selectors).toEqual([]); + expect(sentMsg.rects).toEqual([{ top: 20, left: 10, width: 100, height: 40 }]); expect(res).toMatchObject({ outcome: "continued", tab_id: 5, - resolved_targets: [{ matched: true, ref: "@e1" }], + resolved_targets: [ + { + matched: true, + ref: "@e1", + }, + ], + }); + }); + + it("projects an OOPIF ref through its live content quad for the help overlay", async () => { + const mgr = { + get: (id: string) => + id === "abcd" + ? { + sessionId: "abcd", + agentWindowId: 99, + refStore: { + resolveEntry: () => ({ + backendNodeId: 42, + tabId: 5, + frameId: "child", + cdpSessionId: "child-session", + generation: 1, + }), + }, + borrowedTabs: new Map(), + } + : null, + findByWindowId: (wid: number) => (wid === 99 ? { sessionId: "abcd" } : null), + } as unknown as SessionManager; + const send = vi.fn(async (_tabId, method) => { + if (method === "DOM.getDocument") return { root: { nodeId: 1 } }; + if (method === "DOM.scrollIntoViewIfNeeded") return {}; + if (method === "DOM.getBoxModel") { + return { model: { content: [204, 306, 604, 306, 604, 506, 204, 506] } }; + } + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } }; + } + throw new Error(`unexpected root ${method}`); + }); + const sendToTarget = vi.fn(async (_target, method) => { + if (method === "DOM.scrollIntoViewIfNeeded") return {}; + if (method === "DOM.getContentQuads") { + return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }; + } + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } }; + } + throw new Error(`unexpected child ${method}`); }); + const deps = baseDeps({ + cdp: { + send, + sendToTarget, + getFrameGraph: vi.fn(async () => ({ + rootFrameId: "main", + frames: [ + { frameId: "main", target: { tabId: 5 } }, + { + frameId: "child", + parentFrameId: "main", + ownerBackendNodeId: 99, + target: { tabId: 5, sessionId: "child-session" }, + }, + ], + })), + } as unknown as NonNullable, + }); + + const res = await handleRequestHelp( + mgr, + baseParams({ tab_id: 5, targets: [{ ref: "@e1" }] }), + deps, + ); + + const sentMsg = (deps.sendToTab as ReturnType).mock.calls[0][1]; + expect(sentMsg.rects).toEqual([{ top: 346, left: 224, width: 200, height: 80 }]); + expect(res).toMatchObject({ resolved_targets: [{ matched: true, ref: "@e1" }] }); + expect(sendToTarget).toHaveBeenCalledWith( + { tabId: 5, sessionId: "child-session" }, + "DOM.getContentQuads", + { backendNodeId: 42 }, + ); }); it("reports ref target unmatched when ref does not resolve", async () => { @@ -398,16 +488,15 @@ describe("handleRequestHelp", () => { }); it("reports ref target unmatched when ref is for another tab", async () => { + const refStore = new RefStore(); + refStore.set("e1", 42, { tabId: 4 }); const mgr = { get: (id: string) => id === "abcd" ? { sessionId: "abcd", agentWindowId: 99, - refStore: { - resolve: (ref: string, opts: { tabId?: number }) => - ref === "e1" && opts.tabId === 4 ? 42 : null, - }, + refStore, borrowedTabs: new Map(), } : null, diff --git a/apps/extension/src/tools/__tests__/interaction.test.ts b/apps/extension/src/tools/__tests__/interaction.test.ts index 6f8eeed8..b78877ad 100644 --- a/apps/extension/src/tools/__tests__/interaction.test.ts +++ b/apps/extension/src/tools/__tests__/interaction.test.ts @@ -30,6 +30,9 @@ function makeFakeCdp(handlers: Record unknown>) { const sendImpl = async (tabId: number, method: string, params?: object) => { sent.push({ tabId, method, params }); const h = handlers[method]; + if (!h && method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } }; + } if (!h) throw new Error(`unexpected CDP call ${method}`); return h(params); }; @@ -164,6 +167,63 @@ describe("handleClick", () => { }); }); + it("resolves frame refs in their CDP session and dispatches input in top coordinates", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + const ctx = await sm.start("aa11"); + ctx.refStore.set("e3", 1234, { + tabId: 4, + frameId: "child-frame", + cdpSessionId: "child-session", + }); + const fake = makeFakeCdp({ + "Input.dispatchMouseEvent": () => ({}), + "DOM.scrollIntoViewIfNeeded": () => ({}), + "DOM.getBoxModel": () => ({ + model: { content: [204, 306, 604, 306, 604, 506, 204, 506] }, + }), + }); + fake.cdp.getFrameGraph = vi.fn(async () => ({ + rootFrameId: "main", + frames: [ + { frameId: "main", target: { tabId: 4 } }, + { + frameId: "child-frame", + parentFrameId: "main", + ownerBackendNodeId: 99, + target: { tabId: 4, sessionId: "child-session" }, + }, + ], + })); + const targetCalls: Array<{ sessionId?: string; method: string }> = []; + fake.cdp.sendToTarget = vi.fn(async (target, method) => { + targetCalls.push({ sessionId: target.sessionId, method }); + if (method === "DOM.scrollIntoViewIfNeeded") return {}; + if (method === "DOM.getContentQuads") { + return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }; + } + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } }; + } + throw new Error(`unexpected child CDP call ${method}`); + }) as CdpRunner["sendToTarget"]; + + const res = await handleClick( + sm, + { session_id: "aa11", ref: "@e3" }, + { cdp: fake.cdp, tabsApi: fake.tabsApi }, + ); + + if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); + // The iframe content box starts at (204, 306) after its border and is scaled 2x. + expect(res).toMatchObject({ x: 324, y: 386 }); + expect(targetCalls).toEqual([ + { sessionId: "child-session", method: "DOM.scrollIntoViewIfNeeded" }, + { sessionId: "child-session", method: "DOM.getContentQuads" }, + { sessionId: "child-session", method: "Page.getLayoutMetrics" }, + ]); + expect(fake.sent.filter((call) => call.method === "Input.dispatchMouseEvent")).toHaveLength(3); + }); + it("enables overlay bypass before mouse events when overlay blocks the click point", async () => { const order: string[] = []; const bypassOverlay = vi.fn(async (_tabId: number, enabled: boolean) => { diff --git a/apps/extension/src/tools/__tests__/observation.test.ts b/apps/extension/src/tools/__tests__/observation.test.ts index 1f9b8cb7..a849843c 100644 --- a/apps/extension/src/tools/__tests__/observation.test.ts +++ b/apps/extension/src/tools/__tests__/observation.test.ts @@ -63,6 +63,9 @@ function makeFakeCdp(handlers: Record unknown>) { const send = vi.fn(async (_tabId: number, method: string, params?: object) => { sent.push({ method, params }); const handler = handlers[method]; + if (!handler && method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } }; + } if (!handler) throw new Error(`unexpected CDP call ${method}`); return handler(params); }); @@ -586,6 +589,7 @@ describe("buildVomScene", () => { { frameId: "main", contextScopeId: "main", + target: { tabId: 7 }, domNodes: mainNodes, axNodes: [ axNode("root", 1, "RootWebArea"), @@ -598,6 +602,7 @@ describe("buildVomScene", () => { contextScopeId: "first", parentFrameId: "main", ownerBackendNodeId: 10, + target: { tabId: 7, sessionId: "session-first" }, domNodes: firstNodes, axNodes: [ axNode("root", 100, "RootWebArea"), @@ -609,6 +614,7 @@ describe("buildVomScene", () => { contextScopeId: "second", parentFrameId: "main", ownerBackendNodeId: 20, + target: { tabId: 7, sessionId: "session-second" }, domNodes: secondNodes, axNodes: [ axNode("root", 200, "RootWebArea"), @@ -621,6 +627,7 @@ describe("buildVomScene", () => { contextScopeId: "nested", parentFrameId: "second", ownerBackendNodeId: 210, + target: { tabId: 7, sessionId: "session-nested" }, domNodes: nestedNodes, axNodes: [ axNode("root", 300, "RootWebArea"), @@ -673,6 +680,7 @@ describe("buildVomScene", () => { { frameId: "main", contextScopeId: "main", + target: { tabId: 7 }, domNodes: [], axNodes: [], }, @@ -680,6 +688,7 @@ describe("buildVomScene", () => { frameId: "child", contextScopeId: "child", parentFrameId: "main", + target: { tabId: 7 }, domNodes: [childNode], axNodes: [ { @@ -740,6 +749,7 @@ describe("buildVomScene", () => { { frameId: "main", contextScopeId: "main", + target: { tabId: 7 }, domNodes: mainNodes, axNodes: [ { @@ -761,6 +771,7 @@ describe("buildVomScene", () => { contextScopeId: "child", parentFrameId: "main", ownerBackendNodeId: 10, + target: { tabId: 7, sessionId: "child-session" }, domNodes: [], axNodes: [ { @@ -3407,6 +3418,32 @@ describe("handleGetHtml", () => { expect(deps.send).toHaveBeenCalledTimes(1); }); + it("reads a frame ref through its child CDP session", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + const ctx = await sm.start("aa11"); + ctx.refStore.set("e7", 4242, { + tabId: 4, + frameId: "child-frame", + cdpSessionId: "child-session", + }); + const deps = makeDeps({}); + const sendToTarget = vi.fn(async (target, method, params) => { + expect(target).toEqual({ tabId: 4, sessionId: "child-session" }); + expect(method).toBe("DOM.getOuterHTML"); + expect(params).toEqual({ backendNodeId: 4242 }); + return { outerHTML: "" }; + }); + (deps.cdp as CdpRunner).sendToTarget = sendToTarget as unknown as NonNullable< + CdpRunner["sendToTarget"] + >; + + const res = await handleGetHtml(sm, { session_id: "aa11", ref: "@e7" }, deps); + + if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); + expect(res.html).toBe(""); + expect(deps.send).not.toHaveBeenCalled(); + }); + it("returns not_found when a ref belongs to another tab", async () => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); const ctx = await sm.start("aa11"); diff --git a/apps/extension/src/tools/__tests__/snapshot-ref.test.ts b/apps/extension/src/tools/__tests__/snapshot-ref.test.ts index 07c3451c..cc9459d8 100644 --- a/apps/extension/src/tools/__tests__/snapshot-ref.test.ts +++ b/apps/extension/src/tools/__tests__/snapshot-ref.test.ts @@ -46,6 +46,25 @@ describe("lookupSnapshotRef", () => { }); describe("resolveSnapshotRef", () => { + it("preserves the frame and child session needed to route iframe refs", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + const ctx = await sm.start("aa11"); + ctx.refStore.set("e3", 1234, { + tabId: 4, + frameId: "child-frame", + cdpSessionId: "child-session", + }); + + const expected = { + backendNodeId: 1234, + refKey: "e3", + frameId: "child-frame", + cdpSessionId: "child-session", + }; + expect(lookupSnapshotRef(ctx, "@e3", 4)).toEqual(expected); + expect(resolveSnapshotRef(ctx, "@e3", 4)).toEqual(expected); + }); + it("returns not_found for unknown ref", async () => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); const ctx = await sm.start("aa11"); diff --git a/apps/extension/src/tools/element-geometry.ts b/apps/extension/src/tools/element-geometry.ts index a610f01c..ccfd344f 100644 --- a/apps/extension/src/tools/element-geometry.ts +++ b/apps/extension/src/tools/element-geometry.ts @@ -1,9 +1,15 @@ -// Shared CDP helpers for scrolling nodes into view and computing -// viewport-space bounding rectangles. Used by interaction tools (click -// centre) and observation tools (element screenshot clip). +// Resolve and scroll nodes inside one CDP target. Cross-frame projection +// belongs to frame-geometry.ts. import type { RpcError } from "@/transport/types"; import { rpcError } from "./errors"; +import { + parseCdpQuad, + polygonBounds, + type Region, + rectPolygon, + type ViewportRect, +} from "./geometry"; import { type CdpRunner, isRpcError } from "./shared"; const ELEMENT_NOT_VISIBLE_MESSAGE = @@ -13,45 +19,6 @@ function elementNotVisibleError(): RpcError { return rpcError("permission_denied", "element_not_visible", ELEMENT_NOT_VISIBLE_MESSAGE); } -/** Viewport-space axis-aligned bounding box for CDP `Page.captureScreenshot` clip. */ -export interface ViewportRect { - x: number; - y: number; - width: number; - height: number; -} - -/** - * Compute the axis-aligned bounding box of an 8-double CDP content - * quad / box-model polygon. Exported for unit tests. - */ -export function quadBoundingRect(quad: number[]): ViewportRect | null { - if (quad.length !== 8) return null; - const xs = [quad[0], quad[2], quad[4], quad[6]]; - const ys = [quad[1], quad[3], quad[5], quad[7]]; - const minX = Math.min(...xs); - const maxX = Math.max(...xs); - const minY = Math.min(...ys); - const maxY = Math.max(...ys); - const width = maxX - minX; - const height = maxY - minY; - if (width <= 0 || height <= 0) return null; - return { x: minX, y: minY, width, height }; -} - -function rectToQuad(rect: ViewportRect): number[] { - return [ - rect.x, - rect.y, - rect.x + rect.width, - rect.y, - rect.x + rect.width, - rect.y + rect.height, - rect.x, - rect.y + rect.height, - ]; -} - /** * Resolve `backendNodeId` → CDP `objectId` so we can invoke * `Runtime.callFunctionOn` against the live JS object. @@ -112,48 +79,24 @@ export async function scrollNodeIntoView( } } -/** - * Viewport-space bounding box for `backendNodeId`. Prefers - * `DOM.getContentQuads`, then `DOM.getBoxModel`, then the union of - * visible descendant client rects for zero-size overflow containers. - */ -export async function nodeBoundingRect( - cdp: CdpRunner, - tabId: number, - backendNodeId: number, -): Promise { - const polygon = await visibleContentPolygon(cdp, tabId, backendNodeId); - if (isRpcError(polygon)) return polygon; - // visibleContentPolygon only returns quads with positive area. - return quadBoundingRect(polygon)!; -} - -/** - * Compute the centroid of a CDP content quad. Quads are reported as - * 8 doubles in clockwise order: (x1, y1, x2, y2, x3, y3, x4, y4). - */ -export function quadCentre(quad: number[]): { x: number; y: number } { - const x = (quad[0] + quad[4]) / 2; - const y = (quad[1] + quad[5]) / 2; - return { x, y }; -} - -/** Centre of a `DOM.getBoxModel` `content` polygon (also 8 doubles). */ -export function boxCentre(box: number[]): { x: number; y: number } { - return quadCentre(box); +function validCdpRegion(quads: number[][] | undefined): Region { + return (quads ?? []).flatMap((raw) => { + const polygon = parseCdpQuad(raw); + return polygon && polygonBounds(polygon) ? [polygon] : []; + }); } -async function visibleContentPolygon( +async function resolveVisibleContentRegion( cdp: CdpRunner, tabId: number, backendNodeId: number, -): Promise { +): Promise { try { const quads = await cdp.send<{ quads?: number[][] }>(tabId, "DOM.getContentQuads", { backendNodeId, }); - const quad = quads.quads?.find((q) => q.length === 8 && quadBoundingRect(q) !== null); - if (quad) return quad; + const visible = validCdpRegion(quads.quads); + if (visible.length > 0) return visible; } catch (err) { console.debug("[bsk element-geometry] getContentQuads failed", err); } @@ -162,15 +105,34 @@ async function visibleContentPolygon( backendNodeId, }); const content = box.model?.content; - if (content && content.length === 8 && quadBoundingRect(content)) return content; + const fallback = validCdpRegion(content ? [content] : undefined); + if (fallback.length > 0) return fallback; } catch (err) { console.debug("[bsk element-geometry] getBoxModel failed", err); } const descendantRect = await descendantBoundingRect(cdp, tabId, backendNodeId); - if (descendantRect) return rectToQuad(descendantRect); + if (descendantRect) { + return [ + rectPolygon({ + x: descendantRect.x, + y: descendantRect.y, + w: descendantRect.width, + h: descendantRect.height, + }), + ]; + } return elementNotVisibleError(); } +/** Resolve every visible content region in the node's target-local viewport. */ +export async function nodeContentRegion( + cdp: CdpRunner, + tabId: number, + backendNodeId: number, +): Promise { + return resolveVisibleContentRegion(cdp, tabId, backendNodeId); +} + async function descendantBoundingRect( cdp: CdpRunner, tabId: number, @@ -241,20 +203,3 @@ function parseViewportRect(value: unknown): ViewportRect | null { if (![x, y, width, height].every(Number.isFinite) || width <= 0 || height <= 0) return null; return { x, y, width, height }; } - -/** - * Get the click point for `backendNodeId`. Prefers - * `DOM.getContentQuads` (richer for rotated / transformed elements); - * falls back to `DOM.getBoxModel`, then visible descendant bounds. - * Returns `permission_denied` with "element not visible" when every - * geometry path comes back empty / fails. - */ -export async function nodeCentre( - cdp: CdpRunner, - tabId: number, - backendNodeId: number, -): Promise<{ x: number; y: number } | RpcError> { - const polygon = await visibleContentPolygon(cdp, tabId, backendNodeId); - if (isRpcError(polygon)) return polygon; - return quadCentre(polygon); -} diff --git a/apps/extension/src/tools/frame-geometry.ts b/apps/extension/src/tools/frame-geometry.ts new file mode 100644 index 00000000..ba2be422 --- /dev/null +++ b/apps/extension/src/tools/frame-geometry.ts @@ -0,0 +1,290 @@ +import { type CdpFrame, type CdpFrameGraph, type CdpTarget } from "@/browser-driver/frame-graph"; +import type { RpcError } from "@/transport/types"; +import { nodeContentRegion, scrollNodeIntoView } from "./element-geometry"; +import { + clipPolygon, + type GeometryProjection, + type Point, + type Polygon, + type ProjectiveEdge, + parseCdpQuad, + polygonArea, + polygonCentroid, + projectRegionToViewport, + type Quad, + type Region, + rectPolygon, + regionBounds, + type Size, + type ViewportRect, +} from "./geometry"; +import { type CdpRunner, cdpRunnerForTarget, isRpcError, sendToCdpTarget } from "./shared"; + +export interface NodeAddress { + target: CdpTarget; + backendNodeId: number; + frameId?: string; +} + +export interface ResolvedNodeGeometry { + topVisibleRegions: Region; + topBounds: ViewportRect; + actionPoint: Point; +} + +function geometryError(message: string): RpcError { + return { code: "cdp_failed", message }; +} + +function frameMap(graph: CdpFrameGraph): Map { + return new Map(graph.frames.map((frame) => [frame.frameId, frame])); +} + +function targetRootFrame(byId: Map, frame: CdpFrame): CdpFrame | null { + const seen = new Set(); + let current = frame; + while (current.parentFrameId) { + if (seen.has(current.frameId)) return null; + seen.add(current.frameId); + const parent = byId.get(current.parentFrameId); + if (!parent || parent.target.sessionId !== current.target.sessionId) break; + current = parent; + } + return current; +} + +function frameAncestry(graph: CdpFrameGraph, frameId: string): CdpFrame[] | null { + const byId = frameMap(graph); + const path: CdpFrame[] = []; + const seen = new Set(); + let current = byId.get(frameId); + if (!current) return null; + while (current.parentFrameId) { + if (seen.has(current.frameId)) return null; + seen.add(current.frameId); + path.push(current); + const parent = byId.get(current.parentFrameId); + if (!parent) return null; + current = parent; + } + return path; +} + +async function targetViewport(cdp: CdpRunner, target: CdpTarget): Promise { + const metrics = await sendToCdpTarget<{ + cssLayoutViewport?: { clientWidth?: number; clientHeight?: number }; + layoutViewport?: { clientWidth?: number; clientHeight?: number }; + }>(cdp, target, "Page.getLayoutMetrics", {}); + const viewport = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {}; + const width = viewport.clientWidth ?? 0; + const height = viewport.clientHeight ?? 0; + return width > 0 && height > 0 ? { width, height } : null; +} + +async function ownerContentQuad( + cdp: CdpRunner, + parent: CdpFrame, + ownerBackendNodeId: number, +): Promise { + const result = await sendToCdpTarget<{ model?: { content?: number[] } }>( + cdp, + parent.target, + "DOM.getBoxModel", + { backendNodeId: ownerBackendNodeId }, + ); + return parseCdpQuad(result.model?.content); +} + +async function sameTargetFrameClips( + cdp: CdpRunner, + byId: Map, + frame: CdpFrame, + targetRoot: CdpFrame, +): Promise { + const clips: Polygon[] = []; + let current = frame; + const seen = new Set(); + while (current.frameId !== targetRoot.frameId) { + if (seen.has(current.frameId) || !current.parentFrameId) return null; + seen.add(current.frameId); + const parent = byId.get(current.parentFrameId); + if (!parent || current.ownerBackendNodeId === undefined) return null; + const clip = await ownerContentQuad(cdp, parent, current.ownerBackendNodeId); + if (!clip) return null; + clips.push(clip); + current = parent; + } + return clips; +} + +export async function resolveFrameProjection( + cdp: CdpRunner, + graph: CdpFrameGraph, + frameId: string, +): Promise { + const byId = frameMap(graph); + const frame = byId.get(frameId); + if (!frame) return null; + let targetRoot = targetRootFrame(byId, frame); + if (!targetRoot) return null; + const sourceViewport = await targetViewport(cdp, targetRoot.target); + if (!sourceViewport) return null; + const sourceClips = await sameTargetFrameClips(cdp, byId, frame, targetRoot); + if (!sourceClips) return null; + + const edges: ProjectiveEdge[] = []; + while (targetRoot.parentFrameId) { + const parent = byId.get(targetRoot.parentFrameId); + if (!parent || targetRoot.ownerBackendNodeId === undefined) return null; + const destinationQuad = await ownerContentQuad(cdp, parent, targetRoot.ownerBackendNodeId); + if (!destinationQuad) return null; + const source = + edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target); + if (!source) return null; + const parentTargetRoot = targetRootFrame(byId, parent); + if (!parentTargetRoot) return null; + const destinationClips = await sameTargetFrameClips(cdp, byId, parent, parentTargetRoot); + if (!destinationClips) return null; + edges.push({ sourceViewport: source, destinationQuad, destinationClips }); + targetRoot = parentTargetRoot; + } + + const topViewport = + edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target); + return topViewport ? { sourceClips, edges, topViewport } : null; +} + +async function loadFrameGraph(cdp: CdpRunner, tabId: number): Promise { + if (!cdp.getFrameGraph) return null; + try { + return await cdp.getFrameGraph(tabId); + } catch (error) { + console.debug("[bsk frame-geometry] frame graph resolution failed", error); + return null; + } +} + +async function scrollFrameOwners( + cdp: CdpRunner, + tabId: number, + graph: CdpFrameGraph, + frameId: string, +): Promise { + const byId = frameMap(graph); + const ancestry = frameAncestry(graph, frameId); + if (!ancestry) return geometryError(`could not resolve frame ancestry for ${frameId}`); + for (const child of [...ancestry].reverse()) { + const parent = child.parentFrameId ? byId.get(child.parentFrameId) : undefined; + if (!parent || child.ownerBackendNodeId === undefined) { + return geometryError(`could not resolve frame owner for ${child.frameId}`); + } + const error = await scrollNodeIntoView( + cdpRunnerForTarget(cdp, parent.target), + tabId, + child.ownerBackendNodeId, + ); + if (error) return error; + } + return null; +} + +async function scrollElementWithFrameGraph( + cdp: CdpRunner, + tabId: number, + target: CdpTarget, + backendNodeId: number, + frameId: string | undefined, + graph: CdpFrameGraph | null, +): Promise { + if (frameId) { + if (!graph) return geometryError(`could not resolve frame graph for ${frameId}`); + const error = await scrollFrameOwners(cdp, tabId, graph, frameId); + if (error) return error; + } else if (target.sessionId) { + return geometryError("an OOPIF node address requires frameId"); + } + return scrollNodeIntoView(cdpRunnerForTarget(cdp, target), tabId, backendNodeId); +} + +export async function scrollElementAndFramesIntoView( + cdp: CdpRunner, + tabId: number, + target: CdpTarget, + backendNodeId: number, + frameId?: string, +): Promise { + const graph = frameId ? await loadFrameGraph(cdp, tabId) : null; + return scrollElementWithFrameGraph(cdp, tabId, target, backendNodeId, frameId, graph); +} + +function largestRegion(regions: Region): Polygon | null { + let largest: { polygon: Polygon; area: number } | null = null; + for (const polygon of regions) { + const area = polygonArea(polygon); + if (area <= 0) continue; + if (!largest || area > largest.area) largest = { polygon, area }; + } + return largest?.polygon ?? null; +} + +export async function resolveNodeGeometry( + cdp: CdpRunner, + tabId: number, + address: NodeAddress, + options: { scrollIntoView?: boolean } = {}, +): Promise { + try { + if (address.target.sessionId && !address.frameId) { + return geometryError("an OOPIF node address requires frameId"); + } + const graph = address.frameId ? await loadFrameGraph(cdp, tabId) : null; + if (address.frameId && !graph) { + return geometryError(`could not resolve frame graph for ${address.frameId}`); + } + + if (options.scrollIntoView) { + const scrollError = await scrollElementWithFrameGraph( + cdp, + tabId, + address.target, + address.backendNodeId, + address.frameId, + graph, + ); + if (scrollError) return scrollError; + } + + const localRegion = await nodeContentRegion( + cdpRunnerForTarget(cdp, address.target), + tabId, + address.backendNodeId, + ); + if (isRpcError(localRegion)) return localRegion; + + let topVisibleRegions: Region; + if (address.frameId && graph) { + const projection = await resolveFrameProjection(cdp, graph, address.frameId); + if (!projection) + return geometryError(`could not resolve frame geometry for ${address.frameId}`); + topVisibleRegions = projectRegionToViewport(localRegion, projection); + } else { + const viewport = await targetViewport(cdp, address.target); + if (!viewport) return geometryError("could not resolve top viewport geometry"); + topVisibleRegions = localRegion + .map((polygon) => + clipPolygon(polygon, rectPolygon({ x: 0, y: 0, w: viewport.width, h: viewport.height })), + ) + .filter((polygon) => polygon.length >= 3); + } + + const topBounds = regionBounds(topVisibleRegions); + const actionRegion = largestRegion(topVisibleRegions); + const actionPoint = actionRegion ? polygonCentroid(actionRegion) : null; + if (!topBounds || !actionPoint) { + return { code: "permission_denied", message: "element not visible" }; + } + return { topVisibleRegions, topBounds, actionPoint }; + } catch (error) { + return geometryError(error instanceof Error ? error.message : String(error)); + } +} diff --git a/apps/extension/src/tools/geometry.ts b/apps/extension/src/tools/geometry.ts new file mode 100644 index 00000000..fea8782b --- /dev/null +++ b/apps/extension/src/tools/geometry.ts @@ -0,0 +1,266 @@ +export interface Point { + x: number; + y: number; +} + +export type Polygon = Point[]; +export type Region = Polygon[]; +export type Quad = [Point, Point, Point, Point]; + +export interface Size { + width: number; + height: number; +} + +export interface ViewportRect { + x: number; + y: number; + width: number; + height: number; +} + +export interface ProjectiveEdge { + sourceViewport: Size; + destinationQuad: Quad; + destinationClips?: Polygon[]; +} + +export interface GeometryProjection { + sourceClips: Polygon[]; + edges: ProjectiveEdge[]; + topViewport: Size; +} + +export function parseCdpQuad(raw: number[] | undefined): Quad | null { + if (!raw || raw.length !== 8 || !raw.every(Number.isFinite)) return null; + return [ + { x: raw[0], y: raw[1] }, + { x: raw[2], y: raw[3] }, + { x: raw[4], y: raw[5] }, + { x: raw[6], y: raw[7] }, + ]; +} + +export function rectPolygon(rect: { x: number; y: number; w: number; h: number }): Quad { + return [ + { x: rect.x, y: rect.y }, + { x: rect.x + rect.w, y: rect.y }, + { x: rect.x + rect.w, y: rect.y + rect.h }, + { x: rect.x, y: rect.y + rect.h }, + ]; +} + +export function viewportPolygon(viewport: Size): Quad { + return rectPolygon({ x: 0, y: 0, w: viewport.width, h: viewport.height }); +} + +/** Project a normalized unit-square point into an arbitrary content quad. */ +export function projectUnitPoint(point: Point, quad: Quad): Point { + const [p0, p1, p2, p3] = quad; + const dx1 = p1.x - p2.x; + const dx2 = p3.x - p2.x; + const dx3 = p0.x - p1.x + p2.x - p3.x; + const dy1 = p1.y - p2.y; + const dy2 = p3.y - p2.y; + const dy3 = p0.y - p1.y + p2.y - p3.y; + + let a: number; + let b: number; + let c: number; + let d: number; + let e: number; + let f: number; + let g = 0; + let h = 0; + + if (Math.abs(dx3) < 1e-9 && Math.abs(dy3) < 1e-9) { + a = p1.x - p0.x; + b = p3.x - p0.x; + c = p0.x; + d = p1.y - p0.y; + e = p3.y - p0.y; + f = p0.y; + } else { + const denominator = dx1 * dy2 - dx2 * dy1; + if (Math.abs(denominator) < 1e-9) { + return { + x: p0.x + (p1.x - p0.x) * point.x + (p3.x - p0.x) * point.y, + y: p0.y + (p1.y - p0.y) * point.x + (p3.y - p0.y) * point.y, + }; + } + g = (dx3 * dy2 - dx2 * dy3) / denominator; + h = (dx1 * dy3 - dx3 * dy1) / denominator; + a = p1.x - p0.x + g * p1.x; + b = p3.x - p0.x + h * p3.x; + c = p0.x; + d = p1.y - p0.y + g * p1.y; + e = p3.y - p0.y + h * p3.y; + f = p0.y; + } + + const scale = g * point.x + h * point.y + 1; + return { + x: (a * point.x + b * point.y + c) / scale, + y: (d * point.x + e * point.y + f) / scale, + }; +} + +export function projectPolygon(polygon: Polygon, edge: ProjectiveEdge): Polygon { + const { width, height } = edge.sourceViewport; + if (width <= 0 || height <= 0) return []; + return polygon.map((point) => + projectUnitPoint({ x: point.x / width, y: point.y / height }, edge.destinationQuad), + ); +} + +function polygonSignedArea(points: Polygon): number { + let area = 0; + for (let index = 0; index < points.length; index += 1) { + const current = points[index]; + const next = points[(index + 1) % points.length]; + area += current.x * next.y - next.x * current.y; + } + return area / 2; +} + +function edgeIntersection(start: Point, end: Point, clipStart: Point, clipEnd: Point): Point { + const subjectX = end.x - start.x; + const subjectY = end.y - start.y; + const clipX = clipEnd.x - clipStart.x; + const clipY = clipEnd.y - clipStart.y; + const denominator = subjectX * clipY - subjectY * clipX; + if (Math.abs(denominator) < 1e-9) return end; + const offsetX = clipStart.x - start.x; + const offsetY = clipStart.y - start.y; + const scale = (offsetX * clipY - offsetY * clipX) / denominator; + return { x: start.x + scale * subjectX, y: start.y + scale * subjectY }; +} + +/** Clip a polygon to a convex polygon, preserving every resulting vertex. */ +export function clipPolygon(subject: Polygon, clip: Polygon): Polygon { + if (subject.length < 3 || clip.length < 3) return []; + const orientation = polygonSignedArea(clip) >= 0 ? 1 : -1; + let output = [...subject]; + for (let edge = 0; edge < clip.length && output.length > 0; edge += 1) { + const clipStart = clip[edge]; + const clipEnd = clip[(edge + 1) % clip.length]; + const input = output; + output = []; + const inside = (point: Point) => + orientation * + ((clipEnd.x - clipStart.x) * (point.y - clipStart.y) - + (clipEnd.y - clipStart.y) * (point.x - clipStart.x)) >= + -1e-9; + let start = input[input.length - 1]; + for (const end of input) { + const startInside = inside(start); + const endInside = inside(end); + if (endInside) { + if (!startInside) output.push(edgeIntersection(start, end, clipStart, clipEnd)); + output.push(end); + } else if (startInside) { + output.push(edgeIntersection(start, end, clipStart, clipEnd)); + } + start = end; + } + } + return output; +} + +export function polygonBounds(points: Polygon): ViewportRect | null { + if (points.length < 3) return null; + const xs = points.map((point) => point.x); + const ys = points.map((point) => point.y); + const x = Math.min(...xs); + const y = Math.min(...ys); + const right = Math.max(...xs); + const bottom = Math.max(...ys); + if (right <= x || bottom <= y) return null; + return { x, y, width: right - x, height: bottom - y }; +} + +export function regionBounds(region: Region): ViewportRect | null { + return polygonBounds(region.flat()); +} + +export function polygonArea(points: Polygon): number { + return Math.abs(polygonSignedArea(points)); +} + +export function polygonCentroid(points: Polygon): Point | null { + if (points.length < 3) return null; + let twiceArea = 0; + let x = 0; + let y = 0; + for (let index = 0; index < points.length; index += 1) { + const current = points[index]; + const next = points[(index + 1) % points.length]; + const cross = current.x * next.y - next.x * current.y; + twiceArea += cross; + x += (current.x + next.x) * cross; + y += (current.y + next.y) * cross; + } + if (Math.abs(twiceArea) < 1e-9) return null; + return { x: x / (3 * twiceArea), y: y / (3 * twiceArea) }; +} + +export function projectAndClipRegion( + region: Region, + edges: ProjectiveEdge[], + topViewport: Size, +): Region { + let projected = region; + for (const edge of edges) { + const sourceClip = viewportPolygon(edge.sourceViewport); + projected = projected + .map((polygon) => clipPolygon(polygon, sourceClip)) + .filter((polygon) => polygon.length >= 3) + .map((polygon) => + [edge.destinationQuad, ...(edge.destinationClips ?? [])].reduce( + (current, clip) => clipPolygon(current, clip), + projectPolygon(polygon, edge), + ), + ) + .filter((polygon) => polygon.length >= 3); + } + const topClip = viewportPolygon(topViewport); + return projected + .map((polygon) => clipPolygon(polygon, topClip)) + .filter((polygon) => polygon.length >= 3); +} + +export function projectRegionToViewport(region: Region, projection: GeometryProjection): Region { + const clipped = region + .map((polygon) => + projection.sourceClips.reduce((current, clip) => clipPolygon(current, clip), polygon), + ) + .filter((polygon) => polygon.length >= 3); + return projectAndClipRegion(clipped, projection.edges, projection.topViewport); +} + +export function projectRectToViewport( + rect: { x: number; y: number; w: number; h: number } | null, + projection: GeometryProjection, +): ViewportRect | null { + if (!rect) return null; + return regionBounds(projectRegionToViewport([rectPolygon(rect)], projection)); +} + +export function childFrameProjection( + parent: GeometryProjection, + ownerRectInParent: { x: number; y: number; w: number; h: number }, +): GeometryProjection { + const destinationQuad = rectPolygon(ownerRectInParent); + return { + sourceClips: [], + edges: [ + { + sourceViewport: { width: ownerRectInParent.w, height: ownerRectInParent.h }, + destinationQuad, + destinationClips: parent.sourceClips, + }, + ...parent.edges, + ], + topViewport: parent.topViewport, + }; +} diff --git a/apps/extension/src/tools/human-loop.ts b/apps/extension/src/tools/human-loop.ts index 5c05f046..c50916ec 100644 --- a/apps/extension/src/tools/human-loop.ts +++ b/apps/extension/src/tools/human-loop.ts @@ -8,6 +8,7 @@ import { HELP_REQUEST, type HelpCancelMessage, type HelpFinishMessage, + type HelpHighlightRect, type HelpQueryResponse, type HelpRequestMessage, isHelpAckMessage, @@ -25,6 +26,7 @@ import type { ResolvedTarget, RpcError, } from "@/transport/types"; +import { resolveNodeGeometry } from "./frame-geometry"; import { type CdpRunner, type ChromeTabsApi, @@ -37,7 +39,6 @@ import { import { lookupSnapshotRef } from "./snapshot-ref"; const DEFAULT_HELP_TIMEOUT_MS = 300_000; -const HELP_ATTR = "data-bsk-help"; const HELP_SEND_RETRIES = 3; const HELP_SEND_RETRY_DELAY_MS = 350; const HELP_REARM_DEBOUNCE_MS = 150; @@ -74,6 +75,7 @@ interface ActiveHelpRequest { title?: string; targets: HelpTarget[]; selectors: string[]; + rects: HelpHighlightRect[]; timeoutMs: number; notificationId: string; resolvedTargets?: ResolvedTarget[]; @@ -134,6 +136,7 @@ function helpRequestMessage(help: ActiveHelpRequest, tabId: number): HelpRequest ...(help.title ? { title: help.title } : {}), displayMode: isPrimaryTab ? "full" : "compact", selectors: isPrimaryTab ? help.selectors : [], + rects: isPrimaryTab ? help.rects : [], timeoutMs: help.timeoutMs, }; } @@ -165,33 +168,6 @@ async function sendHelpRequestWithAck( throw lastError ?? new Error("failed to show help overlay"); } -async function tagRefTarget( - cdp: CdpRunner, - tabId: number, - backendNodeId: number, - index: number, -): Promise { - let objectId: string | undefined; - try { - const resolved = await cdp.send<{ object?: { objectId?: string } }>(tabId, "DOM.resolveNode", { - backendNodeId, - }); - objectId = resolved.object?.objectId; - if (!objectId) return null; - await cdp.send(tabId, "Runtime.callFunctionOn", { - objectId, - functionDeclaration: `function(){ this.setAttribute("${HELP_ATTR}", "${index}"); }`, - }); - return `[${HELP_ATTR}="${index}"]`; - } catch { - return null; - } finally { - if (objectId) { - await cdp.send(tabId, "Runtime.releaseObject", { objectId }).catch(() => {}); - } - } -} - async function selectorExists( cdp: CdpRunner, tabId: number, @@ -209,36 +185,22 @@ async function selectorExists( } } -async function clearRefTags(cdp: CdpRunner | undefined, tabId: number): Promise { - if (!cdp) return; - try { - const doc = await cdp.send<{ root?: { nodeId?: number } }>(tabId, "DOM.getDocument", { - depth: 0, - }); - const rootId = doc.root?.nodeId; - if (rootId === undefined) return; - const { nodeIds } = await cdp.send<{ nodeIds: number[] }>(tabId, "DOM.querySelectorAll", { - nodeId: rootId, - selector: `[${HELP_ATTR}]`, - }); - for (const nodeId of nodeIds ?? []) { - await cdp.send(tabId, "DOM.removeAttribute", { nodeId, name: HELP_ATTR }).catch(() => {}); - } - } catch { - // Best-effort cleanup. - } -} - async function resolveHelpTargets( ctx: ReturnType, tabId: number, targets: HelpTarget[], deps: RequestHelpDeps, -): Promise<{ selectors: string[]; resolvedTargets?: ResolvedTarget[] }> { + options: { scrollIntoView: boolean }, +): Promise<{ + selectors: string[]; + rects: HelpHighlightRect[]; + resolvedTargets?: ResolvedTarget[]; +}> { const selectors: string[] = []; + const rects: HelpHighlightRect[] = []; const resolved: ResolvedTarget[] = []; let rootNodeId: number | undefined; - if (deps.cdp && targets.length > 0) { + if (deps.cdp && targets.some((target) => Boolean(target.selector))) { try { const doc = await deps.cdp.send<{ root?: { nodeId?: number } }>(tabId, "DOM.getDocument", { depth: 0, @@ -262,28 +224,55 @@ async function resolveHelpTargets( resolved.push({ matched, selector: tgt.selector }); } else if (tgt.ref) { const looked = ctx ? lookupSnapshotRef(ctx, tgt.ref, tabId) : null; - const backendNodeId = looked?.backendNodeId ?? null; - let sel: string | null = null; - if (backendNodeId !== null && deps.cdp) { - sel = await tagRefTarget(deps.cdp, tabId, backendNodeId, i); + let rect: HelpHighlightRect | null = null; + if (looked && deps.cdp) { + const target = { + tabId, + ...(looked.cdpSessionId ? { sessionId: looked.cdpSessionId } : {}), + }; + const geometry = await resolveNodeGeometry( + deps.cdp, + tabId, + { + target, + backendNodeId: looked.backendNodeId, + ...(looked.frameId ? { frameId: looked.frameId } : {}), + }, + { scrollIntoView: options.scrollIntoView }, + ); + if (!isRpcError(geometry)) { + rect = { + top: geometry.topBounds.y, + left: geometry.topBounds.x, + width: geometry.topBounds.width, + height: geometry.topBounds.height, + }; + } } - if (sel) selectors.push(sel); - resolved.push({ matched: sel !== null, ref: tgt.ref }); + if (rect) rects.push(rect); + resolved.push({ + matched: rect !== null, + ref: tgt.ref, + }); } } - return { selectors, resolvedTargets: resolved.length > 0 ? resolved : undefined }; + return { selectors, rects, resolvedTargets: resolved.length > 0 ? resolved : undefined }; } -async function refreshHelpTargets(help: ActiveHelpRequest): Promise { - await clearRefTags(help.deps.cdp, help.primaryTabId); - const { selectors, resolvedTargets } = await resolveHelpTargets( +async function refreshHelpTargets( + help: ActiveHelpRequest, + options: { scrollIntoView: boolean } = { scrollIntoView: true }, +): Promise { + const { selectors, rects, resolvedTargets } = await resolveHelpTargets( help.ctx, help.primaryTabId, help.targets, help.deps, + options, ); help.selectors = selectors; + help.rects = rects; help.resolvedTargets = resolvedTargets; } @@ -294,12 +283,7 @@ async function cleanupHelp(help: ActiveHelpRequest): Promise { const tabsToCancel = new Set([help.primaryTabId, ...help.overlayTabIds]); await Promise.all( [...tabsToCancel].map((tabId) => - Promise.all([ - clearRefTags(help.deps.cdp, tabId), - help.deps - .sendToTab(tabId, { type: HELP_CANCEL, requestId: help.requestId }) - .catch(() => {}), - ]), + help.deps.sendToTab(tabId, { type: HELP_CANCEL, requestId: help.requestId }).catch(() => {}), ), ); } @@ -476,7 +460,9 @@ function attachHelpRuntimeListener(deps: RequestHelpDeps): () => void { sendResponse({ active: false }); return; } - if (tabId === help.primaryTabId) await refreshHelpTargets(help); + if (tabId === help.primaryTabId) { + await refreshHelpTargets(help, { scrollIntoView: false }); + } help.overlayTabIds.add(tabId); sendResponse({ active: true, @@ -486,6 +472,7 @@ function attachHelpRuntimeListener(deps: RequestHelpDeps): () => void { ...(help.title ? { title: help.title } : {}), displayMode: tabId === help.primaryTabId ? "full" : "compact", selectors: tabId === help.primaryTabId ? help.selectors : [], + rects: tabId === help.primaryTabId ? help.rects : [], timeoutMs: help.timeoutMs, }, }); @@ -686,7 +673,13 @@ export async function handleRequestHelp( if (deps.autoAttachLifecycle !== false) ensureHelpLifecycleListeners(deps); const tabId = target.tabId; const initialTargets = params.targets ?? []; - const { selectors, resolvedTargets } = await resolveHelpTargets(ctx, tabId, initialTargets, deps); + const { selectors, rects, resolvedTargets } = await resolveHelpTargets( + ctx, + tabId, + initialTargets, + deps, + { scrollIntoView: true }, + ); await deps.windows.update(target.windowId, { focused: true }).catch(() => {}); await deps.activateTab(tabId).catch(() => {}); @@ -729,6 +722,7 @@ export async function handleRequestHelp( ...(params.title ? { title: params.title } : {}), targets: initialTargets, selectors, + rects, timeoutMs, notificationId, resolvedTargets, diff --git a/apps/extension/src/tools/interaction.ts b/apps/extension/src/tools/interaction.ts index ee66dac7..182cabdf 100644 --- a/apps/extension/src/tools/interaction.ts +++ b/apps/extension/src/tools/interaction.ts @@ -3,14 +3,15 @@ // // All interaction tools: // 1. Resolve target tab (sandbox: must be inside Agent Window). -// 2. Resolve target element by `ref` (RefStore.resolve with tabId -// binding) or `selector` (DOM.querySelector + describeNode). -// 3. Scroll the node into view, then dispatch the appropriate -// `Input.*` CDP events. +// 2. Resolve target element by `ref` (compound frame/session identity) +// or `selector` (DOM.querySelector + describeNode). +// 3. Scroll the node and its frame owners into view, project its live +// content quad into the top viewport, then dispatch `Input.*` events. // 4. Honour `AbortSignal` so canceled calls don't issue follow-up CDP // commands. import { ChromiumCdp } from "@/browser-driver/chromium-cdp"; +import type { CdpTarget } from "@/browser-driver/frame-graph"; import type { SessionContext, SessionManager } from "@/session-manager/manager"; import type { ClickParams, @@ -28,17 +29,13 @@ import type { SelectResult, } from "@/transport/types"; import { attachDialogs, markDialogCursor } from "./dialogs"; -import { - backendNodeToObject, - boxCentre, - nodeCentre, - quadCentre, - scrollNodeIntoView, -} from "./element-geometry"; +import { backendNodeToObject } from "./element-geometry"; import { rpcError } from "./errors"; +import { resolveNodeGeometry, scrollElementAndFramesIntoView } from "./frame-geometry"; import { type CdpRunner, type ChromeTabsApi, + cdpRunnerForTarget, chromeTabsApi, enforceAgentWindow, isRpcError, @@ -137,7 +134,16 @@ async function resolveBackendNode( target: { tabId: number }, params: { ref?: string; selector?: string }, toolName: string, -): Promise<{ backendNodeId: number; usedRef?: string; usedSelector?: string } | RpcError> { +): Promise< + | { + backendNodeId: number; + cdpTarget: CdpTarget; + frameId?: string; + usedRef?: string; + usedSelector?: string; + } + | RpcError +> { const hasRef = typeof params.ref === "string" && params.ref.length > 0; const hasSelector = typeof params.selector === "string" && params.selector.length > 0; if (hasRef && hasSelector) { @@ -155,7 +161,15 @@ async function resolveBackendNode( if (hasRef) { const resolved = resolveSnapshotRef(ctx, params.ref as string, target.tabId); if (isRpcError(resolved)) return resolved; - return { backendNodeId: resolved.backendNodeId, usedRef: resolved.refKey }; + return { + backendNodeId: resolved.backendNodeId, + cdpTarget: { + tabId: target.tabId, + ...(resolved.cdpSessionId ? { sessionId: resolved.cdpSessionId } : {}), + }, + ...(resolved.frameId ? { frameId: resolved.frameId } : {}), + usedRef: resolved.refKey, + }; } // selector path try { @@ -192,7 +206,7 @@ async function resolveBackendNode( message: "DOM.describeNode returned no backendNodeId", }; } - return { backendNodeId, usedSelector: params.selector }; + return { backendNodeId, cdpTarget: { tabId: target.tabId }, usedSelector: params.selector }; } catch (err) { return { code: "cdp_failed", @@ -228,19 +242,19 @@ export async function handleClick( return { code: "cancelled", message: "click aborted" }; } - try { - deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); - const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId); - if (scrollErr) return scrollErr; - } catch (err) { - return { - code: "cdp_failed", - message: err instanceof Error ? err.message : String(err), - }; - } - - const centre = await nodeCentre(deps.cdp, target.tabId, node.backendNodeId); - if (isRpcError(centre)) return centre; + deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); + const geometry = await resolveNodeGeometry( + deps.cdp, + target.tabId, + { + target: node.cdpTarget, + backendNodeId: node.backendNodeId, + ...(node.frameId ? { frameId: node.frameId } : {}), + }, + { scrollIntoView: true }, + ); + if (isRpcError(geometry)) return geometry; + const centre = geometry.actionPoint; if (throwIfAborted(deps.signal)) { return { code: "cancelled", message: "click aborted" }; @@ -353,19 +367,19 @@ export async function handleHover( const node = await resolveBackendNode(deps.cdp, ctx, target, params, "hover"); if (isRpcError(node)) return node; - try { - deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); - const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId); - if (scrollErr) return scrollErr; - } catch (err) { - return { - code: "cdp_failed", - message: err instanceof Error ? err.message : String(err), - }; - } - - const centre = await nodeCentre(deps.cdp, target.tabId, node.backendNodeId); - if (isRpcError(centre)) return centre; + deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); + const geometry = await resolveNodeGeometry( + deps.cdp, + target.tabId, + { + target: node.cdpTarget, + backendNodeId: node.backendNodeId, + ...(node.frameId ? { frameId: node.frameId } : {}), + }, + { scrollIntoView: true }, + ); + if (isRpcError(geometry)) return geometry; + const centre = geometry.actionPoint; if (throwIfAborted(deps.signal)) { return { code: "cancelled", message: "hover aborted" }; @@ -585,10 +599,11 @@ export async function handleFill( const node = await resolveBackendNode(deps.cdp, ctx, target, params, "fill"); if (isRpcError(node)) return node; + const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget); try { deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); - const described = await deps.cdp.send<{ node?: DescribedNode }>( + const described = await nodeCdp.send<{ node?: DescribedNode }>( target.tabId, "DOM.describeNode", { @@ -602,12 +617,18 @@ export async function handleFill( `element ${described.node?.nodeName ?? "?"} not fillable (need input/textarea/contenteditable)`, ); } - const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId); + const scrollErr = await scrollElementAndFramesIntoView( + deps.cdp, + target.tabId, + node.cdpTarget, + node.backendNodeId, + node.frameId, + ); if (scrollErr) return scrollErr; if (throwIfAborted(deps.signal)) { return { code: "cancelled", message: "fill aborted" }; } - await deps.cdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId }); + await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId }); } catch (err) { return { code: "cdp_failed", @@ -619,7 +640,7 @@ export async function handleFill( return { code: "cancelled", message: "fill aborted" }; } - const objectIdOrErr = await backendNodeToObject(deps.cdp, target.tabId, node.backendNodeId); + const objectIdOrErr = await backendNodeToObject(nodeCdp, target.tabId, node.backendNodeId); if (isRpcError(objectIdOrErr)) return objectIdOrErr; const objectId = objectIdOrErr; const clearBefore = params.clear_before ?? true; @@ -628,7 +649,7 @@ export async function handleFill( if (clearBefore) { // Clear input/textarea value or wipe contenteditable innerText, // then fire `input` so frameworks observe the empty state. - await deps.cdp.send(target.tabId, "Runtime.callFunctionOn", { + await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", { objectId, functionDeclaration: `function() { if (this.isContentEditable) { this.textContent = ''; } @@ -655,7 +676,7 @@ export async function handleFill( return { code: "cancelled", message: "fill aborted" }; } // Fire `input` + `change` so React / Vue controlled inputs commit. - await deps.cdp.send(target.tabId, "Runtime.callFunctionOn", { + await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", { objectId, functionDeclaration: `function() { this.dispatchEvent(new Event('input', { bubbles: true })); @@ -853,14 +874,21 @@ export async function handlePress( if (params.ref || params.selector) { const node = await resolveBackendNode(deps.cdp, ctx, target, params, "press"); if (isRpcError(node)) return node; + const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget); try { deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); - const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId); + const scrollErr = await scrollElementAndFramesIntoView( + deps.cdp, + target.tabId, + node.cdpTarget, + node.backendNodeId, + node.frameId, + ); if (scrollErr) return scrollErr; if (throwIfAborted(deps.signal)) { return { code: "cancelled", message: "press aborted" }; } - await deps.cdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId }); + await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId }); } catch (err) { return { code: "cdp_failed", @@ -965,10 +993,11 @@ export async function handleSelect( const node = await resolveBackendNode(deps.cdp, ctx, target, params, "select"); if (isRpcError(node)) return node; + const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget); try { deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); - const described = await deps.cdp.send<{ node?: DescribedNode }>( + const described = await nodeCdp.send<{ node?: DescribedNode }>( target.tabId, "DOM.describeNode", { backendNodeId: node.backendNodeId }, @@ -991,12 +1020,18 @@ export async function handleSelect( "single-select