feat(interaction): add real mouse-wheel primitive

Author: NianJiuZst

apps/extension/src/tools/__tests__/wheel.test.ts

@@ -0,0 +1,149 @@
+import { describe, expect, it, vi } from "vitest";
+import { SessionManager } from "@/session-manager/manager";
+import type { CdpRunner } from "@/tools/shared";
+import { handleWheel } from "../wheel";
+
+function fakeAgentWindow(ids: number[]) {
+  let index = 0;
+  return {
+    create: vi.fn(async () => {
+      const id = ids[index++];
+      if (id === undefined) throw new Error("ran out of fake ids");
+      return id;
+    }),
+    remove: vi.fn(async () => {}),
+    ensureActiveTab: vi.fn(async () => {}),
+  };
+}
+
+function makeFakeCdp(handlers: Record<string, (params: unknown) => unknown> = {}) {
+  const sent: Array<{ tabId: number; method: string; params?: object }> = [];
+  const cdp: CdpRunner = {
+    send: vi.fn(async (tabId: number, method: string, params?: object) => {
+      sent.push({ tabId, method, params });
+      const handler = handlers[method];
+      if (handler) return handler(params);
+      if (method === "Page.getLayoutMetrics") {
+        return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
+      }
+      if (method === "Input.dispatchMouseEvent") return {};
+      throw new Error(`unexpected CDP call ${method}`);
+    }) as unknown as CdpRunner["send"],
+    trackSessionTab: vi.fn(),
+  };
+  const tabsApi = {
+    get: vi.fn(
+      async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
+    ),
+    query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
+  };
+  return { cdp, tabsApi, sent };
+}
+
+describe("handleWheel", () => {
+  it("dispatches a native mouseWheel event at the viewport centre", async () => {
+    const manager = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
+    await manager.start("aa11");
+    const fake = makeFakeCdp();
+    const bypassOverlay = vi.fn(async () => {});
+
+    const result = await handleWheel(
+      manager,
+      {
+        session_id: "aa11",
+        delta_y: 600,
+        delta_x: -20,
+        modifiers: ["ctrl", "shift"],
+      },
+      { cdp: fake.cdp, tabsApi: fake.tabsApi, bypassOverlay },
+    );
+
+    if ("code" in result) throw new Error(`unexpected error: ${JSON.stringify(result)}`);
+    expect(result).toMatchObject({
+      tab_id: 4,
+      x: 500,
+      y: 400,
+      delta_x: -20,
+      delta_y: 600,
+    });
+    expect(fake.sent).toEqual([
+      { tabId: 4, method: "Page.getLayoutMetrics", params: {} },
+      {
+        tabId: 4,
+        method: "Input.dispatchMouseEvent",
+        params: { type: "mouseMoved", x: 500, y: 400, modifiers: 10 },
+      },
+      {
+        tabId: 4,
+        method: "Input.dispatchMouseEvent",
+        params: {
+          type: "mouseWheel",
+          x: 500,
+          y: 400,
+          deltaX: -20,
+          deltaY: 600,
+          modifiers: 10,
+        },
+      },
+    ]);
+    expect(bypassOverlay.mock.calls).toEqual([
+      [4, true],
+      [4, false],
+    ]);
+  });
+
+  it("uses an element's visible action point for targeted wheel input", async () => {
+    const manager = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
+    const ctx = await manager.start("aa11");
+    ctx.refStore.set("e3", 1234, { tabId: 4 });
+    const fake = makeFakeCdp({
+      "DOM.scrollIntoViewIfNeeded": () => ({}),
+      "DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }),
+    });
+
+    const result = await handleWheel(
+      manager,
+      { session_id: "aa11", ref: "@e3", delta_y: 120 },
+      { cdp: fake.cdp, tabsApi: fake.tabsApi },
+    );
+
+    if ("code" in result) throw new Error(`unexpected error: ${JSON.stringify(result)}`);
+    expect(result).toMatchObject({ used_ref: "e3", x: 60, y: 40, delta_y: 120 });
+    expect(fake.sent.at(-1)).toMatchObject({
+      method: "Input.dispatchMouseEvent",
+      params: { type: "mouseWheel", x: 60, y: 40, deltaY: 120 },
+    });
+  });
+
+  it("rejects a zero-distance wheel without issuing CDP calls", async () => {
+    const manager = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
+    await manager.start("aa11");
+    const fake = makeFakeCdp();
+
+    const result = await handleWheel(
+      manager,
+      { session_id: "aa11", delta_x: 0, delta_y: 0 },
+      { cdp: fake.cdp, tabsApi: fake.tabsApi },
+    );
+
+    expect(result).toMatchObject({ code: "invalid_params" });
+    expect(fake.cdp.send).not.toHaveBeenCalled();
+  });
+
+  it("does not issue CDP calls after an early cancellation", async () => {
+    const manager = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
+    await manager.start("aa11");
+    const fake = makeFakeCdp();
+    const abort = new AbortController();
+    abort.abort();
+
+    const result = await handleWheel(
+      manager,
+      { session_id: "aa11", delta_y: 120 },
+      { cdp: fake.cdp, tabsApi: fake.tabsApi, signal: abort.signal },
+    );
+
+    expect(result).toMatchObject({ code: "cancelled" });
+    expect(fake.cdp.send).not.toHaveBeenCalled();
+  });
+});

apps/extension/src/tools/dispatcher.ts

@@ -29,6 +29,7 @@ import type {
   SelectParams,
   SnapshotParams,
   WaitForNavigationParams,
+  WheelParams,
 } from "@/transport/types";
 import { isRequestFrame } from "@/transport/types";
 import { handleConsole } from "./console";
@@ -75,6 +76,7 @@ import {
   type TabSelectParams,
 } from "./tabs";
 import { handleWaitForNavigation } from "./waits";
+import { handleWheel } from "./wheel";
 import { handleWindowResize, type WindowResizeParams } from "./window";
 
 type DispatcherCdpRunner = CdpRunner &
@@ -471,6 +473,20 @@ export class ToolDispatcher {
         );
         return this.rememberHover((req.params as HoverParams).session_id, result);
       }
+      case "tool.wheel":
+        return this.withHoverReassert(
+          req.params as WheelParams,
+          () =>
+            handleWheel(
+              this.sessions,
+              req.params as WheelParams,
+              this.cdp
+                ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal, bypassOverlay }
+                : undefined,
+            ),
+          { releaseAfter: true },
+          signal,
+        );
       case "tool.fill":
         return this.withHoverReleaseForRequest(
           req.params as FillParams,
@@ -718,6 +734,7 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
     case "tool.reload":
     case "tool.click":
     case "tool.hover":
+    case "tool.wheel":
     case "tool.fill":
     case "tool.press":
     case "tool.select":

apps/extension/src/tools/interaction.ts

@@ -128,7 +128,7 @@ async function wait(ms: number, signal?: AbortSignal): Promise<void> {
  * `RpcError` if the caller supplied neither (or both), or if neither
  * lookup matched.
  */
-async function resolveBackendNode(
+export async function resolveBackendNode(
   cdp: CdpRunner,
   ctx: SessionContext,
   target: { tabId: number },

apps/extension/src/tools/wheel.ts

@@ -0,0 +1,180 @@
+import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
+import type { SessionContext, SessionManager } from "@/session-manager/manager";
+import type { RpcError, WheelParams, WheelResult } from "@/transport/types";
+import { attachDialogs, markDialogCursor } from "./dialogs";
+import { resolveNodeGeometry } from "./frame-geometry";
+import { modifiersBitfield, resolveBackendNode } from "./interaction";
+import {
+  type CdpRunner,
+  type ChromeTabsApi,
+  chromeTabsApi,
+  enforceAgentWindow,
+  isRpcError,
+  lookupSession,
+  resolveTargetTab,
+} from "./shared";
+
+export interface WheelDeps {
+  cdp: CdpRunner;
+  tabsApi: ChromeTabsApi;
+  signal?: AbortSignal;
+  /** Temporarily disable the Agent Window overlay's input blocker. */
+  bypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
+}
+
+interface WheelPoint {
+  x: number;
+  y: number;
+  usedRef?: string;
+  usedSelector?: string;
+}
+
+let defaultDeps: { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } | null = null;
+function getDefaultDeps(): { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } {
+  if (!defaultDeps) defaultDeps = { cdp: new ChromiumCdp(), tabsApi: chromeTabsApi };
+  return defaultDeps;
+}
+
+export async function handleWheel(
+  manager: SessionManager,
+  params: WheelParams,
+  deps: WheelDeps = getDefaultDeps(),
+): Promise<WheelResult | RpcError> {
+  const ctxOrErr = lookupSession(manager, params, "wheel");
+  if (isRpcError(ctxOrErr)) return ctxOrErr;
+  const ctx = ctxOrErr;
+  const deltaX = params.delta_x ?? 0;
+  const deltaY = params.delta_y;
+  if (!Number.isFinite(deltaX) || !Number.isFinite(deltaY)) {
+    return { code: "invalid_params", message: "wheel deltas must be finite numbers" };
+  }
+  if (deltaX === 0 && deltaY === 0) {
+    return { code: "invalid_params", message: "at least one wheel delta must be non-zero" };
+  }
+  if (deps.signal?.aborted) return cancelled();
+
+  const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
+  if (isRpcError(target)) return target;
+  const denied = enforceAgentWindow(ctx, target, "wheel");
+  if (denied) return denied;
+  const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
+  deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
+
+  const point = await resolveWheelPoint(deps.cdp, ctx, target, params);
+  if (isRpcError(point)) return point;
+  if (deps.signal?.aborted) return cancelled();
+
+  let bypassEnabled = false;
+  if (deps.bypassOverlay) {
+    try {
+      await deps.bypassOverlay(target.tabId, true);
+      bypassEnabled = true;
+    } catch (error) {
+      console.debug("[bsk wheel] overlay bypass enable failed", error);
+    }
+  }
+
+  try {
+    if (deps.signal?.aborted) return cancelled();
+    const modifiers = modifiersBitfield(params.modifiers);
+    await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", {
+      type: "mouseMoved",
+      x: point.x,
+      y: point.y,
+      modifiers,
+    });
+    if (deps.signal?.aborted) return cancelled();
+    await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", {
+      type: "mouseWheel",
+      x: point.x,
+      y: point.y,
+      deltaX,
+      deltaY,
+      modifiers,
+    });
+  } catch (error) {
+    return {
+      code: "cdp_failed",
+      message: error instanceof Error ? error.message : String(error),
+    };
+  } finally {
+    if (bypassEnabled && deps.bypassOverlay) {
+      try {
+        await deps.bypassOverlay(target.tabId, false);
+      } catch (error) {
+        console.debug("[bsk wheel] overlay bypass disable failed", error);
+      }
+    }
+  }
+
+  return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
+    tab_id: target.tabId,
+    used_ref: point.usedRef,
+    used_selector: point.usedSelector,
+    x: point.x,
+    y: point.y,
+    delta_x: deltaX,
+    delta_y: deltaY,
+  });
+}
+
+async function resolveWheelPoint(
+  cdp: CdpRunner,
+  ctx: SessionContext,
+  target: { tabId: number },
+  params: WheelParams,
+): Promise<WheelPoint | RpcError> {
+  const hasRef = typeof params.ref === "string" && params.ref.length > 0;
+  const hasSelector = typeof params.selector === "string" && params.selector.length > 0;
+  if (hasRef || hasSelector) {
+    const node = await resolveBackendNode(cdp, ctx, target, params, "wheel");
+    if (isRpcError(node)) return node;
+    const geometry = await resolveNodeGeometry(
+      cdp,
+      target.tabId,
+      {
+        target: node.cdpTarget,
+        backendNodeId: node.backendNodeId,
+        ...(node.frameId ? { frameId: node.frameId } : {}),
+      },
+      { scrollIntoView: true },
+    );
+    if (isRpcError(geometry)) return geometry;
+    return {
+      x: geometry.actionPoint.x,
+      y: geometry.actionPoint.y,
+      usedRef: node.usedRef,
+      usedSelector: node.usedSelector,
+    };
+  }
+
+  try {
+    const metrics = await cdp.send<{
+      cssLayoutViewport?: { clientWidth?: number; clientHeight?: number };
+      layoutViewport?: { clientWidth?: number; clientHeight?: number };
+    }>(target.tabId, "Page.getLayoutMetrics", {});
+    const viewport = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
+    const width = viewport.clientWidth;
+    const height = viewport.clientHeight;
+    if (
+      typeof width !== "number" ||
+      typeof height !== "number" ||
+      !Number.isFinite(width) ||
+      !Number.isFinite(height) ||
+      width <= 0 ||
+      height <= 0
+    ) {
+      return { code: "cdp_failed", message: "Page.getLayoutMetrics returned no viewport size" };
+    }
+    return { x: width / 2, y: height / 2 };
+  } catch (error) {
+    return {
+      code: "cdp_failed",
+      message: error instanceof Error ? error.message : String(error),
+    };
+  }
+}
+
+function cancelled(): RpcError {
+  return { code: "cancelled", message: "wheel aborted" };
+}