feat(interaction): add scroll-to-element primitive

apps/extension/src/tools/__tests__/scroll.test.ts

@@ -0,0 +1,145 @@
+import { describe, expect, it, vi } from "vitest";
+import { SessionManager } from "@/session-manager/manager";
+import type { CdpRunner } from "@/tools/shared";
+import { handleScrollTo } from "../scroll";
+
+function fakeAgentWindow(ids: number[]) {
+  let i = 0;
+  return {
+    create: vi.fn(async () => {
+      const id = ids[i++];
+      if (id === undefined) throw new Error("ran out of fake ids");
+      return id;
+    }),
+    remove: vi.fn(async () => {}),
+    ensureActiveTab: vi.fn(async () => {}),
+  };
+}
+
+function makeFakeCdp(handlers: Record<string, (params: unknown) => unknown>) {
+  const sent: Array<{ tabId: number; method: string; params?: object }> = [];
+  const sendImpl = async (tabId: number, method: string, params?: object) => {
+    sent.push({ tabId, method, params });
+    const handler = handlers[method];
+    if (!handler && method === "Page.getLayoutMetrics") {
+      return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
+    }
+    if (!handler) throw new Error(`unexpected CDP call ${method}`);
+    return handler(params);
+  };
+  const cdp: CdpRunner = {
+    send: vi.fn(sendImpl) as unknown as CdpRunner["send"],
+    trackSessionTab: vi.fn(),
+  };
+  const tabsApi = {
+    get: vi.fn(
+      async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
+    ),
+    query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
+  };
+  return { cdp, tabsApi, sent };
+}
+
+describe("handleScrollTo", () => {
+  it("scrolls a ref into view and returns its visible top-viewport bounds", async () => {
+    const manager = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
+    const ctx = await manager.start("aa11");
+    ctx.refStore.set("e3", 1234, { tabId: 4 });
+    const fake = makeFakeCdp({
+      "DOM.scrollIntoViewIfNeeded": () => ({}),
+      "DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }),
+    });
+
+    const result = await handleScrollTo(
+      manager,
+      { session_id: "aa11", ref: "@e3" },
+      { cdp: fake.cdp, tabsApi: fake.tabsApi },
+    );
+
+    if ("code" in result) throw new Error(`unexpected error: ${JSON.stringify(result)}`);
+    expect(result).toMatchObject({
+      tab_id: 4,
+      used_ref: "e3",
+      x: 10,
+      y: 20,
+      width: 100,
+      height: 40,
+    });
+    expect(fake.sent.map((call) => call.method)).toEqual([
+      "DOM.scrollIntoViewIfNeeded",
+      "DOM.getContentQuads",
+      "Page.getLayoutMetrics",
+    ]);
+  });
+
+  it("scrolls OOPIF refs through their parent frame and CDP session", async () => {
+    const manager = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
+    const ctx = await manager.start("aa11");
+    ctx.refStore.set("e3", 1234, {
+      tabId: 4,
+      frameId: "child-frame",
+      cdpSessionId: "child-session",
+    });
+    const fake = makeFakeCdp({
+      "DOM.scrollIntoViewIfNeeded": () => ({}),
+      "DOM.getBoxModel": () => ({
+        model: { content: [204, 306, 604, 306, 604, 506, 204, 506] },
+      }),
+    });
+    fake.cdp.getFrameGraph = vi.fn(async () => ({
+      rootFrameId: "main",
+      frames: [
+        { frameId: "main", target: { tabId: 4 } },
+        {
+          frameId: "child-frame",
+          parentFrameId: "main",
+          ownerBackendNodeId: 99,
+          target: { tabId: 4, sessionId: "child-session" },
+        },
+      ],
+    }));
+    const targetCalls: Array<{ sessionId?: string; method: string }> = [];
+    fake.cdp.sendToTarget = vi.fn(async (target, method) => {
+      targetCalls.push({ sessionId: target.sessionId, method });
+      if (method === "DOM.scrollIntoViewIfNeeded") return {};
+      if (method === "DOM.getContentQuads") {
+        return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
+      }
+      if (method === "Page.getLayoutMetrics") {
+        return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
+      }
+      throw new Error(`unexpected child CDP call ${method}`);
+    }) as CdpRunner["sendToTarget"];
+
+    const result = await handleScrollTo(
+      manager,
+      { session_id: "aa11", ref: "@e3" },
+      { cdp: fake.cdp, tabsApi: fake.tabsApi },
+    );
+
+    if ("code" in result) throw new Error(`unexpected error: ${JSON.stringify(result)}`);
+    expect(result).toMatchObject({ x: 224, y: 346, width: 200, height: 80 });
+    expect(targetCalls).toEqual([
+      { sessionId: "child-session", method: "DOM.scrollIntoViewIfNeeded" },
+      { sessionId: "child-session", method: "DOM.getContentQuads" },
+      { sessionId: "child-session", method: "Page.getLayoutMetrics" },
+    ]);
+  });
+
+  it("does not issue CDP calls after an early cancellation", async () => {
+    const manager = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
+    await manager.start("aa11");
+    const abort = new AbortController();
+    abort.abort();
+    const fake = makeFakeCdp({});
+
+    const result = await handleScrollTo(
+      manager,
+      { session_id: "aa11", selector: "#target" },
+      { cdp: fake.cdp, tabsApi: fake.tabsApi, signal: abort.signal },
+    );
+
+    expect(result).toMatchObject({ code: "cancelled" });
+    expect(fake.cdp.send).not.toHaveBeenCalled();
+  });
+});

apps/extension/src/tools/dispatcher.ts

@@ -26,6 +26,7 @@ import type {
   ResponseFrame,
   RpcError,
   ScreenshotParams,
+  ScrollToParams,
   SelectParams,
   SnapshotParams,
   WaitForNavigationParams,
@@ -52,6 +53,7 @@ import {
   handleSnapshot,
 } from "./observation";
 import { handleRecordAwait, handleRecordStart, handleRecordStop } from "./record";
+import { handleScrollTo } from "./scroll";
 import {
   handleSessionStart,
   handleSessionStop,
@@ -471,6 +473,17 @@ export class ToolDispatcher {
         );
         return this.rememberHover((req.params as HoverParams).session_id, result);
       }
+      case "tool.scroll_to":
+        return this.withHoverReleaseForRequest(
+          req.params as ScrollToParams,
+          () =>
+            handleScrollTo(
+              this.sessions,
+              req.params as ScrollToParams,
+              this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
+            ),
+          signal,
+        );
       case "tool.fill":
         return this.withHoverReleaseForRequest(
           req.params as FillParams,
@@ -718,6 +731,7 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
     case "tool.reload":
     case "tool.click":
     case "tool.hover":
+    case "tool.scroll_to":
     case "tool.fill":
     case "tool.press":
     case "tool.select":

apps/extension/src/tools/interaction.ts

@@ -128,7 +128,7 @@ async function wait(ms: number, signal?: AbortSignal): Promise<void> {
  * `RpcError` if the caller supplied neither (or both), or if neither
  * lookup matched.
  */
-async function resolveBackendNode(
+export async function resolveBackendNode(
   cdp: CdpRunner,
   ctx: SessionContext,
   target: { tabId: number },

apps/extension/src/tools/scroll.ts

@@ -0,0 +1,69 @@
+import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
+import type { SessionManager } from "@/session-manager/manager";
+import type { RpcError, ScrollToParams, ScrollToResult } from "@/transport/types";
+import { attachDialogs, markDialogCursor } from "./dialogs";
+import { resolveNodeGeometry } from "./frame-geometry";
+import { resolveBackendNode } from "./interaction";
+import {
+  type CdpRunner,
+  type ChromeTabsApi,
+  chromeTabsApi,
+  enforceAgentWindow,
+  isRpcError,
+  lookupSession,
+  resolveTargetTab,
+} from "./shared";
+
+export interface ScrollToDeps {
+  cdp: CdpRunner;
+  tabsApi: ChromeTabsApi;
+  signal?: AbortSignal;
+}
+
+let defaultDeps: { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } | null = null;
+function getDefaultDeps(): { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } {
+  if (!defaultDeps) defaultDeps = { cdp: new ChromiumCdp(), tabsApi: chromeTabsApi };
+  return defaultDeps;
+}
+
+export async function handleScrollTo(
+  manager: SessionManager,
+  params: ScrollToParams,
+  deps: ScrollToDeps = getDefaultDeps(),
+): Promise<ScrollToResult | RpcError> {
+  const ctxOrErr = lookupSession(manager, params, "scroll-to");
+  if (isRpcError(ctxOrErr)) return ctxOrErr;
+  const ctx = ctxOrErr;
+  if (deps.signal?.aborted) return { code: "cancelled", message: "scroll-to aborted" };
+  const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
+  if (isRpcError(target)) return target;
+  const denied = enforceAgentWindow(ctx, target, "scroll-to");
+  if (denied) return denied;
+  const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
+  const node = await resolveBackendNode(deps.cdp, ctx, target, params, "scroll-to");
+  if (isRpcError(node)) return node;
+  if (deps.signal?.aborted) return { code: "cancelled", message: "scroll-to aborted" };
+
+  deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
+  const geometry = await resolveNodeGeometry(
+    deps.cdp,
+    target.tabId,
+    {
+      target: node.cdpTarget,
+      backendNodeId: node.backendNodeId,
+      ...(node.frameId ? { frameId: node.frameId } : {}),
+    },
+    { scrollIntoView: true },
+  );
+  if (isRpcError(geometry)) return geometry;
+
+  return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
+    tab_id: target.tabId,
+    used_ref: node.usedRef,
+    used_selector: node.usedSelector,
+    x: geometry.topBounds.x,
+    y: geometry.topBounds.y,
+    width: geometry.topBounds.width,
+    height: geometry.topBounds.height,
+  });
+}

apps/extension/src/transport/types.ts

@@ -444,6 +444,25 @@ export interface HoverResult {
   dialogs?: JavaScriptDialogInfo[];
 }
 
+export interface ScrollToParams {
+  session_id: string;
+  ref?: string;
+  selector?: string;
+  tab_id?: number;
+  timeout_ms?: number;
+}
+
+export interface ScrollToResult {
+  tab_id: number;
+  used_ref?: string;
+  used_selector?: string;
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+  dialogs?: JavaScriptDialogInfo[];
+}
+
 export interface FillParams {
   session_id: string;
   value: string;