// DOM interaction tools — `tool.click`, `tool.fill`, `tool.press`, and // `tool.select`. // // All interaction tools: // 1. Resolve target tab (sandbox: must be inside Agent Window). // 2. Resolve target element by `ref` (compound frame/session identity) // or `selector` (DOM.querySelector + describeNode). // 3. Scroll the node and its frame owners into view, project its live // content quad into the top viewport, then dispatch `Input.*` events. // 4. Honour `AbortSignal` so canceled calls don't issue follow-up CDP // commands. import { ChromiumCdp } from "@/browser-driver/chromium-cdp"; import type { CdpTarget } from "@/browser-driver/frame-graph"; import type { SessionContext, SessionManager } from "@/session-manager/manager"; import type { ClickParams, ClickResult, FillParams, FillResult, HoverParams, HoverResult, KeyModifier, MouseButton, PressParams, PressResult, RpcError, SelectParams, SelectResult, } from "@/transport/types"; import { attachDialogs, markDialogCursor } from "./dialogs"; import { backendNodeToObject } from "./element-geometry"; import { rpcError } from "./errors"; import { resolveNodeGeometry, scrollElementAndFramesIntoView } from "./frame-geometry"; import { dispatchMouseClick } from "./mouse-input"; import { type CdpRunner, type ChromeTabsApi, cdpRunnerForTarget, chromeTabsApi, enforceAgentWindow, isRpcError, lookupSession, resolveTargetTab, } from "./shared"; import { resolveSnapshotRef } from "./snapshot-ref"; import { resolveSurfacePointerTarget } from "./surface-target"; export interface InteractionDeps { cdp: CdpRunner; tabsApi: ChromeTabsApi; /** Abort hook (full chain wired in M10.2). */ signal?: AbortSignal; defaultTimeoutMs?: number; /** Temporarily disable overlay click blocker during CDP automation. */ bypassOverlay?: (tabId: number, enabled: boolean) => Promise; /** Keep hover hit-testing active for the caller's next observation/action. */ keepOverlayBypassAfterHover?: boolean; } const DEFAULT_TIMEOUT_MS = 30_000; const DEFAULT_HOVER_SETTLE_MS = 200; let defaultDeps: { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } | null = null; function getDefaultDeps(): { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } { if (!defaultDeps) { defaultDeps = { cdp: new ChromiumCdp(), tabsApi: chromeTabsApi }; } return defaultDeps; } /** * Fold a list of `KeyModifier`s into CDP's bit layout (§4 of the * CDP Input domain): alt=1, ctrl=2, meta=4, shift=8. * * Exported for unit tests. */ export function modifiersBitfield(mods: KeyModifier[] | undefined): number { if (!mods) return 0; let bits = 0; for (const m of mods) { switch (m) { case "alt": bits |= 1; break; case "ctrl": bits |= 2; break; case "meta": bits |= 4; break; case "shift": bits |= 8; break; } } return bits; } function throwIfAborted(signal: AbortSignal | undefined): RpcError | null { if (signal?.aborted) { return { code: "cancelled", message: "interaction aborted" }; } return null; } function isAbortLikeError(err: unknown): boolean { return err instanceof DOMException && err.name === "AbortError"; } async function wait(ms: number, signal?: AbortSignal): Promise { if (signal?.aborted) throw new DOMException("aborted", "AbortError"); await new Promise((resolve, reject) => { const cleanup = () => signal?.removeEventListener("abort", onAbort); const timer = setTimeout(() => { cleanup(); resolve(); }, ms); const onAbort = () => { clearTimeout(timer); cleanup(); reject(new DOMException("aborted", "AbortError")); }; signal?.addEventListener("abort", onAbort, { once: true }); }); } /** * Resolve `{ref?, selector?}` into a `backendNodeId`. Returns an * `RpcError` if the caller supplied neither (or both), or if neither * lookup matched. */ async function resolveBackendNode( cdp: CdpRunner, ctx: SessionContext, target: { tabId: number }, params: { ref?: string; selector?: string }, toolName: string, ): Promise< | { backendNodeId: number; cdpTarget: CdpTarget; frameId?: string; usedRef?: string; usedSelector?: string; } | RpcError > { const hasRef = typeof params.ref === "string" && params.ref.length > 0; const hasSelector = typeof params.selector === "string" && params.selector.length > 0; if (hasRef && hasSelector) { return { code: "invalid_params", message: `${toolName}: pass either ref or selector, not both`, }; } if (!hasRef && !hasSelector) { return { code: "invalid_params", message: `${toolName} requires a ref or a selector`, }; } if (hasRef) { const resolved = resolveSnapshotRef(ctx, params.ref as string, target.tabId, "interact"); if (isRpcError(resolved)) return resolved; return { backendNodeId: resolved.backendNodeId, cdpTarget: { tabId: target.tabId, ...(resolved.cdpSessionId ? { sessionId: resolved.cdpSessionId } : {}), }, ...(resolved.frameId ? { frameId: resolved.frameId } : {}), usedRef: resolved.refKey, }; } // selector path try { const doc = await cdp.send<{ root?: { nodeId?: number } }>(target.tabId, "DOM.getDocument", { depth: 0, }); const rootNodeId = doc.root?.nodeId; if (typeof rootNodeId !== "number") { return { code: "cdp_failed", message: "DOM.getDocument returned no root nodeId", }; } const found = await cdp.send<{ nodeId?: number }>(target.tabId, "DOM.querySelector", { nodeId: rootNodeId, selector: params.selector, }); if (typeof found.nodeId !== "number" || found.nodeId === 0) { return rpcError( "not_found", "selector_not_found", `selector ${params.selector} did not match any element`, ); } const described = await cdp.send<{ node?: { backendNodeId?: number } }>( target.tabId, "DOM.describeNode", { nodeId: found.nodeId }, ); const backendNodeId = described.node?.backendNodeId; if (typeof backendNodeId !== "number") { return { code: "cdp_failed", message: "DOM.describeNode returned no backendNodeId", }; } return { backendNodeId, cdpTarget: { tabId: target.tabId }, usedSelector: params.selector }; } catch (err) { return { code: "cdp_failed", message: err instanceof Error ? err.message : String(err), }; } } // --------------------------------------------------------------------------- // tool.click // --------------------------------------------------------------------------- interface ResolvedClickPointerTarget { point: { x: number; y: number }; usedRef?: string; usedSelector?: string; moveSettleMs: number; capture?: { id: string; imageX: number; imageY: number; }; } export async function handleClick( manager: SessionManager, params: ClickParams, deps: InteractionDeps = getDefaultDeps(), ): Promise { const hasSurfacePointParams = params.capture_id !== undefined || params.image_x !== undefined || params.image_y !== undefined; const ctxOrErr = lookupSession(manager, params, "click"); if (isRpcError(ctxOrErr)) return ctxOrErr; const ctx = ctxOrErr; const aborted = throwIfAborted(deps.signal); if (aborted) return aborted; const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi); if (isRpcError(target)) return target; const denied = enforceAgentWindow(ctx, target, "click"); if (denied) return denied; const dialogCursor = markDialogCursor(deps.cdp, target.tabId); const button: MouseButton = params.button ?? "left"; const clickCount = params.click_count ?? 1; if (!Number.isSafeInteger(clickCount) || clickCount < 1) { return { code: "invalid_params", message: "click_count must be a positive integer" }; } const modifiers = modifiersBitfield(params.modifiers); if (throwIfAborted(deps.signal)) { return { code: "cancelled", message: "click aborted" }; } deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); const pointerTarget: ResolvedClickPointerTarget | RpcError = hasSurfacePointParams ? await resolveSurfacePointerTarget( ctx, target, { ref: params.ref, selector: params.selector, captureId: params.capture_id, imageX: params.image_x, imageY: params.image_y, }, { cdp: deps.cdp, signal: deps.signal, }, ) : await (async () => { const node = await resolveBackendNode(deps.cdp, ctx, target, params, "click"); if (isRpcError(node)) return node; const geometry = await resolveNodeGeometry( deps.cdp, target.tabId, { target: node.cdpTarget, backendNodeId: node.backendNodeId, ...(node.frameId ? { frameId: node.frameId } : {}), }, { scrollIntoView: true }, ); if (isRpcError(geometry)) return geometry; return { point: geometry.actionPoint, usedRef: node.usedRef, usedSelector: node.usedSelector, moveSettleMs: 0, }; })(); if (isRpcError(pointerTarget)) return pointerTarget; if (throwIfAborted(deps.signal)) { return { code: "cancelled", message: "click aborted" }; } const overlayBlocking = await checkOverlayAtPoint( deps.cdp, target.tabId, pointerTarget.point.x, pointerTarget.point.y, ); let automationBypassEnabled = false; if (overlayBlocking && deps.bypassOverlay) { try { await deps.bypassOverlay(target.tabId, true); automationBypassEnabled = true; } catch (err) { console.debug("[bsk interaction] overlay bypass enable failed", err); } } try { const dispatch = await dispatchMouseClick(deps.cdp, target.tabId, pointerTarget.point, { button, clickCount, modifiers, signal: deps.signal, moveSettleMs: pointerTarget.moveSettleMs, }); if (dispatch === "cancelled") { return { code: "cancelled", message: "click aborted" }; } } catch (err) { return { code: "cdp_failed", message: err instanceof Error ? err.message : String(err), }; } finally { if (automationBypassEnabled && deps.bypassOverlay && !deps.keepOverlayBypassAfterHover) { try { await deps.bypassOverlay(target.tabId, false); } catch (err) { console.debug("[bsk interaction] overlay bypass disable failed", err); } } } return attachDialogs(deps.cdp, target.tabId, dialogCursor, { tab_id: target.tabId, used_ref: pointerTarget.usedRef, used_selector: pointerTarget.usedSelector, x: pointerTarget.point.x, y: pointerTarget.point.y, ...(pointerTarget.capture ? { capture_id: pointerTarget.capture.id, image_x: pointerTarget.capture.imageX, image_y: pointerTarget.capture.imageY, } : {}), }); } // --------------------------------------------------------------------------- // tool.hover // --------------------------------------------------------------------------- export async function handleHover( manager: SessionManager, params: HoverParams, deps: InteractionDeps = getDefaultDeps(), ): Promise { const ctxOrErr = lookupSession(manager, params, "hover"); if (isRpcError(ctxOrErr)) return ctxOrErr; const ctx = ctxOrErr; const aborted = throwIfAborted(deps.signal); if (aborted) return aborted; const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi); if (isRpcError(target)) return target; const denied = enforceAgentWindow(ctx, target, "hover"); if (denied) return denied; const dialogCursor = markDialogCursor(deps.cdp, target.tabId); const node = await resolveBackendNode(deps.cdp, ctx, target, params, "hover"); if (isRpcError(node)) return node; deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); const geometry = await resolveNodeGeometry( deps.cdp, target.tabId, { target: node.cdpTarget, backendNodeId: node.backendNodeId, ...(node.frameId ? { frameId: node.frameId } : {}), }, { scrollIntoView: true }, ); if (isRpcError(geometry)) return geometry; const centre = geometry.actionPoint; if (throwIfAborted(deps.signal)) { return { code: "cancelled", message: "hover aborted" }; } const modifiers = modifiersBitfield(params.modifiers); const overlayBlocking = await checkOverlayAtPoint(deps.cdp, target.tabId, centre.x, centre.y); let automationBypassEnabled = false; let hoverCompleted = false; if (overlayBlocking && deps.bypassOverlay) { try { await deps.bypassOverlay(target.tabId, true); automationBypassEnabled = true; } catch (err) { console.debug("[bsk interaction] overlay bypass enable failed", err); } } try { await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", { type: "mouseMoved", x: centre.x, y: centre.y, modifiers, }); const settleMs = params.settle_ms ?? DEFAULT_HOVER_SETTLE_MS; if (settleMs > 0) { await wait(settleMs, deps.signal); } hoverCompleted = true; } catch (err) { if (isAbortLikeError(err)) { return { code: "cancelled", message: "hover aborted" }; } return { code: "cdp_failed", message: err instanceof Error ? err.message : String(err), }; } finally { if ( automationBypassEnabled && deps.bypassOverlay && (!deps.keepOverlayBypassAfterHover || !hoverCompleted) ) { try { await deps.bypassOverlay(target.tabId, false); } catch (err) { console.debug("[bsk interaction] overlay bypass disable failed", err); } } } return attachDialogs(deps.cdp, target.tabId, dialogCursor, { tab_id: target.tabId, used_ref: node.usedRef, used_selector: node.usedSelector, x: centre.x, y: centre.y, }); } interface OverlayHitInspection { overlayHostPresent: boolean; overlayHostConnected: boolean; hitIndex: number; } /** * Returns true when the control overlay shadow root has a visible layer * with pointer-events blocking the click point (mirrors intern execClick). */ async function checkOverlayAtPoint( cdp: CdpRunner, tabId: number, x: number, y: number, ): Promise { try { const hitTest = await cdp.send<{ result?: { value?: { overlayHostPresent?: boolean; overlayHostConnected?: boolean; } | null; }; }>(tabId, "Runtime.evaluate", { expression: `(function() { const overlayHost = document.querySelector("[data-bsk-overlay]"); return { overlayHostPresent: !!overlayHost, overlayHostConnected: !!overlayHost?.isConnected, }; })()`, returnByValue: true, }); const hitTarget = hitTest.result?.value ?? null; if (!hitTarget?.overlayHostPresent || !hitTarget.overlayHostConnected) { return false; } const inspection = await cdp.send<{ result?: { value?: OverlayHitInspection | null }; }>(tabId, "Runtime.evaluate", { expression: `(function() { const overlayHost = document.querySelector("[data-bsk-overlay]"); const shadowRoot = overlayHost instanceof HTMLElement ? overlayHost.shadowRoot : null; const overlays = Array.from(shadowRoot?.querySelectorAll("*") ?? []); const overlayDetails = overlays .map((node) => { if (!(node instanceof HTMLElement)) return null; const style = window.getComputedStyle(node); const rect = node.getBoundingClientRect(); return { display: style.display, pointerEvents: style.pointerEvents, connected: node.isConnected, rect: { x: rect.x, y: rect.y, width: rect.width, height: rect.height }, }; }) .filter((node) => { if (!node) return false; return ( node.connected && node.display !== "none" && node.pointerEvents !== "none" && node.rect.width > 0 && node.rect.height > 0 ); }); const hitIndex = overlayDetails.findIndex((node) => { const withinX = ${x} >= node.rect.x && ${x} <= node.rect.x + node.rect.width; const withinY = ${y} >= node.rect.y && ${y} <= node.rect.y + node.rect.height; return withinX && withinY; }); return { overlayHostPresent: true, overlayHostConnected: true, hitIndex, }; })()`, returnByValue: true, }); const value = inspection.result?.value; return (value?.hitIndex ?? -1) >= 0; } catch (err) { console.debug("[bsk interaction] overlay hit-test failed", err); return false; } } /** * Describes the subset of `DOM.describeNode`'s `node` we inspect to * decide whether the target is fillable. `attributes` is the CDP * flat-array form (`[name, value, name, value, ...]`). */ interface DescribedNode { nodeName?: string; attributes?: string[]; } /** * Decide whether a node can receive `tool.fill`: native `` / * `