// DOM interaction tools — click, hover, focus/blur, fill, press, and select. // // All interaction tools: // 1. Resolve target tab (sandbox: must be inside Agent Window). // 2. Resolve target element by `ref` (compound frame/session identity) // or `selector` (DOM.querySelector + describeNode). // 3. Scroll the node and its frame owners into view, project its live // content quad into the top viewport, then dispatch `Input.*` events. // 4. Honour `AbortSignal` so canceled calls don't issue follow-up CDP // commands. import { ChromiumCdp } from "@/browser-driver/chromium-cdp"; import type { CdpTarget } from "@/browser-driver/frame-graph"; import type { SessionContext, SessionManager } from "@/session-manager/manager"; import type { BlurParams, BlurResult, ClickParams, ClickResult, FillParams, FillResult, FocusParams, FocusResult, HoverParams, HoverResult, KeyModifier, MouseButton, PressParams, PressResult, RpcError, SelectParams, SelectResult, } from "@/transport/types"; import { attachDialogs, markDialogCursor } from "./dialogs"; import { backendNodeToObject } from "./element-geometry"; import { rpcError } from "./errors"; import { resolveNodeGeometry, scrollElementAndFramesIntoView } from "./frame-geometry"; import { type CdpRunner, type ChromeTabsApi, cdpRunnerForTarget, chromeTabsApi, enforceAgentWindow, isRpcError, lookupSession, type ResolvedTargetTab, resolveTargetTab, } from "./shared"; import { resolveSnapshotRef } from "./snapshot-ref"; export interface InteractionDeps { cdp: CdpRunner; tabsApi: ChromeTabsApi; /** Abort hook (full chain wired in M10.2). */ signal?: AbortSignal; defaultTimeoutMs?: number; /** Temporarily disable overlay click blocker during CDP automation. */ bypassOverlay?: (tabId: number, enabled: boolean) => Promise; /** Keep hover hit-testing active for the caller's next observation/action. */ keepOverlayBypassAfterHover?: boolean; } export interface ResolvedActionTarget { tab: ResolvedTargetTab; backendNodeId: number; cdpTarget: CdpTarget; frameId?: string; usedRef?: string; usedSelector?: string; } const DEFAULT_TIMEOUT_MS = 30_000; const DEFAULT_HOVER_SETTLE_MS = 200; let defaultDeps: { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } | null = null; function getDefaultDeps(): { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } { if (!defaultDeps) { defaultDeps = { cdp: new ChromiumCdp(), tabsApi: chromeTabsApi }; } return defaultDeps; } /** * Fold a list of `KeyModifier`s into CDP's bit layout (§4 of the * CDP Input domain): alt=1, ctrl=2, meta=4, shift=8. * * Exported for unit tests. */ export function modifiersBitfield(mods: KeyModifier[] | undefined): number { if (!mods) return 0; let bits = 0; for (const m of mods) { switch (m) { case "alt": bits |= 1; break; case "ctrl": bits |= 2; break; case "meta": bits |= 4; break; case "shift": bits |= 8; break; } } return bits; } function throwIfAborted(signal: AbortSignal | undefined): RpcError | null { if (signal?.aborted) { return { code: "cancelled", message: "interaction aborted" }; } return null; } function isAbortLikeError(err: unknown): boolean { return err instanceof DOMException && err.name === "AbortError"; } async function wait(ms: number, signal?: AbortSignal): Promise { if (signal?.aborted) throw new DOMException("aborted", "AbortError"); await new Promise((resolve, reject) => { const cleanup = () => signal?.removeEventListener("abort", onAbort); const timer = setTimeout(() => { cleanup(); resolve(); }, ms); const onAbort = () => { clearTimeout(timer); cleanup(); reject(new DOMException("aborted", "AbortError")); }; signal?.addEventListener("abort", onAbort, { once: true }); }); } /** * Resolve `{ref?, selector?}` into a `backendNodeId`. Returns an * `RpcError` if the caller supplied neither (or both), or if neither * lookup matched. */ export async function resolveBackendNode( cdp: CdpRunner, ctx: SessionContext, target: { tabId: number }, params: { ref?: string; selector?: string }, toolName: string, ): Promise< | { backendNodeId: number; cdpTarget: CdpTarget; frameId?: string; usedRef?: string; usedSelector?: string; } | RpcError > { const hasRef = typeof params.ref === "string" && params.ref.length > 0; const hasSelector = typeof params.selector === "string" && params.selector.length > 0; if (hasRef && hasSelector) { return { code: "invalid_params", message: `${toolName}: pass either ref or selector, not both`, }; } if (!hasRef && !hasSelector) { return { code: "invalid_params", message: `${toolName} requires a ref or a selector`, }; } if (hasRef) { const resolved = resolveSnapshotRef(ctx, params.ref as string, target.tabId); if (isRpcError(resolved)) return resolved; return { backendNodeId: resolved.backendNodeId, cdpTarget: { tabId: target.tabId, ...(resolved.cdpSessionId ? { sessionId: resolved.cdpSessionId } : {}), }, ...(resolved.frameId ? { frameId: resolved.frameId } : {}), usedRef: resolved.refKey, }; } // selector path try { // Selector lookup itself attaches CDP, even when no element is found. cdp.trackSessionTab?.(ctx.sessionId, target.tabId); const doc = await cdp.send<{ root?: { nodeId?: number } }>(target.tabId, "DOM.getDocument", { depth: 0, }); const rootNodeId = doc.root?.nodeId; if (typeof rootNodeId !== "number") { return { code: "cdp_failed", message: "DOM.getDocument returned no root nodeId", }; } const found = await cdp.send<{ nodeId?: number }>(target.tabId, "DOM.querySelector", { nodeId: rootNodeId, selector: params.selector, }); if (typeof found.nodeId !== "number" || found.nodeId === 0) { return rpcError( "not_found", "selector_not_found", `selector ${params.selector} did not match any element`, ); } const described = await cdp.send<{ node?: { backendNodeId?: number } }>( target.tabId, "DOM.describeNode", { nodeId: found.nodeId }, ); const backendNodeId = described.node?.backendNodeId; if (typeof backendNodeId !== "number") { return { code: "cdp_failed", message: "DOM.describeNode returned no backendNodeId", }; } return { backendNodeId, cdpTarget: { tabId: target.tabId }, usedSelector: params.selector }; } catch (err) { return { code: "cdp_failed", message: err instanceof Error ? err.message : String(err), }; } } // Check the target's own root, then its hosts: closed shadow roots are not // reachable via host.shadowRoot. DOM focus remains meaningful in background tabs. const FOCUS_CHECK = `function() { if (!this.isConnected) return false; let element = this; while (element) { const root = element.getRootNode(); if (root.activeElement !== element) return false; element = root.host; } return true; }`; const BLUR_TARGET = `function() { if (!this.isConnected) throw new Error('blur target is detached'); const wasFocused = (${FOCUS_CHECK}).call(this); if (typeof this.blur !== 'function') return { ok: false, was_focused: wasFocused }; this.blur(); return { ok: true, was_focused: wasFocused }; }`; interface FocusScriptReply { result?: { value?: T }; exceptionDetails?: { text?: string; exception?: { description?: string } }; } function checkFocusAbort(signal: AbortSignal | undefined): void { if (signal?.aborted) throw new DOMException("interaction aborted", "AbortError"); } // Only focus/blur use this view. Guard every CDP boundary, including commands // inside selector resolution and frame scrolling, without changing other tools. function focusCdp(cdp: CdpRunner, signal: AbortSignal | undefined): CdpRunner { const send = (target: CdpTarget, method: string, params?: object): Promise => { checkFocusAbort(signal); return cdpRunnerForTarget(cdp, target).send(target.tabId, method, params); }; return { send: (tabId, method, params) => send({ tabId }, method, params), sendToTarget: send, trackSessionTab: cdp.trackSessionTab?.bind(cdp), getFrameGraph: cdp.getFrameGraph ? (tabId) => { checkFocusAbort(signal); return cdp.getFrameGraph!(tabId); } : undefined, }; } // --------------------------------------------------------------------------- // tool.focus / tool.blur // --------------------------------------------------------------------------- export function handleFocus( manager: SessionManager, params: FocusParams, deps: InteractionDeps = getDefaultDeps(), ): Promise { return changeFocus(manager, params, deps, "focus"); } export function handleBlur( manager: SessionManager, params: BlurParams, deps: InteractionDeps = getDefaultDeps(), ): Promise { return changeFocus(manager, params, deps, "blur"); } function changeFocus( manager: SessionManager, params: FocusParams, deps: InteractionDeps, action: "focus", ): Promise; function changeFocus( manager: SessionManager, params: BlurParams, deps: InteractionDeps, action: "blur", ): Promise; async function changeFocus( manager: SessionManager, params: FocusParams | BlurParams, deps: InteractionDeps, action: "focus" | "blur", ): Promise { const ctx = lookupSession(manager, params, action); if (isRpcError(ctx)) return ctx; const cdp = focusCdp(deps.cdp, deps.signal); let objectId: string | undefined; let cleanupCdp: CdpRunner | undefined; let tabId: number | undefined; try { checkFocusAbort(deps.signal); const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi); checkFocusAbort(deps.signal); if (isRpcError(target)) return target; const denied = enforceAgentWindow(ctx, target, action); if (denied) return denied; tabId = target.tabId; const dialogCursor = markDialogCursor(deps.cdp, tabId); const node = await resolveBackendNode(cdp, ctx, target, params, action); checkFocusAbort(deps.signal); if (isRpcError(node)) return node; const nodeCdp = cdpRunnerForTarget(cdp, node.cdpTarget); cleanupCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget); deps.cdp.trackSessionTab?.(ctx.sessionId, tabId); if (action === "focus") { const scrollErr = await scrollElementAndFramesIntoView( cdp, tabId, node.cdpTarget, node.backendNodeId, node.frameId, ); checkFocusAbort(deps.signal); if (scrollErr) return scrollErr; } const resolved = await backendNodeToObject(nodeCdp, tabId, node.backendNodeId); // Keep the handle before checking cancellation so finally can release it. if (!isRpcError(resolved)) objectId = resolved; checkFocusAbort(deps.signal); if (isRpcError(resolved)) return resolved; const runScript = async (functionDeclaration: string): Promise => { const reply = await nodeCdp.send>( target.tabId, "Runtime.callFunctionOn", { objectId, functionDeclaration, returnByValue: true, }, ); checkFocusAbort(deps.signal); if (reply.exceptionDetails) { const details = reply.exceptionDetails; throw new Error( `${action} script failed: ${details.exception?.description ?? details.text ?? "unknown exception"}`, ); } return reply.result?.value; }; let wasFocused: boolean | undefined; if (action === "focus") { await nodeCdp.send(tabId, "DOM.focus", { backendNodeId: node.backendNodeId }); checkFocusAbort(deps.signal); } else { const mutation = await runScript<{ ok: boolean; was_focused: boolean }>(BLUR_TARGET); if (typeof mutation?.ok !== "boolean" || typeof mutation.was_focused !== "boolean") { throw new Error("blur script returned an unexpected result"); } if (!mutation.ok) { return { code: "invalid_params", message: "target element does not support blur()" }; } wasFocused = mutation.was_focused; } // Use a separate call so microtasks from focus/blur handlers finish before // we report success. In particular, a blur handler can restore focus. const focused = await runScript(FOCUS_CHECK); if (typeof focused !== "boolean") { throw new Error(`${action} verification returned an unexpected result`); } if (focused !== (action === "focus")) { throw new Error( action === "focus" ? "target element did not become focused" : "target element remained focused after blur()", ); } return attachDialogs(deps.cdp, tabId, dialogCursor, { tab_id: tabId, used_ref: node.usedRef, used_selector: node.usedSelector, focused, ...(action === "blur" ? { was_focused: wasFocused! } : {}), }); } catch (err) { return ( throwIfAborted(deps.signal) ?? { code: "cdp_failed", message: err instanceof Error ? err.message : String(err), } ); } finally { if (objectId !== undefined && cleanupCdp && tabId !== undefined) { // Cleanup must also run after cancellation. Navigation may have already // disposed the object, which must not replace the operation's result. try { await cleanupCdp.send(tabId, "Runtime.releaseObject", { objectId }); } catch { // The target or execution context may no longer exist. } } } } export async function resolveActionTarget( cdp: CdpRunner, ctx: SessionContext, target: ResolvedTargetTab, params: { ref?: string; selector?: string }, toolName: string, ): Promise { const node = await resolveBackendNode(cdp, ctx, target, params, toolName); return isRpcError(node) ? node : { tab: target, ...node }; } // --------------------------------------------------------------------------- // tool.click // --------------------------------------------------------------------------- export async function handleClick( manager: SessionManager, params: ClickParams, deps: InteractionDeps = getDefaultDeps(), ): Promise { const ctxOrErr = lookupSession(manager, params, "click"); if (isRpcError(ctxOrErr)) return ctxOrErr; const ctx = ctxOrErr; const aborted = throwIfAborted(deps.signal); if (aborted) return aborted; const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi); if (isRpcError(target)) return target; const denied = enforceAgentWindow(ctx, target, "click"); if (denied) return denied; const resolved = await resolveActionTarget(deps.cdp, ctx, target, params, "click"); if (isRpcError(resolved)) return resolved; return clickResolvedTarget(ctx, resolved, params, deps); } export async function clickResolvedTarget( ctx: SessionContext, resolved: ResolvedActionTarget, params: Pick, deps: InteractionDeps, ): Promise { const { tab: target } = resolved; const dialogCursor = markDialogCursor(deps.cdp, target.tabId); if (throwIfAborted(deps.signal)) { return { code: "cancelled", message: "click aborted" }; } deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); const geometry = await resolveNodeGeometry( deps.cdp, target.tabId, { target: resolved.cdpTarget, backendNodeId: resolved.backendNodeId, ...(resolved.frameId ? { frameId: resolved.frameId } : {}), }, { scrollIntoView: true }, ); if (isRpcError(geometry)) return geometry; const centre = geometry.actionPoint; if (throwIfAborted(deps.signal)) { return { code: "cancelled", message: "click aborted" }; } const button: MouseButton = params.button ?? "left"; const clickCount = params.click_count ?? 1; if (clickCount < 1) { return { code: "invalid_params", message: "click_count must be greater than zero" }; } const modifiers = modifiersBitfield(params.modifiers); const overlayBlocking = await checkOverlayAtPoint(deps.cdp, target.tabId, centre.x, centre.y); let automationBypassEnabled = false; if (overlayBlocking && deps.bypassOverlay) { try { await deps.bypassOverlay(target.tabId, true); automationBypassEnabled = true; } catch (err) { console.debug("[bsk interaction] overlay bypass enable failed", err); } } try { // Move first so hover state activates, then press → release. await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", { type: "mouseMoved", x: centre.x, y: centre.y, modifiers, }); if (throwIfAborted(deps.signal)) { return { code: "cancelled", message: "click aborted" }; } await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", { type: "mousePressed", x: centre.x, y: centre.y, button, clickCount, modifiers, }); if (throwIfAborted(deps.signal)) { try { await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", { type: "mouseReleased", x: centre.x, y: centre.y, button, clickCount, modifiers, }); } catch (err) { console.debug("[bsk interaction] best-effort mouseReleased after abort failed", err); } return { code: "cancelled", message: "click aborted" }; } await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", { type: "mouseReleased", x: centre.x, y: centre.y, button, clickCount, modifiers, }); } catch (err) { return { code: "cdp_failed", message: err instanceof Error ? err.message : String(err), }; } finally { if (automationBypassEnabled && deps.bypassOverlay && !deps.keepOverlayBypassAfterHover) { try { await deps.bypassOverlay(target.tabId, false); } catch (err) { console.debug("[bsk interaction] overlay bypass disable failed", err); } } } return attachDialogs(deps.cdp, target.tabId, dialogCursor, { tab_id: target.tabId, used_ref: resolved.usedRef, used_selector: resolved.usedSelector, x: centre.x, y: centre.y, }); } // --------------------------------------------------------------------------- // tool.hover // --------------------------------------------------------------------------- export async function handleHover( manager: SessionManager, params: HoverParams, deps: InteractionDeps = getDefaultDeps(), ): Promise { const ctxOrErr = lookupSession(manager, params, "hover"); if (isRpcError(ctxOrErr)) return ctxOrErr; const ctx = ctxOrErr; const aborted = throwIfAborted(deps.signal); if (aborted) return aborted; const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi); if (isRpcError(target)) return target; const denied = enforceAgentWindow(ctx, target, "hover"); if (denied) return denied; const dialogCursor = markDialogCursor(deps.cdp, target.tabId); const node = await resolveBackendNode(deps.cdp, ctx, target, params, "hover"); if (isRpcError(node)) return node; deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); const geometry = await resolveNodeGeometry( deps.cdp, target.tabId, { target: node.cdpTarget, backendNodeId: node.backendNodeId, ...(node.frameId ? { frameId: node.frameId } : {}), }, { scrollIntoView: true }, ); if (isRpcError(geometry)) return geometry; const centre = geometry.actionPoint; if (throwIfAborted(deps.signal)) { return { code: "cancelled", message: "hover aborted" }; } const modifiers = modifiersBitfield(params.modifiers); const overlayBlocking = await checkOverlayAtPoint(deps.cdp, target.tabId, centre.x, centre.y); let automationBypassEnabled = false; let hoverCompleted = false; if (overlayBlocking && deps.bypassOverlay) { try { await deps.bypassOverlay(target.tabId, true); automationBypassEnabled = true; } catch (err) { console.debug("[bsk interaction] overlay bypass enable failed", err); } } try { await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", { type: "mouseMoved", x: centre.x, y: centre.y, modifiers, }); const settleMs = params.settle_ms ?? DEFAULT_HOVER_SETTLE_MS; if (settleMs > 0) { await wait(settleMs, deps.signal); } hoverCompleted = true; } catch (err) { if (isAbortLikeError(err)) { return { code: "cancelled", message: "hover aborted" }; } return { code: "cdp_failed", message: err instanceof Error ? err.message : String(err), }; } finally { if ( automationBypassEnabled && deps.bypassOverlay && (!deps.keepOverlayBypassAfterHover || !hoverCompleted) ) { try { await deps.bypassOverlay(target.tabId, false); } catch (err) { console.debug("[bsk interaction] overlay bypass disable failed", err); } } } return attachDialogs(deps.cdp, target.tabId, dialogCursor, { tab_id: target.tabId, used_ref: node.usedRef, used_selector: node.usedSelector, x: centre.x, y: centre.y, }); } interface OverlayHitInspection { overlayHostPresent: boolean; overlayHostConnected: boolean; hitIndex: number; } /** * Returns true when the control overlay shadow root has a visible layer * with pointer-events blocking the click point (mirrors intern execClick). */ async function checkOverlayAtPoint( cdp: CdpRunner, tabId: number, x: number, y: number, ): Promise { try { const hitTest = await cdp.send<{ result?: { value?: { overlayHostPresent?: boolean; overlayHostConnected?: boolean; } | null; }; }>(tabId, "Runtime.evaluate", { expression: `(function() { const overlayHost = document.querySelector("[data-bsk-overlay]"); return { overlayHostPresent: !!overlayHost, overlayHostConnected: !!overlayHost?.isConnected, }; })()`, returnByValue: true, }); const hitTarget = hitTest.result?.value ?? null; if (!hitTarget?.overlayHostPresent || !hitTarget.overlayHostConnected) { return false; } const inspection = await cdp.send<{ result?: { value?: OverlayHitInspection | null }; }>(tabId, "Runtime.evaluate", { expression: `(function() { const overlayHost = document.querySelector("[data-bsk-overlay]"); const shadowRoot = overlayHost instanceof HTMLElement ? overlayHost.shadowRoot : null; const overlays = Array.from(shadowRoot?.querySelectorAll("*") ?? []); const overlayDetails = overlays .map((node) => { if (!(node instanceof HTMLElement)) return null; const style = window.getComputedStyle(node); const rect = node.getBoundingClientRect(); return { display: style.display, pointerEvents: style.pointerEvents, connected: node.isConnected, rect: { x: rect.x, y: rect.y, width: rect.width, height: rect.height }, }; }) .filter((node) => { if (!node) return false; return ( node.connected && node.display !== "none" && node.pointerEvents !== "none" && node.rect.width > 0 && node.rect.height > 0 ); }); const hitIndex = overlayDetails.findIndex((node) => { const withinX = ${x} >= node.rect.x && ${x} <= node.rect.x + node.rect.width; const withinY = ${y} >= node.rect.y && ${y} <= node.rect.y + node.rect.height; return withinX && withinY; }); return { overlayHostPresent: true, overlayHostConnected: true, hitIndex, }; })()`, returnByValue: true, }); const value = inspection.result?.value; return (value?.hitIndex ?? -1) >= 0; } catch (err) { console.debug("[bsk interaction] overlay hit-test failed", err); return false; } } /** * Describes the subset of `DOM.describeNode`'s `node` we inspect to * decide whether the target is fillable. `attributes` is the CDP * flat-array form (`[name, value, name, value, ...]`). */ interface DescribedNode { nodeName?: string; attributes?: string[]; } /** * Decide whether a node can receive `tool.fill`: native `` / * `