import type { CdpFrameGraph } from "@/browser-driver/frame-graph"; import type { SessionManager } from "@/session-manager/manager"; import type { ClickParams, ClickResult, RpcError } from "@/transport/types"; import { attachDialogs, markDialogCursor } from "./dialogs"; import { rpcError } from "./errors"; import { resolveNodeGeometry } from "./frame-geometry"; import { dispatchMouseClick } from "./mouse-input"; import { type CdpRunner, type ChromeTabsApi, enforceAgentWindow, isRpcError, lookupSession, normaliseRef, resolveTargetTab, } from "./shared"; import { resolveSnapshotRef } from "./snapshot-ref"; import { mapImagePointToViewport, pointInRegion, sameRect, surfaceVisibleRect, } from "./surface-coordinate"; export interface SurfaceCaptureEnvironment { navigationIdentity: string; viewportSignature: string; frameProjectionSignature: string; } export interface SurfacePointActionDeps { cdp: CdpRunner; tabsApi: ChromeTabsApi; signal?: AbortSignal; bypassOverlay?: (tabId: number, enabled: boolean) => Promise; } // Canvas-style controls commonly update their pointer hit-test on the next // animation frame. Leave a scheduling boundary after moving and before pressing. const SURFACE_POINTER_MOVE_SETTLE_MS = 32; function stale(message: string): RpcError { return rpcError("permission_denied", "surface_capture_stale", message); } function finiteObject(value: unknown): unknown { if (Array.isArray(value)) return value.map(finiteObject); if (typeof value !== "object" || value === null) return value; return Object.fromEntries( Object.entries(value) .filter( ([, entry]) => entry !== undefined && (typeof entry !== "number" || Number.isFinite(entry)), ) .sort(([a], [b]) => a.localeCompare(b)) .map(([key, entry]) => [key, finiteObject(entry)]), ); } function framePathSignature(graph: CdpFrameGraph, frameId: string | undefined): string | null { if (!frameId) return "top"; const frames = new Map(graph.frames.map((frame) => [frame.frameId, frame])); const path: Array<{ frameId: string; parentFrameId?: string; ownerBackendNodeId?: number; targetSessionId?: string; }> = []; const seen = new Set(); let current = frames.get(frameId); while (current) { if (seen.has(current.frameId)) return null; seen.add(current.frameId); path.push({ frameId: current.frameId, ...(current.parentFrameId ? { parentFrameId: current.parentFrameId } : {}), ...(current.ownerBackendNodeId !== undefined ? { ownerBackendNodeId: current.ownerBackendNodeId } : {}), ...(current.target.sessionId ? { targetSessionId: current.target.sessionId } : {}), }); if (!current.parentFrameId) return JSON.stringify(path); const parent = frames.get(current.parentFrameId); if (!parent) return null; current = parent; } return null; } export async function captureSurfaceEnvironment( cdp: CdpRunner, tabId: number, frameId: string | undefined, ): Promise { try { const [tree, metrics, graph] = await Promise.all([ cdp.send<{ frameTree?: { frame?: { id?: string; loaderId?: string; url?: string } }; }>(tabId, "Page.getFrameTree", {}), cdp.send>(tabId, "Page.getLayoutMetrics", {}), frameId && cdp.getFrameGraph ? cdp.getFrameGraph(tabId) : Promise.resolve(null), ]); const root = tree.frameTree?.frame; if (!root?.id || !root.loaderId) { return { code: "cdp_failed", message: "could not identify the current navigation" }; } const frameProjectionSignature = frameId ? graph ? framePathSignature(graph, frameId) : null : "top"; if (!frameProjectionSignature) { return { code: "cdp_failed", message: `could not identify frame projection for ${frameId}` }; } return { navigationIdentity: JSON.stringify({ id: root.id, loaderId: root.loaderId, url: root.url }), viewportSignature: JSON.stringify( finiteObject({ cssLayoutViewport: metrics.cssLayoutViewport, cssVisualViewport: metrics.cssVisualViewport, }), ), frameProjectionSignature, }; } catch (error) { return { code: "cdp_failed", message: error instanceof Error ? error.message : String(error) }; } } function validatePointParams(params: ClickParams): RpcError | null { if (!params.capture_id || typeof params.capture_id !== "string") { return { code: "invalid_params", message: "surface point click requires capture_id" }; } if (typeof params.image_x !== "number" || !Number.isFinite(params.image_x)) { return { code: "invalid_params", message: "image_x must be a finite number" }; } if (typeof params.image_y !== "number" || !Number.isFinite(params.image_y)) { return { code: "invalid_params", message: "image_y must be a finite number" }; } if (!params.ref || params.selector) { return { code: "invalid_params", message: "surface point click requires one Surface ref and does not accept a selector", }; } if ((params.button ?? "left") !== "left" || (params.click_count ?? 1) !== 1) { return { code: "invalid_params", message: "surface point click currently supports one left click only", }; } if (params.modifiers && params.modifiers.length > 0) { return { code: "invalid_params", message: "surface point click does not accept modifiers" }; } return null; } function captureConsumeError(reason: "not_found" | "expired" | "consumed"): RpcError { const code = reason === "not_found" ? "not_found" : "permission_denied"; return rpcError(code, `surface_capture_${reason}`, `surface capture is ${reason}`); } function aborted(signal: AbortSignal | undefined): RpcError | null { return signal?.aborted ? { code: "cancelled", message: "surface point click aborted" } : null; } export async function handleSurfacePointClick( manager: SessionManager, params: ClickParams, deps: SurfacePointActionDeps, ): Promise { const invalid = validatePointParams(params); if (invalid) return invalid; const ctxOrError = lookupSession(manager, params, "click"); if (isRpcError(ctxOrError)) return ctxOrError; const ctx = ctxOrError; const earlyAbort = aborted(deps.signal); if (earlyAbort) return earlyAbort; const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi); if (isRpcError(target)) return target; const denied = enforceAgentWindow(ctx, target, "click"); if (denied) return denied; const consumed = ctx.surfaceCaptures.consume(params.capture_id as string); if (!consumed.ok) return captureConsumeError(consumed.reason); const capture = consumed.capture; const postConsumeAbort = aborted(deps.signal); if (postConsumeAbort) return postConsumeAbort; if (capture.sessionId !== ctx.sessionId || capture.tabId !== target.tabId) { return stale("surface capture belongs to a different session or tab"); } if (normaliseRef(params.ref as string) !== capture.surface.ref) { return stale("surface ref does not match the capture"); } const node = resolveSnapshotRef(ctx, params.ref as string, target.tabId, "screenshot"); if (isRpcError(node)) return stale(node.message); if ( node.kind !== "surface" || node.backendNodeId !== capture.surface.backendNodeId || node.frameId !== capture.surface.frameId || node.generation !== capture.surface.observationGeneration ) { return stale("surface observation generation or node identity changed"); } deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); try { await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url); } catch (error) { return { code: "cdp_failed", message: error instanceof Error ? error.message : String(error) }; } const environment = await captureSurfaceEnvironment(deps.cdp, target.tabId, node.frameId); if (isRpcError(environment)) return environment; if ( environment.navigationIdentity !== capture.navigationIdentity || environment.viewportSignature !== capture.viewportSignature || environment.frameProjectionSignature !== capture.frameProjectionSignature ) { return stale("navigation, viewport, zoom, scroll, or frame projection changed"); } const geometry = await resolveNodeGeometry( deps.cdp, target.tabId, { target: { tabId: target.tabId, ...(node.cdpSessionId ? { sessionId: node.cdpSessionId } : {}), }, backendNodeId: node.backendNodeId, ...(node.frameId ? { frameId: node.frameId } : {}), }, { scrollIntoView: false }, ); if (isRpcError(geometry)) return stale(geometry.message); const currentRect = surfaceVisibleRect(geometry.topBounds, node.visibleRect); if (!currentRect || !sameRect(currentRect, capture.topViewportRect)) { return stale("surface visible region changed"); } const point = mapImagePointToViewport( capture.topViewportRect, capture.imageWidth, capture.imageHeight, params.image_x as number, params.image_y as number, ); if (!point || !pointInRegion(point, geometry.topVisibleRegions)) { return rpcError( "invalid_params", "surface_coordinate_invalid", "image coordinate is outside the captured Surface", ); } const dialogCursor = markDialogCursor(deps.cdp, target.tabId); let bypassEnabled = false; try { if (deps.bypassOverlay) { await deps.bypassOverlay(target.tabId, true); bypassEnabled = true; } const dispatch = await dispatchMouseClick(deps.cdp, target.tabId, point, { button: "left", clickCount: 1, modifiers: 0, signal: deps.signal, moveSettleMs: SURFACE_POINTER_MOVE_SETTLE_MS, }); if (dispatch === "cancelled") { return { code: "cancelled", message: "surface point click aborted" }; } } catch (error) { return { code: "cdp_failed", message: error instanceof Error ? error.message : String(error) }; } finally { if (bypassEnabled && deps.bypassOverlay) { try { await deps.bypassOverlay(target.tabId, false); } catch (error) { console.debug("[bsk surface-point] overlay bypass disable failed", error); } } } return attachDialogs(deps.cdp, target.tabId, dialogCursor, { tab_id: target.tabId, used_ref: capture.surface.ref, x: point.x, y: point.y, capture_id: capture.id, image_x: params.image_x, image_y: params.image_y, }); }