import type { CdpFrame, CdpFrameGraph } from "@/browser-driver/frame-graph"; import { type GeometryProjection, projectRectToViewport } from "../geometry"; import { GeometryContext } from "../geometry/frame-context"; import { type CdpRunner, cdpRunnerForTarget, sendToCdpTarget } from "../shared"; import { type CapturedNode, type CapturedViewModel, captureViewModel } from "./capture"; import { buildFrameDocuments, type FrameAxBatch, type FrameDocument, type FrameOwnedAxNode, } from "./frame-document"; export type FrameAxNode = FrameOwnedAxNode; export type CapturedFrameDocument = FrameDocument; function throwIfAborted(signal?: AbortSignal): void { if (signal?.aborted) throw new DOMException("observation aborted", "AbortError"); } function isAbortError(err: unknown): boolean { return err instanceof Error && err.name === "AbortError"; } function transformFrameNodes( nodes: CapturedNode[], projection: GeometryProjection | null, ): CapturedNode[] { return nodes.map((node) => ({ ...node, rect: projection ? (() => { const bounds = projectRectToViewport(node.rect, projection); return bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null; })() : null, })); } async function captureMissingFrameDocuments( cdp: CdpRunner, tabId: number, graph: CdpFrameGraph, captured: CapturedViewModel, geometry: GeometryContext, signal?: AbortSignal, ): Promise { if (!captured.frameNodes) captured.frameNodes = new Map(); if (!captured.frameOwnerBackendNodeIds) captured.frameOwnerBackendNodeIds = new Map(); if (!captured.frameParentIds) captured.frameParentIds = new Map(); for (const frame of graph.frames) { if (!frame.target.sessionId || captured.frameNodes.has(frame.frameId)) continue; try { const child = await captureViewModel(cdpRunnerForTarget(cdp, frame.target), tabId, { signal, geometry, target: frame.target, }); if (child.frameExcludedBackendNodeIds?.size) { captured.frameExcludedBackendNodeIds ??= new Map(); for (const [id, excluded] of child.frameExcludedBackendNodeIds) captured.frameExcludedBackendNodeIds.set(id, excluded); } if (child.frameGeometryIssues?.length) { (captured.frameGeometryIssues ??= []).push(...child.frameGeometryIssues); } let projection: GeometryProjection | null = null; try { projection = await geometry.targetProjection(frame.frameId); } catch (err) { if (isAbortError(err)) throw err; console.debug("[bsk observation] child frame geometry projection failed", { frameId: frame.frameId, err, }); } const rootFrameId = child.rootFrameId ?? frame.frameId; const childFrames = child.frameNodes ?? new Map(); if (!childFrames.has(rootFrameId)) childFrames.set(rootFrameId, child.nodes); if (rootFrameId !== frame.frameId && !childFrames.has(frame.frameId)) { childFrames.set( frame.frameId, child.nodes.map((node) => ({ ...node, frameId: frame.frameId })), ); } for (const [childFrameId, nodes] of childFrames) { captured.frameNodes.set(childFrameId, transformFrameNodes(nodes, projection)); } if (frame.ownerBackendNodeId !== undefined) { captured.iframeNodes.set( frame.ownerBackendNodeId, captured.frameNodes.get(frame.frameId) ?? [], ); captured.frameOwnerBackendNodeIds.set(frame.frameId, frame.ownerBackendNodeId); } if (frame.parentFrameId) captured.frameParentIds.set(frame.frameId, frame.parentFrameId); for (const [childFrameId, ownerBackendNodeId] of child.frameOwnerBackendNodeIds ?? []) { captured.frameOwnerBackendNodeIds.set(childFrameId, ownerBackendNodeId); } for (const [childFrameId, parentFrameId] of child.frameParentIds ?? []) { captured.frameParentIds.set(childFrameId, parentFrameId); } } catch (err) { if (isAbortError(err)) throw err; console.debug("[bsk observation] child frame DOM capture failed", { frameId: frame.frameId, err, }); } } } async function captureAxTrees( cdp: CdpRunner, tabId: number, discoveredFrames: CdpFrame[], captured: CapturedViewModel, signal?: AbortSignal, ): Promise[]> { let frames = [...discoveredFrames]; const capturedFrameIds = [...(captured.frameNodes?.keys() ?? [])]; if (frames.length === 0) { const rootFrameId = captured.rootFrameId ?? capturedFrameIds[0] ?? "root"; frames = [{ frameId: rootFrameId, target: { tabId } }]; } for (const frameId of capturedFrameIds) { if (!frames.some((frame) => frame.frameId === frameId)) { frames.push({ frameId, target: { tabId } }); } } let firstFailure: unknown; const trees = await Promise.all( frames.map(async (frame): Promise> => { try { throwIfAborted(signal); await sendToCdpTarget(cdp, frame.target, "Accessibility.enable", {}); const result = await sendToCdpTarget<{ nodes?: T[] }>( cdp, frame.target, "Accessibility.getFullAXTree", frame.frameId === "root" ? {} : { frameId: frame.frameId }, ); throwIfAborted(signal); return { frame, nodes: result.nodes ?? [], }; } catch (err) { if (isAbortError(err)) throw err; firstFailure ??= err; console.debug("[bsk observation] frame AX capture failed", { frameId: frame.frameId, err, }); return { frame, nodes: [] }; } }), ); const capturedNodeCount = [...captured.iframeNodes.values()].reduce( (count, nodes) => count + nodes.length, captured.nodes.length, ); if (firstFailure && capturedNodeCount === 0 && trees.every((tree) => tree.nodes.length === 0)) { throw firstFailure; } return trees; } export async function captureFrameData( cdp: CdpRunner, tabId: number, captured: CapturedViewModel, signal?: AbortSignal, geometry = new GeometryContext(cdp, tabId, undefined, signal), ): Promise[]> { const graph = await geometry.graph(); const frames = graph?.frames ?? []; if (graph) await captureMissingFrameDocuments(cdp, tabId, graph, captured, geometry, signal); throwIfAborted(signal); const batches = await captureAxTrees(cdp, tabId, frames, captured, signal); return buildFrameDocuments(graph, batches, captured, signal); }