mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
804 lines
27 KiB
TypeScript
804 lines
27 KiB
TypeScript
// CDP capture adapter: DOMSnapshot.captureSnapshot + Page.getLayoutMetrics
|
|
// → CapturedNode[] + Viewport. This is the ONLY VOM module that touches
|
|
// raw CDP. The captureSnapshot reply is columnar (parallel arrays + a
|
|
// shared string table); we request exactly three computed styles so the
|
|
// `styles` columns are [position, pointer-events, cursor] in that order.
|
|
|
|
import type { Rect, Viewport } from "@browser-skill/vom";
|
|
import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
|
|
import type { CdpRunner } from "../shared";
|
|
|
|
const REQUESTED_STYLES = ["position", "pointer-events", "cursor"] as const;
|
|
const STYLE_COL = Object.fromEntries(
|
|
REQUESTED_STYLES.map((name, index) => [name, index]),
|
|
) as Record<(typeof REQUESTED_STYLES)[number], number>;
|
|
|
|
export interface CapturedNode {
|
|
backendNodeId: number;
|
|
parentBackendNodeId: number | null;
|
|
/** Owning iframe backend node id; `null` for the top-level document. */
|
|
ownerFrameBackendNodeId?: number | null;
|
|
tag: string;
|
|
attrs: Record<string, string>;
|
|
/** Top-level viewport-relative CSS px, clipped to the owning frame viewport. */
|
|
rect: Rect | null;
|
|
/** Frame-local viewport-relative CSS px before top-level translation/clipping. */
|
|
localRect?: Rect | null;
|
|
paintOrder: number;
|
|
position: string;
|
|
pointerEvents: string;
|
|
/**
|
|
* computed `cursor`. `cursor: pointer` is the strongest CDP-free signal
|
|
* that a non-semantic element (a `<div>`/`<span>` with a click handler)
|
|
* is actually an interactive control — used by the adapter to surface
|
|
* custom buttons/checkboxes the AX tree drops as `generic`. Optional like
|
|
* `textContent`: the live parser always sets it, hand-built fixtures may not.
|
|
*/
|
|
cursor?: string;
|
|
textContent?: string;
|
|
formState?: "empty" | "filled" | "default";
|
|
formValue?: string;
|
|
formDefaultValue?: string;
|
|
formPlaceholder?: string;
|
|
}
|
|
|
|
export type CapturedIframeNodes = Map<number, CapturedNode[]>;
|
|
|
|
export interface CapturedSurfaceProbe {
|
|
triggerLabel: string;
|
|
triggerAction: "hover" | "focus" | string;
|
|
subItems: string[];
|
|
}
|
|
|
|
export interface CapturedViewModel {
|
|
nodes: CapturedNode[];
|
|
viewport: Viewport;
|
|
iframeNodes: CapturedIframeNodes;
|
|
surfaceProbes?: CapturedSurfaceProbe[];
|
|
/** Backend node ids belonging to the agent overlay host + its shadow subtree. */
|
|
excludedBackendNodeIds: Set<number>;
|
|
}
|
|
|
|
export interface CaptureViewModelOptions {
|
|
conditionalSurfaceProbe?: boolean;
|
|
}
|
|
|
|
/** Sparse array format Chrome uses for infrequently-set per-node fields. */
|
|
interface SparseArray {
|
|
index: number[];
|
|
value: number[];
|
|
}
|
|
|
|
interface SnapshotDocument {
|
|
scrollOffsetX?: number;
|
|
scrollOffsetY?: number;
|
|
nodes?: {
|
|
parentIndex?: number[];
|
|
nodeName?: number[];
|
|
backendNodeId?: number[];
|
|
attributes?: number[][];
|
|
/**
|
|
* Per-node text value (index into strings), set for `#text` / CDATA nodes.
|
|
* Element nodes carry -1. Same length as `backendNodeId`.
|
|
*/
|
|
nodeValue?: number[];
|
|
/** Maps node array index → index into `documents[]` for frame content. */
|
|
contentDocumentIndex?: SparseArray;
|
|
};
|
|
layout?: {
|
|
nodeIndex?: number[];
|
|
styles?: number[][];
|
|
bounds?: number[][];
|
|
paintOrders?: number[];
|
|
};
|
|
}
|
|
|
|
interface SnapshotReply {
|
|
strings?: string[];
|
|
documents?: SnapshotDocument[];
|
|
}
|
|
|
|
interface LayoutMetricsReply {
|
|
cssLayoutViewport?: {
|
|
clientWidth?: number;
|
|
clientHeight?: number;
|
|
pageX?: number;
|
|
pageY?: number;
|
|
};
|
|
layoutViewport?: { clientWidth?: number; clientHeight?: number; pageX?: number; pageY?: number };
|
|
}
|
|
|
|
function isFormControlTag(tag: string): boolean {
|
|
return tag === "input" || tag === "textarea" || tag === "select";
|
|
}
|
|
|
|
const MAX_FORM_ENRICH_CONTROLS = 250;
|
|
|
|
interface CapturedFormState {
|
|
value?: string;
|
|
defaultValue?: string;
|
|
placeholder?: string;
|
|
state?: "empty" | "filled" | "default";
|
|
sensitive?: boolean;
|
|
}
|
|
|
|
interface CapturedFrameFormState {
|
|
controls: CapturedFormState[];
|
|
childFrames: CapturedFrameFormState[];
|
|
}
|
|
|
|
function formStateBatchExpression(maxControls: number): string {
|
|
return `(() => {
|
|
const maxControls = ${JSON.stringify(maxControls)};
|
|
let remaining = maxControls;
|
|
const controlSelector = "input,textarea,select";
|
|
const controlState = (el) => {
|
|
const tag = el.tagName.toLowerCase();
|
|
const type = tag === "input" ? String(el.type || "text").toLowerCase() : tag;
|
|
const sensitive = type === "password" || type === "credit-card";
|
|
const rawValue = typeof el.value === "string" ? el.value : "";
|
|
const defaultValue = typeof el.defaultValue === "string" ? el.defaultValue : "";
|
|
const placeholder = typeof el.placeholder === "string" ? el.placeholder : "";
|
|
const state = rawValue === "" ? "empty" : rawValue === defaultValue ? "default" : "filled";
|
|
return {
|
|
state,
|
|
sensitive,
|
|
defaultValue,
|
|
placeholder,
|
|
...(sensitive ? {} : { value: rawValue }),
|
|
};
|
|
};
|
|
const collect = (doc) => {
|
|
const controls = [];
|
|
for (const el of Array.from(doc.querySelectorAll(controlSelector))) {
|
|
if (remaining <= 0) break;
|
|
controls.push(controlState(el));
|
|
remaining -= 1;
|
|
}
|
|
const childFrames = [];
|
|
for (const frame of Array.from(doc.querySelectorAll("iframe"))) {
|
|
let childDoc = null;
|
|
try { childDoc = frame.contentDocument; } catch { childDoc = null; }
|
|
childFrames.push(childDoc && remaining > 0 ? collect(childDoc) : { controls: [], childFrames: [] });
|
|
}
|
|
return { controls, childFrames };
|
|
};
|
|
return collect(document);
|
|
})()`;
|
|
}
|
|
|
|
function flattenFrameFormStates(
|
|
frame: CapturedFrameFormState | undefined,
|
|
out: CapturedFormState[][] = [],
|
|
): CapturedFormState[][] {
|
|
if (!frame) return out;
|
|
out.push(Array.isArray(frame.controls) ? frame.controls : []);
|
|
for (const child of Array.isArray(frame.childFrames) ? frame.childFrames : []) {
|
|
flattenFrameFormStates(child, out);
|
|
}
|
|
return out;
|
|
}
|
|
|
|
function applyFormStates(nodes: CapturedNode[], states: CapturedFormState[]): void {
|
|
let index = 0;
|
|
for (const node of nodes) {
|
|
if (!isFormControlTag(node.tag)) continue;
|
|
const state = states[index];
|
|
index += 1;
|
|
if (!state) continue;
|
|
node.formState = state.state;
|
|
node.formDefaultValue = state.defaultValue ?? "";
|
|
node.formPlaceholder = state.placeholder ?? "";
|
|
if (!state.sensitive && state.value !== undefined) {
|
|
node.formValue = state.value;
|
|
}
|
|
}
|
|
}
|
|
|
|
async function enrichFormControlStates(
|
|
cdp: CdpRunner,
|
|
tabId: number,
|
|
frameNodeGroups: CapturedNode[][],
|
|
): Promise<void> {
|
|
const hasControls = frameNodeGroups.some((nodes) =>
|
|
nodes.some((node) => isFormControlTag(node.tag)),
|
|
);
|
|
if (!hasControls) return;
|
|
try {
|
|
const result = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
|
expression: formStateBatchExpression(MAX_FORM_ENRICH_CONTROLS),
|
|
returnByValue: true,
|
|
});
|
|
const frameStates = flattenFrameFormStates(runtimeValue<CapturedFrameFormState>(result));
|
|
for (let i = 0; i < frameNodeGroups.length; i += 1) {
|
|
applyFormStates(frameNodeGroups[i], frameStates[i] ?? []);
|
|
}
|
|
} catch {
|
|
// Best-effort enrichment. DOMSnapshot/AX data still carries the nodes.
|
|
}
|
|
}
|
|
|
|
interface RuntimeEvaluateReply {
|
|
result?: {
|
|
value?: unknown;
|
|
};
|
|
}
|
|
|
|
interface HoverCandidate {
|
|
triggerSel: string;
|
|
affectedSub: string;
|
|
label: string;
|
|
x: number;
|
|
y: number;
|
|
}
|
|
|
|
interface CdpDomNode {
|
|
backendNodeId?: number;
|
|
children?: CdpDomNode[];
|
|
shadowRoots?: CdpDomNode[];
|
|
}
|
|
|
|
interface ParseDocumentResult {
|
|
nodes: CapturedNode[];
|
|
excludedBackendNodeIds: Set<number>;
|
|
}
|
|
|
|
interface FrameContext {
|
|
ownerFrameBackendNodeId: number | null;
|
|
originInTop: { x: number; y: number };
|
|
clipRectInTop: Rect;
|
|
scrollX: number;
|
|
scrollY: number;
|
|
}
|
|
|
|
function str(strings: string[], idx: number | undefined): string {
|
|
if (idx === undefined || idx < 0) return "";
|
|
return strings[idx] ?? "";
|
|
}
|
|
|
|
function devicePixelRatio(metrics: LayoutMetricsReply): number {
|
|
const layoutW = metrics.layoutViewport?.clientWidth ?? 0;
|
|
const cssW = metrics.cssLayoutViewport?.clientWidth ?? 0;
|
|
if (!layoutW || !cssW) return 1;
|
|
const dpr = layoutW / cssW;
|
|
if (!Number.isFinite(dpr) || dpr <= 0) return 1;
|
|
return dpr >= 1 ? dpr : 1;
|
|
}
|
|
|
|
function viewportRect(viewport: Viewport): Rect {
|
|
return { x: 0, y: 0, w: Math.max(0, viewport.width), h: Math.max(0, viewport.height) };
|
|
}
|
|
|
|
function intersectRects(a: Rect, b: Rect): Rect | null {
|
|
const x1 = Math.max(a.x, b.x);
|
|
const y1 = Math.max(a.y, b.y);
|
|
const x2 = Math.min(a.x + a.w, b.x + b.w);
|
|
const y2 = Math.min(a.y + a.h, b.y + b.h);
|
|
if (x2 <= x1 || y2 <= y1) return null;
|
|
return { x: x1, y: y1, w: x2 - x1, h: y2 - y1 };
|
|
}
|
|
|
|
function rectInTop(localRect: Rect, context: FrameContext): Rect | null {
|
|
const translated = {
|
|
x: context.originInTop.x + localRect.x,
|
|
y: context.originInTop.y + localRect.y,
|
|
w: localRect.w,
|
|
h: localRect.h,
|
|
};
|
|
return intersectRects(translated, context.clipRectInTop);
|
|
}
|
|
|
|
function unclipRectInTop(localRect: Rect, context: FrameContext): Rect {
|
|
return {
|
|
x: context.originInTop.x + localRect.x,
|
|
y: context.originInTop.y + localRect.y,
|
|
w: localRect.w,
|
|
h: localRect.h,
|
|
};
|
|
}
|
|
|
|
function sparseIndexMap(sparse: SparseArray | undefined): Map<number, number> {
|
|
const out = new Map<number, number>();
|
|
if (!sparse?.index || !sparse.value) return out;
|
|
for (let i = 0; i < sparse.index.length; i++) {
|
|
const nodeIndex = sparse.index[i];
|
|
const docIndex = sparse.value[i];
|
|
if (nodeIndex !== undefined && docIndex !== undefined) out.set(nodeIndex, docIndex);
|
|
}
|
|
return out;
|
|
}
|
|
|
|
function collectBackendIdsFromDomNode(node: CdpDomNode | undefined, out: Set<number>): void {
|
|
if (!node) return;
|
|
if (typeof node.backendNodeId === "number") {
|
|
out.add(node.backendNodeId);
|
|
}
|
|
for (const child of node.children ?? []) {
|
|
collectBackendIdsFromDomNode(child, out);
|
|
}
|
|
for (const shadow of node.shadowRoots ?? []) {
|
|
collectBackendIdsFromDomNode(shadow, out);
|
|
}
|
|
}
|
|
|
|
const MAX_HOVER_PROBE_MS = 2_000;
|
|
const MAX_HOVER_TRIGGERS = 8;
|
|
const HOVER_SETTLE_MS = 200;
|
|
|
|
function runtimeValue<T>(reply: RuntimeEvaluateReply): T | undefined {
|
|
return reply.result?.value as T | undefined;
|
|
}
|
|
|
|
function hoverCssScanExpression(): string {
|
|
return `(() => {
|
|
const visibilityProps = ["display", "visibility", "opacity", "maxHeight", "height", "overflow"];
|
|
const pairs = [];
|
|
const seenRules = new Set();
|
|
for (const sheet of Array.from(document.styleSheets)) {
|
|
let rules;
|
|
try { rules = sheet.cssRules; } catch { continue; }
|
|
if (!rules) continue;
|
|
for (const rule of Array.from(rules)) {
|
|
if (rule.type !== 1 || !rule.selectorText || !rule.style) continue;
|
|
const selectorText = String(rule.selectorText);
|
|
if (!selectorText.includes(":hover")) continue;
|
|
if (!visibilityProps.some((prop) => rule.style[prop])) continue;
|
|
for (const rawPart of selectorText.split(",")) {
|
|
const part = rawPart.trim();
|
|
const hoverIndex = part.indexOf(":hover");
|
|
if (hoverIndex < 0) continue;
|
|
const triggerSel = part.slice(0, hoverIndex).trim();
|
|
const affectedSub = part.slice(hoverIndex + 6).trim();
|
|
if (!triggerSel) continue;
|
|
const key = triggerSel + "||" + affectedSub;
|
|
if (seenRules.has(key)) continue;
|
|
seenRules.add(key);
|
|
pairs.push({ triggerSel, affectedSub });
|
|
}
|
|
}
|
|
}
|
|
|
|
const candidates = [];
|
|
const seenLabels = new Set();
|
|
const isUnsafeTrigger = (el) => {
|
|
if (!(el instanceof HTMLElement)) return true;
|
|
const tag = el.tagName.toLowerCase();
|
|
if (["input", "textarea", "select", "option"].includes(tag)) return true;
|
|
if (el.isContentEditable) return true;
|
|
if (el.hasAttribute("disabled") || el.hasAttribute("inert")) return true;
|
|
if ((el.getAttribute("aria-disabled") || "").toLowerCase() === "true") return true;
|
|
return false;
|
|
};
|
|
for (const pair of pairs) {
|
|
let elements;
|
|
try { elements = Array.from(document.querySelectorAll(pair.triggerSel)); } catch { continue; }
|
|
for (const el of elements) {
|
|
if (!(el instanceof Element)) continue;
|
|
if (isUnsafeTrigger(el)) continue;
|
|
const rect = el.getBoundingClientRect();
|
|
const style = getComputedStyle(el);
|
|
if (rect.width <= 0 || rect.height <= 0 || style.visibility === "hidden" || style.display === "none" || style.pointerEvents === "none") continue;
|
|
const label = (el.textContent || "").replace(/\\s+/g, " ").trim();
|
|
if (!label || seenLabels.has(label)) continue;
|
|
seenLabels.add(label);
|
|
candidates.push({
|
|
triggerSel: pair.triggerSel,
|
|
affectedSub: pair.affectedSub,
|
|
label,
|
|
x: rect.left + rect.width / 2,
|
|
y: rect.top + rect.height / 2,
|
|
});
|
|
if (candidates.length >= ${MAX_HOVER_TRIGGERS}) return candidates;
|
|
}
|
|
}
|
|
return candidates;
|
|
})()`;
|
|
}
|
|
|
|
function hoverCollectExpression(candidate: HoverCandidate): string {
|
|
return `(() => {
|
|
const triggerSel = ${JSON.stringify(candidate.triggerSel)};
|
|
const rawAffectedSub = ${JSON.stringify(candidate.affectedSub)};
|
|
const x = ${JSON.stringify(candidate.x)};
|
|
const y = ${JSON.stringify(candidate.y)};
|
|
const hit = document.elementFromPoint(x, y);
|
|
const trigger = hit instanceof Element ? hit.closest(triggerSel) : null;
|
|
if (!trigger) return [];
|
|
|
|
const affectedSub = rawAffectedSub.replace(/^[>+~]\\s*/, "").trim();
|
|
let targets = [];
|
|
if (!affectedSub) {
|
|
targets = [trigger];
|
|
} else {
|
|
try { targets = Array.from(trigger.querySelectorAll(affectedSub)); } catch { targets = []; }
|
|
}
|
|
const items = [];
|
|
const seen = new Set();
|
|
const push = (value) => {
|
|
const text = String(value || "").replace(/\\s+/g, " ").trim();
|
|
if (!text || seen.has(text)) return;
|
|
seen.add(text);
|
|
items.push(text);
|
|
};
|
|
for (const target of targets) {
|
|
if (!(target instanceof Element)) continue;
|
|
const style = getComputedStyle(target);
|
|
if (style.display === "none" || style.visibility === "hidden" || style.opacity === "0") continue;
|
|
const clickable = target.querySelectorAll('a, button, [role="menuitem"], [role="option"]');
|
|
if (clickable.length > 0) {
|
|
for (const child of Array.from(clickable)) push(child.textContent);
|
|
} else {
|
|
push(target.textContent);
|
|
}
|
|
}
|
|
return items.slice(0, 12);
|
|
})()`;
|
|
}
|
|
|
|
async function wait(ms: number): Promise<void> {
|
|
await new Promise((resolve) => setTimeout(resolve, ms));
|
|
}
|
|
|
|
async function clearHover(cdp: CdpRunner, tabId: number): Promise<void> {
|
|
await cdp
|
|
.send(tabId, "Input.dispatchMouseEvent", {
|
|
type: "mouseMoved",
|
|
x: -10,
|
|
y: -10,
|
|
})
|
|
.catch(() =>
|
|
cdp.send(tabId, "Input.dispatchMouseEvent", { type: "mouseMoved", x: 0, y: 0 }).catch(() => {
|
|
// best effort
|
|
}),
|
|
);
|
|
}
|
|
|
|
async function probeHoverSurfaces(cdp: CdpRunner, tabId: number): Promise<CapturedSurfaceProbe[]> {
|
|
const started = Date.now();
|
|
try {
|
|
const scan = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
|
expression: hoverCssScanExpression(),
|
|
returnByValue: true,
|
|
});
|
|
const candidates = runtimeValue<HoverCandidate[]>(scan) ?? [];
|
|
const results: CapturedSurfaceProbe[] = [];
|
|
const seen = new Set<string>();
|
|
for (const candidate of candidates.slice(0, MAX_HOVER_TRIGGERS)) {
|
|
if (Date.now() - started > MAX_HOVER_PROBE_MS) break;
|
|
if (!candidate.label || seen.has(candidate.label)) continue;
|
|
try {
|
|
await cdp.send(tabId, "Input.dispatchMouseEvent", {
|
|
type: "mouseMoved",
|
|
x: candidate.x,
|
|
y: candidate.y,
|
|
});
|
|
await wait(HOVER_SETTLE_MS);
|
|
const collected = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
|
expression: hoverCollectExpression(candidate),
|
|
returnByValue: true,
|
|
});
|
|
const subItems = (runtimeValue<string[]>(collected) ?? [])
|
|
.map((item) => item.replace(/\s+/g, " ").trim())
|
|
.filter(Boolean);
|
|
if (subItems.length === 0) continue;
|
|
seen.add(candidate.label);
|
|
results.push({
|
|
triggerLabel: candidate.label,
|
|
triggerAction: "hover",
|
|
subItems,
|
|
});
|
|
} catch {
|
|
continue;
|
|
} finally {
|
|
await clearHover(cdp, tabId);
|
|
}
|
|
}
|
|
return results;
|
|
} catch (err) {
|
|
console.debug("[bsk capture] hover surface probe failed", err);
|
|
return [];
|
|
}
|
|
}
|
|
|
|
/**
|
|
* When DOMSnapshot is unavailable, locate the marked overlay host via CDP and
|
|
* collect every backendNodeId in its pierced subtree (open shadow included).
|
|
*/
|
|
export async function collectOverlayExcludedBackendIds(
|
|
cdp: CdpRunner,
|
|
tabId: number,
|
|
): Promise<Set<number>> {
|
|
const excluded = new Set<number>();
|
|
try {
|
|
const doc = await cdp.send<{ root?: { nodeId?: number } }>(tabId, "DOM.getDocument", {
|
|
depth: 0,
|
|
pierce: true,
|
|
});
|
|
const rootNodeId = doc.root?.nodeId;
|
|
if (typeof rootNodeId !== "number") return excluded;
|
|
|
|
const found = await cdp.send<{ nodeId?: number }>(tabId, "DOM.querySelector", {
|
|
nodeId: rootNodeId,
|
|
selector: OVERLAY_HOST_SELECTOR,
|
|
});
|
|
if (typeof found.nodeId !== "number" || found.nodeId === 0) return excluded;
|
|
|
|
const described = await cdp.send<{ node?: CdpDomNode }>(tabId, "DOM.describeNode", {
|
|
nodeId: found.nodeId,
|
|
depth: -1,
|
|
pierce: true,
|
|
});
|
|
collectBackendIdsFromDomNode(described.node, excluded);
|
|
} catch (err) {
|
|
console.debug("[bsk capture] overlay exclusion fallback failed", err);
|
|
}
|
|
return excluded;
|
|
}
|
|
|
|
/**
|
|
* Parse one DOMSnapshot document into CapturedNode[].
|
|
*
|
|
* @param doc - the raw DOMSnapshot document object
|
|
* @param strings - the shared string table for the whole snapshot
|
|
* @param dpr - device-pixel-ratio from Page.getLayoutMetrics
|
|
* @param context - frame coordinate context; output rects are top viewport-relative
|
|
*/
|
|
function parseDocumentNodes(
|
|
doc: SnapshotDocument,
|
|
strings: string[],
|
|
dpr: number,
|
|
context: FrameContext,
|
|
): ParseDocumentResult {
|
|
const dn = doc.nodes;
|
|
const dl = doc.layout;
|
|
if (!dn?.backendNodeId) {
|
|
return { nodes: [], excludedBackendNodeIds: new Set() };
|
|
}
|
|
|
|
const count = dn.backendNodeId.length;
|
|
const layoutByNode = new Map<number, number>();
|
|
(dl?.nodeIndex ?? []).forEach((nodeIdx, layoutIdx) => layoutByNode.set(nodeIdx, layoutIdx));
|
|
|
|
// The agent's own overlay (WXT shadow host, marked with OVERLAY_HOST_MARKER_ATTR)
|
|
// is injected into the page and its open shadow root
|
|
// is inlined by DOMSnapshot as descendants of the host. Its fixed
|
|
// full-viewport click-blocker would otherwise dominate occlusion detection
|
|
// and hide the real page, so mark the host + its whole subtree excluded.
|
|
// parentIndex always references an earlier node, so one forward pass suffices.
|
|
const excluded = new Array<boolean>(count).fill(false);
|
|
const excludedBackendNodeIds = new Set<number>();
|
|
for (let n = 0; n < count; n++) {
|
|
const parentIdx = dn.parentIndex?.[n] ?? -1;
|
|
const inherited = parentIdx >= 0 && excluded[parentIdx];
|
|
let isOverlayHost = inherited;
|
|
if (!isOverlayHost) {
|
|
const pairs = dn.attributes?.[n] ?? [];
|
|
const attrNames: string[] = [];
|
|
for (let a = 0; a + 1 < pairs.length; a += 2) {
|
|
attrNames.push(str(strings, pairs[a]));
|
|
}
|
|
isOverlayHost = isOverlayHostNode(str(strings, dn.nodeName?.[n]), attrNames);
|
|
}
|
|
if (!isOverlayHost) continue;
|
|
excluded[n] = true;
|
|
const backendNodeId = dn.backendNodeId[n];
|
|
if (backendNodeId !== undefined) {
|
|
excludedBackendNodeIds.add(backendNodeId);
|
|
}
|
|
}
|
|
|
|
const posCol = STYLE_COL.position;
|
|
const peCol = STYLE_COL["pointer-events"];
|
|
const cursorCol = STYLE_COL.cursor;
|
|
|
|
// Collect visible text from #text child nodes so element CapturedNodes
|
|
// carry a textContent value usable as a button/link label fallback.
|
|
// nodeValue is a parallel array: string index for text nodes, -1 otherwise.
|
|
const nodeTextContent = new Map<number, string>();
|
|
if (dn.nodeValue) {
|
|
for (let n = 0; n < count; n++) {
|
|
const nvIdx = dn.nodeValue[n] ?? -1;
|
|
if (nvIdx < 0) continue;
|
|
const text = str(strings, nvIdx).trim();
|
|
if (!text) continue;
|
|
const parentIdx = dn.parentIndex?.[n] ?? -1;
|
|
if (parentIdx >= 0) {
|
|
const existing = nodeTextContent.get(parentIdx);
|
|
nodeTextContent.set(parentIdx, existing ? `${existing} ${text}` : text);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Build CapturedNodes.
|
|
const nodes: CapturedNode[] = [];
|
|
for (let n = 0; n < count; n++) {
|
|
if (excluded[n]) continue;
|
|
const backendNodeId = dn.backendNodeId[n];
|
|
const parentIdx = dn.parentIndex?.[n] ?? -1;
|
|
const parentBackendNodeId = parentIdx >= 0 ? (dn.backendNodeId[parentIdx] ?? null) : null;
|
|
const tag = str(strings, dn.nodeName?.[n]).toLowerCase();
|
|
|
|
const attrs: Record<string, string> = {};
|
|
const pairs = dn.attributes?.[n] ?? [];
|
|
for (let a = 0; a + 1 < pairs.length; a += 2) {
|
|
attrs[str(strings, pairs[a]).toLowerCase()] = str(strings, pairs[a + 1]);
|
|
}
|
|
|
|
let rect: Rect | null = null;
|
|
let localRect: Rect | null = null;
|
|
let paintOrder = 0;
|
|
let position = "static";
|
|
let pointerEvents = "auto";
|
|
let cursor = "auto";
|
|
const li = layoutByNode.get(n);
|
|
if (li !== undefined) {
|
|
const b = dl?.bounds?.[li];
|
|
if (b && b.length >= 4 && b[2] > 0 && b[3] > 0) {
|
|
localRect = {
|
|
x: b[0] / dpr - context.scrollX,
|
|
y: b[1] / dpr - context.scrollY,
|
|
w: b[2] / dpr,
|
|
h: b[3] / dpr,
|
|
};
|
|
rect = rectInTop(localRect, context);
|
|
}
|
|
paintOrder = dl?.paintOrders?.[li] ?? 0;
|
|
const styleRow = dl?.styles?.[li] ?? [];
|
|
position = str(strings, styleRow[posCol]) || "static";
|
|
pointerEvents = str(strings, styleRow[peCol]) || "auto";
|
|
cursor = str(strings, styleRow[cursorCol]) || "auto";
|
|
}
|
|
|
|
// Skip non-element nodes (#text, #cdata-section, etc.) — they carry no
|
|
// geometry and are never queried by callers.
|
|
if (tag.startsWith("#")) continue;
|
|
|
|
const textContent = nodeTextContent.get(n);
|
|
|
|
nodes.push({
|
|
backendNodeId,
|
|
parentBackendNodeId,
|
|
ownerFrameBackendNodeId: context.ownerFrameBackendNodeId,
|
|
tag,
|
|
attrs,
|
|
rect,
|
|
localRect,
|
|
paintOrder,
|
|
position,
|
|
pointerEvents,
|
|
cursor,
|
|
textContent,
|
|
});
|
|
}
|
|
return { nodes, excludedBackendNodeIds };
|
|
}
|
|
|
|
interface ParseFrameDocumentsResult {
|
|
iframeNodes: CapturedIframeNodes;
|
|
excludedBackendNodeIds: Set<number>;
|
|
}
|
|
|
|
function parseChildFrameDocuments(
|
|
documents: SnapshotDocument[],
|
|
strings: string[],
|
|
dpr: number,
|
|
parentDocIndex: number,
|
|
parentNodes: CapturedNode[],
|
|
parentContext: FrameContext,
|
|
visited = new Set<number>(),
|
|
): ParseFrameDocumentsResult {
|
|
const iframeNodes: CapturedIframeNodes = new Map();
|
|
const excludedBackendNodeIds = new Set<number>();
|
|
const parentDoc = documents[parentDocIndex];
|
|
const cdi = sparseIndexMap(parentDoc?.nodes?.contentDocumentIndex);
|
|
if (cdi.size === 0) return { iframeNodes, excludedBackendNodeIds };
|
|
|
|
const parentBackendIds = parentDoc?.nodes?.backendNodeId ?? [];
|
|
const parentNodeByBackendId = new Map(parentNodes.map((node) => [node.backendNodeId, node]));
|
|
|
|
for (const [nodeArrayIdx, childDocIndex] of cdi) {
|
|
if (visited.has(childDocIndex)) continue;
|
|
const childDoc = documents[childDocIndex];
|
|
if (!childDoc) continue;
|
|
const iframeBackendId = parentBackendIds[nodeArrayIdx];
|
|
if (iframeBackendId === undefined) continue;
|
|
const iframeNode = parentNodeByBackendId.get(iframeBackendId);
|
|
if (!iframeNode?.localRect) continue;
|
|
|
|
const iframeRectInTop = unclipRectInTop(iframeNode.localRect, parentContext);
|
|
const childClip = intersectRects(iframeRectInTop, parentContext.clipRectInTop);
|
|
if (!childClip) {
|
|
iframeNodes.set(iframeBackendId, []);
|
|
continue;
|
|
}
|
|
|
|
const childContext: FrameContext = {
|
|
ownerFrameBackendNodeId: iframeBackendId,
|
|
originInTop: { x: iframeRectInTop.x, y: iframeRectInTop.y },
|
|
clipRectInTop: childClip,
|
|
scrollX: childDoc.scrollOffsetX ?? 0,
|
|
scrollY: childDoc.scrollOffsetY ?? 0,
|
|
};
|
|
const nextVisited = new Set(visited);
|
|
nextVisited.add(childDocIndex);
|
|
const parsed = parseDocumentNodes(childDoc, strings, dpr, childContext);
|
|
iframeNodes.set(iframeBackendId, parsed.nodes);
|
|
for (const id of parsed.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
|
|
|
|
const nested = parseChildFrameDocuments(
|
|
documents,
|
|
strings,
|
|
dpr,
|
|
childDocIndex,
|
|
parsed.nodes,
|
|
childContext,
|
|
nextVisited,
|
|
);
|
|
for (const [nestedFrameId, nestedNodes] of nested.iframeNodes) {
|
|
iframeNodes.set(nestedFrameId, nestedNodes);
|
|
}
|
|
for (const id of nested.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
|
|
}
|
|
|
|
return { iframeNodes, excludedBackendNodeIds };
|
|
}
|
|
|
|
export async function captureViewModel(
|
|
cdp: CdpRunner,
|
|
tabId: number,
|
|
options: CaptureViewModelOptions = {},
|
|
): Promise<CapturedViewModel> {
|
|
const metrics = await cdp.send<LayoutMetricsReply>(tabId, "Page.getLayoutMetrics", {});
|
|
const dpr = devicePixelRatio(metrics);
|
|
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
|
const viewport: Viewport = {
|
|
width: vpSrc.clientWidth ?? 0,
|
|
height: vpSrc.clientHeight ?? 0,
|
|
};
|
|
const scrollX = vpSrc.pageX ?? 0;
|
|
const scrollY = vpSrc.pageY ?? 0;
|
|
|
|
await cdp.send(tabId, "DOMSnapshot.enable", {});
|
|
const snap = await cdp.send<SnapshotReply>(tabId, "DOMSnapshot.captureSnapshot", {
|
|
computedStyles: REQUESTED_STYLES,
|
|
includePaintOrder: true,
|
|
includeDOMRects: true,
|
|
});
|
|
|
|
const strings = snap.strings ?? [];
|
|
const documents = snap.documents ?? [];
|
|
const doc0 = documents[0];
|
|
if (!doc0?.nodes?.backendNodeId) {
|
|
return {
|
|
nodes: [],
|
|
viewport,
|
|
iframeNodes: new Map(),
|
|
surfaceProbes: [],
|
|
excludedBackendNodeIds: new Set(),
|
|
};
|
|
}
|
|
|
|
const topContext: FrameContext = {
|
|
ownerFrameBackendNodeId: null,
|
|
originInTop: { x: 0, y: 0 },
|
|
clipRectInTop: viewportRect(viewport),
|
|
scrollX,
|
|
scrollY,
|
|
};
|
|
const mainParsed = parseDocumentNodes(doc0, strings, dpr, topContext);
|
|
const nodes = mainParsed.nodes;
|
|
const excludedBackendNodeIds = new Set(mainParsed.excludedBackendNodeIds);
|
|
|
|
const frameParsed = parseChildFrameDocuments(documents, strings, dpr, 0, nodes, topContext);
|
|
const iframeNodes = frameParsed.iframeNodes;
|
|
for (const id of frameParsed.excludedBackendNodeIds) {
|
|
excludedBackendNodeIds.add(id);
|
|
}
|
|
|
|
await enrichFormControlStates(cdp, tabId, [nodes, ...iframeNodes.values()]);
|
|
|
|
const surfaceProbes = options.conditionalSurfaceProbe ? await probeHoverSurfaces(cdp, tabId) : [];
|
|
|
|
return { nodes, viewport, iframeNodes, surfaceProbes, excludedBackendNodeIds };
|
|
}
|