mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat(vom): vom base algorithm and infrastructure
This commit is contained in:
Ljy-0827
File diff suppressed because it is too large
Load Diff
@@ -3,6 +3,7 @@
|
||||
// tab (defaulting to the Agent Window's active tab when omitted) and
|
||||
// returns a payload that mirrors the bsk-protocol Rust structs.
|
||||
|
||||
import { renderVom, type VomNode, type VomScene } from "@browser-skill/vom";
|
||||
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
||||
import type { SessionManager } from "@/session-manager/manager";
|
||||
import type {
|
||||
@@ -26,6 +27,12 @@ import {
|
||||
normaliseRef as sharedNormaliseRef,
|
||||
} from "./shared";
|
||||
import { resolveSnapshotRef } from "./snapshot-ref";
|
||||
import {
|
||||
type CapturedNode,
|
||||
type CapturedViewModel,
|
||||
captureViewModel,
|
||||
collectOverlayExcludedBackendIds,
|
||||
} from "./vom/capture";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Shared helpers (legacy aliases — observation.ts kept exporting these
|
||||
@@ -230,176 +237,462 @@ export interface CdpAxNode {
|
||||
childIds?: string[];
|
||||
}
|
||||
|
||||
const INTERACTIVE_ROLES = new Set([
|
||||
function axValue(field?: { value?: string | number | boolean }): string | undefined {
|
||||
const value = field?.value;
|
||||
return value === undefined ? undefined : String(value);
|
||||
}
|
||||
|
||||
function axString(field?: { value?: string | number | boolean }): string | undefined {
|
||||
const value = axValue(field)?.replace(/\s+/g, " ").trim();
|
||||
return value ? value : undefined;
|
||||
}
|
||||
|
||||
function normalizeTag(tag: string | undefined): string {
|
||||
return tag?.toLowerCase() ?? "";
|
||||
}
|
||||
|
||||
function isModalSignal(
|
||||
axNode: CdpAxNode | undefined,
|
||||
capturedNode: CapturedNode | undefined,
|
||||
): boolean {
|
||||
const role = axString(axNode?.role)?.toLowerCase();
|
||||
if (role === "dialog" || role === "alertdialog") return true;
|
||||
if (normalizeTag(capturedNode?.tag) === "dialog") return true;
|
||||
|
||||
const attrs = capturedNode?.attrs ?? {};
|
||||
return (attrs["aria-modal"] ?? "").toLowerCase() === "true" || attrs.role === "dialog";
|
||||
}
|
||||
|
||||
function isSensitive(capturedNode: CapturedNode | undefined): boolean {
|
||||
const attrs = capturedNode?.attrs ?? {};
|
||||
return (attrs.type ?? "").toLowerCase() === "password";
|
||||
}
|
||||
|
||||
function findAxAncestor<T>(
|
||||
axNode: CdpAxNode,
|
||||
axById: Map<string, CdpAxNode>,
|
||||
select: (node: CdpAxNode) => T | undefined,
|
||||
): T | undefined {
|
||||
let parentId = axNode.parentId;
|
||||
while (parentId) {
|
||||
const parent = axById.get(parentId);
|
||||
if (!parent) break;
|
||||
const hit = select(parent);
|
||||
if (hit !== undefined) return hit;
|
||||
parentId = parent.parentId;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function nearestBackendParent(axNode: CdpAxNode, axById: Map<string, CdpAxNode>): number | null {
|
||||
return (
|
||||
findAxAncestor(axNode, axById, (parent) =>
|
||||
typeof parent.backendDOMNodeId === "number" ? parent.backendDOMNodeId : undefined,
|
||||
) ?? null
|
||||
);
|
||||
}
|
||||
|
||||
const IFRAME_RENDERABLE_TAGS = new Set(["input", "button", "a", "select", "textarea"]);
|
||||
|
||||
function iframeRoleFor(node: CapturedNode): string | undefined {
|
||||
const tag = normalizeTag(node.tag);
|
||||
if (tag === "input" || tag === "textarea") return "textbox";
|
||||
if (tag === "button") return "button";
|
||||
if (tag === "a") return "link";
|
||||
if (tag === "select") return "combobox";
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function iframeNameFor(node: CapturedNode): string | undefined {
|
||||
const tag = normalizeTag(node.tag);
|
||||
const ariaLabel = node.attrs["aria-label"]?.replace(/\s+/g, " ").trim();
|
||||
if (ariaLabel) return ariaLabel;
|
||||
|
||||
const text = node.textContent?.replace(/\s+/g, " ").trim();
|
||||
if (text) return text;
|
||||
|
||||
if (tag === "input" || tag === "textarea") {
|
||||
const placeholder = node.attrs.placeholder?.replace(/\s+/g, " ").trim();
|
||||
if (placeholder) return placeholder;
|
||||
}
|
||||
|
||||
const id = node.attrs.id?.replace(/\s+/g, " ").trim();
|
||||
return id ? id : undefined;
|
||||
}
|
||||
|
||||
function isRenderableIframeControl(node: CapturedNode): boolean {
|
||||
const tag = normalizeTag(node.tag);
|
||||
if (!IFRAME_RENDERABLE_TAGS.has(tag)) return false;
|
||||
if (!node.rect) return false;
|
||||
if (node.pointerEvents === "none") return false;
|
||||
if ((node.attrs.type ?? "").toLowerCase() === "hidden") return false;
|
||||
|
||||
const name = iframeNameFor(node);
|
||||
return tag !== "a" || name !== undefined;
|
||||
}
|
||||
|
||||
function capturedIframeNameFor(node: CapturedNode): string | undefined {
|
||||
const ariaLabel = node.attrs["aria-label"]?.replace(/\s+/g, " ").trim();
|
||||
if (ariaLabel) return ariaLabel;
|
||||
|
||||
const title = node.attrs.title?.replace(/\s+/g, " ").trim();
|
||||
if (title) return title;
|
||||
|
||||
const id = node.attrs.id?.replace(/\s+/g, " ").trim();
|
||||
return id ? id : undefined;
|
||||
}
|
||||
|
||||
const FORM_CONTROL_TAGS = new Set(["input", "textarea", "select"]);
|
||||
|
||||
/** AX roles we treat as non-semantic wrappers eligible for clickable promotion. */
|
||||
const PROMOTABLE_ROLES = new Set(["", "generic", "none", "presentation"]);
|
||||
|
||||
/**
|
||||
* Interactive AX roles. Mirrors `packages/vom`'s INTERACTIVE_ROLES — kept
|
||||
* local so the package stays a pure projection (design §2). Used to tell an
|
||||
* atomic custom control (icon button / toggle) apart from a clickable
|
||||
* *container* (card/row) that wraps real controls we must not collapse.
|
||||
*/
|
||||
const INTERACTIVE_ROLES_LC = new Set([
|
||||
"button",
|
||||
"link",
|
||||
"checkbox",
|
||||
"radio",
|
||||
"textbox",
|
||||
"combobox",
|
||||
"listbox",
|
||||
"option",
|
||||
"switch",
|
||||
"tab",
|
||||
"link",
|
||||
"menuitem",
|
||||
"menuitemcheckbox",
|
||||
"menuitemradio",
|
||||
"option",
|
||||
"radio",
|
||||
"scrollbar",
|
||||
"searchbox",
|
||||
"slider",
|
||||
"spinbutton",
|
||||
"scrollbar",
|
||||
"switch",
|
||||
"tab",
|
||||
"textbox",
|
||||
"treeitem",
|
||||
]);
|
||||
|
||||
const STRUCTURAL_ROLES = new Set([
|
||||
"heading",
|
||||
"main",
|
||||
"navigation",
|
||||
"banner",
|
||||
"contentinfo",
|
||||
"complementary",
|
||||
"form",
|
||||
"search",
|
||||
"region",
|
||||
"article",
|
||||
"list",
|
||||
"listitem",
|
||||
"table",
|
||||
"row",
|
||||
"cell",
|
||||
"rowheader",
|
||||
"columnheader",
|
||||
"dialog",
|
||||
"alertdialog",
|
||||
"img",
|
||||
"figure",
|
||||
"section",
|
||||
"RootWebArea",
|
||||
"WebArea",
|
||||
]);
|
||||
|
||||
const SKIP_ROLES = new Set(["generic", "none", "presentation", "InlineTextBox"]);
|
||||
|
||||
/**
|
||||
* Decide whether an aria node should appear in the rendered tree.
|
||||
* Exported for unit tests.
|
||||
*/
|
||||
export function shouldRender(node: CdpAxNode): boolean {
|
||||
if (node.ignored) return false;
|
||||
const role = node.role?.value ?? "";
|
||||
if (!role) return false;
|
||||
if (SKIP_ROLES.has(role)) return false;
|
||||
const name = node.name?.value ?? "";
|
||||
if (INTERACTIVE_ROLES.has(role)) return true;
|
||||
if (STRUCTURAL_ROLES.has(role)) return true;
|
||||
return name.trim().length > 0;
|
||||
function cleanAttr(value: string | undefined): string | undefined {
|
||||
const trimmed = value?.replace(/\s+/g, " ").trim();
|
||||
return trimmed ? trimmed : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Approximate-token estimator (~4 chars / token, GPT-style). Good
|
||||
* enough for `max_tokens` budgets.
|
||||
* Name for a native form control. VOM models the *perceived* viewport
|
||||
* (spec §1): an empty field displays its placeholder, so when the field has
|
||||
* no value we prefer the placeholder over an accessible name that pages
|
||||
* frequently pollute by wrapping the `<input>` in a `<label>` that also
|
||||
* holds prefixes/buttons (e.g. xiaohongshu's "+86" / "获取验证码"). Filled
|
||||
* fields keep the accessible name and surface their value separately.
|
||||
*/
|
||||
function estimateTokens(s: string): number {
|
||||
return Math.ceil(s.length / 4);
|
||||
function formControlName(
|
||||
captured: CapturedNode | undefined,
|
||||
axName: string | undefined,
|
||||
axValue: string | undefined,
|
||||
): string | undefined {
|
||||
const placeholder = cleanAttr(captured?.attrs.placeholder);
|
||||
const ariaLabel = cleanAttr(captured?.attrs["aria-label"]);
|
||||
const title = cleanAttr(captured?.attrs.title);
|
||||
const hasValue = axValue !== undefined && axValue !== "";
|
||||
if (!hasValue && placeholder) return placeholder;
|
||||
return axName ?? ariaLabel ?? placeholder ?? title;
|
||||
}
|
||||
|
||||
interface RenderedSnapshot {
|
||||
text: string;
|
||||
refs: Array<{ ref: string; backendNodeId: number }>;
|
||||
truncated: boolean;
|
||||
/** Strip a sprite/file reference like `#close` or `icon-close.svg` to `close`. */
|
||||
function iconKeyword(href: string | undefined): string | undefined {
|
||||
if (!href) return undefined;
|
||||
let s = href.trim();
|
||||
const hash = s.lastIndexOf("#");
|
||||
if (hash >= 0) s = s.slice(hash + 1);
|
||||
s = s.split("/").pop() ?? s;
|
||||
s = s.replace(/\.(svg|png|webp|gif|jpe?g)$/i, "");
|
||||
s = s.replace(/^(icons?[-_]?|ic[-_]|svg[-_])/i, "");
|
||||
s = s.replace(/[-_]+/g, " ").trim();
|
||||
return s ? s : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert an `Accessibility.getFullAXTree` result into the
|
||||
* `@e<N>`-tagged indented text plus the ref → backendNodeId map for
|
||||
* the session's RefStore. Exported for unit tests.
|
||||
*/
|
||||
export function renderAxTree(
|
||||
nodes: CdpAxNode[],
|
||||
opts: { maxDepth?: number; maxTokens?: number } = {},
|
||||
): RenderedSnapshot {
|
||||
const byId = new Map<string, CdpAxNode>();
|
||||
const childParent = new Map<string, string>();
|
||||
for (const n of nodes) {
|
||||
byId.set(n.nodeId, n);
|
||||
}
|
||||
// Detect roots: nodes whose `parentId` is undefined OR not in the
|
||||
// returned set. CDP sometimes ships orphan branches.
|
||||
for (const n of nodes) {
|
||||
if (n.parentId && byId.has(n.parentId)) {
|
||||
childParent.set(n.nodeId, n.parentId);
|
||||
/** First icon-derived label found in a clickable element's captured subtree. */
|
||||
function iconHint(
|
||||
root: CapturedNode,
|
||||
capChildren: Map<number, CapturedNode[]>,
|
||||
): string | undefined {
|
||||
const queue: Array<{ node: CapturedNode; depth: number }> = (
|
||||
capChildren.get(root.backendNodeId) ?? []
|
||||
).map((node) => ({ node, depth: 1 }));
|
||||
while (queue.length > 0) {
|
||||
const { node, depth } = queue.shift() as { node: CapturedNode; depth: number };
|
||||
const labelled =
|
||||
cleanAttr(node.attrs["aria-label"]) ??
|
||||
cleanAttr(node.attrs.title) ??
|
||||
cleanAttr(node.attrs.alt);
|
||||
if (labelled) return labelled;
|
||||
if (node.tag === "use") {
|
||||
const kw = iconKeyword(node.attrs["xlink:href"] ?? node.attrs.href);
|
||||
if (kw) return kw;
|
||||
}
|
||||
if (depth < 4) {
|
||||
for (const child of capChildren.get(node.backendNodeId) ?? []) {
|
||||
queue.push({ node: child, depth: depth + 1 });
|
||||
}
|
||||
}
|
||||
}
|
||||
const roots = nodes.filter((n) => !childParent.has(n.nodeId));
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const lines: string[] = [];
|
||||
const refs: Array<{ ref: string; backendNodeId: number }> = [];
|
||||
const maxDepth = opts.maxDepth ?? Number.POSITIVE_INFINITY;
|
||||
const maxTokens = opts.maxTokens ?? Number.POSITIVE_INFINITY;
|
||||
let truncated = false;
|
||||
let tokenTruncated = false;
|
||||
let tokenBudget = 0;
|
||||
let nextRef = 1;
|
||||
/** Best-effort name for a promoted clickable: own labels → text → icon. */
|
||||
function clickableName(
|
||||
captured: CapturedNode,
|
||||
capChildren: Map<number, CapturedNode[]>,
|
||||
): string | undefined {
|
||||
return (
|
||||
cleanAttr(captured.attrs["aria-label"]) ??
|
||||
cleanAttr(captured.attrs.title) ??
|
||||
cleanAttr(captured.textContent) ??
|
||||
iconHint(captured, capChildren)
|
||||
);
|
||||
}
|
||||
|
||||
const walk = (node: CdpAxNode, depth: number, ancestorRendered: boolean): void => {
|
||||
if (tokenTruncated) return;
|
||||
const renderThis = shouldRender(node) && depth <= maxDepth;
|
||||
if (renderThis) {
|
||||
const role = node.role?.value ?? "";
|
||||
const name = node.name?.value ?? "";
|
||||
let line = `${" ".repeat(Math.min(depth, 32))}`;
|
||||
let ref: string | null = null;
|
||||
if (typeof node.backendDOMNodeId === "number") {
|
||||
ref = `e${nextRef}`;
|
||||
line += `@${ref} `;
|
||||
}
|
||||
line += role;
|
||||
if (name.length > 0) {
|
||||
const cleaned = name.replace(/\s+/g, " ").trim();
|
||||
line += ` ${JSON.stringify(cleaned)}`;
|
||||
}
|
||||
const value = node.value?.value;
|
||||
if (typeof value === "string" && value.length > 0 && value !== name) {
|
||||
line += ` =${JSON.stringify(value.slice(0, 200))}`;
|
||||
}
|
||||
const lineTokens = estimateTokens(line) + 1; // +1 for newline
|
||||
if (tokenBudget + lineTokens > maxTokens) {
|
||||
truncated = true;
|
||||
tokenTruncated = true;
|
||||
return;
|
||||
}
|
||||
tokenBudget += lineTokens;
|
||||
if (ref && typeof node.backendDOMNodeId === "number") {
|
||||
refs.push({ ref, backendNodeId: node.backendDOMNodeId });
|
||||
nextRef += 1;
|
||||
}
|
||||
lines.push(line);
|
||||
}
|
||||
const nextDepth = renderThis ? depth + 1 : ancestorRendered ? depth : depth;
|
||||
if (depth + 1 > maxDepth && renderThis && (node.childIds?.length ?? 0) > 0) {
|
||||
// Children would exceed depth cap — note truncation flag without
|
||||
// bailing on siblings elsewhere in the tree.
|
||||
truncated = true;
|
||||
return;
|
||||
}
|
||||
for (const cid of node.childIds ?? []) {
|
||||
const child = byId.get(cid);
|
||||
if (!child) continue;
|
||||
if (tokenTruncated) return;
|
||||
walk(child, nextDepth, ancestorRendered || renderThis);
|
||||
function rectCoverage(
|
||||
rect: { x: number; y: number; w: number; h: number } | null,
|
||||
vw: number,
|
||||
vh: number,
|
||||
): number {
|
||||
if (!rect || vw <= 0 || vh <= 0) return 0;
|
||||
const ix = Math.max(0, Math.min(rect.x + rect.w, vw) - Math.max(rect.x, 0));
|
||||
const iy = Math.max(0, Math.min(rect.y + rect.h, vh) - Math.max(rect.y, 0));
|
||||
const overlap = ix * iy;
|
||||
return overlap <= 0 ? 0 : Math.min(1, overlap / (vw * vh));
|
||||
}
|
||||
|
||||
/**
|
||||
* Promote custom clickable controls the AX tree drops as `generic`/unnamed
|
||||
* (icon buttons, CSS checkboxes, "send code" spans …) to a referenceable
|
||||
* `button` so the snapshot can act on them. Guards keep noise out: must be
|
||||
* `cursor: pointer`, have a box, not be a viewport-scale overlay, and not
|
||||
* wrap any real interactive descendant (so clickable cards or rows stay
|
||||
* containers and their inner controls keep their own refs).
|
||||
*/
|
||||
function promoteClickableControls(
|
||||
nodes: VomNode[],
|
||||
capturedById: Map<number, CapturedNode>,
|
||||
capturedNodes: CapturedNode[],
|
||||
vw: number,
|
||||
vh: number,
|
||||
): void {
|
||||
// Snapshot original roles + parent/child links before any mutation so the
|
||||
// guards below are immune to roles we flip to "button" mid-pass.
|
||||
const originalRoleLc = new Map<number, string>(
|
||||
nodes.map((node) => [node.id, (node.role ?? "").toLowerCase()]),
|
||||
);
|
||||
const parentOf = new Map<number, number | null>(nodes.map((node) => [node.id, node.parentId]));
|
||||
const semChildren = new Map<number, VomNode[]>();
|
||||
for (const node of nodes) {
|
||||
if (node.parentId === null) continue;
|
||||
const siblings = semChildren.get(node.parentId);
|
||||
if (siblings) siblings.push(node);
|
||||
else semChildren.set(node.parentId, [node]);
|
||||
}
|
||||
|
||||
const capChildren = new Map<number, CapturedNode[]>();
|
||||
for (const cap of capturedNodes) {
|
||||
if (cap.parentBackendNodeId === null) continue;
|
||||
const siblings = capChildren.get(cap.parentBackendNodeId);
|
||||
if (siblings) siblings.push(cap);
|
||||
else capChildren.set(cap.parentBackendNodeId, [cap]);
|
||||
}
|
||||
|
||||
// A custom clickable: a non-semantic, pointer-cursor box small enough to be
|
||||
// a control rather than a viewport-scale overlay.
|
||||
const candidates = new Set<number>();
|
||||
for (const node of nodes) {
|
||||
if (!PROMOTABLE_ROLES.has(originalRoleLc.get(node.id) ?? "")) continue;
|
||||
const cap = capturedById.get(node.id);
|
||||
if (!cap || cap.cursor !== "pointer" || !node.rect) continue;
|
||||
if (rectCoverage(node.rect, vw, vh) > 0.5) continue;
|
||||
candidates.add(node.id);
|
||||
}
|
||||
|
||||
// wraps a *real* interactive control → it's a clickable container, leave it
|
||||
// (and its inner refs) alone.
|
||||
const wrapsInteractive = (id: number): boolean => {
|
||||
const stack = [...(semChildren.get(id) ?? [])];
|
||||
while (stack.length > 0) {
|
||||
const node = stack.pop() as VomNode;
|
||||
if (INTERACTIVE_ROLES_LC.has(originalRoleLc.get(node.id) ?? "")) return true;
|
||||
const kids = semChildren.get(node.id);
|
||||
if (kids) stack.push(...kids);
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
for (const r of roots) {
|
||||
walk(r, 0, false);
|
||||
if (tokenTruncated) break;
|
||||
// only the outermost clickable in a nested chain becomes the control, so an
|
||||
// icon inside a custom button doesn't render as a button-in-a-button.
|
||||
const hasCandidateAncestor = (id: number): boolean => {
|
||||
let parent = parentOf.get(id) ?? null;
|
||||
while (parent !== null) {
|
||||
if (candidates.has(parent)) return true;
|
||||
parent = parentOf.get(parent) ?? null;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
for (const node of nodes) {
|
||||
if (!candidates.has(node.id)) continue;
|
||||
if (wrapsInteractive(node.id) || hasCandidateAncestor(node.id)) continue;
|
||||
const cap = capturedById.get(node.id);
|
||||
if (!cap) continue;
|
||||
|
||||
node.role = "button";
|
||||
if (!cleanAttr(node.name)) {
|
||||
const derived = clickableName(cap, capChildren);
|
||||
if (derived) node.name = derived;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function vomNodeFromCaptured(capturedNode: CapturedNode, parentId: number | null): VomNode {
|
||||
const sensitive = isSensitive(capturedNode);
|
||||
const value = sensitive ? (capturedNode.attrs.value ?? "") : undefined;
|
||||
const tag = normalizeTag(capturedNode.tag);
|
||||
const node: VomNode = {
|
||||
id: capturedNode.backendNodeId,
|
||||
parentId,
|
||||
tag,
|
||||
rect: capturedNode.rect,
|
||||
paintOrder: capturedNode.paintOrder,
|
||||
position: capturedNode.position || "static",
|
||||
pointerEvents: capturedNode.pointerEvents || "auto",
|
||||
modal: isModalSignal(undefined, capturedNode),
|
||||
sensitive,
|
||||
...(value !== undefined ? { value } : {}),
|
||||
};
|
||||
if (tag === "iframe") {
|
||||
node.role = "Iframe";
|
||||
const name = capturedIframeNameFor(capturedNode);
|
||||
if (name) node.name = name;
|
||||
}
|
||||
return node;
|
||||
}
|
||||
|
||||
function axNodeInOverlaySubtree(
|
||||
axNode: CdpAxNode,
|
||||
axById: Map<string, CdpAxNode>,
|
||||
excludedBackendNodeIds: Set<number>,
|
||||
): boolean {
|
||||
if (
|
||||
typeof axNode.backendDOMNodeId === "number" &&
|
||||
excludedBackendNodeIds.has(axNode.backendDOMNodeId)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
return (
|
||||
findAxAncestor(axNode, axById, (parent) => {
|
||||
const parentBackendId = parent.backendDOMNodeId;
|
||||
return typeof parentBackendId === "number" && excludedBackendNodeIds.has(parentBackendId)
|
||||
? true
|
||||
: undefined;
|
||||
}) === true
|
||||
);
|
||||
}
|
||||
|
||||
export function buildVomScene(axNodes: CdpAxNode[], captured: CapturedViewModel): VomScene {
|
||||
const capturedByBackendId = new Map<number, CapturedNode>();
|
||||
for (const node of captured.nodes) {
|
||||
capturedByBackendId.set(node.backendNodeId, node);
|
||||
}
|
||||
|
||||
return {
|
||||
text: lines.join("\n"),
|
||||
refs,
|
||||
truncated,
|
||||
};
|
||||
const axById = new Map<string, CdpAxNode>();
|
||||
for (const node of axNodes) {
|
||||
axById.set(node.nodeId, node);
|
||||
}
|
||||
|
||||
const excludedBackendNodeIds = captured.excludedBackendNodeIds;
|
||||
const seenBackendIds = new Set<number>();
|
||||
const nodes: VomNode[] = [];
|
||||
|
||||
for (const axNode of axNodes) {
|
||||
if (axNode.ignored || typeof axNode.backendDOMNodeId !== "number") continue;
|
||||
if (axNodeInOverlaySubtree(axNode, axById, excludedBackendNodeIds)) continue;
|
||||
|
||||
const capturedNode = capturedByBackendId.get(axNode.backendDOMNodeId);
|
||||
const role = axString(axNode.role);
|
||||
const value = axValue(axNode.value);
|
||||
const name = FORM_CONTROL_TAGS.has(normalizeTag(capturedNode?.tag))
|
||||
? formControlName(capturedNode, axString(axNode.name), value)
|
||||
: axString(axNode.name);
|
||||
const vomNode: VomNode = {
|
||||
id: axNode.backendDOMNodeId,
|
||||
parentId: nearestBackendParent(axNode, axById),
|
||||
tag: normalizeTag(capturedNode?.tag),
|
||||
rect: capturedNode?.rect ?? null,
|
||||
paintOrder: capturedNode?.paintOrder ?? 0,
|
||||
position: capturedNode?.position || "static",
|
||||
pointerEvents: capturedNode?.pointerEvents || "auto",
|
||||
modal: isModalSignal(axNode, capturedNode),
|
||||
sensitive: isSensitive(capturedNode),
|
||||
};
|
||||
if (role) vomNode.role = role;
|
||||
if (name) vomNode.name = name;
|
||||
if (value !== undefined) vomNode.value = value;
|
||||
|
||||
// For link nodes, attach external hostname so the renderer can annotate it.
|
||||
// Relative hrefs and same-origin hrefs are omitted to avoid noise.
|
||||
if (role === "link" && capturedNode?.attrs.href) {
|
||||
try {
|
||||
const target = new URL(capturedNode.attrs.href, window.location.href);
|
||||
if (target.origin !== window.location.origin) {
|
||||
vomNode.href = target.hostname;
|
||||
}
|
||||
} catch {
|
||||
// Relative or non-URL href (e.g. javascript:) — skip
|
||||
}
|
||||
}
|
||||
|
||||
seenBackendIds.add(axNode.backendDOMNodeId);
|
||||
nodes.push(vomNode);
|
||||
}
|
||||
|
||||
for (const capturedNode of captured.nodes) {
|
||||
if (seenBackendIds.has(capturedNode.backendNodeId)) continue;
|
||||
if (excludedBackendNodeIds.has(capturedNode.backendNodeId)) continue;
|
||||
seenBackendIds.add(capturedNode.backendNodeId);
|
||||
nodes.push(vomNodeFromCaptured(capturedNode, capturedNode.parentBackendNodeId));
|
||||
}
|
||||
|
||||
for (const [iframeBackendId, iframeNodes] of captured.iframeNodes) {
|
||||
for (const iframeNode of iframeNodes) {
|
||||
if (
|
||||
seenBackendIds.has(iframeNode.backendNodeId) ||
|
||||
excludedBackendNodeIds.has(iframeNode.backendNodeId) ||
|
||||
!isRenderableIframeControl(iframeNode)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const role = iframeRoleFor(iframeNode);
|
||||
const name = iframeNameFor(iframeNode);
|
||||
if (!role || (!name && normalizeTag(iframeNode.tag) === "a")) continue;
|
||||
|
||||
seenBackendIds.add(iframeNode.backendNodeId);
|
||||
const vomNode: VomNode = {
|
||||
...vomNodeFromCaptured(iframeNode, iframeBackendId),
|
||||
role,
|
||||
};
|
||||
if (name) vomNode.name = name;
|
||||
nodes.push(vomNode);
|
||||
}
|
||||
}
|
||||
|
||||
promoteClickableControls(
|
||||
nodes,
|
||||
capturedByBackendId,
|
||||
captured.nodes,
|
||||
captured.viewport.width,
|
||||
captured.viewport.height,
|
||||
);
|
||||
|
||||
return { viewport: captured.viewport, nodes };
|
||||
}
|
||||
|
||||
export interface SnapshotDeps {
|
||||
@@ -514,6 +807,42 @@ export async function handleGetHtml(
|
||||
}
|
||||
}
|
||||
|
||||
function emptyCapturedViewModel(viewport = { width: 0, height: 0 }): CapturedViewModel {
|
||||
return { viewport, nodes: [], iframeNodes: new Map(), excludedBackendNodeIds: new Set() };
|
||||
}
|
||||
|
||||
interface LayoutMetricsViewportReply {
|
||||
cssLayoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||
layoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||
}
|
||||
|
||||
async function fallbackCapturedViewModel(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
): Promise<CapturedViewModel> {
|
||||
let viewport = { width: 0, height: 0 };
|
||||
try {
|
||||
const metrics = await cdp.send<LayoutMetricsViewportReply>(tabId, "Page.getLayoutMetrics", {});
|
||||
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||
viewport = {
|
||||
width: vpSrc.clientWidth ?? 0,
|
||||
height: vpSrc.clientHeight ?? 0,
|
||||
};
|
||||
} catch {
|
||||
// viewport stays zero-sized
|
||||
}
|
||||
const excludedBackendNodeIds = await collectOverlayExcludedBackendIds(cdp, tabId);
|
||||
return { ...emptyCapturedViewModel(viewport), excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
async function captureForVom(cdp: CdpRunner, tabId: number): Promise<CapturedViewModel> {
|
||||
try {
|
||||
return await captureViewModel(cdp, tabId);
|
||||
} catch {
|
||||
return fallbackCapturedViewModel(cdp, tabId);
|
||||
}
|
||||
}
|
||||
|
||||
export async function handleSnapshot(
|
||||
manager: SessionManager,
|
||||
params: SnapshotParams,
|
||||
@@ -534,11 +863,10 @@ export async function handleSnapshot(
|
||||
"Accessibility.getFullAXTree",
|
||||
{},
|
||||
);
|
||||
const rendered = renderAxTree(result.nodes ?? [], {
|
||||
maxDepth: params.max_depth,
|
||||
maxTokens: params.max_tokens,
|
||||
});
|
||||
// Reset the session-scoped ref-store for this fresh snapshot.
|
||||
const axNodes = result.nodes ?? [];
|
||||
const captured = await captureForVom(deps.cdp, target.tabId);
|
||||
const scene = buildVomScene(axNodes, captured);
|
||||
const rendered = renderVom(scene, { maxDepth: params.max_depth, maxTokens: params.max_tokens });
|
||||
ctx.refStore.replace(
|
||||
rendered.refs.map(
|
||||
(r) => [r.ref, { backendNodeId: r.backendNodeId, tabId: target.tabId }] as const,
|
||||
|
||||
@@ -0,0 +1,630 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "../../../lib/overlay-bridge";
|
||||
import { captureViewModel, collectOverlayExcludedBackendIds } from "../capture";
|
||||
|
||||
// Minimal but format-accurate captureSnapshot reply: a body with one
|
||||
// fixed full-screen overlay div carrying a password input.
|
||||
function fakeSnapshotReply() {
|
||||
// strings table
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
"div",
|
||||
"input",
|
||||
"position",
|
||||
"fixed",
|
||||
"static",
|
||||
"pointer-events",
|
||||
"auto",
|
||||
"type",
|
||||
"password",
|
||||
];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
return {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 2],
|
||||
nodeType: [1, 1, 1, 1],
|
||||
nodeName: [i("html"), i("body"), i("div"), i("input")],
|
||||
backendNodeId: [10, 11, 12, 13],
|
||||
attributes: [[], [], [], [i("type"), i("password")]],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2, 3],
|
||||
// styles columns follow requested computedStyles order: [position, pointer-events]
|
||||
styles: [
|
||||
[i("static"), i("auto")],
|
||||
[i("fixed"), i("auto")],
|
||||
[i("static"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 4000],
|
||||
[0, 0, 1000, 800],
|
||||
[400, 300, 200, 40],
|
||||
],
|
||||
paintOrders: [0, 50, 51],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function makeCdp(snapshot: unknown) {
|
||||
return {
|
||||
send: vi.fn(async (_tab: number, method: string) => {
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
|
||||
}
|
||||
throw new Error(`unexpected ${method}`);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
}
|
||||
|
||||
describe("captureViewModel", () => {
|
||||
it("parses nodes, attrs, rects, paint order and styles", async () => {
|
||||
const { nodes, viewport, iframeNodes } = await captureViewModel(
|
||||
makeCdp(fakeSnapshotReply()),
|
||||
4,
|
||||
);
|
||||
expect(iframeNodes.size).toBe(0);
|
||||
expect(viewport).toEqual({ width: 1000, height: 800 });
|
||||
const div = nodes.find((n) => n.backendNodeId === 12);
|
||||
expect(div).toMatchObject({ tag: "div", position: "fixed", parentBackendNodeId: 11 });
|
||||
expect(div?.rect).toEqual({ x: 0, y: 0, w: 1000, h: 800 });
|
||||
expect(div?.paintOrder).toBe(50);
|
||||
const input = nodes.find((n) => n.backendNodeId === 13);
|
||||
expect(input?.attrs).toEqual({ type: "password" });
|
||||
});
|
||||
|
||||
it("excludes the agent's own overlay shadow host and its inlined shadow subtree", async () => {
|
||||
// DOMSnapshot inlines an open shadow root's content as descendants of the
|
||||
// host. The agent's WXT overlay host carries a fixed full-viewport
|
||||
// click-blocker + the "中断" button. Capturing it makes the snapshot detect
|
||||
// the agent's own mask as a blocking layer and occlude the real page, so the
|
||||
// whole subtree must be dropped.
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
"div",
|
||||
OVERLAY_HOST_NAME,
|
||||
"button",
|
||||
OVERLAY_HOST_MARKER_ATTR,
|
||||
"position",
|
||||
"fixed",
|
||||
"static",
|
||||
"pointer-events",
|
||||
"auto",
|
||||
"cursor",
|
||||
"pointer",
|
||||
];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const snapshot = {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 0, 3, 4],
|
||||
nodeName: [i("html"), i("body"), i("div"), i(OVERLAY_HOST_NAME), i("div"), i("button")],
|
||||
backendNodeId: [10, 11, 12, 13, 14, 15],
|
||||
attributes: [[], [], [], [i(OVERLAY_HOST_MARKER_ATTR), -1], [], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1, 2, 3, 4, 5],
|
||||
styles: [
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("fixed"), i("auto"), i("auto")],
|
||||
[i("fixed"), i("auto"), i("pointer")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 4000],
|
||||
[0, 0, 1000, 800],
|
||||
[10, 10, 200, 40],
|
||||
[0, 0, 0, 0],
|
||||
[0, 0, 1000, 800],
|
||||
[400, 750, 80, 40],
|
||||
],
|
||||
paintOrders: [0, 1, 2, 3, 9, 10],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const { nodes } = await captureViewModel(makeCdp(snapshot), 4);
|
||||
const ids = nodes.map((n) => n.backendNodeId);
|
||||
// Real page nodes survive.
|
||||
expect(ids).toContain(12);
|
||||
// Overlay host + its inlined shadow subtree are gone.
|
||||
expect(ids).not.toContain(13);
|
||||
expect(ids).not.toContain(14);
|
||||
expect(ids).not.toContain(15);
|
||||
});
|
||||
|
||||
it("returns excludedBackendNodeIds for the overlay host subtree", async () => {
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
"div",
|
||||
OVERLAY_HOST_NAME,
|
||||
"button",
|
||||
OVERLAY_HOST_MARKER_ATTR,
|
||||
"position",
|
||||
"fixed",
|
||||
"static",
|
||||
"pointer-events",
|
||||
"auto",
|
||||
"cursor",
|
||||
"pointer",
|
||||
];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const snapshot = {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 0, 3, 4],
|
||||
nodeName: [i("html"), i("body"), i("div"), i(OVERLAY_HOST_NAME), i("div"), i("button")],
|
||||
backendNodeId: [10, 11, 12, 13, 14, 15],
|
||||
attributes: [[], [], [], [i(OVERLAY_HOST_MARKER_ATTR), -1], [], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1, 2, 3, 4, 5],
|
||||
styles: [
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("fixed"), i("auto"), i("auto")],
|
||||
[i("fixed"), i("auto"), i("pointer")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 4000],
|
||||
[0, 0, 1000, 800],
|
||||
[10, 10, 200, 40],
|
||||
[0, 0, 0, 0],
|
||||
[0, 0, 1000, 800],
|
||||
[400, 750, 80, 40],
|
||||
],
|
||||
paintOrders: [0, 1, 2, 3, 9, 10],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const { excludedBackendNodeIds } = await captureViewModel(makeCdp(snapshot), 4);
|
||||
expect(excludedBackendNodeIds).toEqual(new Set([13, 14, 15]));
|
||||
});
|
||||
|
||||
it("excludes overlay host matched by tag name only", async () => {
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
OVERLAY_HOST_NAME,
|
||||
"position",
|
||||
"static",
|
||||
"pointer-events",
|
||||
"auto",
|
||||
"cursor",
|
||||
"auto",
|
||||
];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const snapshot = {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i(OVERLAY_HOST_NAME)],
|
||||
backendNodeId: [10, 11, 13],
|
||||
attributes: [[], [], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2],
|
||||
styles: [
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[0, 0, 100, 40],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const { nodes, excludedBackendNodeIds } = await captureViewModel(makeCdp(snapshot), 4);
|
||||
expect(nodes.map((n) => n.backendNodeId)).not.toContain(13);
|
||||
expect(excludedBackendNodeIds).toEqual(new Set([13]));
|
||||
});
|
||||
|
||||
it("excludes overlay host matched by marker attribute only", async () => {
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
"div",
|
||||
OVERLAY_HOST_MARKER_ATTR,
|
||||
"position",
|
||||
"static",
|
||||
"pointer-events",
|
||||
"auto",
|
||||
"cursor",
|
||||
"auto",
|
||||
];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const snapshot = {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("div")],
|
||||
backendNodeId: [10, 11, 13],
|
||||
attributes: [[], [], [i(OVERLAY_HOST_MARKER_ATTR), -1]],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2],
|
||||
styles: [
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[0, 0, 100, 40],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const { nodes, excludedBackendNodeIds } = await captureViewModel(makeCdp(snapshot), 4);
|
||||
expect(nodes.map((n) => n.backendNodeId)).not.toContain(13);
|
||||
expect(excludedBackendNodeIds).toEqual(new Set([13]));
|
||||
});
|
||||
|
||||
it("parses the computed cursor from the third style column", async () => {
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
"div",
|
||||
"position",
|
||||
"static",
|
||||
"fixed",
|
||||
"pointer-events",
|
||||
"auto",
|
||||
"cursor",
|
||||
"pointer",
|
||||
];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const snapshot = {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("div")],
|
||||
backendNodeId: [10, 11, 12],
|
||||
attributes: [[], [], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2],
|
||||
// styles columns: [position, pointer-events, cursor]
|
||||
styles: [
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("fixed"), i("auto"), i("pointer")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[10, 10, 40, 40],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
const { nodes } = await captureViewModel(makeCdp(snapshot), 4);
|
||||
expect(nodes.find((n) => n.backendNodeId === 12)?.cursor).toBe("pointer");
|
||||
expect(nodes.find((n) => n.backendNodeId === 11)?.cursor).toBe("auto");
|
||||
});
|
||||
|
||||
it("subtracts scroll offset to make rects viewport-relative", async () => {
|
||||
const cdp = {
|
||||
send: vi.fn(async (_t: number, method: string) => {
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return fakeSnapshotReply();
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 200 },
|
||||
};
|
||||
}
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
const { nodes } = await captureViewModel(cdp, 4);
|
||||
// overlay div bounds y=0 minus scroll pageY=200 → y=-200
|
||||
expect(nodes.find((n) => n.backendNodeId === 12)?.rect?.y).toBe(-200);
|
||||
});
|
||||
|
||||
it("normalizes device-pixel bounds by devicePixelRatio", async () => {
|
||||
const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const snapshot = {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("div")],
|
||||
backendNodeId: [10, 11, 12],
|
||||
attributes: [[], [], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2],
|
||||
styles: [
|
||||
[i("static"), i("auto")],
|
||||
[i("fixed"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 2000, 3200],
|
||||
[0, 0, 2000, 1600],
|
||||
],
|
||||
paintOrders: [0, 50],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
const cdp = {
|
||||
send: vi.fn(async (_t: number, method: string) => {
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
layoutViewport: { clientWidth: 2000, clientHeight: 1600 },
|
||||
};
|
||||
}
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
const { nodes } = await captureViewModel(cdp, 4);
|
||||
expect(nodes.find((n) => n.backendNodeId === 12)?.rect).toEqual({
|
||||
x: 0,
|
||||
y: 0,
|
||||
w: 1000,
|
||||
h: 800,
|
||||
});
|
||||
});
|
||||
|
||||
it("extracts textContent from #text child nodes via nodeValue", async () => {
|
||||
// Simulates a button with text "登录" and an <a> with text "忘记密码".
|
||||
// nodeValue is a parallel array: element nodes = -1, text nodes = string index.
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
"button",
|
||||
"a",
|
||||
"position",
|
||||
"fixed",
|
||||
"static",
|
||||
"pointer-events",
|
||||
"auto",
|
||||
"登录",
|
||||
"忘记密码",
|
||||
];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const snapshot = {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
scrollOffsetX: 0,
|
||||
scrollOffsetY: 0,
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 2, 1, 4],
|
||||
nodeName: [i("html"), i("body"), i("button"), i("#text"), i("a"), i("#text")],
|
||||
backendNodeId: [10, 11, 12, 13, 14, 15],
|
||||
attributes: [[], [], [], [], [], []],
|
||||
nodeValue: [-1, -1, -1, i("登录"), -1, i("忘记密码")],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2, 4],
|
||||
styles: [
|
||||
[i("static"), i("auto")],
|
||||
[i("static"), i("auto")],
|
||||
[i("static"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[100, 300, 200, 60],
|
||||
[100, 380, 120, 36],
|
||||
],
|
||||
paintOrders: [0, 10, 11],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
const cdp = {
|
||||
send: vi.fn(async (_t: number, method: string) => {
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
};
|
||||
}
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
const { nodes } = await captureViewModel(cdp, 4);
|
||||
const btn = nodes.find((n) => n.backendNodeId === 12);
|
||||
expect(btn?.textContent).toBe("登录");
|
||||
const link = nodes.find((n) => n.backendNodeId === 14);
|
||||
expect(link?.textContent).toBe("忘记密码");
|
||||
// Nodes without text children should have no textContent
|
||||
const body = nodes.find((n) => n.backendNodeId === 11);
|
||||
expect(body?.textContent).toBeUndefined();
|
||||
});
|
||||
|
||||
it("parses iframe sub-documents and returns iframeNodes keyed by iframe backendNodeId", async () => {
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
"div",
|
||||
"iframe",
|
||||
"input",
|
||||
"position",
|
||||
"fixed",
|
||||
"static",
|
||||
"pointer-events",
|
||||
"auto",
|
||||
"type",
|
||||
"text",
|
||||
"src",
|
||||
"https://passport.jd.com/login",
|
||||
];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const snapshot = {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
scrollOffsetX: 0,
|
||||
scrollOffsetY: 0,
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 2],
|
||||
nodeName: [i("html"), i("body"), i("div"), i("iframe")],
|
||||
backendNodeId: [10, 11, 12, 13],
|
||||
attributes: [[], [], [], [i("src"), i("https://passport.jd.com/login")]],
|
||||
// Sparse: node at array-index 3 (the iframe, backendNodeId=13) → documents[1]
|
||||
contentDocumentIndex: { index: [3], value: [1] },
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2, 3],
|
||||
styles: [
|
||||
[i("static"), i("auto")],
|
||||
[i("fixed"), i("auto")],
|
||||
[i("static"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 4000],
|
||||
[0, 0, 1000, 800],
|
||||
[100, 300, 800, 400],
|
||||
],
|
||||
paintOrders: [0, 50, 51],
|
||||
},
|
||||
},
|
||||
// documents[1]: the iframe's sub-document with a login input
|
||||
{
|
||||
scrollOffsetX: 0,
|
||||
scrollOffsetY: 0,
|
||||
nodes: {
|
||||
parentIndex: [-1, 0],
|
||||
nodeName: [i("body"), i("input")],
|
||||
backendNodeId: [100, 101],
|
||||
attributes: [[], [i("type"), i("text")]],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1],
|
||||
styles: [[i("static"), i("auto")]],
|
||||
bounds: [[0, 0, 200, 40]],
|
||||
paintOrders: [0],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
const cdp = {
|
||||
send: vi.fn(async (_t: number, method: string) => {
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
};
|
||||
}
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
const { nodes, iframeNodes } = await captureViewModel(cdp, 4);
|
||||
// Main frame has 4 nodes; iframe is included
|
||||
expect(nodes.find((n) => n.backendNodeId === 13)?.tag).toBe("iframe");
|
||||
// iframeNodes keyed by the <iframe> element's backendNodeId=13
|
||||
expect(iframeNodes.size).toBe(1);
|
||||
expect(iframeNodes.has(13)).toBe(true);
|
||||
const subNodes = iframeNodes.get(13)!;
|
||||
expect(subNodes.find((n) => n.backendNodeId === 101)?.tag).toBe("input");
|
||||
expect(subNodes.find((n) => n.backendNodeId === 101)?.attrs.type).toBe("text");
|
||||
});
|
||||
|
||||
it("normalizes bounds then subtracts CSS scroll at dpr>1", async () => {
|
||||
const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const snapshot = {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("div")],
|
||||
backendNodeId: [10, 11, 12],
|
||||
attributes: [[], [], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2],
|
||||
styles: [
|
||||
[i("static"), i("auto")],
|
||||
[i("fixed"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 2000, 3200],
|
||||
[0, 400, 2000, 1600],
|
||||
],
|
||||
paintOrders: [0, 50],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
const cdp = {
|
||||
send: vi.fn(async (_t: number, method: string) => {
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 100 },
|
||||
layoutViewport: { clientWidth: 2000, clientHeight: 1600 },
|
||||
};
|
||||
}
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
const { nodes } = await captureViewModel(cdp, 4);
|
||||
expect(nodes.find((n) => n.backendNodeId === 12)?.rect?.y).toBe(400 / 2 - 100);
|
||||
});
|
||||
|
||||
it("collectOverlayExcludedBackendIds walks the pierced overlay host subtree", async () => {
|
||||
const cdp = {
|
||||
send: vi.fn(async (_t: number, method: string) => {
|
||||
if (method === "DOM.getDocument") return { root: { nodeId: 1 } };
|
||||
if (method === "DOM.querySelector") return { nodeId: 99 };
|
||||
if (method === "DOM.describeNode") {
|
||||
return {
|
||||
node: {
|
||||
backendNodeId: 200,
|
||||
shadowRoots: [
|
||||
{ backendNodeId: 201, children: [{ backendNodeId: 202, children: [] }] },
|
||||
],
|
||||
},
|
||||
};
|
||||
}
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
const excluded = await collectOverlayExcludedBackendIds(cdp, 4);
|
||||
expect(excluded).toEqual(new Set([200, 201, 202]));
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,356 @@
|
||||
// CDP capture adapter: DOMSnapshot.captureSnapshot + Page.getLayoutMetrics
|
||||
// → CapturedNode[] + Viewport. This is the ONLY VOM module that touches
|
||||
// raw CDP. The captureSnapshot reply is columnar (parallel arrays + a
|
||||
// shared string table); we request exactly three computed styles so the
|
||||
// `styles` columns are [position, pointer-events, cursor] in that order.
|
||||
|
||||
import type { Rect, Viewport } from "@browser-skill/vom";
|
||||
import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
|
||||
import type { CdpRunner } from "../shared";
|
||||
|
||||
const REQUESTED_STYLES = ["position", "pointer-events", "cursor"] as const;
|
||||
const STYLE_COL = Object.fromEntries(
|
||||
REQUESTED_STYLES.map((name, index) => [name, index]),
|
||||
) as Record<(typeof REQUESTED_STYLES)[number], number>;
|
||||
|
||||
export interface CapturedNode {
|
||||
backendNodeId: number;
|
||||
parentBackendNodeId: number | null;
|
||||
tag: string;
|
||||
attrs: Record<string, string>;
|
||||
rect: Rect | null;
|
||||
paintOrder: number;
|
||||
position: string;
|
||||
pointerEvents: string;
|
||||
/**
|
||||
* computed `cursor`. `cursor: pointer` is the strongest CDP-free signal
|
||||
* that a non-semantic element (a `<div>`/`<span>` with a click handler)
|
||||
* is actually an interactive control — used by the adapter to surface
|
||||
* custom buttons/checkboxes the AX tree drops as `generic`. Optional like
|
||||
* `textContent`: the live parser always sets it, hand-built fixtures may not.
|
||||
*/
|
||||
cursor?: string;
|
||||
textContent?: string;
|
||||
}
|
||||
|
||||
export type CapturedIframeNodes = Map<number, CapturedNode[]>;
|
||||
|
||||
export interface CapturedViewModel {
|
||||
nodes: CapturedNode[];
|
||||
viewport: Viewport;
|
||||
iframeNodes: CapturedIframeNodes;
|
||||
/** Backend node ids belonging to the agent overlay host + its shadow subtree. */
|
||||
excludedBackendNodeIds: Set<number>;
|
||||
}
|
||||
|
||||
/** Sparse array format Chrome uses for infrequently-set per-node fields. */
|
||||
interface SparseArray {
|
||||
index: number[];
|
||||
value: number[];
|
||||
}
|
||||
|
||||
interface SnapshotDocument {
|
||||
scrollOffsetX?: number;
|
||||
scrollOffsetY?: number;
|
||||
nodes?: {
|
||||
parentIndex?: number[];
|
||||
nodeName?: number[];
|
||||
backendNodeId?: number[];
|
||||
attributes?: number[][];
|
||||
/**
|
||||
* Per-node text value (index into strings), set for `#text` / CDATA nodes.
|
||||
* Element nodes carry -1. Same length as `backendNodeId`.
|
||||
*/
|
||||
nodeValue?: number[];
|
||||
/** Maps node array index → index into `documents[]` for frame content. */
|
||||
contentDocumentIndex?: SparseArray;
|
||||
};
|
||||
layout?: {
|
||||
nodeIndex?: number[];
|
||||
styles?: number[][];
|
||||
bounds?: number[][];
|
||||
paintOrders?: number[];
|
||||
};
|
||||
}
|
||||
|
||||
interface SnapshotReply {
|
||||
strings?: string[];
|
||||
documents?: SnapshotDocument[];
|
||||
}
|
||||
|
||||
interface LayoutMetricsReply {
|
||||
cssLayoutViewport?: {
|
||||
clientWidth?: number;
|
||||
clientHeight?: number;
|
||||
pageX?: number;
|
||||
pageY?: number;
|
||||
};
|
||||
layoutViewport?: { clientWidth?: number; clientHeight?: number; pageX?: number; pageY?: number };
|
||||
}
|
||||
|
||||
interface CdpDomNode {
|
||||
backendNodeId?: number;
|
||||
children?: CdpDomNode[];
|
||||
shadowRoots?: CdpDomNode[];
|
||||
}
|
||||
|
||||
interface ParseDocumentResult {
|
||||
nodes: CapturedNode[];
|
||||
excludedBackendNodeIds: Set<number>;
|
||||
}
|
||||
|
||||
function str(strings: string[], idx: number | undefined): string {
|
||||
if (idx === undefined || idx < 0) return "";
|
||||
return strings[idx] ?? "";
|
||||
}
|
||||
|
||||
function devicePixelRatio(metrics: LayoutMetricsReply): number {
|
||||
const layoutW = metrics.layoutViewport?.clientWidth ?? 0;
|
||||
const cssW = metrics.cssLayoutViewport?.clientWidth ?? 0;
|
||||
if (!layoutW || !cssW) return 1;
|
||||
const dpr = layoutW / cssW;
|
||||
if (!Number.isFinite(dpr) || dpr <= 0) return 1;
|
||||
return dpr >= 1 ? dpr : 1;
|
||||
}
|
||||
|
||||
function collectBackendIdsFromDomNode(node: CdpDomNode | undefined, out: Set<number>): void {
|
||||
if (!node) return;
|
||||
if (typeof node.backendNodeId === "number") {
|
||||
out.add(node.backendNodeId);
|
||||
}
|
||||
for (const child of node.children ?? []) {
|
||||
collectBackendIdsFromDomNode(child, out);
|
||||
}
|
||||
for (const shadow of node.shadowRoots ?? []) {
|
||||
collectBackendIdsFromDomNode(shadow, out);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* When DOMSnapshot is unavailable, locate the marked overlay host via CDP and
|
||||
* collect every backendNodeId in its pierced subtree (open shadow included).
|
||||
*/
|
||||
export async function collectOverlayExcludedBackendIds(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
): Promise<Set<number>> {
|
||||
const excluded = new Set<number>();
|
||||
try {
|
||||
const doc = await cdp.send<{ root?: { nodeId?: number } }>(tabId, "DOM.getDocument", {
|
||||
depth: 0,
|
||||
pierce: true,
|
||||
});
|
||||
const rootNodeId = doc.root?.nodeId;
|
||||
if (typeof rootNodeId !== "number") return excluded;
|
||||
|
||||
const found = await cdp.send<{ nodeId?: number }>(tabId, "DOM.querySelector", {
|
||||
nodeId: rootNodeId,
|
||||
selector: OVERLAY_HOST_SELECTOR,
|
||||
});
|
||||
if (typeof found.nodeId !== "number" || found.nodeId === 0) return excluded;
|
||||
|
||||
const described = await cdp.send<{ node?: CdpDomNode }>(tabId, "DOM.describeNode", {
|
||||
nodeId: found.nodeId,
|
||||
depth: -1,
|
||||
pierce: true,
|
||||
});
|
||||
collectBackendIdsFromDomNode(described.node, excluded);
|
||||
} catch (err) {
|
||||
console.debug("[bsk capture] overlay exclusion fallback failed", err);
|
||||
}
|
||||
return excluded;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse one DOMSnapshot document into CapturedNode[].
|
||||
*
|
||||
* @param doc - the raw DOMSnapshot document object
|
||||
* @param strings - the shared string table for the whole snapshot
|
||||
* @param dpr - device-pixel-ratio from Page.getLayoutMetrics
|
||||
* @param scrollX - horizontal scroll offset to subtract (CSS px)
|
||||
* @param scrollY - vertical scroll offset to subtract (CSS px)
|
||||
*/
|
||||
function parseDocumentNodes(
|
||||
doc: SnapshotDocument,
|
||||
strings: string[],
|
||||
dpr: number,
|
||||
scrollX: number,
|
||||
scrollY: number,
|
||||
): ParseDocumentResult {
|
||||
const dn = doc.nodes;
|
||||
const dl = doc.layout;
|
||||
if (!dn?.backendNodeId) {
|
||||
return { nodes: [], excludedBackendNodeIds: new Set() };
|
||||
}
|
||||
|
||||
const count = dn.backendNodeId.length;
|
||||
const layoutByNode = new Map<number, number>();
|
||||
(dl?.nodeIndex ?? []).forEach((nodeIdx, layoutIdx) => layoutByNode.set(nodeIdx, layoutIdx));
|
||||
|
||||
// The agent's own overlay (WXT shadow host, marked with OVERLAY_HOST_MARKER_ATTR)
|
||||
// is injected into the page and its open shadow root
|
||||
// is inlined by DOMSnapshot as descendants of the host. Its fixed
|
||||
// full-viewport click-blocker would otherwise dominate occlusion detection
|
||||
// and hide the real page, so mark the host + its whole subtree excluded.
|
||||
// parentIndex always references an earlier node, so one forward pass suffices.
|
||||
const excluded = new Array<boolean>(count).fill(false);
|
||||
const excludedBackendNodeIds = new Set<number>();
|
||||
for (let n = 0; n < count; n++) {
|
||||
const parentIdx = dn.parentIndex?.[n] ?? -1;
|
||||
const inherited = parentIdx >= 0 && excluded[parentIdx];
|
||||
let isOverlayHost = inherited;
|
||||
if (!isOverlayHost) {
|
||||
const pairs = dn.attributes?.[n] ?? [];
|
||||
const attrNames: string[] = [];
|
||||
for (let a = 0; a + 1 < pairs.length; a += 2) {
|
||||
attrNames.push(str(strings, pairs[a]));
|
||||
}
|
||||
isOverlayHost = isOverlayHostNode(str(strings, dn.nodeName?.[n]), attrNames);
|
||||
}
|
||||
if (!isOverlayHost) continue;
|
||||
excluded[n] = true;
|
||||
const backendNodeId = dn.backendNodeId[n];
|
||||
if (backendNodeId !== undefined) {
|
||||
excludedBackendNodeIds.add(backendNodeId);
|
||||
}
|
||||
}
|
||||
|
||||
const posCol = STYLE_COL.position;
|
||||
const peCol = STYLE_COL["pointer-events"];
|
||||
const cursorCol = STYLE_COL.cursor;
|
||||
|
||||
// Collect visible text from #text child nodes so element CapturedNodes
|
||||
// carry a textContent value usable as a button/link label fallback.
|
||||
// nodeValue is a parallel array: string index for text nodes, -1 otherwise.
|
||||
const nodeTextContent = new Map<number, string>();
|
||||
if (dn.nodeValue) {
|
||||
for (let n = 0; n < count; n++) {
|
||||
const nvIdx = dn.nodeValue[n] ?? -1;
|
||||
if (nvIdx < 0) continue;
|
||||
const text = str(strings, nvIdx).trim();
|
||||
if (!text) continue;
|
||||
const parentIdx = dn.parentIndex?.[n] ?? -1;
|
||||
if (parentIdx >= 0) {
|
||||
const existing = nodeTextContent.get(parentIdx);
|
||||
nodeTextContent.set(parentIdx, existing ? `${existing} ${text}` : text);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Build CapturedNodes.
|
||||
const nodes: CapturedNode[] = [];
|
||||
for (let n = 0; n < count; n++) {
|
||||
if (excluded[n]) continue;
|
||||
const backendNodeId = dn.backendNodeId[n];
|
||||
const parentIdx = dn.parentIndex?.[n] ?? -1;
|
||||
const parentBackendNodeId = parentIdx >= 0 ? (dn.backendNodeId[parentIdx] ?? null) : null;
|
||||
const tag = str(strings, dn.nodeName?.[n]).toLowerCase();
|
||||
|
||||
const attrs: Record<string, string> = {};
|
||||
const pairs = dn.attributes?.[n] ?? [];
|
||||
for (let a = 0; a + 1 < pairs.length; a += 2) {
|
||||
attrs[str(strings, pairs[a]).toLowerCase()] = str(strings, pairs[a + 1]);
|
||||
}
|
||||
|
||||
let rect: Rect | null = null;
|
||||
let paintOrder = 0;
|
||||
let position = "static";
|
||||
let pointerEvents = "auto";
|
||||
let cursor = "auto";
|
||||
const li = layoutByNode.get(n);
|
||||
if (li !== undefined) {
|
||||
const b = dl?.bounds?.[li];
|
||||
if (b && b.length >= 4 && b[2] > 0 && b[3] > 0) {
|
||||
rect = {
|
||||
x: b[0] / dpr - scrollX,
|
||||
y: b[1] / dpr - scrollY,
|
||||
w: b[2] / dpr,
|
||||
h: b[3] / dpr,
|
||||
};
|
||||
}
|
||||
paintOrder = dl?.paintOrders?.[li] ?? 0;
|
||||
const styleRow = dl?.styles?.[li] ?? [];
|
||||
position = str(strings, styleRow[posCol]) || "static";
|
||||
pointerEvents = str(strings, styleRow[peCol]) || "auto";
|
||||
cursor = str(strings, styleRow[cursorCol]) || "auto";
|
||||
}
|
||||
|
||||
// Skip non-element nodes (#text, #cdata-section, etc.) — they carry no
|
||||
// geometry and are never queried by callers.
|
||||
if (tag.startsWith("#")) continue;
|
||||
|
||||
const textContent = nodeTextContent.get(n);
|
||||
|
||||
nodes.push({
|
||||
backendNodeId,
|
||||
parentBackendNodeId,
|
||||
tag,
|
||||
attrs,
|
||||
rect,
|
||||
paintOrder,
|
||||
position,
|
||||
pointerEvents,
|
||||
cursor,
|
||||
textContent,
|
||||
});
|
||||
}
|
||||
return { nodes, excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
export async function captureViewModel(cdp: CdpRunner, tabId: number): Promise<CapturedViewModel> {
|
||||
const metrics = await cdp.send<LayoutMetricsReply>(tabId, "Page.getLayoutMetrics", {});
|
||||
const dpr = devicePixelRatio(metrics);
|
||||
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||
const viewport: Viewport = {
|
||||
width: vpSrc.clientWidth ?? 0,
|
||||
height: vpSrc.clientHeight ?? 0,
|
||||
};
|
||||
const scrollX = vpSrc.pageX ?? 0;
|
||||
const scrollY = vpSrc.pageY ?? 0;
|
||||
|
||||
await cdp.send(tabId, "DOMSnapshot.enable", {});
|
||||
const snap = await cdp.send<SnapshotReply>(tabId, "DOMSnapshot.captureSnapshot", {
|
||||
computedStyles: REQUESTED_STYLES,
|
||||
includePaintOrder: true,
|
||||
includeDOMRects: true,
|
||||
});
|
||||
|
||||
const strings = snap.strings ?? [];
|
||||
const documents = snap.documents ?? [];
|
||||
const doc0 = documents[0];
|
||||
if (!doc0?.nodes?.backendNodeId) {
|
||||
return { nodes: [], viewport, iframeNodes: new Map(), excludedBackendNodeIds: new Set() };
|
||||
}
|
||||
|
||||
const mainParsed = parseDocumentNodes(doc0, strings, dpr, scrollX, scrollY);
|
||||
const nodes = mainParsed.nodes;
|
||||
const excludedBackendNodeIds = new Set(mainParsed.excludedBackendNodeIds);
|
||||
|
||||
// Build a map from node-array-index → sub-document-index using the sparse
|
||||
// contentDocumentIndex field. Chrome emits this only for <iframe> nodes.
|
||||
const iframeNodes: CapturedIframeNodes = new Map();
|
||||
if (documents.length > 1) {
|
||||
const cdi = doc0.nodes?.contentDocumentIndex;
|
||||
if (cdi?.index && cdi.value) {
|
||||
for (let k = 0; k < cdi.index.length; k++) {
|
||||
const nodeArrayIdx = cdi.index[k];
|
||||
const docIdx = cdi.value[k];
|
||||
const subDoc = documents[docIdx];
|
||||
if (!subDoc) continue;
|
||||
const iframeBid = doc0.nodes?.backendNodeId?.[nodeArrayIdx];
|
||||
if (iframeBid === undefined) continue;
|
||||
// Sub-documents have their own scroll offsets; bounds are relative
|
||||
// to the iframe's own coordinate space (not the main viewport).
|
||||
const subScrollX = subDoc.scrollOffsetX ?? 0;
|
||||
const subScrollY = subDoc.scrollOffsetY ?? 0;
|
||||
const subParsed = parseDocumentNodes(subDoc, strings, dpr, subScrollX, subScrollY);
|
||||
iframeNodes.set(iframeBid, subParsed.nodes);
|
||||
for (const id of subParsed.excludedBackendNodeIds) {
|
||||
excludedBackendNodeIds.add(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { nodes, viewport, iframeNodes, excludedBackendNodeIds };
|
||||
}
|
||||
Reference in New Issue
Block a user