mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat(vom): vom base algorithm and infrastructure
This commit is contained in:
Ljy-0827
@@ -3,6 +3,7 @@
|
||||
// tab (defaulting to the Agent Window's active tab when omitted) and
|
||||
// returns a payload that mirrors the bsk-protocol Rust structs.
|
||||
|
||||
import { renderVom, type VomNode, type VomScene } from "@browser-skill/vom";
|
||||
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
||||
import type { SessionManager } from "@/session-manager/manager";
|
||||
import type {
|
||||
@@ -26,6 +27,12 @@ import {
|
||||
normaliseRef as sharedNormaliseRef,
|
||||
} from "./shared";
|
||||
import { resolveSnapshotRef } from "./snapshot-ref";
|
||||
import {
|
||||
type CapturedNode,
|
||||
type CapturedViewModel,
|
||||
captureViewModel,
|
||||
collectOverlayExcludedBackendIds,
|
||||
} from "./vom/capture";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Shared helpers (legacy aliases — observation.ts kept exporting these
|
||||
@@ -230,176 +237,462 @@ export interface CdpAxNode {
|
||||
childIds?: string[];
|
||||
}
|
||||
|
||||
const INTERACTIVE_ROLES = new Set([
|
||||
function axValue(field?: { value?: string | number | boolean }): string | undefined {
|
||||
const value = field?.value;
|
||||
return value === undefined ? undefined : String(value);
|
||||
}
|
||||
|
||||
function axString(field?: { value?: string | number | boolean }): string | undefined {
|
||||
const value = axValue(field)?.replace(/\s+/g, " ").trim();
|
||||
return value ? value : undefined;
|
||||
}
|
||||
|
||||
function normalizeTag(tag: string | undefined): string {
|
||||
return tag?.toLowerCase() ?? "";
|
||||
}
|
||||
|
||||
function isModalSignal(
|
||||
axNode: CdpAxNode | undefined,
|
||||
capturedNode: CapturedNode | undefined,
|
||||
): boolean {
|
||||
const role = axString(axNode?.role)?.toLowerCase();
|
||||
if (role === "dialog" || role === "alertdialog") return true;
|
||||
if (normalizeTag(capturedNode?.tag) === "dialog") return true;
|
||||
|
||||
const attrs = capturedNode?.attrs ?? {};
|
||||
return (attrs["aria-modal"] ?? "").toLowerCase() === "true" || attrs.role === "dialog";
|
||||
}
|
||||
|
||||
function isSensitive(capturedNode: CapturedNode | undefined): boolean {
|
||||
const attrs = capturedNode?.attrs ?? {};
|
||||
return (attrs.type ?? "").toLowerCase() === "password";
|
||||
}
|
||||
|
||||
function findAxAncestor<T>(
|
||||
axNode: CdpAxNode,
|
||||
axById: Map<string, CdpAxNode>,
|
||||
select: (node: CdpAxNode) => T | undefined,
|
||||
): T | undefined {
|
||||
let parentId = axNode.parentId;
|
||||
while (parentId) {
|
||||
const parent = axById.get(parentId);
|
||||
if (!parent) break;
|
||||
const hit = select(parent);
|
||||
if (hit !== undefined) return hit;
|
||||
parentId = parent.parentId;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function nearestBackendParent(axNode: CdpAxNode, axById: Map<string, CdpAxNode>): number | null {
|
||||
return (
|
||||
findAxAncestor(axNode, axById, (parent) =>
|
||||
typeof parent.backendDOMNodeId === "number" ? parent.backendDOMNodeId : undefined,
|
||||
) ?? null
|
||||
);
|
||||
}
|
||||
|
||||
const IFRAME_RENDERABLE_TAGS = new Set(["input", "button", "a", "select", "textarea"]);
|
||||
|
||||
function iframeRoleFor(node: CapturedNode): string | undefined {
|
||||
const tag = normalizeTag(node.tag);
|
||||
if (tag === "input" || tag === "textarea") return "textbox";
|
||||
if (tag === "button") return "button";
|
||||
if (tag === "a") return "link";
|
||||
if (tag === "select") return "combobox";
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function iframeNameFor(node: CapturedNode): string | undefined {
|
||||
const tag = normalizeTag(node.tag);
|
||||
const ariaLabel = node.attrs["aria-label"]?.replace(/\s+/g, " ").trim();
|
||||
if (ariaLabel) return ariaLabel;
|
||||
|
||||
const text = node.textContent?.replace(/\s+/g, " ").trim();
|
||||
if (text) return text;
|
||||
|
||||
if (tag === "input" || tag === "textarea") {
|
||||
const placeholder = node.attrs.placeholder?.replace(/\s+/g, " ").trim();
|
||||
if (placeholder) return placeholder;
|
||||
}
|
||||
|
||||
const id = node.attrs.id?.replace(/\s+/g, " ").trim();
|
||||
return id ? id : undefined;
|
||||
}
|
||||
|
||||
function isRenderableIframeControl(node: CapturedNode): boolean {
|
||||
const tag = normalizeTag(node.tag);
|
||||
if (!IFRAME_RENDERABLE_TAGS.has(tag)) return false;
|
||||
if (!node.rect) return false;
|
||||
if (node.pointerEvents === "none") return false;
|
||||
if ((node.attrs.type ?? "").toLowerCase() === "hidden") return false;
|
||||
|
||||
const name = iframeNameFor(node);
|
||||
return tag !== "a" || name !== undefined;
|
||||
}
|
||||
|
||||
function capturedIframeNameFor(node: CapturedNode): string | undefined {
|
||||
const ariaLabel = node.attrs["aria-label"]?.replace(/\s+/g, " ").trim();
|
||||
if (ariaLabel) return ariaLabel;
|
||||
|
||||
const title = node.attrs.title?.replace(/\s+/g, " ").trim();
|
||||
if (title) return title;
|
||||
|
||||
const id = node.attrs.id?.replace(/\s+/g, " ").trim();
|
||||
return id ? id : undefined;
|
||||
}
|
||||
|
||||
const FORM_CONTROL_TAGS = new Set(["input", "textarea", "select"]);
|
||||
|
||||
/** AX roles we treat as non-semantic wrappers eligible for clickable promotion. */
|
||||
const PROMOTABLE_ROLES = new Set(["", "generic", "none", "presentation"]);
|
||||
|
||||
/**
|
||||
* Interactive AX roles. Mirrors `packages/vom`'s INTERACTIVE_ROLES — kept
|
||||
* local so the package stays a pure projection (design §2). Used to tell an
|
||||
* atomic custom control (icon button / toggle) apart from a clickable
|
||||
* *container* (card/row) that wraps real controls we must not collapse.
|
||||
*/
|
||||
const INTERACTIVE_ROLES_LC = new Set([
|
||||
"button",
|
||||
"link",
|
||||
"checkbox",
|
||||
"radio",
|
||||
"textbox",
|
||||
"combobox",
|
||||
"listbox",
|
||||
"option",
|
||||
"switch",
|
||||
"tab",
|
||||
"link",
|
||||
"menuitem",
|
||||
"menuitemcheckbox",
|
||||
"menuitemradio",
|
||||
"option",
|
||||
"radio",
|
||||
"scrollbar",
|
||||
"searchbox",
|
||||
"slider",
|
||||
"spinbutton",
|
||||
"scrollbar",
|
||||
"switch",
|
||||
"tab",
|
||||
"textbox",
|
||||
"treeitem",
|
||||
]);
|
||||
|
||||
const STRUCTURAL_ROLES = new Set([
|
||||
"heading",
|
||||
"main",
|
||||
"navigation",
|
||||
"banner",
|
||||
"contentinfo",
|
||||
"complementary",
|
||||
"form",
|
||||
"search",
|
||||
"region",
|
||||
"article",
|
||||
"list",
|
||||
"listitem",
|
||||
"table",
|
||||
"row",
|
||||
"cell",
|
||||
"rowheader",
|
||||
"columnheader",
|
||||
"dialog",
|
||||
"alertdialog",
|
||||
"img",
|
||||
"figure",
|
||||
"section",
|
||||
"RootWebArea",
|
||||
"WebArea",
|
||||
]);
|
||||
|
||||
const SKIP_ROLES = new Set(["generic", "none", "presentation", "InlineTextBox"]);
|
||||
|
||||
/**
|
||||
* Decide whether an aria node should appear in the rendered tree.
|
||||
* Exported for unit tests.
|
||||
*/
|
||||
export function shouldRender(node: CdpAxNode): boolean {
|
||||
if (node.ignored) return false;
|
||||
const role = node.role?.value ?? "";
|
||||
if (!role) return false;
|
||||
if (SKIP_ROLES.has(role)) return false;
|
||||
const name = node.name?.value ?? "";
|
||||
if (INTERACTIVE_ROLES.has(role)) return true;
|
||||
if (STRUCTURAL_ROLES.has(role)) return true;
|
||||
return name.trim().length > 0;
|
||||
function cleanAttr(value: string | undefined): string | undefined {
|
||||
const trimmed = value?.replace(/\s+/g, " ").trim();
|
||||
return trimmed ? trimmed : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Approximate-token estimator (~4 chars / token, GPT-style). Good
|
||||
* enough for `max_tokens` budgets.
|
||||
* Name for a native form control. VOM models the *perceived* viewport
|
||||
* (spec §1): an empty field displays its placeholder, so when the field has
|
||||
* no value we prefer the placeholder over an accessible name that pages
|
||||
* frequently pollute by wrapping the `<input>` in a `<label>` that also
|
||||
* holds prefixes/buttons (e.g. xiaohongshu's "+86" / "获取验证码"). Filled
|
||||
* fields keep the accessible name and surface their value separately.
|
||||
*/
|
||||
function estimateTokens(s: string): number {
|
||||
return Math.ceil(s.length / 4);
|
||||
function formControlName(
|
||||
captured: CapturedNode | undefined,
|
||||
axName: string | undefined,
|
||||
axValue: string | undefined,
|
||||
): string | undefined {
|
||||
const placeholder = cleanAttr(captured?.attrs.placeholder);
|
||||
const ariaLabel = cleanAttr(captured?.attrs["aria-label"]);
|
||||
const title = cleanAttr(captured?.attrs.title);
|
||||
const hasValue = axValue !== undefined && axValue !== "";
|
||||
if (!hasValue && placeholder) return placeholder;
|
||||
return axName ?? ariaLabel ?? placeholder ?? title;
|
||||
}
|
||||
|
||||
interface RenderedSnapshot {
|
||||
text: string;
|
||||
refs: Array<{ ref: string; backendNodeId: number }>;
|
||||
truncated: boolean;
|
||||
/** Strip a sprite/file reference like `#close` or `icon-close.svg` to `close`. */
|
||||
function iconKeyword(href: string | undefined): string | undefined {
|
||||
if (!href) return undefined;
|
||||
let s = href.trim();
|
||||
const hash = s.lastIndexOf("#");
|
||||
if (hash >= 0) s = s.slice(hash + 1);
|
||||
s = s.split("/").pop() ?? s;
|
||||
s = s.replace(/\.(svg|png|webp|gif|jpe?g)$/i, "");
|
||||
s = s.replace(/^(icons?[-_]?|ic[-_]|svg[-_])/i, "");
|
||||
s = s.replace(/[-_]+/g, " ").trim();
|
||||
return s ? s : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert an `Accessibility.getFullAXTree` result into the
|
||||
* `@e<N>`-tagged indented text plus the ref → backendNodeId map for
|
||||
* the session's RefStore. Exported for unit tests.
|
||||
*/
|
||||
export function renderAxTree(
|
||||
nodes: CdpAxNode[],
|
||||
opts: { maxDepth?: number; maxTokens?: number } = {},
|
||||
): RenderedSnapshot {
|
||||
const byId = new Map<string, CdpAxNode>();
|
||||
const childParent = new Map<string, string>();
|
||||
for (const n of nodes) {
|
||||
byId.set(n.nodeId, n);
|
||||
}
|
||||
// Detect roots: nodes whose `parentId` is undefined OR not in the
|
||||
// returned set. CDP sometimes ships orphan branches.
|
||||
for (const n of nodes) {
|
||||
if (n.parentId && byId.has(n.parentId)) {
|
||||
childParent.set(n.nodeId, n.parentId);
|
||||
/** First icon-derived label found in a clickable element's captured subtree. */
|
||||
function iconHint(
|
||||
root: CapturedNode,
|
||||
capChildren: Map<number, CapturedNode[]>,
|
||||
): string | undefined {
|
||||
const queue: Array<{ node: CapturedNode; depth: number }> = (
|
||||
capChildren.get(root.backendNodeId) ?? []
|
||||
).map((node) => ({ node, depth: 1 }));
|
||||
while (queue.length > 0) {
|
||||
const { node, depth } = queue.shift() as { node: CapturedNode; depth: number };
|
||||
const labelled =
|
||||
cleanAttr(node.attrs["aria-label"]) ??
|
||||
cleanAttr(node.attrs.title) ??
|
||||
cleanAttr(node.attrs.alt);
|
||||
if (labelled) return labelled;
|
||||
if (node.tag === "use") {
|
||||
const kw = iconKeyword(node.attrs["xlink:href"] ?? node.attrs.href);
|
||||
if (kw) return kw;
|
||||
}
|
||||
if (depth < 4) {
|
||||
for (const child of capChildren.get(node.backendNodeId) ?? []) {
|
||||
queue.push({ node: child, depth: depth + 1 });
|
||||
}
|
||||
}
|
||||
}
|
||||
const roots = nodes.filter((n) => !childParent.has(n.nodeId));
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const lines: string[] = [];
|
||||
const refs: Array<{ ref: string; backendNodeId: number }> = [];
|
||||
const maxDepth = opts.maxDepth ?? Number.POSITIVE_INFINITY;
|
||||
const maxTokens = opts.maxTokens ?? Number.POSITIVE_INFINITY;
|
||||
let truncated = false;
|
||||
let tokenTruncated = false;
|
||||
let tokenBudget = 0;
|
||||
let nextRef = 1;
|
||||
/** Best-effort name for a promoted clickable: own labels → text → icon. */
|
||||
function clickableName(
|
||||
captured: CapturedNode,
|
||||
capChildren: Map<number, CapturedNode[]>,
|
||||
): string | undefined {
|
||||
return (
|
||||
cleanAttr(captured.attrs["aria-label"]) ??
|
||||
cleanAttr(captured.attrs.title) ??
|
||||
cleanAttr(captured.textContent) ??
|
||||
iconHint(captured, capChildren)
|
||||
);
|
||||
}
|
||||
|
||||
const walk = (node: CdpAxNode, depth: number, ancestorRendered: boolean): void => {
|
||||
if (tokenTruncated) return;
|
||||
const renderThis = shouldRender(node) && depth <= maxDepth;
|
||||
if (renderThis) {
|
||||
const role = node.role?.value ?? "";
|
||||
const name = node.name?.value ?? "";
|
||||
let line = `${" ".repeat(Math.min(depth, 32))}`;
|
||||
let ref: string | null = null;
|
||||
if (typeof node.backendDOMNodeId === "number") {
|
||||
ref = `e${nextRef}`;
|
||||
line += `@${ref} `;
|
||||
}
|
||||
line += role;
|
||||
if (name.length > 0) {
|
||||
const cleaned = name.replace(/\s+/g, " ").trim();
|
||||
line += ` ${JSON.stringify(cleaned)}`;
|
||||
}
|
||||
const value = node.value?.value;
|
||||
if (typeof value === "string" && value.length > 0 && value !== name) {
|
||||
line += ` =${JSON.stringify(value.slice(0, 200))}`;
|
||||
}
|
||||
const lineTokens = estimateTokens(line) + 1; // +1 for newline
|
||||
if (tokenBudget + lineTokens > maxTokens) {
|
||||
truncated = true;
|
||||
tokenTruncated = true;
|
||||
return;
|
||||
}
|
||||
tokenBudget += lineTokens;
|
||||
if (ref && typeof node.backendDOMNodeId === "number") {
|
||||
refs.push({ ref, backendNodeId: node.backendDOMNodeId });
|
||||
nextRef += 1;
|
||||
}
|
||||
lines.push(line);
|
||||
}
|
||||
const nextDepth = renderThis ? depth + 1 : ancestorRendered ? depth : depth;
|
||||
if (depth + 1 > maxDepth && renderThis && (node.childIds?.length ?? 0) > 0) {
|
||||
// Children would exceed depth cap — note truncation flag without
|
||||
// bailing on siblings elsewhere in the tree.
|
||||
truncated = true;
|
||||
return;
|
||||
}
|
||||
for (const cid of node.childIds ?? []) {
|
||||
const child = byId.get(cid);
|
||||
if (!child) continue;
|
||||
if (tokenTruncated) return;
|
||||
walk(child, nextDepth, ancestorRendered || renderThis);
|
||||
function rectCoverage(
|
||||
rect: { x: number; y: number; w: number; h: number } | null,
|
||||
vw: number,
|
||||
vh: number,
|
||||
): number {
|
||||
if (!rect || vw <= 0 || vh <= 0) return 0;
|
||||
const ix = Math.max(0, Math.min(rect.x + rect.w, vw) - Math.max(rect.x, 0));
|
||||
const iy = Math.max(0, Math.min(rect.y + rect.h, vh) - Math.max(rect.y, 0));
|
||||
const overlap = ix * iy;
|
||||
return overlap <= 0 ? 0 : Math.min(1, overlap / (vw * vh));
|
||||
}
|
||||
|
||||
/**
|
||||
* Promote custom clickable controls the AX tree drops as `generic`/unnamed
|
||||
* (icon buttons, CSS checkboxes, "send code" spans …) to a referenceable
|
||||
* `button` so the snapshot can act on them. Guards keep noise out: must be
|
||||
* `cursor: pointer`, have a box, not be a viewport-scale overlay, and not
|
||||
* wrap any real interactive descendant (so clickable cards or rows stay
|
||||
* containers and their inner controls keep their own refs).
|
||||
*/
|
||||
function promoteClickableControls(
|
||||
nodes: VomNode[],
|
||||
capturedById: Map<number, CapturedNode>,
|
||||
capturedNodes: CapturedNode[],
|
||||
vw: number,
|
||||
vh: number,
|
||||
): void {
|
||||
// Snapshot original roles + parent/child links before any mutation so the
|
||||
// guards below are immune to roles we flip to "button" mid-pass.
|
||||
const originalRoleLc = new Map<number, string>(
|
||||
nodes.map((node) => [node.id, (node.role ?? "").toLowerCase()]),
|
||||
);
|
||||
const parentOf = new Map<number, number | null>(nodes.map((node) => [node.id, node.parentId]));
|
||||
const semChildren = new Map<number, VomNode[]>();
|
||||
for (const node of nodes) {
|
||||
if (node.parentId === null) continue;
|
||||
const siblings = semChildren.get(node.parentId);
|
||||
if (siblings) siblings.push(node);
|
||||
else semChildren.set(node.parentId, [node]);
|
||||
}
|
||||
|
||||
const capChildren = new Map<number, CapturedNode[]>();
|
||||
for (const cap of capturedNodes) {
|
||||
if (cap.parentBackendNodeId === null) continue;
|
||||
const siblings = capChildren.get(cap.parentBackendNodeId);
|
||||
if (siblings) siblings.push(cap);
|
||||
else capChildren.set(cap.parentBackendNodeId, [cap]);
|
||||
}
|
||||
|
||||
// A custom clickable: a non-semantic, pointer-cursor box small enough to be
|
||||
// a control rather than a viewport-scale overlay.
|
||||
const candidates = new Set<number>();
|
||||
for (const node of nodes) {
|
||||
if (!PROMOTABLE_ROLES.has(originalRoleLc.get(node.id) ?? "")) continue;
|
||||
const cap = capturedById.get(node.id);
|
||||
if (!cap || cap.cursor !== "pointer" || !node.rect) continue;
|
||||
if (rectCoverage(node.rect, vw, vh) > 0.5) continue;
|
||||
candidates.add(node.id);
|
||||
}
|
||||
|
||||
// wraps a *real* interactive control → it's a clickable container, leave it
|
||||
// (and its inner refs) alone.
|
||||
const wrapsInteractive = (id: number): boolean => {
|
||||
const stack = [...(semChildren.get(id) ?? [])];
|
||||
while (stack.length > 0) {
|
||||
const node = stack.pop() as VomNode;
|
||||
if (INTERACTIVE_ROLES_LC.has(originalRoleLc.get(node.id) ?? "")) return true;
|
||||
const kids = semChildren.get(node.id);
|
||||
if (kids) stack.push(...kids);
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
for (const r of roots) {
|
||||
walk(r, 0, false);
|
||||
if (tokenTruncated) break;
|
||||
// only the outermost clickable in a nested chain becomes the control, so an
|
||||
// icon inside a custom button doesn't render as a button-in-a-button.
|
||||
const hasCandidateAncestor = (id: number): boolean => {
|
||||
let parent = parentOf.get(id) ?? null;
|
||||
while (parent !== null) {
|
||||
if (candidates.has(parent)) return true;
|
||||
parent = parentOf.get(parent) ?? null;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
for (const node of nodes) {
|
||||
if (!candidates.has(node.id)) continue;
|
||||
if (wrapsInteractive(node.id) || hasCandidateAncestor(node.id)) continue;
|
||||
const cap = capturedById.get(node.id);
|
||||
if (!cap) continue;
|
||||
|
||||
node.role = "button";
|
||||
if (!cleanAttr(node.name)) {
|
||||
const derived = clickableName(cap, capChildren);
|
||||
if (derived) node.name = derived;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function vomNodeFromCaptured(capturedNode: CapturedNode, parentId: number | null): VomNode {
|
||||
const sensitive = isSensitive(capturedNode);
|
||||
const value = sensitive ? (capturedNode.attrs.value ?? "") : undefined;
|
||||
const tag = normalizeTag(capturedNode.tag);
|
||||
const node: VomNode = {
|
||||
id: capturedNode.backendNodeId,
|
||||
parentId,
|
||||
tag,
|
||||
rect: capturedNode.rect,
|
||||
paintOrder: capturedNode.paintOrder,
|
||||
position: capturedNode.position || "static",
|
||||
pointerEvents: capturedNode.pointerEvents || "auto",
|
||||
modal: isModalSignal(undefined, capturedNode),
|
||||
sensitive,
|
||||
...(value !== undefined ? { value } : {}),
|
||||
};
|
||||
if (tag === "iframe") {
|
||||
node.role = "Iframe";
|
||||
const name = capturedIframeNameFor(capturedNode);
|
||||
if (name) node.name = name;
|
||||
}
|
||||
return node;
|
||||
}
|
||||
|
||||
function axNodeInOverlaySubtree(
|
||||
axNode: CdpAxNode,
|
||||
axById: Map<string, CdpAxNode>,
|
||||
excludedBackendNodeIds: Set<number>,
|
||||
): boolean {
|
||||
if (
|
||||
typeof axNode.backendDOMNodeId === "number" &&
|
||||
excludedBackendNodeIds.has(axNode.backendDOMNodeId)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
return (
|
||||
findAxAncestor(axNode, axById, (parent) => {
|
||||
const parentBackendId = parent.backendDOMNodeId;
|
||||
return typeof parentBackendId === "number" && excludedBackendNodeIds.has(parentBackendId)
|
||||
? true
|
||||
: undefined;
|
||||
}) === true
|
||||
);
|
||||
}
|
||||
|
||||
export function buildVomScene(axNodes: CdpAxNode[], captured: CapturedViewModel): VomScene {
|
||||
const capturedByBackendId = new Map<number, CapturedNode>();
|
||||
for (const node of captured.nodes) {
|
||||
capturedByBackendId.set(node.backendNodeId, node);
|
||||
}
|
||||
|
||||
return {
|
||||
text: lines.join("\n"),
|
||||
refs,
|
||||
truncated,
|
||||
};
|
||||
const axById = new Map<string, CdpAxNode>();
|
||||
for (const node of axNodes) {
|
||||
axById.set(node.nodeId, node);
|
||||
}
|
||||
|
||||
const excludedBackendNodeIds = captured.excludedBackendNodeIds;
|
||||
const seenBackendIds = new Set<number>();
|
||||
const nodes: VomNode[] = [];
|
||||
|
||||
for (const axNode of axNodes) {
|
||||
if (axNode.ignored || typeof axNode.backendDOMNodeId !== "number") continue;
|
||||
if (axNodeInOverlaySubtree(axNode, axById, excludedBackendNodeIds)) continue;
|
||||
|
||||
const capturedNode = capturedByBackendId.get(axNode.backendDOMNodeId);
|
||||
const role = axString(axNode.role);
|
||||
const value = axValue(axNode.value);
|
||||
const name = FORM_CONTROL_TAGS.has(normalizeTag(capturedNode?.tag))
|
||||
? formControlName(capturedNode, axString(axNode.name), value)
|
||||
: axString(axNode.name);
|
||||
const vomNode: VomNode = {
|
||||
id: axNode.backendDOMNodeId,
|
||||
parentId: nearestBackendParent(axNode, axById),
|
||||
tag: normalizeTag(capturedNode?.tag),
|
||||
rect: capturedNode?.rect ?? null,
|
||||
paintOrder: capturedNode?.paintOrder ?? 0,
|
||||
position: capturedNode?.position || "static",
|
||||
pointerEvents: capturedNode?.pointerEvents || "auto",
|
||||
modal: isModalSignal(axNode, capturedNode),
|
||||
sensitive: isSensitive(capturedNode),
|
||||
};
|
||||
if (role) vomNode.role = role;
|
||||
if (name) vomNode.name = name;
|
||||
if (value !== undefined) vomNode.value = value;
|
||||
|
||||
// For link nodes, attach external hostname so the renderer can annotate it.
|
||||
// Relative hrefs and same-origin hrefs are omitted to avoid noise.
|
||||
if (role === "link" && capturedNode?.attrs.href) {
|
||||
try {
|
||||
const target = new URL(capturedNode.attrs.href, window.location.href);
|
||||
if (target.origin !== window.location.origin) {
|
||||
vomNode.href = target.hostname;
|
||||
}
|
||||
} catch {
|
||||
// Relative or non-URL href (e.g. javascript:) — skip
|
||||
}
|
||||
}
|
||||
|
||||
seenBackendIds.add(axNode.backendDOMNodeId);
|
||||
nodes.push(vomNode);
|
||||
}
|
||||
|
||||
for (const capturedNode of captured.nodes) {
|
||||
if (seenBackendIds.has(capturedNode.backendNodeId)) continue;
|
||||
if (excludedBackendNodeIds.has(capturedNode.backendNodeId)) continue;
|
||||
seenBackendIds.add(capturedNode.backendNodeId);
|
||||
nodes.push(vomNodeFromCaptured(capturedNode, capturedNode.parentBackendNodeId));
|
||||
}
|
||||
|
||||
for (const [iframeBackendId, iframeNodes] of captured.iframeNodes) {
|
||||
for (const iframeNode of iframeNodes) {
|
||||
if (
|
||||
seenBackendIds.has(iframeNode.backendNodeId) ||
|
||||
excludedBackendNodeIds.has(iframeNode.backendNodeId) ||
|
||||
!isRenderableIframeControl(iframeNode)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const role = iframeRoleFor(iframeNode);
|
||||
const name = iframeNameFor(iframeNode);
|
||||
if (!role || (!name && normalizeTag(iframeNode.tag) === "a")) continue;
|
||||
|
||||
seenBackendIds.add(iframeNode.backendNodeId);
|
||||
const vomNode: VomNode = {
|
||||
...vomNodeFromCaptured(iframeNode, iframeBackendId),
|
||||
role,
|
||||
};
|
||||
if (name) vomNode.name = name;
|
||||
nodes.push(vomNode);
|
||||
}
|
||||
}
|
||||
|
||||
promoteClickableControls(
|
||||
nodes,
|
||||
capturedByBackendId,
|
||||
captured.nodes,
|
||||
captured.viewport.width,
|
||||
captured.viewport.height,
|
||||
);
|
||||
|
||||
return { viewport: captured.viewport, nodes };
|
||||
}
|
||||
|
||||
export interface SnapshotDeps {
|
||||
@@ -514,6 +807,42 @@ export async function handleGetHtml(
|
||||
}
|
||||
}
|
||||
|
||||
function emptyCapturedViewModel(viewport = { width: 0, height: 0 }): CapturedViewModel {
|
||||
return { viewport, nodes: [], iframeNodes: new Map(), excludedBackendNodeIds: new Set() };
|
||||
}
|
||||
|
||||
interface LayoutMetricsViewportReply {
|
||||
cssLayoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||
layoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||
}
|
||||
|
||||
async function fallbackCapturedViewModel(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
): Promise<CapturedViewModel> {
|
||||
let viewport = { width: 0, height: 0 };
|
||||
try {
|
||||
const metrics = await cdp.send<LayoutMetricsViewportReply>(tabId, "Page.getLayoutMetrics", {});
|
||||
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||
viewport = {
|
||||
width: vpSrc.clientWidth ?? 0,
|
||||
height: vpSrc.clientHeight ?? 0,
|
||||
};
|
||||
} catch {
|
||||
// viewport stays zero-sized
|
||||
}
|
||||
const excludedBackendNodeIds = await collectOverlayExcludedBackendIds(cdp, tabId);
|
||||
return { ...emptyCapturedViewModel(viewport), excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
async function captureForVom(cdp: CdpRunner, tabId: number): Promise<CapturedViewModel> {
|
||||
try {
|
||||
return await captureViewModel(cdp, tabId);
|
||||
} catch {
|
||||
return fallbackCapturedViewModel(cdp, tabId);
|
||||
}
|
||||
}
|
||||
|
||||
export async function handleSnapshot(
|
||||
manager: SessionManager,
|
||||
params: SnapshotParams,
|
||||
@@ -534,11 +863,10 @@ export async function handleSnapshot(
|
||||
"Accessibility.getFullAXTree",
|
||||
{},
|
||||
);
|
||||
const rendered = renderAxTree(result.nodes ?? [], {
|
||||
maxDepth: params.max_depth,
|
||||
maxTokens: params.max_tokens,
|
||||
});
|
||||
// Reset the session-scoped ref-store for this fresh snapshot.
|
||||
const axNodes = result.nodes ?? [];
|
||||
const captured = await captureForVom(deps.cdp, target.tabId);
|
||||
const scene = buildVomScene(axNodes, captured);
|
||||
const rendered = renderVom(scene, { maxDepth: params.max_depth, maxTokens: params.max_tokens });
|
||||
ctx.refStore.replace(
|
||||
rendered.refs.map(
|
||||
(r) => [r.ref, { backendNodeId: r.backendNodeId, tabId: target.tabId }] as const,
|
||||
|
||||
Reference in New Issue
Block a user