mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat(vom): vom base algorithm and infrastructure
This commit is contained in:
Ljy-0827
@@ -14,6 +14,7 @@
|
||||
"dependencies": {
|
||||
"@browser-skill/i18n": "workspace:*",
|
||||
"@browser-skill/ui": "workspace:*",
|
||||
"@browser-skill/vom": "workspace:*",
|
||||
"@remixicon/react": "^4.9.0",
|
||||
"react": "^19.0.0",
|
||||
"react-dom": "^19.0.0"
|
||||
|
||||
@@ -28,6 +28,8 @@ import {
|
||||
import {
|
||||
isOverlayAgentOverlayResetMessage,
|
||||
OVERLAY_AUTOMATION_BYPASS,
|
||||
OVERLAY_HOST_MARKER_ATTR,
|
||||
OVERLAY_HOST_NAME,
|
||||
OVERLAY_MSG_WHO_AM_I,
|
||||
type OverlayAgentOverlayResetMessage,
|
||||
type OverlayAutomationBypassMessage,
|
||||
@@ -67,13 +69,13 @@ export default defineContentScript({
|
||||
let remountInProgress = false;
|
||||
|
||||
const ui = await createShadowRootUi(ctx, {
|
||||
name: "browser-skill-overlay",
|
||||
name: OVERLAY_HOST_NAME,
|
||||
position: "inline",
|
||||
anchor: "html",
|
||||
css: overlayCss,
|
||||
onMount(container, _shadow, shadowHost) {
|
||||
shadowHost.setAttribute("aria-hidden", "true");
|
||||
shadowHost.setAttribute("data-bsk-overlay", "");
|
||||
shadowHost.setAttribute(OVERLAY_HOST_MARKER_ATTR, "");
|
||||
overlayHost = shadowHost;
|
||||
hostLossReported = false;
|
||||
const app = document.createElement("div");
|
||||
|
||||
@@ -16,6 +16,37 @@
|
||||
export const OVERLAY_MSG_WHO_AM_I = "overlay.who_am_i";
|
||||
export const OVERLAY_MSG_INTERRUPT = "overlay.interrupt";
|
||||
|
||||
/**
|
||||
* WXT shadow-host element name (`createShadowRootUi({ name })`) and the marker
|
||||
* attribute set on the host for the agent's own overlay UI. The VOM capture
|
||||
* adapter uses these to skip the overlay host and its shadow subtree.
|
||||
*/
|
||||
export const OVERLAY_HOST_NAME = "browser-skill-overlay";
|
||||
export const OVERLAY_HOST_MARKER_ATTR = "data-bsk-overlay";
|
||||
|
||||
/** CSS selector for the WXT shadow host (tag + marker attribute). */
|
||||
export const OVERLAY_HOST_SELECTOR = `${OVERLAY_HOST_NAME}, [${OVERLAY_HOST_MARKER_ATTR}]`;
|
||||
|
||||
export function isOverlayHostElementName(tagName: string): boolean {
|
||||
return tagName.toLowerCase() === OVERLAY_HOST_NAME;
|
||||
}
|
||||
|
||||
export function isOverlayHostMarkerAttribute(attrName: string): boolean {
|
||||
return attrName.toLowerCase() === OVERLAY_HOST_MARKER_ATTR;
|
||||
}
|
||||
|
||||
export function isOverlayHostNode(tagName: string, attributeNames?: Iterable<string>): boolean {
|
||||
if (isOverlayHostElementName(tagName)) return true;
|
||||
if (!attributeNames) return false;
|
||||
for (const name of attributeNames) {
|
||||
if (isOverlayHostMarkerAttribute(name)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Page-world `document.querySelector(...)` for the overlay shadow host. */
|
||||
export const OVERLAY_HOST_LOOKUP_EXPR = `document.querySelector(${JSON.stringify(OVERLAY_HOST_SELECTOR)})`;
|
||||
|
||||
export interface OverlayWhoAmIRequest {
|
||||
kind: typeof OVERLAY_MSG_WHO_AM_I;
|
||||
tabId?: number;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -3,6 +3,7 @@
|
||||
// tab (defaulting to the Agent Window's active tab when omitted) and
|
||||
// returns a payload that mirrors the bsk-protocol Rust structs.
|
||||
|
||||
import { renderVom, type VomNode, type VomScene } from "@browser-skill/vom";
|
||||
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
||||
import type { SessionManager } from "@/session-manager/manager";
|
||||
import type {
|
||||
@@ -26,6 +27,12 @@ import {
|
||||
normaliseRef as sharedNormaliseRef,
|
||||
} from "./shared";
|
||||
import { resolveSnapshotRef } from "./snapshot-ref";
|
||||
import {
|
||||
type CapturedNode,
|
||||
type CapturedViewModel,
|
||||
captureViewModel,
|
||||
collectOverlayExcludedBackendIds,
|
||||
} from "./vom/capture";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Shared helpers (legacy aliases — observation.ts kept exporting these
|
||||
@@ -230,176 +237,462 @@ export interface CdpAxNode {
|
||||
childIds?: string[];
|
||||
}
|
||||
|
||||
const INTERACTIVE_ROLES = new Set([
|
||||
function axValue(field?: { value?: string | number | boolean }): string | undefined {
|
||||
const value = field?.value;
|
||||
return value === undefined ? undefined : String(value);
|
||||
}
|
||||
|
||||
function axString(field?: { value?: string | number | boolean }): string | undefined {
|
||||
const value = axValue(field)?.replace(/\s+/g, " ").trim();
|
||||
return value ? value : undefined;
|
||||
}
|
||||
|
||||
function normalizeTag(tag: string | undefined): string {
|
||||
return tag?.toLowerCase() ?? "";
|
||||
}
|
||||
|
||||
function isModalSignal(
|
||||
axNode: CdpAxNode | undefined,
|
||||
capturedNode: CapturedNode | undefined,
|
||||
): boolean {
|
||||
const role = axString(axNode?.role)?.toLowerCase();
|
||||
if (role === "dialog" || role === "alertdialog") return true;
|
||||
if (normalizeTag(capturedNode?.tag) === "dialog") return true;
|
||||
|
||||
const attrs = capturedNode?.attrs ?? {};
|
||||
return (attrs["aria-modal"] ?? "").toLowerCase() === "true" || attrs.role === "dialog";
|
||||
}
|
||||
|
||||
function isSensitive(capturedNode: CapturedNode | undefined): boolean {
|
||||
const attrs = capturedNode?.attrs ?? {};
|
||||
return (attrs.type ?? "").toLowerCase() === "password";
|
||||
}
|
||||
|
||||
function findAxAncestor<T>(
|
||||
axNode: CdpAxNode,
|
||||
axById: Map<string, CdpAxNode>,
|
||||
select: (node: CdpAxNode) => T | undefined,
|
||||
): T | undefined {
|
||||
let parentId = axNode.parentId;
|
||||
while (parentId) {
|
||||
const parent = axById.get(parentId);
|
||||
if (!parent) break;
|
||||
const hit = select(parent);
|
||||
if (hit !== undefined) return hit;
|
||||
parentId = parent.parentId;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function nearestBackendParent(axNode: CdpAxNode, axById: Map<string, CdpAxNode>): number | null {
|
||||
return (
|
||||
findAxAncestor(axNode, axById, (parent) =>
|
||||
typeof parent.backendDOMNodeId === "number" ? parent.backendDOMNodeId : undefined,
|
||||
) ?? null
|
||||
);
|
||||
}
|
||||
|
||||
const IFRAME_RENDERABLE_TAGS = new Set(["input", "button", "a", "select", "textarea"]);
|
||||
|
||||
function iframeRoleFor(node: CapturedNode): string | undefined {
|
||||
const tag = normalizeTag(node.tag);
|
||||
if (tag === "input" || tag === "textarea") return "textbox";
|
||||
if (tag === "button") return "button";
|
||||
if (tag === "a") return "link";
|
||||
if (tag === "select") return "combobox";
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function iframeNameFor(node: CapturedNode): string | undefined {
|
||||
const tag = normalizeTag(node.tag);
|
||||
const ariaLabel = node.attrs["aria-label"]?.replace(/\s+/g, " ").trim();
|
||||
if (ariaLabel) return ariaLabel;
|
||||
|
||||
const text = node.textContent?.replace(/\s+/g, " ").trim();
|
||||
if (text) return text;
|
||||
|
||||
if (tag === "input" || tag === "textarea") {
|
||||
const placeholder = node.attrs.placeholder?.replace(/\s+/g, " ").trim();
|
||||
if (placeholder) return placeholder;
|
||||
}
|
||||
|
||||
const id = node.attrs.id?.replace(/\s+/g, " ").trim();
|
||||
return id ? id : undefined;
|
||||
}
|
||||
|
||||
function isRenderableIframeControl(node: CapturedNode): boolean {
|
||||
const tag = normalizeTag(node.tag);
|
||||
if (!IFRAME_RENDERABLE_TAGS.has(tag)) return false;
|
||||
if (!node.rect) return false;
|
||||
if (node.pointerEvents === "none") return false;
|
||||
if ((node.attrs.type ?? "").toLowerCase() === "hidden") return false;
|
||||
|
||||
const name = iframeNameFor(node);
|
||||
return tag !== "a" || name !== undefined;
|
||||
}
|
||||
|
||||
function capturedIframeNameFor(node: CapturedNode): string | undefined {
|
||||
const ariaLabel = node.attrs["aria-label"]?.replace(/\s+/g, " ").trim();
|
||||
if (ariaLabel) return ariaLabel;
|
||||
|
||||
const title = node.attrs.title?.replace(/\s+/g, " ").trim();
|
||||
if (title) return title;
|
||||
|
||||
const id = node.attrs.id?.replace(/\s+/g, " ").trim();
|
||||
return id ? id : undefined;
|
||||
}
|
||||
|
||||
const FORM_CONTROL_TAGS = new Set(["input", "textarea", "select"]);
|
||||
|
||||
/** AX roles we treat as non-semantic wrappers eligible for clickable promotion. */
|
||||
const PROMOTABLE_ROLES = new Set(["", "generic", "none", "presentation"]);
|
||||
|
||||
/**
|
||||
* Interactive AX roles. Mirrors `packages/vom`'s INTERACTIVE_ROLES — kept
|
||||
* local so the package stays a pure projection (design §2). Used to tell an
|
||||
* atomic custom control (icon button / toggle) apart from a clickable
|
||||
* *container* (card/row) that wraps real controls we must not collapse.
|
||||
*/
|
||||
const INTERACTIVE_ROLES_LC = new Set([
|
||||
"button",
|
||||
"link",
|
||||
"checkbox",
|
||||
"radio",
|
||||
"textbox",
|
||||
"combobox",
|
||||
"listbox",
|
||||
"option",
|
||||
"switch",
|
||||
"tab",
|
||||
"link",
|
||||
"menuitem",
|
||||
"menuitemcheckbox",
|
||||
"menuitemradio",
|
||||
"option",
|
||||
"radio",
|
||||
"scrollbar",
|
||||
"searchbox",
|
||||
"slider",
|
||||
"spinbutton",
|
||||
"scrollbar",
|
||||
"switch",
|
||||
"tab",
|
||||
"textbox",
|
||||
"treeitem",
|
||||
]);
|
||||
|
||||
const STRUCTURAL_ROLES = new Set([
|
||||
"heading",
|
||||
"main",
|
||||
"navigation",
|
||||
"banner",
|
||||
"contentinfo",
|
||||
"complementary",
|
||||
"form",
|
||||
"search",
|
||||
"region",
|
||||
"article",
|
||||
"list",
|
||||
"listitem",
|
||||
"table",
|
||||
"row",
|
||||
"cell",
|
||||
"rowheader",
|
||||
"columnheader",
|
||||
"dialog",
|
||||
"alertdialog",
|
||||
"img",
|
||||
"figure",
|
||||
"section",
|
||||
"RootWebArea",
|
||||
"WebArea",
|
||||
]);
|
||||
|
||||
const SKIP_ROLES = new Set(["generic", "none", "presentation", "InlineTextBox"]);
|
||||
|
||||
/**
|
||||
* Decide whether an aria node should appear in the rendered tree.
|
||||
* Exported for unit tests.
|
||||
*/
|
||||
export function shouldRender(node: CdpAxNode): boolean {
|
||||
if (node.ignored) return false;
|
||||
const role = node.role?.value ?? "";
|
||||
if (!role) return false;
|
||||
if (SKIP_ROLES.has(role)) return false;
|
||||
const name = node.name?.value ?? "";
|
||||
if (INTERACTIVE_ROLES.has(role)) return true;
|
||||
if (STRUCTURAL_ROLES.has(role)) return true;
|
||||
return name.trim().length > 0;
|
||||
function cleanAttr(value: string | undefined): string | undefined {
|
||||
const trimmed = value?.replace(/\s+/g, " ").trim();
|
||||
return trimmed ? trimmed : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Approximate-token estimator (~4 chars / token, GPT-style). Good
|
||||
* enough for `max_tokens` budgets.
|
||||
* Name for a native form control. VOM models the *perceived* viewport
|
||||
* (spec §1): an empty field displays its placeholder, so when the field has
|
||||
* no value we prefer the placeholder over an accessible name that pages
|
||||
* frequently pollute by wrapping the `<input>` in a `<label>` that also
|
||||
* holds prefixes/buttons (e.g. xiaohongshu's "+86" / "获取验证码"). Filled
|
||||
* fields keep the accessible name and surface their value separately.
|
||||
*/
|
||||
function estimateTokens(s: string): number {
|
||||
return Math.ceil(s.length / 4);
|
||||
function formControlName(
|
||||
captured: CapturedNode | undefined,
|
||||
axName: string | undefined,
|
||||
axValue: string | undefined,
|
||||
): string | undefined {
|
||||
const placeholder = cleanAttr(captured?.attrs.placeholder);
|
||||
const ariaLabel = cleanAttr(captured?.attrs["aria-label"]);
|
||||
const title = cleanAttr(captured?.attrs.title);
|
||||
const hasValue = axValue !== undefined && axValue !== "";
|
||||
if (!hasValue && placeholder) return placeholder;
|
||||
return axName ?? ariaLabel ?? placeholder ?? title;
|
||||
}
|
||||
|
||||
interface RenderedSnapshot {
|
||||
text: string;
|
||||
refs: Array<{ ref: string; backendNodeId: number }>;
|
||||
truncated: boolean;
|
||||
/** Strip a sprite/file reference like `#close` or `icon-close.svg` to `close`. */
|
||||
function iconKeyword(href: string | undefined): string | undefined {
|
||||
if (!href) return undefined;
|
||||
let s = href.trim();
|
||||
const hash = s.lastIndexOf("#");
|
||||
if (hash >= 0) s = s.slice(hash + 1);
|
||||
s = s.split("/").pop() ?? s;
|
||||
s = s.replace(/\.(svg|png|webp|gif|jpe?g)$/i, "");
|
||||
s = s.replace(/^(icons?[-_]?|ic[-_]|svg[-_])/i, "");
|
||||
s = s.replace(/[-_]+/g, " ").trim();
|
||||
return s ? s : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert an `Accessibility.getFullAXTree` result into the
|
||||
* `@e<N>`-tagged indented text plus the ref → backendNodeId map for
|
||||
* the session's RefStore. Exported for unit tests.
|
||||
*/
|
||||
export function renderAxTree(
|
||||
nodes: CdpAxNode[],
|
||||
opts: { maxDepth?: number; maxTokens?: number } = {},
|
||||
): RenderedSnapshot {
|
||||
const byId = new Map<string, CdpAxNode>();
|
||||
const childParent = new Map<string, string>();
|
||||
for (const n of nodes) {
|
||||
byId.set(n.nodeId, n);
|
||||
}
|
||||
// Detect roots: nodes whose `parentId` is undefined OR not in the
|
||||
// returned set. CDP sometimes ships orphan branches.
|
||||
for (const n of nodes) {
|
||||
if (n.parentId && byId.has(n.parentId)) {
|
||||
childParent.set(n.nodeId, n.parentId);
|
||||
/** First icon-derived label found in a clickable element's captured subtree. */
|
||||
function iconHint(
|
||||
root: CapturedNode,
|
||||
capChildren: Map<number, CapturedNode[]>,
|
||||
): string | undefined {
|
||||
const queue: Array<{ node: CapturedNode; depth: number }> = (
|
||||
capChildren.get(root.backendNodeId) ?? []
|
||||
).map((node) => ({ node, depth: 1 }));
|
||||
while (queue.length > 0) {
|
||||
const { node, depth } = queue.shift() as { node: CapturedNode; depth: number };
|
||||
const labelled =
|
||||
cleanAttr(node.attrs["aria-label"]) ??
|
||||
cleanAttr(node.attrs.title) ??
|
||||
cleanAttr(node.attrs.alt);
|
||||
if (labelled) return labelled;
|
||||
if (node.tag === "use") {
|
||||
const kw = iconKeyword(node.attrs["xlink:href"] ?? node.attrs.href);
|
||||
if (kw) return kw;
|
||||
}
|
||||
if (depth < 4) {
|
||||
for (const child of capChildren.get(node.backendNodeId) ?? []) {
|
||||
queue.push({ node: child, depth: depth + 1 });
|
||||
}
|
||||
}
|
||||
}
|
||||
const roots = nodes.filter((n) => !childParent.has(n.nodeId));
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const lines: string[] = [];
|
||||
const refs: Array<{ ref: string; backendNodeId: number }> = [];
|
||||
const maxDepth = opts.maxDepth ?? Number.POSITIVE_INFINITY;
|
||||
const maxTokens = opts.maxTokens ?? Number.POSITIVE_INFINITY;
|
||||
let truncated = false;
|
||||
let tokenTruncated = false;
|
||||
let tokenBudget = 0;
|
||||
let nextRef = 1;
|
||||
/** Best-effort name for a promoted clickable: own labels → text → icon. */
|
||||
function clickableName(
|
||||
captured: CapturedNode,
|
||||
capChildren: Map<number, CapturedNode[]>,
|
||||
): string | undefined {
|
||||
return (
|
||||
cleanAttr(captured.attrs["aria-label"]) ??
|
||||
cleanAttr(captured.attrs.title) ??
|
||||
cleanAttr(captured.textContent) ??
|
||||
iconHint(captured, capChildren)
|
||||
);
|
||||
}
|
||||
|
||||
const walk = (node: CdpAxNode, depth: number, ancestorRendered: boolean): void => {
|
||||
if (tokenTruncated) return;
|
||||
const renderThis = shouldRender(node) && depth <= maxDepth;
|
||||
if (renderThis) {
|
||||
const role = node.role?.value ?? "";
|
||||
const name = node.name?.value ?? "";
|
||||
let line = `${" ".repeat(Math.min(depth, 32))}`;
|
||||
let ref: string | null = null;
|
||||
if (typeof node.backendDOMNodeId === "number") {
|
||||
ref = `e${nextRef}`;
|
||||
line += `@${ref} `;
|
||||
}
|
||||
line += role;
|
||||
if (name.length > 0) {
|
||||
const cleaned = name.replace(/\s+/g, " ").trim();
|
||||
line += ` ${JSON.stringify(cleaned)}`;
|
||||
}
|
||||
const value = node.value?.value;
|
||||
if (typeof value === "string" && value.length > 0 && value !== name) {
|
||||
line += ` =${JSON.stringify(value.slice(0, 200))}`;
|
||||
}
|
||||
const lineTokens = estimateTokens(line) + 1; // +1 for newline
|
||||
if (tokenBudget + lineTokens > maxTokens) {
|
||||
truncated = true;
|
||||
tokenTruncated = true;
|
||||
return;
|
||||
}
|
||||
tokenBudget += lineTokens;
|
||||
if (ref && typeof node.backendDOMNodeId === "number") {
|
||||
refs.push({ ref, backendNodeId: node.backendDOMNodeId });
|
||||
nextRef += 1;
|
||||
}
|
||||
lines.push(line);
|
||||
}
|
||||
const nextDepth = renderThis ? depth + 1 : ancestorRendered ? depth : depth;
|
||||
if (depth + 1 > maxDepth && renderThis && (node.childIds?.length ?? 0) > 0) {
|
||||
// Children would exceed depth cap — note truncation flag without
|
||||
// bailing on siblings elsewhere in the tree.
|
||||
truncated = true;
|
||||
return;
|
||||
}
|
||||
for (const cid of node.childIds ?? []) {
|
||||
const child = byId.get(cid);
|
||||
if (!child) continue;
|
||||
if (tokenTruncated) return;
|
||||
walk(child, nextDepth, ancestorRendered || renderThis);
|
||||
function rectCoverage(
|
||||
rect: { x: number; y: number; w: number; h: number } | null,
|
||||
vw: number,
|
||||
vh: number,
|
||||
): number {
|
||||
if (!rect || vw <= 0 || vh <= 0) return 0;
|
||||
const ix = Math.max(0, Math.min(rect.x + rect.w, vw) - Math.max(rect.x, 0));
|
||||
const iy = Math.max(0, Math.min(rect.y + rect.h, vh) - Math.max(rect.y, 0));
|
||||
const overlap = ix * iy;
|
||||
return overlap <= 0 ? 0 : Math.min(1, overlap / (vw * vh));
|
||||
}
|
||||
|
||||
/**
|
||||
* Promote custom clickable controls the AX tree drops as `generic`/unnamed
|
||||
* (icon buttons, CSS checkboxes, "send code" spans …) to a referenceable
|
||||
* `button` so the snapshot can act on them. Guards keep noise out: must be
|
||||
* `cursor: pointer`, have a box, not be a viewport-scale overlay, and not
|
||||
* wrap any real interactive descendant (so clickable cards or rows stay
|
||||
* containers and their inner controls keep their own refs).
|
||||
*/
|
||||
function promoteClickableControls(
|
||||
nodes: VomNode[],
|
||||
capturedById: Map<number, CapturedNode>,
|
||||
capturedNodes: CapturedNode[],
|
||||
vw: number,
|
||||
vh: number,
|
||||
): void {
|
||||
// Snapshot original roles + parent/child links before any mutation so the
|
||||
// guards below are immune to roles we flip to "button" mid-pass.
|
||||
const originalRoleLc = new Map<number, string>(
|
||||
nodes.map((node) => [node.id, (node.role ?? "").toLowerCase()]),
|
||||
);
|
||||
const parentOf = new Map<number, number | null>(nodes.map((node) => [node.id, node.parentId]));
|
||||
const semChildren = new Map<number, VomNode[]>();
|
||||
for (const node of nodes) {
|
||||
if (node.parentId === null) continue;
|
||||
const siblings = semChildren.get(node.parentId);
|
||||
if (siblings) siblings.push(node);
|
||||
else semChildren.set(node.parentId, [node]);
|
||||
}
|
||||
|
||||
const capChildren = new Map<number, CapturedNode[]>();
|
||||
for (const cap of capturedNodes) {
|
||||
if (cap.parentBackendNodeId === null) continue;
|
||||
const siblings = capChildren.get(cap.parentBackendNodeId);
|
||||
if (siblings) siblings.push(cap);
|
||||
else capChildren.set(cap.parentBackendNodeId, [cap]);
|
||||
}
|
||||
|
||||
// A custom clickable: a non-semantic, pointer-cursor box small enough to be
|
||||
// a control rather than a viewport-scale overlay.
|
||||
const candidates = new Set<number>();
|
||||
for (const node of nodes) {
|
||||
if (!PROMOTABLE_ROLES.has(originalRoleLc.get(node.id) ?? "")) continue;
|
||||
const cap = capturedById.get(node.id);
|
||||
if (!cap || cap.cursor !== "pointer" || !node.rect) continue;
|
||||
if (rectCoverage(node.rect, vw, vh) > 0.5) continue;
|
||||
candidates.add(node.id);
|
||||
}
|
||||
|
||||
// wraps a *real* interactive control → it's a clickable container, leave it
|
||||
// (and its inner refs) alone.
|
||||
const wrapsInteractive = (id: number): boolean => {
|
||||
const stack = [...(semChildren.get(id) ?? [])];
|
||||
while (stack.length > 0) {
|
||||
const node = stack.pop() as VomNode;
|
||||
if (INTERACTIVE_ROLES_LC.has(originalRoleLc.get(node.id) ?? "")) return true;
|
||||
const kids = semChildren.get(node.id);
|
||||
if (kids) stack.push(...kids);
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
for (const r of roots) {
|
||||
walk(r, 0, false);
|
||||
if (tokenTruncated) break;
|
||||
// only the outermost clickable in a nested chain becomes the control, so an
|
||||
// icon inside a custom button doesn't render as a button-in-a-button.
|
||||
const hasCandidateAncestor = (id: number): boolean => {
|
||||
let parent = parentOf.get(id) ?? null;
|
||||
while (parent !== null) {
|
||||
if (candidates.has(parent)) return true;
|
||||
parent = parentOf.get(parent) ?? null;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
for (const node of nodes) {
|
||||
if (!candidates.has(node.id)) continue;
|
||||
if (wrapsInteractive(node.id) || hasCandidateAncestor(node.id)) continue;
|
||||
const cap = capturedById.get(node.id);
|
||||
if (!cap) continue;
|
||||
|
||||
node.role = "button";
|
||||
if (!cleanAttr(node.name)) {
|
||||
const derived = clickableName(cap, capChildren);
|
||||
if (derived) node.name = derived;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function vomNodeFromCaptured(capturedNode: CapturedNode, parentId: number | null): VomNode {
|
||||
const sensitive = isSensitive(capturedNode);
|
||||
const value = sensitive ? (capturedNode.attrs.value ?? "") : undefined;
|
||||
const tag = normalizeTag(capturedNode.tag);
|
||||
const node: VomNode = {
|
||||
id: capturedNode.backendNodeId,
|
||||
parentId,
|
||||
tag,
|
||||
rect: capturedNode.rect,
|
||||
paintOrder: capturedNode.paintOrder,
|
||||
position: capturedNode.position || "static",
|
||||
pointerEvents: capturedNode.pointerEvents || "auto",
|
||||
modal: isModalSignal(undefined, capturedNode),
|
||||
sensitive,
|
||||
...(value !== undefined ? { value } : {}),
|
||||
};
|
||||
if (tag === "iframe") {
|
||||
node.role = "Iframe";
|
||||
const name = capturedIframeNameFor(capturedNode);
|
||||
if (name) node.name = name;
|
||||
}
|
||||
return node;
|
||||
}
|
||||
|
||||
function axNodeInOverlaySubtree(
|
||||
axNode: CdpAxNode,
|
||||
axById: Map<string, CdpAxNode>,
|
||||
excludedBackendNodeIds: Set<number>,
|
||||
): boolean {
|
||||
if (
|
||||
typeof axNode.backendDOMNodeId === "number" &&
|
||||
excludedBackendNodeIds.has(axNode.backendDOMNodeId)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
return (
|
||||
findAxAncestor(axNode, axById, (parent) => {
|
||||
const parentBackendId = parent.backendDOMNodeId;
|
||||
return typeof parentBackendId === "number" && excludedBackendNodeIds.has(parentBackendId)
|
||||
? true
|
||||
: undefined;
|
||||
}) === true
|
||||
);
|
||||
}
|
||||
|
||||
export function buildVomScene(axNodes: CdpAxNode[], captured: CapturedViewModel): VomScene {
|
||||
const capturedByBackendId = new Map<number, CapturedNode>();
|
||||
for (const node of captured.nodes) {
|
||||
capturedByBackendId.set(node.backendNodeId, node);
|
||||
}
|
||||
|
||||
return {
|
||||
text: lines.join("\n"),
|
||||
refs,
|
||||
truncated,
|
||||
};
|
||||
const axById = new Map<string, CdpAxNode>();
|
||||
for (const node of axNodes) {
|
||||
axById.set(node.nodeId, node);
|
||||
}
|
||||
|
||||
const excludedBackendNodeIds = captured.excludedBackendNodeIds;
|
||||
const seenBackendIds = new Set<number>();
|
||||
const nodes: VomNode[] = [];
|
||||
|
||||
for (const axNode of axNodes) {
|
||||
if (axNode.ignored || typeof axNode.backendDOMNodeId !== "number") continue;
|
||||
if (axNodeInOverlaySubtree(axNode, axById, excludedBackendNodeIds)) continue;
|
||||
|
||||
const capturedNode = capturedByBackendId.get(axNode.backendDOMNodeId);
|
||||
const role = axString(axNode.role);
|
||||
const value = axValue(axNode.value);
|
||||
const name = FORM_CONTROL_TAGS.has(normalizeTag(capturedNode?.tag))
|
||||
? formControlName(capturedNode, axString(axNode.name), value)
|
||||
: axString(axNode.name);
|
||||
const vomNode: VomNode = {
|
||||
id: axNode.backendDOMNodeId,
|
||||
parentId: nearestBackendParent(axNode, axById),
|
||||
tag: normalizeTag(capturedNode?.tag),
|
||||
rect: capturedNode?.rect ?? null,
|
||||
paintOrder: capturedNode?.paintOrder ?? 0,
|
||||
position: capturedNode?.position || "static",
|
||||
pointerEvents: capturedNode?.pointerEvents || "auto",
|
||||
modal: isModalSignal(axNode, capturedNode),
|
||||
sensitive: isSensitive(capturedNode),
|
||||
};
|
||||
if (role) vomNode.role = role;
|
||||
if (name) vomNode.name = name;
|
||||
if (value !== undefined) vomNode.value = value;
|
||||
|
||||
// For link nodes, attach external hostname so the renderer can annotate it.
|
||||
// Relative hrefs and same-origin hrefs are omitted to avoid noise.
|
||||
if (role === "link" && capturedNode?.attrs.href) {
|
||||
try {
|
||||
const target = new URL(capturedNode.attrs.href, window.location.href);
|
||||
if (target.origin !== window.location.origin) {
|
||||
vomNode.href = target.hostname;
|
||||
}
|
||||
} catch {
|
||||
// Relative or non-URL href (e.g. javascript:) — skip
|
||||
}
|
||||
}
|
||||
|
||||
seenBackendIds.add(axNode.backendDOMNodeId);
|
||||
nodes.push(vomNode);
|
||||
}
|
||||
|
||||
for (const capturedNode of captured.nodes) {
|
||||
if (seenBackendIds.has(capturedNode.backendNodeId)) continue;
|
||||
if (excludedBackendNodeIds.has(capturedNode.backendNodeId)) continue;
|
||||
seenBackendIds.add(capturedNode.backendNodeId);
|
||||
nodes.push(vomNodeFromCaptured(capturedNode, capturedNode.parentBackendNodeId));
|
||||
}
|
||||
|
||||
for (const [iframeBackendId, iframeNodes] of captured.iframeNodes) {
|
||||
for (const iframeNode of iframeNodes) {
|
||||
if (
|
||||
seenBackendIds.has(iframeNode.backendNodeId) ||
|
||||
excludedBackendNodeIds.has(iframeNode.backendNodeId) ||
|
||||
!isRenderableIframeControl(iframeNode)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const role = iframeRoleFor(iframeNode);
|
||||
const name = iframeNameFor(iframeNode);
|
||||
if (!role || (!name && normalizeTag(iframeNode.tag) === "a")) continue;
|
||||
|
||||
seenBackendIds.add(iframeNode.backendNodeId);
|
||||
const vomNode: VomNode = {
|
||||
...vomNodeFromCaptured(iframeNode, iframeBackendId),
|
||||
role,
|
||||
};
|
||||
if (name) vomNode.name = name;
|
||||
nodes.push(vomNode);
|
||||
}
|
||||
}
|
||||
|
||||
promoteClickableControls(
|
||||
nodes,
|
||||
capturedByBackendId,
|
||||
captured.nodes,
|
||||
captured.viewport.width,
|
||||
captured.viewport.height,
|
||||
);
|
||||
|
||||
return { viewport: captured.viewport, nodes };
|
||||
}
|
||||
|
||||
export interface SnapshotDeps {
|
||||
@@ -514,6 +807,42 @@ export async function handleGetHtml(
|
||||
}
|
||||
}
|
||||
|
||||
function emptyCapturedViewModel(viewport = { width: 0, height: 0 }): CapturedViewModel {
|
||||
return { viewport, nodes: [], iframeNodes: new Map(), excludedBackendNodeIds: new Set() };
|
||||
}
|
||||
|
||||
interface LayoutMetricsViewportReply {
|
||||
cssLayoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||
layoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||
}
|
||||
|
||||
async function fallbackCapturedViewModel(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
): Promise<CapturedViewModel> {
|
||||
let viewport = { width: 0, height: 0 };
|
||||
try {
|
||||
const metrics = await cdp.send<LayoutMetricsViewportReply>(tabId, "Page.getLayoutMetrics", {});
|
||||
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||
viewport = {
|
||||
width: vpSrc.clientWidth ?? 0,
|
||||
height: vpSrc.clientHeight ?? 0,
|
||||
};
|
||||
} catch {
|
||||
// viewport stays zero-sized
|
||||
}
|
||||
const excludedBackendNodeIds = await collectOverlayExcludedBackendIds(cdp, tabId);
|
||||
return { ...emptyCapturedViewModel(viewport), excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
async function captureForVom(cdp: CdpRunner, tabId: number): Promise<CapturedViewModel> {
|
||||
try {
|
||||
return await captureViewModel(cdp, tabId);
|
||||
} catch {
|
||||
return fallbackCapturedViewModel(cdp, tabId);
|
||||
}
|
||||
}
|
||||
|
||||
export async function handleSnapshot(
|
||||
manager: SessionManager,
|
||||
params: SnapshotParams,
|
||||
@@ -534,11 +863,10 @@ export async function handleSnapshot(
|
||||
"Accessibility.getFullAXTree",
|
||||
{},
|
||||
);
|
||||
const rendered = renderAxTree(result.nodes ?? [], {
|
||||
maxDepth: params.max_depth,
|
||||
maxTokens: params.max_tokens,
|
||||
});
|
||||
// Reset the session-scoped ref-store for this fresh snapshot.
|
||||
const axNodes = result.nodes ?? [];
|
||||
const captured = await captureForVom(deps.cdp, target.tabId);
|
||||
const scene = buildVomScene(axNodes, captured);
|
||||
const rendered = renderVom(scene, { maxDepth: params.max_depth, maxTokens: params.max_tokens });
|
||||
ctx.refStore.replace(
|
||||
rendered.refs.map(
|
||||
(r) => [r.ref, { backendNodeId: r.backendNodeId, tabId: target.tabId }] as const,
|
||||
|
||||
@@ -0,0 +1,630 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "../../../lib/overlay-bridge";
|
||||
import { captureViewModel, collectOverlayExcludedBackendIds } from "../capture";
|
||||
|
||||
// Minimal but format-accurate captureSnapshot reply: a body with one
|
||||
// fixed full-screen overlay div carrying a password input.
|
||||
function fakeSnapshotReply() {
|
||||
// strings table
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
"div",
|
||||
"input",
|
||||
"position",
|
||||
"fixed",
|
||||
"static",
|
||||
"pointer-events",
|
||||
"auto",
|
||||
"type",
|
||||
"password",
|
||||
];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
return {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 2],
|
||||
nodeType: [1, 1, 1, 1],
|
||||
nodeName: [i("html"), i("body"), i("div"), i("input")],
|
||||
backendNodeId: [10, 11, 12, 13],
|
||||
attributes: [[], [], [], [i("type"), i("password")]],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2, 3],
|
||||
// styles columns follow requested computedStyles order: [position, pointer-events]
|
||||
styles: [
|
||||
[i("static"), i("auto")],
|
||||
[i("fixed"), i("auto")],
|
||||
[i("static"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 4000],
|
||||
[0, 0, 1000, 800],
|
||||
[400, 300, 200, 40],
|
||||
],
|
||||
paintOrders: [0, 50, 51],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function makeCdp(snapshot: unknown) {
|
||||
return {
|
||||
send: vi.fn(async (_tab: number, method: string) => {
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
|
||||
}
|
||||
throw new Error(`unexpected ${method}`);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
}
|
||||
|
||||
describe("captureViewModel", () => {
|
||||
it("parses nodes, attrs, rects, paint order and styles", async () => {
|
||||
const { nodes, viewport, iframeNodes } = await captureViewModel(
|
||||
makeCdp(fakeSnapshotReply()),
|
||||
4,
|
||||
);
|
||||
expect(iframeNodes.size).toBe(0);
|
||||
expect(viewport).toEqual({ width: 1000, height: 800 });
|
||||
const div = nodes.find((n) => n.backendNodeId === 12);
|
||||
expect(div).toMatchObject({ tag: "div", position: "fixed", parentBackendNodeId: 11 });
|
||||
expect(div?.rect).toEqual({ x: 0, y: 0, w: 1000, h: 800 });
|
||||
expect(div?.paintOrder).toBe(50);
|
||||
const input = nodes.find((n) => n.backendNodeId === 13);
|
||||
expect(input?.attrs).toEqual({ type: "password" });
|
||||
});
|
||||
|
||||
it("excludes the agent's own overlay shadow host and its inlined shadow subtree", async () => {
|
||||
// DOMSnapshot inlines an open shadow root's content as descendants of the
|
||||
// host. The agent's WXT overlay host carries a fixed full-viewport
|
||||
// click-blocker + the "中断" button. Capturing it makes the snapshot detect
|
||||
// the agent's own mask as a blocking layer and occlude the real page, so the
|
||||
// whole subtree must be dropped.
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
"div",
|
||||
OVERLAY_HOST_NAME,
|
||||
"button",
|
||||
OVERLAY_HOST_MARKER_ATTR,
|
||||
"position",
|
||||
"fixed",
|
||||
"static",
|
||||
"pointer-events",
|
||||
"auto",
|
||||
"cursor",
|
||||
"pointer",
|
||||
];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const snapshot = {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 0, 3, 4],
|
||||
nodeName: [i("html"), i("body"), i("div"), i(OVERLAY_HOST_NAME), i("div"), i("button")],
|
||||
backendNodeId: [10, 11, 12, 13, 14, 15],
|
||||
attributes: [[], [], [], [i(OVERLAY_HOST_MARKER_ATTR), -1], [], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1, 2, 3, 4, 5],
|
||||
styles: [
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("fixed"), i("auto"), i("auto")],
|
||||
[i("fixed"), i("auto"), i("pointer")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 4000],
|
||||
[0, 0, 1000, 800],
|
||||
[10, 10, 200, 40],
|
||||
[0, 0, 0, 0],
|
||||
[0, 0, 1000, 800],
|
||||
[400, 750, 80, 40],
|
||||
],
|
||||
paintOrders: [0, 1, 2, 3, 9, 10],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const { nodes } = await captureViewModel(makeCdp(snapshot), 4);
|
||||
const ids = nodes.map((n) => n.backendNodeId);
|
||||
// Real page nodes survive.
|
||||
expect(ids).toContain(12);
|
||||
// Overlay host + its inlined shadow subtree are gone.
|
||||
expect(ids).not.toContain(13);
|
||||
expect(ids).not.toContain(14);
|
||||
expect(ids).not.toContain(15);
|
||||
});
|
||||
|
||||
it("returns excludedBackendNodeIds for the overlay host subtree", async () => {
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
"div",
|
||||
OVERLAY_HOST_NAME,
|
||||
"button",
|
||||
OVERLAY_HOST_MARKER_ATTR,
|
||||
"position",
|
||||
"fixed",
|
||||
"static",
|
||||
"pointer-events",
|
||||
"auto",
|
||||
"cursor",
|
||||
"pointer",
|
||||
];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const snapshot = {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 0, 3, 4],
|
||||
nodeName: [i("html"), i("body"), i("div"), i(OVERLAY_HOST_NAME), i("div"), i("button")],
|
||||
backendNodeId: [10, 11, 12, 13, 14, 15],
|
||||
attributes: [[], [], [], [i(OVERLAY_HOST_MARKER_ATTR), -1], [], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1, 2, 3, 4, 5],
|
||||
styles: [
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("fixed"), i("auto"), i("auto")],
|
||||
[i("fixed"), i("auto"), i("pointer")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 4000],
|
||||
[0, 0, 1000, 800],
|
||||
[10, 10, 200, 40],
|
||||
[0, 0, 0, 0],
|
||||
[0, 0, 1000, 800],
|
||||
[400, 750, 80, 40],
|
||||
],
|
||||
paintOrders: [0, 1, 2, 3, 9, 10],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const { excludedBackendNodeIds } = await captureViewModel(makeCdp(snapshot), 4);
|
||||
expect(excludedBackendNodeIds).toEqual(new Set([13, 14, 15]));
|
||||
});
|
||||
|
||||
it("excludes overlay host matched by tag name only", async () => {
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
OVERLAY_HOST_NAME,
|
||||
"position",
|
||||
"static",
|
||||
"pointer-events",
|
||||
"auto",
|
||||
"cursor",
|
||||
"auto",
|
||||
];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const snapshot = {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i(OVERLAY_HOST_NAME)],
|
||||
backendNodeId: [10, 11, 13],
|
||||
attributes: [[], [], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2],
|
||||
styles: [
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[0, 0, 100, 40],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const { nodes, excludedBackendNodeIds } = await captureViewModel(makeCdp(snapshot), 4);
|
||||
expect(nodes.map((n) => n.backendNodeId)).not.toContain(13);
|
||||
expect(excludedBackendNodeIds).toEqual(new Set([13]));
|
||||
});
|
||||
|
||||
it("excludes overlay host matched by marker attribute only", async () => {
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
"div",
|
||||
OVERLAY_HOST_MARKER_ATTR,
|
||||
"position",
|
||||
"static",
|
||||
"pointer-events",
|
||||
"auto",
|
||||
"cursor",
|
||||
"auto",
|
||||
];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const snapshot = {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("div")],
|
||||
backendNodeId: [10, 11, 13],
|
||||
attributes: [[], [], [i(OVERLAY_HOST_MARKER_ATTR), -1]],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2],
|
||||
styles: [
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[0, 0, 100, 40],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const { nodes, excludedBackendNodeIds } = await captureViewModel(makeCdp(snapshot), 4);
|
||||
expect(nodes.map((n) => n.backendNodeId)).not.toContain(13);
|
||||
expect(excludedBackendNodeIds).toEqual(new Set([13]));
|
||||
});
|
||||
|
||||
it("parses the computed cursor from the third style column", async () => {
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
"div",
|
||||
"position",
|
||||
"static",
|
||||
"fixed",
|
||||
"pointer-events",
|
||||
"auto",
|
||||
"cursor",
|
||||
"pointer",
|
||||
];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const snapshot = {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("div")],
|
||||
backendNodeId: [10, 11, 12],
|
||||
attributes: [[], [], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2],
|
||||
// styles columns: [position, pointer-events, cursor]
|
||||
styles: [
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("fixed"), i("auto"), i("pointer")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[10, 10, 40, 40],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
const { nodes } = await captureViewModel(makeCdp(snapshot), 4);
|
||||
expect(nodes.find((n) => n.backendNodeId === 12)?.cursor).toBe("pointer");
|
||||
expect(nodes.find((n) => n.backendNodeId === 11)?.cursor).toBe("auto");
|
||||
});
|
||||
|
||||
it("subtracts scroll offset to make rects viewport-relative", async () => {
|
||||
const cdp = {
|
||||
send: vi.fn(async (_t: number, method: string) => {
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return fakeSnapshotReply();
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 200 },
|
||||
};
|
||||
}
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
const { nodes } = await captureViewModel(cdp, 4);
|
||||
// overlay div bounds y=0 minus scroll pageY=200 → y=-200
|
||||
expect(nodes.find((n) => n.backendNodeId === 12)?.rect?.y).toBe(-200);
|
||||
});
|
||||
|
||||
it("normalizes device-pixel bounds by devicePixelRatio", async () => {
|
||||
const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const snapshot = {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("div")],
|
||||
backendNodeId: [10, 11, 12],
|
||||
attributes: [[], [], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2],
|
||||
styles: [
|
||||
[i("static"), i("auto")],
|
||||
[i("fixed"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 2000, 3200],
|
||||
[0, 0, 2000, 1600],
|
||||
],
|
||||
paintOrders: [0, 50],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
const cdp = {
|
||||
send: vi.fn(async (_t: number, method: string) => {
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
layoutViewport: { clientWidth: 2000, clientHeight: 1600 },
|
||||
};
|
||||
}
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
const { nodes } = await captureViewModel(cdp, 4);
|
||||
expect(nodes.find((n) => n.backendNodeId === 12)?.rect).toEqual({
|
||||
x: 0,
|
||||
y: 0,
|
||||
w: 1000,
|
||||
h: 800,
|
||||
});
|
||||
});
|
||||
|
||||
it("extracts textContent from #text child nodes via nodeValue", async () => {
|
||||
// Simulates a button with text "登录" and an <a> with text "忘记密码".
|
||||
// nodeValue is a parallel array: element nodes = -1, text nodes = string index.
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
"button",
|
||||
"a",
|
||||
"position",
|
||||
"fixed",
|
||||
"static",
|
||||
"pointer-events",
|
||||
"auto",
|
||||
"登录",
|
||||
"忘记密码",
|
||||
];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const snapshot = {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
scrollOffsetX: 0,
|
||||
scrollOffsetY: 0,
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 2, 1, 4],
|
||||
nodeName: [i("html"), i("body"), i("button"), i("#text"), i("a"), i("#text")],
|
||||
backendNodeId: [10, 11, 12, 13, 14, 15],
|
||||
attributes: [[], [], [], [], [], []],
|
||||
nodeValue: [-1, -1, -1, i("登录"), -1, i("忘记密码")],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2, 4],
|
||||
styles: [
|
||||
[i("static"), i("auto")],
|
||||
[i("static"), i("auto")],
|
||||
[i("static"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[100, 300, 200, 60],
|
||||
[100, 380, 120, 36],
|
||||
],
|
||||
paintOrders: [0, 10, 11],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
const cdp = {
|
||||
send: vi.fn(async (_t: number, method: string) => {
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
};
|
||||
}
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
const { nodes } = await captureViewModel(cdp, 4);
|
||||
const btn = nodes.find((n) => n.backendNodeId === 12);
|
||||
expect(btn?.textContent).toBe("登录");
|
||||
const link = nodes.find((n) => n.backendNodeId === 14);
|
||||
expect(link?.textContent).toBe("忘记密码");
|
||||
// Nodes without text children should have no textContent
|
||||
const body = nodes.find((n) => n.backendNodeId === 11);
|
||||
expect(body?.textContent).toBeUndefined();
|
||||
});
|
||||
|
||||
it("parses iframe sub-documents and returns iframeNodes keyed by iframe backendNodeId", async () => {
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
"div",
|
||||
"iframe",
|
||||
"input",
|
||||
"position",
|
||||
"fixed",
|
||||
"static",
|
||||
"pointer-events",
|
||||
"auto",
|
||||
"type",
|
||||
"text",
|
||||
"src",
|
||||
"https://passport.jd.com/login",
|
||||
];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const snapshot = {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
scrollOffsetX: 0,
|
||||
scrollOffsetY: 0,
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 2],
|
||||
nodeName: [i("html"), i("body"), i("div"), i("iframe")],
|
||||
backendNodeId: [10, 11, 12, 13],
|
||||
attributes: [[], [], [], [i("src"), i("https://passport.jd.com/login")]],
|
||||
// Sparse: node at array-index 3 (the iframe, backendNodeId=13) → documents[1]
|
||||
contentDocumentIndex: { index: [3], value: [1] },
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2, 3],
|
||||
styles: [
|
||||
[i("static"), i("auto")],
|
||||
[i("fixed"), i("auto")],
|
||||
[i("static"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 4000],
|
||||
[0, 0, 1000, 800],
|
||||
[100, 300, 800, 400],
|
||||
],
|
||||
paintOrders: [0, 50, 51],
|
||||
},
|
||||
},
|
||||
// documents[1]: the iframe's sub-document with a login input
|
||||
{
|
||||
scrollOffsetX: 0,
|
||||
scrollOffsetY: 0,
|
||||
nodes: {
|
||||
parentIndex: [-1, 0],
|
||||
nodeName: [i("body"), i("input")],
|
||||
backendNodeId: [100, 101],
|
||||
attributes: [[], [i("type"), i("text")]],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1],
|
||||
styles: [[i("static"), i("auto")]],
|
||||
bounds: [[0, 0, 200, 40]],
|
||||
paintOrders: [0],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
const cdp = {
|
||||
send: vi.fn(async (_t: number, method: string) => {
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
};
|
||||
}
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
const { nodes, iframeNodes } = await captureViewModel(cdp, 4);
|
||||
// Main frame has 4 nodes; iframe is included
|
||||
expect(nodes.find((n) => n.backendNodeId === 13)?.tag).toBe("iframe");
|
||||
// iframeNodes keyed by the <iframe> element's backendNodeId=13
|
||||
expect(iframeNodes.size).toBe(1);
|
||||
expect(iframeNodes.has(13)).toBe(true);
|
||||
const subNodes = iframeNodes.get(13)!;
|
||||
expect(subNodes.find((n) => n.backendNodeId === 101)?.tag).toBe("input");
|
||||
expect(subNodes.find((n) => n.backendNodeId === 101)?.attrs.type).toBe("text");
|
||||
});
|
||||
|
||||
it("normalizes bounds then subtracts CSS scroll at dpr>1", async () => {
|
||||
const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const snapshot = {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("div")],
|
||||
backendNodeId: [10, 11, 12],
|
||||
attributes: [[], [], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2],
|
||||
styles: [
|
||||
[i("static"), i("auto")],
|
||||
[i("fixed"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 2000, 3200],
|
||||
[0, 400, 2000, 1600],
|
||||
],
|
||||
paintOrders: [0, 50],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
const cdp = {
|
||||
send: vi.fn(async (_t: number, method: string) => {
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 100 },
|
||||
layoutViewport: { clientWidth: 2000, clientHeight: 1600 },
|
||||
};
|
||||
}
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
const { nodes } = await captureViewModel(cdp, 4);
|
||||
expect(nodes.find((n) => n.backendNodeId === 12)?.rect?.y).toBe(400 / 2 - 100);
|
||||
});
|
||||
|
||||
it("collectOverlayExcludedBackendIds walks the pierced overlay host subtree", async () => {
|
||||
const cdp = {
|
||||
send: vi.fn(async (_t: number, method: string) => {
|
||||
if (method === "DOM.getDocument") return { root: { nodeId: 1 } };
|
||||
if (method === "DOM.querySelector") return { nodeId: 99 };
|
||||
if (method === "DOM.describeNode") {
|
||||
return {
|
||||
node: {
|
||||
backendNodeId: 200,
|
||||
shadowRoots: [
|
||||
{ backendNodeId: 201, children: [{ backendNodeId: 202, children: [] }] },
|
||||
],
|
||||
},
|
||||
};
|
||||
}
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
const excluded = await collectOverlayExcludedBackendIds(cdp, 4);
|
||||
expect(excluded).toEqual(new Set([200, 201, 202]));
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,356 @@
|
||||
// CDP capture adapter: DOMSnapshot.captureSnapshot + Page.getLayoutMetrics
|
||||
// → CapturedNode[] + Viewport. This is the ONLY VOM module that touches
|
||||
// raw CDP. The captureSnapshot reply is columnar (parallel arrays + a
|
||||
// shared string table); we request exactly three computed styles so the
|
||||
// `styles` columns are [position, pointer-events, cursor] in that order.
|
||||
|
||||
import type { Rect, Viewport } from "@browser-skill/vom";
|
||||
import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
|
||||
import type { CdpRunner } from "../shared";
|
||||
|
||||
const REQUESTED_STYLES = ["position", "pointer-events", "cursor"] as const;
|
||||
const STYLE_COL = Object.fromEntries(
|
||||
REQUESTED_STYLES.map((name, index) => [name, index]),
|
||||
) as Record<(typeof REQUESTED_STYLES)[number], number>;
|
||||
|
||||
export interface CapturedNode {
|
||||
backendNodeId: number;
|
||||
parentBackendNodeId: number | null;
|
||||
tag: string;
|
||||
attrs: Record<string, string>;
|
||||
rect: Rect | null;
|
||||
paintOrder: number;
|
||||
position: string;
|
||||
pointerEvents: string;
|
||||
/**
|
||||
* computed `cursor`. `cursor: pointer` is the strongest CDP-free signal
|
||||
* that a non-semantic element (a `<div>`/`<span>` with a click handler)
|
||||
* is actually an interactive control — used by the adapter to surface
|
||||
* custom buttons/checkboxes the AX tree drops as `generic`. Optional like
|
||||
* `textContent`: the live parser always sets it, hand-built fixtures may not.
|
||||
*/
|
||||
cursor?: string;
|
||||
textContent?: string;
|
||||
}
|
||||
|
||||
export type CapturedIframeNodes = Map<number, CapturedNode[]>;
|
||||
|
||||
export interface CapturedViewModel {
|
||||
nodes: CapturedNode[];
|
||||
viewport: Viewport;
|
||||
iframeNodes: CapturedIframeNodes;
|
||||
/** Backend node ids belonging to the agent overlay host + its shadow subtree. */
|
||||
excludedBackendNodeIds: Set<number>;
|
||||
}
|
||||
|
||||
/** Sparse array format Chrome uses for infrequently-set per-node fields. */
|
||||
interface SparseArray {
|
||||
index: number[];
|
||||
value: number[];
|
||||
}
|
||||
|
||||
interface SnapshotDocument {
|
||||
scrollOffsetX?: number;
|
||||
scrollOffsetY?: number;
|
||||
nodes?: {
|
||||
parentIndex?: number[];
|
||||
nodeName?: number[];
|
||||
backendNodeId?: number[];
|
||||
attributes?: number[][];
|
||||
/**
|
||||
* Per-node text value (index into strings), set for `#text` / CDATA nodes.
|
||||
* Element nodes carry -1. Same length as `backendNodeId`.
|
||||
*/
|
||||
nodeValue?: number[];
|
||||
/** Maps node array index → index into `documents[]` for frame content. */
|
||||
contentDocumentIndex?: SparseArray;
|
||||
};
|
||||
layout?: {
|
||||
nodeIndex?: number[];
|
||||
styles?: number[][];
|
||||
bounds?: number[][];
|
||||
paintOrders?: number[];
|
||||
};
|
||||
}
|
||||
|
||||
interface SnapshotReply {
|
||||
strings?: string[];
|
||||
documents?: SnapshotDocument[];
|
||||
}
|
||||
|
||||
interface LayoutMetricsReply {
|
||||
cssLayoutViewport?: {
|
||||
clientWidth?: number;
|
||||
clientHeight?: number;
|
||||
pageX?: number;
|
||||
pageY?: number;
|
||||
};
|
||||
layoutViewport?: { clientWidth?: number; clientHeight?: number; pageX?: number; pageY?: number };
|
||||
}
|
||||
|
||||
interface CdpDomNode {
|
||||
backendNodeId?: number;
|
||||
children?: CdpDomNode[];
|
||||
shadowRoots?: CdpDomNode[];
|
||||
}
|
||||
|
||||
interface ParseDocumentResult {
|
||||
nodes: CapturedNode[];
|
||||
excludedBackendNodeIds: Set<number>;
|
||||
}
|
||||
|
||||
function str(strings: string[], idx: number | undefined): string {
|
||||
if (idx === undefined || idx < 0) return "";
|
||||
return strings[idx] ?? "";
|
||||
}
|
||||
|
||||
function devicePixelRatio(metrics: LayoutMetricsReply): number {
|
||||
const layoutW = metrics.layoutViewport?.clientWidth ?? 0;
|
||||
const cssW = metrics.cssLayoutViewport?.clientWidth ?? 0;
|
||||
if (!layoutW || !cssW) return 1;
|
||||
const dpr = layoutW / cssW;
|
||||
if (!Number.isFinite(dpr) || dpr <= 0) return 1;
|
||||
return dpr >= 1 ? dpr : 1;
|
||||
}
|
||||
|
||||
function collectBackendIdsFromDomNode(node: CdpDomNode | undefined, out: Set<number>): void {
|
||||
if (!node) return;
|
||||
if (typeof node.backendNodeId === "number") {
|
||||
out.add(node.backendNodeId);
|
||||
}
|
||||
for (const child of node.children ?? []) {
|
||||
collectBackendIdsFromDomNode(child, out);
|
||||
}
|
||||
for (const shadow of node.shadowRoots ?? []) {
|
||||
collectBackendIdsFromDomNode(shadow, out);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* When DOMSnapshot is unavailable, locate the marked overlay host via CDP and
|
||||
* collect every backendNodeId in its pierced subtree (open shadow included).
|
||||
*/
|
||||
export async function collectOverlayExcludedBackendIds(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
): Promise<Set<number>> {
|
||||
const excluded = new Set<number>();
|
||||
try {
|
||||
const doc = await cdp.send<{ root?: { nodeId?: number } }>(tabId, "DOM.getDocument", {
|
||||
depth: 0,
|
||||
pierce: true,
|
||||
});
|
||||
const rootNodeId = doc.root?.nodeId;
|
||||
if (typeof rootNodeId !== "number") return excluded;
|
||||
|
||||
const found = await cdp.send<{ nodeId?: number }>(tabId, "DOM.querySelector", {
|
||||
nodeId: rootNodeId,
|
||||
selector: OVERLAY_HOST_SELECTOR,
|
||||
});
|
||||
if (typeof found.nodeId !== "number" || found.nodeId === 0) return excluded;
|
||||
|
||||
const described = await cdp.send<{ node?: CdpDomNode }>(tabId, "DOM.describeNode", {
|
||||
nodeId: found.nodeId,
|
||||
depth: -1,
|
||||
pierce: true,
|
||||
});
|
||||
collectBackendIdsFromDomNode(described.node, excluded);
|
||||
} catch (err) {
|
||||
console.debug("[bsk capture] overlay exclusion fallback failed", err);
|
||||
}
|
||||
return excluded;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse one DOMSnapshot document into CapturedNode[].
|
||||
*
|
||||
* @param doc - the raw DOMSnapshot document object
|
||||
* @param strings - the shared string table for the whole snapshot
|
||||
* @param dpr - device-pixel-ratio from Page.getLayoutMetrics
|
||||
* @param scrollX - horizontal scroll offset to subtract (CSS px)
|
||||
* @param scrollY - vertical scroll offset to subtract (CSS px)
|
||||
*/
|
||||
function parseDocumentNodes(
|
||||
doc: SnapshotDocument,
|
||||
strings: string[],
|
||||
dpr: number,
|
||||
scrollX: number,
|
||||
scrollY: number,
|
||||
): ParseDocumentResult {
|
||||
const dn = doc.nodes;
|
||||
const dl = doc.layout;
|
||||
if (!dn?.backendNodeId) {
|
||||
return { nodes: [], excludedBackendNodeIds: new Set() };
|
||||
}
|
||||
|
||||
const count = dn.backendNodeId.length;
|
||||
const layoutByNode = new Map<number, number>();
|
||||
(dl?.nodeIndex ?? []).forEach((nodeIdx, layoutIdx) => layoutByNode.set(nodeIdx, layoutIdx));
|
||||
|
||||
// The agent's own overlay (WXT shadow host, marked with OVERLAY_HOST_MARKER_ATTR)
|
||||
// is injected into the page and its open shadow root
|
||||
// is inlined by DOMSnapshot as descendants of the host. Its fixed
|
||||
// full-viewport click-blocker would otherwise dominate occlusion detection
|
||||
// and hide the real page, so mark the host + its whole subtree excluded.
|
||||
// parentIndex always references an earlier node, so one forward pass suffices.
|
||||
const excluded = new Array<boolean>(count).fill(false);
|
||||
const excludedBackendNodeIds = new Set<number>();
|
||||
for (let n = 0; n < count; n++) {
|
||||
const parentIdx = dn.parentIndex?.[n] ?? -1;
|
||||
const inherited = parentIdx >= 0 && excluded[parentIdx];
|
||||
let isOverlayHost = inherited;
|
||||
if (!isOverlayHost) {
|
||||
const pairs = dn.attributes?.[n] ?? [];
|
||||
const attrNames: string[] = [];
|
||||
for (let a = 0; a + 1 < pairs.length; a += 2) {
|
||||
attrNames.push(str(strings, pairs[a]));
|
||||
}
|
||||
isOverlayHost = isOverlayHostNode(str(strings, dn.nodeName?.[n]), attrNames);
|
||||
}
|
||||
if (!isOverlayHost) continue;
|
||||
excluded[n] = true;
|
||||
const backendNodeId = dn.backendNodeId[n];
|
||||
if (backendNodeId !== undefined) {
|
||||
excludedBackendNodeIds.add(backendNodeId);
|
||||
}
|
||||
}
|
||||
|
||||
const posCol = STYLE_COL.position;
|
||||
const peCol = STYLE_COL["pointer-events"];
|
||||
const cursorCol = STYLE_COL.cursor;
|
||||
|
||||
// Collect visible text from #text child nodes so element CapturedNodes
|
||||
// carry a textContent value usable as a button/link label fallback.
|
||||
// nodeValue is a parallel array: string index for text nodes, -1 otherwise.
|
||||
const nodeTextContent = new Map<number, string>();
|
||||
if (dn.nodeValue) {
|
||||
for (let n = 0; n < count; n++) {
|
||||
const nvIdx = dn.nodeValue[n] ?? -1;
|
||||
if (nvIdx < 0) continue;
|
||||
const text = str(strings, nvIdx).trim();
|
||||
if (!text) continue;
|
||||
const parentIdx = dn.parentIndex?.[n] ?? -1;
|
||||
if (parentIdx >= 0) {
|
||||
const existing = nodeTextContent.get(parentIdx);
|
||||
nodeTextContent.set(parentIdx, existing ? `${existing} ${text}` : text);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Build CapturedNodes.
|
||||
const nodes: CapturedNode[] = [];
|
||||
for (let n = 0; n < count; n++) {
|
||||
if (excluded[n]) continue;
|
||||
const backendNodeId = dn.backendNodeId[n];
|
||||
const parentIdx = dn.parentIndex?.[n] ?? -1;
|
||||
const parentBackendNodeId = parentIdx >= 0 ? (dn.backendNodeId[parentIdx] ?? null) : null;
|
||||
const tag = str(strings, dn.nodeName?.[n]).toLowerCase();
|
||||
|
||||
const attrs: Record<string, string> = {};
|
||||
const pairs = dn.attributes?.[n] ?? [];
|
||||
for (let a = 0; a + 1 < pairs.length; a += 2) {
|
||||
attrs[str(strings, pairs[a]).toLowerCase()] = str(strings, pairs[a + 1]);
|
||||
}
|
||||
|
||||
let rect: Rect | null = null;
|
||||
let paintOrder = 0;
|
||||
let position = "static";
|
||||
let pointerEvents = "auto";
|
||||
let cursor = "auto";
|
||||
const li = layoutByNode.get(n);
|
||||
if (li !== undefined) {
|
||||
const b = dl?.bounds?.[li];
|
||||
if (b && b.length >= 4 && b[2] > 0 && b[3] > 0) {
|
||||
rect = {
|
||||
x: b[0] / dpr - scrollX,
|
||||
y: b[1] / dpr - scrollY,
|
||||
w: b[2] / dpr,
|
||||
h: b[3] / dpr,
|
||||
};
|
||||
}
|
||||
paintOrder = dl?.paintOrders?.[li] ?? 0;
|
||||
const styleRow = dl?.styles?.[li] ?? [];
|
||||
position = str(strings, styleRow[posCol]) || "static";
|
||||
pointerEvents = str(strings, styleRow[peCol]) || "auto";
|
||||
cursor = str(strings, styleRow[cursorCol]) || "auto";
|
||||
}
|
||||
|
||||
// Skip non-element nodes (#text, #cdata-section, etc.) — they carry no
|
||||
// geometry and are never queried by callers.
|
||||
if (tag.startsWith("#")) continue;
|
||||
|
||||
const textContent = nodeTextContent.get(n);
|
||||
|
||||
nodes.push({
|
||||
backendNodeId,
|
||||
parentBackendNodeId,
|
||||
tag,
|
||||
attrs,
|
||||
rect,
|
||||
paintOrder,
|
||||
position,
|
||||
pointerEvents,
|
||||
cursor,
|
||||
textContent,
|
||||
});
|
||||
}
|
||||
return { nodes, excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
export async function captureViewModel(cdp: CdpRunner, tabId: number): Promise<CapturedViewModel> {
|
||||
const metrics = await cdp.send<LayoutMetricsReply>(tabId, "Page.getLayoutMetrics", {});
|
||||
const dpr = devicePixelRatio(metrics);
|
||||
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||
const viewport: Viewport = {
|
||||
width: vpSrc.clientWidth ?? 0,
|
||||
height: vpSrc.clientHeight ?? 0,
|
||||
};
|
||||
const scrollX = vpSrc.pageX ?? 0;
|
||||
const scrollY = vpSrc.pageY ?? 0;
|
||||
|
||||
await cdp.send(tabId, "DOMSnapshot.enable", {});
|
||||
const snap = await cdp.send<SnapshotReply>(tabId, "DOMSnapshot.captureSnapshot", {
|
||||
computedStyles: REQUESTED_STYLES,
|
||||
includePaintOrder: true,
|
||||
includeDOMRects: true,
|
||||
});
|
||||
|
||||
const strings = snap.strings ?? [];
|
||||
const documents = snap.documents ?? [];
|
||||
const doc0 = documents[0];
|
||||
if (!doc0?.nodes?.backendNodeId) {
|
||||
return { nodes: [], viewport, iframeNodes: new Map(), excludedBackendNodeIds: new Set() };
|
||||
}
|
||||
|
||||
const mainParsed = parseDocumentNodes(doc0, strings, dpr, scrollX, scrollY);
|
||||
const nodes = mainParsed.nodes;
|
||||
const excludedBackendNodeIds = new Set(mainParsed.excludedBackendNodeIds);
|
||||
|
||||
// Build a map from node-array-index → sub-document-index using the sparse
|
||||
// contentDocumentIndex field. Chrome emits this only for <iframe> nodes.
|
||||
const iframeNodes: CapturedIframeNodes = new Map();
|
||||
if (documents.length > 1) {
|
||||
const cdi = doc0.nodes?.contentDocumentIndex;
|
||||
if (cdi?.index && cdi.value) {
|
||||
for (let k = 0; k < cdi.index.length; k++) {
|
||||
const nodeArrayIdx = cdi.index[k];
|
||||
const docIdx = cdi.value[k];
|
||||
const subDoc = documents[docIdx];
|
||||
if (!subDoc) continue;
|
||||
const iframeBid = doc0.nodes?.backendNodeId?.[nodeArrayIdx];
|
||||
if (iframeBid === undefined) continue;
|
||||
// Sub-documents have their own scroll offsets; bounds are relative
|
||||
// to the iframe's own coordinate space (not the main viewport).
|
||||
const subScrollX = subDoc.scrollOffsetX ?? 0;
|
||||
const subScrollY = subDoc.scrollOffsetY ?? 0;
|
||||
const subParsed = parseDocumentNodes(subDoc, strings, dpr, subScrollX, subScrollY);
|
||||
iframeNodes.set(iframeBid, subParsed.nodes);
|
||||
for (const id of subParsed.excludedBackendNodeIds) {
|
||||
excludedBackendNodeIds.add(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { nodes, viewport, iframeNodes, excludedBackendNodeIds };
|
||||
}
|
||||
@@ -8,6 +8,7 @@
|
||||
"ext:build:zip": "pnpm --filter @browser-skill/extension zip",
|
||||
"ext:dev": "pnpm --filter @browser-skill/extension dev",
|
||||
"ext:test": "pnpm --filter @browser-skill/extension test",
|
||||
"vom:test": "pnpm --filter @browser-skill/vom test",
|
||||
"lint": "biome check . && stylelint \"**/*.css\"",
|
||||
"format": "biome format --write ."
|
||||
},
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"name": "@browser-skill/vom",
|
||||
"private": true,
|
||||
"version": "0.1.0",
|
||||
"type": "module",
|
||||
"exports": {
|
||||
".": {
|
||||
"import": "./src/index.ts",
|
||||
"default": "./src/index.ts"
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
"test": "vitest run",
|
||||
"test:watch": "vitest"
|
||||
},
|
||||
"devDependencies": {
|
||||
"typescript": "^5.8.0",
|
||||
"vitest": "^4.1.6"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { coverage, detectBlockingLayer } from "../layers";
|
||||
import type { Viewport, VomNode } from "../types";
|
||||
|
||||
const VP: Viewport = { width: 1000, height: 800 };
|
||||
|
||||
function node(p: Partial<VomNode> & { id: number }): VomNode {
|
||||
const { id, parentId, tag, rect, paintOrder, position, pointerEvents, ...rest } = p;
|
||||
|
||||
return {
|
||||
id,
|
||||
parentId: parentId ?? null,
|
||||
tag: tag ?? "div",
|
||||
rect: rect ?? null,
|
||||
paintOrder: paintOrder ?? 0,
|
||||
position: position ?? "static",
|
||||
pointerEvents: pointerEvents ?? "auto",
|
||||
...rest,
|
||||
};
|
||||
}
|
||||
|
||||
describe("coverage", () => {
|
||||
it("returns the clamped viewport-overlap fraction", () => {
|
||||
expect(coverage({ x: 0, y: 0, w: 1000, h: 800 }, VP)).toBeCloseTo(1);
|
||||
expect(coverage({ x: 0, y: 0, w: 500, h: 800 }, VP)).toBeCloseTo(0.5);
|
||||
expect(coverage({ x: -100, y: 0, w: 200, h: 800 }, VP)).toBeCloseTo(0.1);
|
||||
});
|
||||
|
||||
it("returns 0 for null rect, invalid viewport, or off-viewport rect", () => {
|
||||
expect(coverage(null, VP)).toBe(0);
|
||||
expect(coverage({ x: 0, y: 0, w: 100, h: 100 }, { width: 0, height: 800 })).toBe(0);
|
||||
expect(coverage({ x: 2000, y: 0, w: 100, h: 100 }, VP)).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("detectBlockingLayer", () => {
|
||||
it("detects a CSS blocker and includes the whole top paint-order band", () => {
|
||||
const layer = detectBlockingLayer(
|
||||
[
|
||||
node({ id: 1, tag: "body", rect: { x: 0, y: 0, w: 1000, h: 4000 }, paintOrder: 0 }),
|
||||
node({
|
||||
id: 2,
|
||||
parentId: 1,
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 30,
|
||||
position: "fixed",
|
||||
}),
|
||||
node({
|
||||
id: 3,
|
||||
parentId: 1,
|
||||
role: "dialog",
|
||||
name: "提示",
|
||||
tag: "dialog",
|
||||
rect: { x: 300, y: 250, w: 400, h: 300 },
|
||||
paintOrder: 40,
|
||||
position: "fixed",
|
||||
}),
|
||||
node({
|
||||
id: 4,
|
||||
parentId: 3,
|
||||
role: "button",
|
||||
name: "关闭",
|
||||
tag: "button",
|
||||
rect: { x: 640, y: 260, w: 40, h: 40 },
|
||||
paintOrder: 41,
|
||||
}),
|
||||
],
|
||||
VP,
|
||||
);
|
||||
|
||||
expect(layer).not.toBeNull();
|
||||
expect(layer?.rootId).toBe(2);
|
||||
expect(layer?.kind).toBe("modal");
|
||||
expect(layer?.coverage).toBeCloseTo(1);
|
||||
expect([...(layer?.members ?? [])].sort((a, b) => a - b)).toEqual([2, 3, 4]);
|
||||
});
|
||||
|
||||
it("classifies a near-full blocker without modal signals or controls as a mask", () => {
|
||||
const layer = detectBlockingLayer(
|
||||
[
|
||||
node({ id: 1, tag: "body", rect: { x: 0, y: 0, w: 1000, h: 800 } }),
|
||||
node({
|
||||
id: 2,
|
||||
parentId: 1,
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 10,
|
||||
position: "fixed",
|
||||
}),
|
||||
],
|
||||
VP,
|
||||
);
|
||||
|
||||
expect(layer?.kind).toBe("mask");
|
||||
});
|
||||
|
||||
it("treats explicit modal nodes as modal even below the generic coverage threshold", () => {
|
||||
const layer = detectBlockingLayer(
|
||||
[
|
||||
node({ id: 1, tag: "body", rect: { x: 0, y: 0, w: 1000, h: 800 } }),
|
||||
node({
|
||||
id: 2,
|
||||
parentId: 1,
|
||||
tag: "dialog",
|
||||
role: "dialog",
|
||||
modal: true,
|
||||
rect: { x: 300, y: 200, w: 400, h: 300 },
|
||||
paintOrder: 90,
|
||||
position: "fixed",
|
||||
}),
|
||||
],
|
||||
VP,
|
||||
);
|
||||
|
||||
expect(layer?.rootId).toBe(2);
|
||||
expect(layer?.kind).toBe("modal");
|
||||
});
|
||||
|
||||
it("ignores small toasts and pointer-events:none covers", () => {
|
||||
expect(
|
||||
detectBlockingLayer(
|
||||
[
|
||||
node({ id: 1, tag: "body", rect: { x: 0, y: 0, w: 1000, h: 800 } }),
|
||||
node({
|
||||
id: 2,
|
||||
parentId: 1,
|
||||
rect: { x: 800, y: 720, w: 180, h: 60 },
|
||||
paintOrder: 99,
|
||||
position: "fixed",
|
||||
}),
|
||||
],
|
||||
VP,
|
||||
),
|
||||
).toBeNull();
|
||||
|
||||
expect(
|
||||
detectBlockingLayer(
|
||||
[
|
||||
node({ id: 1, tag: "body", rect: { x: 0, y: 0, w: 1000, h: 800 } }),
|
||||
node({
|
||||
id: 2,
|
||||
parentId: 1,
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 40,
|
||||
position: "fixed",
|
||||
pointerEvents: "none",
|
||||
}),
|
||||
],
|
||||
VP,
|
||||
),
|
||||
).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,304 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { renderVom } from "../render";
|
||||
import type { VomNode, VomScene } from "../types";
|
||||
|
||||
function node(p: Partial<VomNode> & { id: number }): VomNode {
|
||||
const { id, parentId, tag, rect, paintOrder, position, pointerEvents, ...rest } = p;
|
||||
|
||||
return {
|
||||
id,
|
||||
parentId: parentId ?? null,
|
||||
tag: tag ?? "div",
|
||||
rect: rect ?? null,
|
||||
paintOrder: paintOrder ?? 0,
|
||||
position: position ?? "static",
|
||||
pointerEvents: pointerEvents ?? "auto",
|
||||
...rest,
|
||||
};
|
||||
}
|
||||
|
||||
function scene(nodes: VomNode[]): VomScene {
|
||||
return {
|
||||
viewport: { width: 1280, height: 720 },
|
||||
nodes,
|
||||
};
|
||||
}
|
||||
|
||||
describe("renderVom single-layer page", () => {
|
||||
it("always emits a complete @vom single-layer document when there is no blocker", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
node({ id: 1, role: "RootWebArea", name: "Shop" }),
|
||||
node({ id: 2, parentId: 1, role: "navigation", name: "主菜单" }),
|
||||
node({ id: 3, parentId: 2, role: "link", name: "首页", tag: "a" }),
|
||||
node({ id: 4, parentId: 1, role: "main" }),
|
||||
node({ id: 5, parentId: 4, role: "heading", name: "今日推荐" }),
|
||||
node({ id: 6, parentId: 4, role: "button", name: "加入购物车", tag: "button" }),
|
||||
]),
|
||||
);
|
||||
|
||||
expect(out.text).toBe(
|
||||
[
|
||||
"@vom 1",
|
||||
"@view 1280x720",
|
||||
"@layers 1 focus=L1",
|
||||
"L1 page",
|
||||
' RootWebArea "Shop"',
|
||||
' navigation "主菜单"',
|
||||
' @e1 link "首页"',
|
||||
" main",
|
||||
' heading "今日推荐"',
|
||||
' @e2 button "加入购物车"',
|
||||
].join("\n"),
|
||||
);
|
||||
expect(out.refs).toEqual([
|
||||
{ ref: "e1", backendNodeId: 3 },
|
||||
{ ref: "e2", backendNodeId: 6 },
|
||||
]);
|
||||
expect(out.truncated).toBe(false);
|
||||
});
|
||||
|
||||
it("does not assign refs to structural or display-only nodes", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
node({ id: 1, role: "RootWebArea", name: "Doc" }),
|
||||
node({ id: 2, parentId: 1, role: "heading", name: "Title" }),
|
||||
node({ id: 3, parentId: 1, role: "img", name: "Logo" }),
|
||||
node({ id: 4, parentId: 1, role: "button", name: "Continue", tag: "button" }),
|
||||
]),
|
||||
);
|
||||
|
||||
expect(out.text).toContain(' RootWebArea "Doc"');
|
||||
expect(out.text).toContain(' heading "Title"');
|
||||
expect(out.text).toContain(' img "Logo"');
|
||||
expect(out.text).toContain(' @e1 button "Continue"');
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]);
|
||||
});
|
||||
|
||||
it("assigns refs to listbox controls", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
node({ id: 1, role: "RootWebArea", name: "Doc" }),
|
||||
node({ id: 2, parentId: 1, role: "listbox", name: "Choices" }),
|
||||
]),
|
||||
);
|
||||
|
||||
expect(out.text).toContain(' @e1 listbox "Choices"');
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||
});
|
||||
|
||||
it("renders additional structural roles without assigning refs", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
node({ id: 1, role: "WebArea" }),
|
||||
node({ id: 2, parentId: 1, role: "Iframe" }),
|
||||
node({ id: 3, parentId: 1, role: "alertdialog" }),
|
||||
node({ id: 4, parentId: 3, role: "section" }),
|
||||
node({ id: 5, parentId: 4, role: "button", name: "Close", tag: "button" }),
|
||||
]),
|
||||
);
|
||||
|
||||
expect(out.text).toContain(" WebArea");
|
||||
expect(out.text).toContain(" Iframe");
|
||||
expect(out.text).toContain(" alertdialog");
|
||||
expect(out.text).toContain(" section");
|
||||
expect(out.text).toContain(' @e1 button "Close"');
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 5 }]);
|
||||
});
|
||||
|
||||
it("skips generic nodes without adding an extra indentation level", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
node({ id: 1, role: "RootWebArea", name: "Doc" }),
|
||||
node({ id: 2, parentId: 1, role: "generic" }),
|
||||
node({ id: 3, parentId: 2, role: "button", name: "OK", tag: "button" }),
|
||||
]),
|
||||
);
|
||||
|
||||
expect(out.text).not.toContain("generic");
|
||||
expect(out.text).toContain(' @e1 button "OK"');
|
||||
});
|
||||
|
||||
it("masks sensitive values and never emits the clear text value", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
node({ id: 1, role: "RootWebArea", name: "Login" }),
|
||||
node({
|
||||
id: 2,
|
||||
parentId: 1,
|
||||
role: "textbox",
|
||||
name: "密码",
|
||||
value: "hunter2",
|
||||
tag: "input",
|
||||
sensitive: true,
|
||||
}),
|
||||
]),
|
||||
);
|
||||
|
||||
expect(out.text).toContain('@e1 textbox "密码" ="•••"');
|
||||
expect(out.text).not.toContain("hunter2");
|
||||
});
|
||||
|
||||
it("masks sensitive textboxes even when the value is missing", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
node({ id: 1, role: "RootWebArea", name: "Login" }),
|
||||
node({
|
||||
id: 2,
|
||||
parentId: 1,
|
||||
role: "textbox",
|
||||
name: "密码",
|
||||
tag: "input",
|
||||
sensitive: true,
|
||||
}),
|
||||
]),
|
||||
);
|
||||
|
||||
expect(out.text).toContain('@e1 textbox "密码" ="•••"');
|
||||
});
|
||||
|
||||
it("continues rendering later siblings after a branch exceeds maxDepth", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
node({ id: 1, role: "RootWebArea", name: "Doc" }),
|
||||
node({ id: 2, parentId: 1, role: "section" }),
|
||||
node({ id: 3, parentId: 2, role: "heading", name: "Too deep" }),
|
||||
node({ id: 4, parentId: 1, role: "button", name: "Later", tag: "button" }),
|
||||
]),
|
||||
{ maxDepth: 2 },
|
||||
);
|
||||
|
||||
expect(out.truncated).toBe(true);
|
||||
expect(out.text).toContain(' RootWebArea "Doc"');
|
||||
expect(out.text).toContain(" section");
|
||||
expect(out.text).not.toContain("Too deep");
|
||||
expect(out.text).toContain(' @e1 button "Later"');
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]);
|
||||
});
|
||||
|
||||
it("truncates when maxTokens is exceeded while keeping refs in sync with rendered text", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
node({ id: 1, role: "RootWebArea", name: "Doc" }),
|
||||
node({ id: 2, parentId: 1, role: "button", name: "First", tag: "button" }),
|
||||
node({ id: 3, parentId: 1, role: "button", name: "Second", tag: "button" }),
|
||||
]),
|
||||
{ maxTokens: 22 },
|
||||
);
|
||||
|
||||
expect(out.truncated).toBe(true);
|
||||
const renderedRefs = new Set(Array.from(out.text.matchAll(/@(e\d+)/g), (m) => m[1]));
|
||||
expect(out.refs.every((r) => renderedRefs.has(r.ref))).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("renderVom double-layer page", () => {
|
||||
it("renders a blocking modal first and folds the base page into an occluded summary", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
node({
|
||||
id: 1,
|
||||
role: "RootWebArea",
|
||||
name: "Shop",
|
||||
tag: "body",
|
||||
rect: { x: 0, y: 0, w: 1000, h: 3000 },
|
||||
}),
|
||||
node({ id: 10, parentId: 1, role: "button", name: "底层按钮", tag: "button" }),
|
||||
node({
|
||||
id: 2,
|
||||
parentId: 1,
|
||||
tag: "div",
|
||||
rect: { x: 0, y: 0, w: 1280, h: 720 },
|
||||
paintOrder: 50,
|
||||
position: "fixed",
|
||||
pointerEvents: "auto",
|
||||
}),
|
||||
node({
|
||||
id: 3,
|
||||
parentId: 1,
|
||||
role: "dialog",
|
||||
name: "提示",
|
||||
modal: true,
|
||||
rect: { x: 400, y: 200, w: 480, h: 320 },
|
||||
paintOrder: 60,
|
||||
position: "fixed",
|
||||
}),
|
||||
node({
|
||||
id: 4,
|
||||
parentId: 3,
|
||||
role: "textbox",
|
||||
name: "密码",
|
||||
value: "secret",
|
||||
tag: "input",
|
||||
sensitive: true,
|
||||
}),
|
||||
node({ id: 5, parentId: 3, role: "button", name: "确认", tag: "button" }),
|
||||
]),
|
||||
);
|
||||
|
||||
expect(out.text).toBe(
|
||||
[
|
||||
"@vom 1",
|
||||
"@view 1280x720",
|
||||
"@layers 2 focus=L1",
|
||||
"L1 modal cover=100%",
|
||||
' dialog "提示"',
|
||||
' @e1 textbox "密码" ="•••"',
|
||||
' @e2 button "确认"',
|
||||
"L2 page … occluded by L1 (~2 nodes, not actionable)",
|
||||
].join("\n"),
|
||||
);
|
||||
expect(out.text).not.toContain("底层按钮");
|
||||
expect(out.refs).toEqual([
|
||||
{ ref: "e1", backendNodeId: 4 },
|
||||
{ ref: "e2", backendNodeId: 5 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("renders iframe descendants inside the top layer through parentId without iframe-specific renderer code", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
node({
|
||||
id: 1,
|
||||
role: "RootWebArea",
|
||||
name: "JD",
|
||||
tag: "body",
|
||||
rect: { x: 0, y: 0, w: 1000, h: 3000 },
|
||||
}),
|
||||
node({
|
||||
id: 2,
|
||||
parentId: 1,
|
||||
tag: "div",
|
||||
rect: { x: 0, y: 0, w: 1280, h: 720 },
|
||||
paintOrder: 50,
|
||||
position: "fixed",
|
||||
}),
|
||||
node({
|
||||
id: 3,
|
||||
parentId: 2,
|
||||
role: "Iframe",
|
||||
name: "登录框",
|
||||
tag: "iframe",
|
||||
rect: { x: 100, y: 240, w: 800, h: 400 },
|
||||
paintOrder: 51,
|
||||
}),
|
||||
node({ id: 101, parentId: 3, role: "textbox", name: "请输入手机号", tag: "input" }),
|
||||
node({
|
||||
id: 102,
|
||||
parentId: 3,
|
||||
role: "textbox",
|
||||
name: "密码",
|
||||
value: "secret",
|
||||
tag: "input",
|
||||
sensitive: true,
|
||||
}),
|
||||
]),
|
||||
);
|
||||
|
||||
expect(out.text).toContain("L1 modal cover=100%");
|
||||
expect(out.text).toContain(' Iframe "登录框"');
|
||||
expect(out.text).toContain(' @e1 textbox "请输入手机号"');
|
||||
expect(out.text).toContain(' @e2 textbox "密码" ="•••"');
|
||||
expect(out.refs.map((r) => r.backendNodeId)).toEqual([101, 102]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,11 @@
|
||||
export { renderVom } from "./render";
|
||||
export type {
|
||||
BlockingLayer,
|
||||
LayerKind,
|
||||
Rect,
|
||||
Viewport,
|
||||
VomNode,
|
||||
VomOptions,
|
||||
VomResult,
|
||||
VomScene,
|
||||
} from "./types";
|
||||
@@ -0,0 +1,79 @@
|
||||
import type { BlockingLayer, Rect, Viewport, VomNode } from "./types";
|
||||
|
||||
/** Default fraction of the viewport an overlay must cover to block. */
|
||||
export const BLOCK_COVERAGE_THRESHOLD = 0.6;
|
||||
/** At/above this coverage with no inputs we call it an opaque mask. */
|
||||
export const MASK_COVERAGE_THRESHOLD = 0.9;
|
||||
|
||||
const POSITIONED = new Set(["fixed", "absolute", "sticky"]);
|
||||
const FORM_TAGS = new Set(["input", "textarea", "select"]);
|
||||
|
||||
export function coverage(rect: Rect | null, vp: Viewport): number {
|
||||
if (!rect || vp.width <= 0 || vp.height <= 0) return 0;
|
||||
const ix = Math.max(0, Math.min(rect.x + rect.w, vp.width) - Math.max(rect.x, 0));
|
||||
const iy = Math.max(0, Math.min(rect.y + rect.h, vp.height) - Math.max(rect.y, 0));
|
||||
const overlap = ix * iy;
|
||||
if (overlap <= 0) return 0;
|
||||
return Math.min(1, overlap / (vp.width * vp.height));
|
||||
}
|
||||
|
||||
function isBlockingCandidate(node: VomNode): boolean {
|
||||
if (node.pointerEvents === "none") return false;
|
||||
return POSITIONED.has(node.position);
|
||||
}
|
||||
|
||||
function hasModalFeature(node: VomNode): boolean {
|
||||
return (
|
||||
node.modal === true ||
|
||||
node.tag === "dialog" ||
|
||||
node.role === "dialog" ||
|
||||
node.role === "alertdialog"
|
||||
);
|
||||
}
|
||||
|
||||
function classifyLayer(
|
||||
nodes: VomNode[],
|
||||
members: Set<number>,
|
||||
blockerCoverage: number,
|
||||
): BlockingLayer["kind"] {
|
||||
for (const node of nodes) {
|
||||
if (!members.has(node.id)) continue;
|
||||
if (hasModalFeature(node)) return "modal";
|
||||
if (FORM_TAGS.has(node.tag)) return "modal";
|
||||
if (node.tag === "iframe") return "modal";
|
||||
}
|
||||
return blockerCoverage >= MASK_COVERAGE_THRESHOLD ? "mask" : "modal";
|
||||
}
|
||||
|
||||
export function detectBlockingLayer(nodes: VomNode[], vp: Viewport): BlockingLayer | null {
|
||||
let blocker: { node: VomNode; coverage: number } | null = null;
|
||||
|
||||
for (const node of nodes) {
|
||||
if (!isBlockingCandidate(node)) continue;
|
||||
const cov = coverage(node.rect, vp);
|
||||
const qualifies = cov >= BLOCK_COVERAGE_THRESHOLD || (hasModalFeature(node) && cov >= 0.15);
|
||||
if (!qualifies) continue;
|
||||
if (
|
||||
blocker === null ||
|
||||
cov > blocker.coverage ||
|
||||
(cov === blocker.coverage && node.paintOrder > blocker.node.paintOrder)
|
||||
) {
|
||||
blocker = { node, coverage: cov };
|
||||
}
|
||||
}
|
||||
|
||||
if (!blocker) return null;
|
||||
|
||||
const threshold = blocker.node.paintOrder;
|
||||
const members = new Set<number>();
|
||||
for (const node of nodes) {
|
||||
if (node.paintOrder >= threshold) members.add(node.id);
|
||||
}
|
||||
|
||||
return {
|
||||
rootId: blocker.node.id,
|
||||
kind: classifyLayer(nodes, members, blocker.coverage),
|
||||
coverage: blocker.coverage,
|
||||
members,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,265 @@
|
||||
import { detectBlockingLayer } from "./layers";
|
||||
import type { BlockingLayer, VomNode, VomOptions, VomResult, VomScene } from "./types";
|
||||
|
||||
const SKIP_ROLES = new Set(["generic", "none", "presentation", "inlinetextbox"]);
|
||||
|
||||
const INTERACTIVE_ROLES = new Set([
|
||||
"button",
|
||||
"checkbox",
|
||||
"combobox",
|
||||
"listbox",
|
||||
"link",
|
||||
"menuitem",
|
||||
"menuitemcheckbox",
|
||||
"menuitemradio",
|
||||
"option",
|
||||
"radio",
|
||||
"scrollbar",
|
||||
"searchbox",
|
||||
"slider",
|
||||
"spinbutton",
|
||||
"switch",
|
||||
"tab",
|
||||
"textbox",
|
||||
"treeitem",
|
||||
]);
|
||||
|
||||
const STRUCTURAL_ROLES = new Set([
|
||||
"alert",
|
||||
"alertdialog",
|
||||
"article",
|
||||
"banner",
|
||||
"cell",
|
||||
"columnheader",
|
||||
"complementary",
|
||||
"contentinfo",
|
||||
"definition",
|
||||
"dialog",
|
||||
"directory",
|
||||
"document",
|
||||
"feed",
|
||||
"figure",
|
||||
"form",
|
||||
"grid",
|
||||
"group",
|
||||
"heading",
|
||||
"iframe",
|
||||
"img",
|
||||
"list",
|
||||
"listitem",
|
||||
"main",
|
||||
"navigation",
|
||||
"paragraph",
|
||||
"region",
|
||||
"rootwebarea",
|
||||
"row",
|
||||
"rowgroup",
|
||||
"rowheader",
|
||||
"search",
|
||||
"section",
|
||||
"status",
|
||||
"table",
|
||||
"term",
|
||||
"toolbar",
|
||||
"tree",
|
||||
"webarea",
|
||||
]);
|
||||
|
||||
const SENSITIVE_MASK = "•••";
|
||||
const DEFAULT_MAX_DEPTH = Number.POSITIVE_INFINITY;
|
||||
const DEFAULT_MAX_TOKENS = Number.POSITIVE_INFINITY;
|
||||
const MAX_VALUE_LENGTH = 200;
|
||||
|
||||
function estimateTokens(text: string): number {
|
||||
return Math.ceil(text.length / 4);
|
||||
}
|
||||
|
||||
function cleaned(value: string | undefined): string | undefined {
|
||||
const trimmed = value?.replace(/\s+/g, " ").trim();
|
||||
return trimmed ? trimmed : undefined;
|
||||
}
|
||||
|
||||
function normalizedRole(node: VomNode): string {
|
||||
return node.role?.toLowerCase() ?? "";
|
||||
}
|
||||
|
||||
function shouldRender(node: VomNode): boolean {
|
||||
if (!node.role) return false;
|
||||
|
||||
const role = normalizedRole(node);
|
||||
if (SKIP_ROLES.has(role)) return false;
|
||||
if (INTERACTIVE_ROLES.has(role)) return true;
|
||||
if (STRUCTURAL_ROLES.has(role)) return true;
|
||||
|
||||
return cleaned(node.name) !== undefined;
|
||||
}
|
||||
|
||||
function shouldReference(node: VomNode): boolean {
|
||||
return INTERACTIVE_ROLES.has(normalizedRole(node));
|
||||
}
|
||||
|
||||
function buildChildren(nodes: VomNode[]): Map<number | null, VomNode[]> {
|
||||
const children = new Map<number | null, VomNode[]>();
|
||||
const ids = new Set(nodes.map((node) => node.id));
|
||||
|
||||
for (const node of nodes) {
|
||||
const parentId = node.parentId !== null && ids.has(node.parentId) ? node.parentId : null;
|
||||
const siblings = children.get(parentId) ?? [];
|
||||
siblings.push(node);
|
||||
children.set(parentId, siblings);
|
||||
}
|
||||
|
||||
return children;
|
||||
}
|
||||
|
||||
function renderNodeLine(node: VomNode, depth: number, ref: string | undefined): string {
|
||||
let line = `${" ".repeat(depth)}${ref ? `@${ref} ` : ""}${node.role}`;
|
||||
|
||||
const name = cleaned(node.name);
|
||||
if (name) line += ` ${JSON.stringify(name)}`;
|
||||
|
||||
// For link nodes with an external href, annotate so the agent can
|
||||
// distinguish external navigation from same-origin links.
|
||||
if (node.role === "link" && node.href) {
|
||||
line += ` [→ ${node.href}]`;
|
||||
}
|
||||
|
||||
const value = node.sensitive ? SENSITIVE_MASK : cleaned(node.value)?.slice(0, MAX_VALUE_LENGTH);
|
||||
if (value !== undefined) line += ` =${JSON.stringify(value)}`;
|
||||
|
||||
return line;
|
||||
}
|
||||
|
||||
interface RenderState {
|
||||
lines: string[];
|
||||
refs: Array<{ ref: string; backendNodeId: number }>;
|
||||
nextRef: number;
|
||||
tokens: number;
|
||||
maxDepth: number;
|
||||
maxTokens: number;
|
||||
truncated: boolean;
|
||||
stopped: boolean;
|
||||
}
|
||||
|
||||
function renderTree(
|
||||
children: Map<number | null, VomNode[]>,
|
||||
parentId: number | null,
|
||||
depth: number,
|
||||
state: RenderState,
|
||||
): void {
|
||||
if (state.stopped) return;
|
||||
|
||||
const nodes = children.get(parentId) ?? [];
|
||||
for (const node of nodes) {
|
||||
if (state.stopped) return;
|
||||
|
||||
if (!shouldRender(node)) {
|
||||
renderTree(children, node.id, depth, state);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (depth > state.maxDepth) {
|
||||
state.truncated = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
const ref = shouldReference(node) ? `e${state.nextRef}` : undefined;
|
||||
const line = renderNodeLine(node, depth, ref);
|
||||
const nextTokens = state.tokens + estimateTokens(line);
|
||||
if (nextTokens > state.maxTokens) {
|
||||
state.truncated = true;
|
||||
state.stopped = true;
|
||||
return;
|
||||
}
|
||||
|
||||
state.lines.push(line);
|
||||
state.tokens = nextTokens;
|
||||
if (ref) {
|
||||
state.refs.push({ ref, backendNodeId: node.id });
|
||||
state.nextRef += 1;
|
||||
}
|
||||
|
||||
renderTree(children, node.id, depth + 1, state);
|
||||
}
|
||||
}
|
||||
|
||||
function renderNodes(nodes: VomNode[], options: VomOptions, initialLines: string[]): RenderState {
|
||||
const state: RenderState = {
|
||||
lines: [...initialLines],
|
||||
refs: [],
|
||||
nextRef: 1,
|
||||
tokens: estimateTokens(initialLines.join("\n")),
|
||||
maxDepth: options.maxDepth ?? DEFAULT_MAX_DEPTH,
|
||||
maxTokens: options.maxTokens ?? DEFAULT_MAX_TOKENS,
|
||||
truncated: false,
|
||||
stopped: false,
|
||||
};
|
||||
|
||||
renderTree(buildChildren(nodes), null, 1, state);
|
||||
|
||||
return state;
|
||||
}
|
||||
|
||||
function renderPageOcclusionLine(hiddenNodeCount: number): string {
|
||||
return `L2 page … occluded by L1 (~${hiddenNodeCount} nodes, not actionable)`;
|
||||
}
|
||||
|
||||
function collectDescendantsOfMembers(nodes: VomNode[], members: Set<number>): Set<number> {
|
||||
const included = new Set(members);
|
||||
let changed = true;
|
||||
|
||||
while (changed) {
|
||||
changed = false;
|
||||
for (const node of nodes) {
|
||||
if (included.has(node.id)) continue;
|
||||
if (node.parentId !== null && included.has(node.parentId)) {
|
||||
included.add(node.id);
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return included;
|
||||
}
|
||||
|
||||
function countRenderable(nodes: VomNode[]): number {
|
||||
return nodes.filter(shouldRender).length;
|
||||
}
|
||||
|
||||
function renderDoubleLayer(scene: VomScene, layer: BlockingLayer, options: VomOptions): VomResult {
|
||||
const included = collectDescendantsOfMembers(scene.nodes, layer.members);
|
||||
const visibleNodes = scene.nodes.filter((node) => included.has(node.id));
|
||||
const hiddenCount = countRenderable(scene.nodes.filter((node) => !included.has(node.id)));
|
||||
const header = [
|
||||
"@vom 1",
|
||||
`@view ${scene.viewport.width}x${scene.viewport.height}`,
|
||||
"@layers 2 focus=L1",
|
||||
`L1 ${layer.kind} cover=${Math.round(layer.coverage * 100)}%`,
|
||||
];
|
||||
const state = renderNodes(visibleNodes, options, header);
|
||||
state.lines.push(renderPageOcclusionLine(hiddenCount));
|
||||
|
||||
return {
|
||||
text: state.lines.join("\n"),
|
||||
refs: state.refs,
|
||||
truncated: state.truncated,
|
||||
};
|
||||
}
|
||||
|
||||
export function renderVom(scene: VomScene, options: VomOptions = {}): VomResult {
|
||||
const layer = detectBlockingLayer(scene.nodes, scene.viewport);
|
||||
if (layer) return renderDoubleLayer(scene, layer, options);
|
||||
|
||||
const state = renderNodes(scene.nodes, options, [
|
||||
"@vom 1",
|
||||
`@view ${scene.viewport.width}x${scene.viewport.height}`,
|
||||
"@layers 1 focus=L1",
|
||||
"L1 page",
|
||||
]);
|
||||
|
||||
return {
|
||||
text: state.lines.join("\n"),
|
||||
refs: state.refs,
|
||||
truncated: state.truncated,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
/** Viewport-relative layout box, CSS pixels. */
|
||||
export interface Rect {
|
||||
x: number;
|
||||
y: number;
|
||||
w: number;
|
||||
h: number;
|
||||
}
|
||||
|
||||
export interface Viewport {
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
export type LayerKind = "page" | "modal" | "mask";
|
||||
|
||||
/**
|
||||
* CDP-free data slice consumed by packages/vom.
|
||||
* The extension joins AX semantics and DOMSnapshot geometry by backendNodeId
|
||||
* before constructing these nodes.
|
||||
*/
|
||||
export interface VomNode {
|
||||
id: number;
|
||||
parentId: number | null;
|
||||
|
||||
role?: string;
|
||||
name?: string;
|
||||
value?: string;
|
||||
href?: string; // hostname of external link target; omitted for same-origin links
|
||||
|
||||
tag: string;
|
||||
rect: Rect | null;
|
||||
paintOrder: number;
|
||||
position: string;
|
||||
pointerEvents: string;
|
||||
|
||||
modal?: boolean;
|
||||
sensitive?: boolean;
|
||||
}
|
||||
|
||||
export interface VomScene {
|
||||
viewport: Viewport;
|
||||
nodes: VomNode[];
|
||||
}
|
||||
|
||||
export interface VomOptions {
|
||||
maxDepth?: number;
|
||||
maxTokens?: number;
|
||||
}
|
||||
|
||||
export interface VomResult {
|
||||
text: string;
|
||||
refs: Array<{ ref: string; backendNodeId: number }>;
|
||||
truncated: boolean;
|
||||
}
|
||||
|
||||
export interface BlockingLayer {
|
||||
rootId: number;
|
||||
kind: Exclude<LayerKind, "page">;
|
||||
coverage: number;
|
||||
members: Set<number>;
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ESNext",
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "Bundler",
|
||||
"noEmit": true,
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"resolveJsonModule": true,
|
||||
"strict": true,
|
||||
"skipLibCheck": true
|
||||
},
|
||||
"include": ["src/**/*"]
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
import { defineConfig } from "vitest/config";
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
environment: "node",
|
||||
include: ["src/**/*.test.ts"],
|
||||
},
|
||||
});
|
||||
Generated
+12
@@ -26,6 +26,9 @@ importers:
|
||||
'@browser-skill/ui':
|
||||
specifier: workspace:*
|
||||
version: link:../../packages/ui
|
||||
'@browser-skill/vom':
|
||||
specifier: workspace:*
|
||||
version: link:../../packages/vom
|
||||
'@remixicon/react':
|
||||
specifier: ^4.9.0
|
||||
version: 4.9.0(react@19.2.6)
|
||||
@@ -122,6 +125,15 @@ importers:
|
||||
specifier: ^19.2.3
|
||||
version: 19.2.3(@types/react@19.2.14)
|
||||
|
||||
packages/vom:
|
||||
devDependencies:
|
||||
typescript:
|
||||
specifier: ^5.8.0
|
||||
version: 5.9.3
|
||||
vitest:
|
||||
specifier: ^4.1.6
|
||||
version: 4.1.6(@types/node@25.8.0)(happy-dom@15.11.7)(vite@8.0.13(@types/node@25.8.0)(esbuild@0.27.7)(jiti@2.7.0))
|
||||
|
||||
packages:
|
||||
|
||||
'@1natsu/wait-element@4.2.0':
|
||||
|
||||
Reference in New Issue
Block a user