mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat(vom): vom base algorithm and infrastructure
This commit is contained in:
Ljy-0827
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"name": "@browser-skill/vom",
|
||||
"private": true,
|
||||
"version": "0.1.0",
|
||||
"type": "module",
|
||||
"exports": {
|
||||
".": {
|
||||
"import": "./src/index.ts",
|
||||
"default": "./src/index.ts"
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
"test": "vitest run",
|
||||
"test:watch": "vitest"
|
||||
},
|
||||
"devDependencies": {
|
||||
"typescript": "^5.8.0",
|
||||
"vitest": "^4.1.6"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { coverage, detectBlockingLayer } from "../layers";
|
||||
import type { Viewport, VomNode } from "../types";
|
||||
|
||||
const VP: Viewport = { width: 1000, height: 800 };
|
||||
|
||||
function node(p: Partial<VomNode> & { id: number }): VomNode {
|
||||
const { id, parentId, tag, rect, paintOrder, position, pointerEvents, ...rest } = p;
|
||||
|
||||
return {
|
||||
id,
|
||||
parentId: parentId ?? null,
|
||||
tag: tag ?? "div",
|
||||
rect: rect ?? null,
|
||||
paintOrder: paintOrder ?? 0,
|
||||
position: position ?? "static",
|
||||
pointerEvents: pointerEvents ?? "auto",
|
||||
...rest,
|
||||
};
|
||||
}
|
||||
|
||||
describe("coverage", () => {
|
||||
it("returns the clamped viewport-overlap fraction", () => {
|
||||
expect(coverage({ x: 0, y: 0, w: 1000, h: 800 }, VP)).toBeCloseTo(1);
|
||||
expect(coverage({ x: 0, y: 0, w: 500, h: 800 }, VP)).toBeCloseTo(0.5);
|
||||
expect(coverage({ x: -100, y: 0, w: 200, h: 800 }, VP)).toBeCloseTo(0.1);
|
||||
});
|
||||
|
||||
it("returns 0 for null rect, invalid viewport, or off-viewport rect", () => {
|
||||
expect(coverage(null, VP)).toBe(0);
|
||||
expect(coverage({ x: 0, y: 0, w: 100, h: 100 }, { width: 0, height: 800 })).toBe(0);
|
||||
expect(coverage({ x: 2000, y: 0, w: 100, h: 100 }, VP)).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("detectBlockingLayer", () => {
|
||||
it("detects a CSS blocker and includes the whole top paint-order band", () => {
|
||||
const layer = detectBlockingLayer(
|
||||
[
|
||||
node({ id: 1, tag: "body", rect: { x: 0, y: 0, w: 1000, h: 4000 }, paintOrder: 0 }),
|
||||
node({
|
||||
id: 2,
|
||||
parentId: 1,
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 30,
|
||||
position: "fixed",
|
||||
}),
|
||||
node({
|
||||
id: 3,
|
||||
parentId: 1,
|
||||
role: "dialog",
|
||||
name: "提示",
|
||||
tag: "dialog",
|
||||
rect: { x: 300, y: 250, w: 400, h: 300 },
|
||||
paintOrder: 40,
|
||||
position: "fixed",
|
||||
}),
|
||||
node({
|
||||
id: 4,
|
||||
parentId: 3,
|
||||
role: "button",
|
||||
name: "关闭",
|
||||
tag: "button",
|
||||
rect: { x: 640, y: 260, w: 40, h: 40 },
|
||||
paintOrder: 41,
|
||||
}),
|
||||
],
|
||||
VP,
|
||||
);
|
||||
|
||||
expect(layer).not.toBeNull();
|
||||
expect(layer?.rootId).toBe(2);
|
||||
expect(layer?.kind).toBe("modal");
|
||||
expect(layer?.coverage).toBeCloseTo(1);
|
||||
expect([...(layer?.members ?? [])].sort((a, b) => a - b)).toEqual([2, 3, 4]);
|
||||
});
|
||||
|
||||
it("classifies a near-full blocker without modal signals or controls as a mask", () => {
|
||||
const layer = detectBlockingLayer(
|
||||
[
|
||||
node({ id: 1, tag: "body", rect: { x: 0, y: 0, w: 1000, h: 800 } }),
|
||||
node({
|
||||
id: 2,
|
||||
parentId: 1,
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 10,
|
||||
position: "fixed",
|
||||
}),
|
||||
],
|
||||
VP,
|
||||
);
|
||||
|
||||
expect(layer?.kind).toBe("mask");
|
||||
});
|
||||
|
||||
it("treats explicit modal nodes as modal even below the generic coverage threshold", () => {
|
||||
const layer = detectBlockingLayer(
|
||||
[
|
||||
node({ id: 1, tag: "body", rect: { x: 0, y: 0, w: 1000, h: 800 } }),
|
||||
node({
|
||||
id: 2,
|
||||
parentId: 1,
|
||||
tag: "dialog",
|
||||
role: "dialog",
|
||||
modal: true,
|
||||
rect: { x: 300, y: 200, w: 400, h: 300 },
|
||||
paintOrder: 90,
|
||||
position: "fixed",
|
||||
}),
|
||||
],
|
||||
VP,
|
||||
);
|
||||
|
||||
expect(layer?.rootId).toBe(2);
|
||||
expect(layer?.kind).toBe("modal");
|
||||
});
|
||||
|
||||
it("ignores small toasts and pointer-events:none covers", () => {
|
||||
expect(
|
||||
detectBlockingLayer(
|
||||
[
|
||||
node({ id: 1, tag: "body", rect: { x: 0, y: 0, w: 1000, h: 800 } }),
|
||||
node({
|
||||
id: 2,
|
||||
parentId: 1,
|
||||
rect: { x: 800, y: 720, w: 180, h: 60 },
|
||||
paintOrder: 99,
|
||||
position: "fixed",
|
||||
}),
|
||||
],
|
||||
VP,
|
||||
),
|
||||
).toBeNull();
|
||||
|
||||
expect(
|
||||
detectBlockingLayer(
|
||||
[
|
||||
node({ id: 1, tag: "body", rect: { x: 0, y: 0, w: 1000, h: 800 } }),
|
||||
node({
|
||||
id: 2,
|
||||
parentId: 1,
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 40,
|
||||
position: "fixed",
|
||||
pointerEvents: "none",
|
||||
}),
|
||||
],
|
||||
VP,
|
||||
),
|
||||
).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,304 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { renderVom } from "../render";
|
||||
import type { VomNode, VomScene } from "../types";
|
||||
|
||||
function node(p: Partial<VomNode> & { id: number }): VomNode {
|
||||
const { id, parentId, tag, rect, paintOrder, position, pointerEvents, ...rest } = p;
|
||||
|
||||
return {
|
||||
id,
|
||||
parentId: parentId ?? null,
|
||||
tag: tag ?? "div",
|
||||
rect: rect ?? null,
|
||||
paintOrder: paintOrder ?? 0,
|
||||
position: position ?? "static",
|
||||
pointerEvents: pointerEvents ?? "auto",
|
||||
...rest,
|
||||
};
|
||||
}
|
||||
|
||||
function scene(nodes: VomNode[]): VomScene {
|
||||
return {
|
||||
viewport: { width: 1280, height: 720 },
|
||||
nodes,
|
||||
};
|
||||
}
|
||||
|
||||
describe("renderVom single-layer page", () => {
|
||||
it("always emits a complete @vom single-layer document when there is no blocker", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
node({ id: 1, role: "RootWebArea", name: "Shop" }),
|
||||
node({ id: 2, parentId: 1, role: "navigation", name: "主菜单" }),
|
||||
node({ id: 3, parentId: 2, role: "link", name: "首页", tag: "a" }),
|
||||
node({ id: 4, parentId: 1, role: "main" }),
|
||||
node({ id: 5, parentId: 4, role: "heading", name: "今日推荐" }),
|
||||
node({ id: 6, parentId: 4, role: "button", name: "加入购物车", tag: "button" }),
|
||||
]),
|
||||
);
|
||||
|
||||
expect(out.text).toBe(
|
||||
[
|
||||
"@vom 1",
|
||||
"@view 1280x720",
|
||||
"@layers 1 focus=L1",
|
||||
"L1 page",
|
||||
' RootWebArea "Shop"',
|
||||
' navigation "主菜单"',
|
||||
' @e1 link "首页"',
|
||||
" main",
|
||||
' heading "今日推荐"',
|
||||
' @e2 button "加入购物车"',
|
||||
].join("\n"),
|
||||
);
|
||||
expect(out.refs).toEqual([
|
||||
{ ref: "e1", backendNodeId: 3 },
|
||||
{ ref: "e2", backendNodeId: 6 },
|
||||
]);
|
||||
expect(out.truncated).toBe(false);
|
||||
});
|
||||
|
||||
it("does not assign refs to structural or display-only nodes", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
node({ id: 1, role: "RootWebArea", name: "Doc" }),
|
||||
node({ id: 2, parentId: 1, role: "heading", name: "Title" }),
|
||||
node({ id: 3, parentId: 1, role: "img", name: "Logo" }),
|
||||
node({ id: 4, parentId: 1, role: "button", name: "Continue", tag: "button" }),
|
||||
]),
|
||||
);
|
||||
|
||||
expect(out.text).toContain(' RootWebArea "Doc"');
|
||||
expect(out.text).toContain(' heading "Title"');
|
||||
expect(out.text).toContain(' img "Logo"');
|
||||
expect(out.text).toContain(' @e1 button "Continue"');
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]);
|
||||
});
|
||||
|
||||
it("assigns refs to listbox controls", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
node({ id: 1, role: "RootWebArea", name: "Doc" }),
|
||||
node({ id: 2, parentId: 1, role: "listbox", name: "Choices" }),
|
||||
]),
|
||||
);
|
||||
|
||||
expect(out.text).toContain(' @e1 listbox "Choices"');
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||
});
|
||||
|
||||
it("renders additional structural roles without assigning refs", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
node({ id: 1, role: "WebArea" }),
|
||||
node({ id: 2, parentId: 1, role: "Iframe" }),
|
||||
node({ id: 3, parentId: 1, role: "alertdialog" }),
|
||||
node({ id: 4, parentId: 3, role: "section" }),
|
||||
node({ id: 5, parentId: 4, role: "button", name: "Close", tag: "button" }),
|
||||
]),
|
||||
);
|
||||
|
||||
expect(out.text).toContain(" WebArea");
|
||||
expect(out.text).toContain(" Iframe");
|
||||
expect(out.text).toContain(" alertdialog");
|
||||
expect(out.text).toContain(" section");
|
||||
expect(out.text).toContain(' @e1 button "Close"');
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 5 }]);
|
||||
});
|
||||
|
||||
it("skips generic nodes without adding an extra indentation level", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
node({ id: 1, role: "RootWebArea", name: "Doc" }),
|
||||
node({ id: 2, parentId: 1, role: "generic" }),
|
||||
node({ id: 3, parentId: 2, role: "button", name: "OK", tag: "button" }),
|
||||
]),
|
||||
);
|
||||
|
||||
expect(out.text).not.toContain("generic");
|
||||
expect(out.text).toContain(' @e1 button "OK"');
|
||||
});
|
||||
|
||||
it("masks sensitive values and never emits the clear text value", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
node({ id: 1, role: "RootWebArea", name: "Login" }),
|
||||
node({
|
||||
id: 2,
|
||||
parentId: 1,
|
||||
role: "textbox",
|
||||
name: "密码",
|
||||
value: "hunter2",
|
||||
tag: "input",
|
||||
sensitive: true,
|
||||
}),
|
||||
]),
|
||||
);
|
||||
|
||||
expect(out.text).toContain('@e1 textbox "密码" ="•••"');
|
||||
expect(out.text).not.toContain("hunter2");
|
||||
});
|
||||
|
||||
it("masks sensitive textboxes even when the value is missing", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
node({ id: 1, role: "RootWebArea", name: "Login" }),
|
||||
node({
|
||||
id: 2,
|
||||
parentId: 1,
|
||||
role: "textbox",
|
||||
name: "密码",
|
||||
tag: "input",
|
||||
sensitive: true,
|
||||
}),
|
||||
]),
|
||||
);
|
||||
|
||||
expect(out.text).toContain('@e1 textbox "密码" ="•••"');
|
||||
});
|
||||
|
||||
it("continues rendering later siblings after a branch exceeds maxDepth", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
node({ id: 1, role: "RootWebArea", name: "Doc" }),
|
||||
node({ id: 2, parentId: 1, role: "section" }),
|
||||
node({ id: 3, parentId: 2, role: "heading", name: "Too deep" }),
|
||||
node({ id: 4, parentId: 1, role: "button", name: "Later", tag: "button" }),
|
||||
]),
|
||||
{ maxDepth: 2 },
|
||||
);
|
||||
|
||||
expect(out.truncated).toBe(true);
|
||||
expect(out.text).toContain(' RootWebArea "Doc"');
|
||||
expect(out.text).toContain(" section");
|
||||
expect(out.text).not.toContain("Too deep");
|
||||
expect(out.text).toContain(' @e1 button "Later"');
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]);
|
||||
});
|
||||
|
||||
it("truncates when maxTokens is exceeded while keeping refs in sync with rendered text", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
node({ id: 1, role: "RootWebArea", name: "Doc" }),
|
||||
node({ id: 2, parentId: 1, role: "button", name: "First", tag: "button" }),
|
||||
node({ id: 3, parentId: 1, role: "button", name: "Second", tag: "button" }),
|
||||
]),
|
||||
{ maxTokens: 22 },
|
||||
);
|
||||
|
||||
expect(out.truncated).toBe(true);
|
||||
const renderedRefs = new Set(Array.from(out.text.matchAll(/@(e\d+)/g), (m) => m[1]));
|
||||
expect(out.refs.every((r) => renderedRefs.has(r.ref))).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("renderVom double-layer page", () => {
|
||||
it("renders a blocking modal first and folds the base page into an occluded summary", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
node({
|
||||
id: 1,
|
||||
role: "RootWebArea",
|
||||
name: "Shop",
|
||||
tag: "body",
|
||||
rect: { x: 0, y: 0, w: 1000, h: 3000 },
|
||||
}),
|
||||
node({ id: 10, parentId: 1, role: "button", name: "底层按钮", tag: "button" }),
|
||||
node({
|
||||
id: 2,
|
||||
parentId: 1,
|
||||
tag: "div",
|
||||
rect: { x: 0, y: 0, w: 1280, h: 720 },
|
||||
paintOrder: 50,
|
||||
position: "fixed",
|
||||
pointerEvents: "auto",
|
||||
}),
|
||||
node({
|
||||
id: 3,
|
||||
parentId: 1,
|
||||
role: "dialog",
|
||||
name: "提示",
|
||||
modal: true,
|
||||
rect: { x: 400, y: 200, w: 480, h: 320 },
|
||||
paintOrder: 60,
|
||||
position: "fixed",
|
||||
}),
|
||||
node({
|
||||
id: 4,
|
||||
parentId: 3,
|
||||
role: "textbox",
|
||||
name: "密码",
|
||||
value: "secret",
|
||||
tag: "input",
|
||||
sensitive: true,
|
||||
}),
|
||||
node({ id: 5, parentId: 3, role: "button", name: "确认", tag: "button" }),
|
||||
]),
|
||||
);
|
||||
|
||||
expect(out.text).toBe(
|
||||
[
|
||||
"@vom 1",
|
||||
"@view 1280x720",
|
||||
"@layers 2 focus=L1",
|
||||
"L1 modal cover=100%",
|
||||
' dialog "提示"',
|
||||
' @e1 textbox "密码" ="•••"',
|
||||
' @e2 button "确认"',
|
||||
"L2 page … occluded by L1 (~2 nodes, not actionable)",
|
||||
].join("\n"),
|
||||
);
|
||||
expect(out.text).not.toContain("底层按钮");
|
||||
expect(out.refs).toEqual([
|
||||
{ ref: "e1", backendNodeId: 4 },
|
||||
{ ref: "e2", backendNodeId: 5 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("renders iframe descendants inside the top layer through parentId without iframe-specific renderer code", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
node({
|
||||
id: 1,
|
||||
role: "RootWebArea",
|
||||
name: "JD",
|
||||
tag: "body",
|
||||
rect: { x: 0, y: 0, w: 1000, h: 3000 },
|
||||
}),
|
||||
node({
|
||||
id: 2,
|
||||
parentId: 1,
|
||||
tag: "div",
|
||||
rect: { x: 0, y: 0, w: 1280, h: 720 },
|
||||
paintOrder: 50,
|
||||
position: "fixed",
|
||||
}),
|
||||
node({
|
||||
id: 3,
|
||||
parentId: 2,
|
||||
role: "Iframe",
|
||||
name: "登录框",
|
||||
tag: "iframe",
|
||||
rect: { x: 100, y: 240, w: 800, h: 400 },
|
||||
paintOrder: 51,
|
||||
}),
|
||||
node({ id: 101, parentId: 3, role: "textbox", name: "请输入手机号", tag: "input" }),
|
||||
node({
|
||||
id: 102,
|
||||
parentId: 3,
|
||||
role: "textbox",
|
||||
name: "密码",
|
||||
value: "secret",
|
||||
tag: "input",
|
||||
sensitive: true,
|
||||
}),
|
||||
]),
|
||||
);
|
||||
|
||||
expect(out.text).toContain("L1 modal cover=100%");
|
||||
expect(out.text).toContain(' Iframe "登录框"');
|
||||
expect(out.text).toContain(' @e1 textbox "请输入手机号"');
|
||||
expect(out.text).toContain(' @e2 textbox "密码" ="•••"');
|
||||
expect(out.refs.map((r) => r.backendNodeId)).toEqual([101, 102]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,11 @@
|
||||
export { renderVom } from "./render";
|
||||
export type {
|
||||
BlockingLayer,
|
||||
LayerKind,
|
||||
Rect,
|
||||
Viewport,
|
||||
VomNode,
|
||||
VomOptions,
|
||||
VomResult,
|
||||
VomScene,
|
||||
} from "./types";
|
||||
@@ -0,0 +1,79 @@
|
||||
import type { BlockingLayer, Rect, Viewport, VomNode } from "./types";
|
||||
|
||||
/** Default fraction of the viewport an overlay must cover to block. */
|
||||
export const BLOCK_COVERAGE_THRESHOLD = 0.6;
|
||||
/** At/above this coverage with no inputs we call it an opaque mask. */
|
||||
export const MASK_COVERAGE_THRESHOLD = 0.9;
|
||||
|
||||
const POSITIONED = new Set(["fixed", "absolute", "sticky"]);
|
||||
const FORM_TAGS = new Set(["input", "textarea", "select"]);
|
||||
|
||||
export function coverage(rect: Rect | null, vp: Viewport): number {
|
||||
if (!rect || vp.width <= 0 || vp.height <= 0) return 0;
|
||||
const ix = Math.max(0, Math.min(rect.x + rect.w, vp.width) - Math.max(rect.x, 0));
|
||||
const iy = Math.max(0, Math.min(rect.y + rect.h, vp.height) - Math.max(rect.y, 0));
|
||||
const overlap = ix * iy;
|
||||
if (overlap <= 0) return 0;
|
||||
return Math.min(1, overlap / (vp.width * vp.height));
|
||||
}
|
||||
|
||||
function isBlockingCandidate(node: VomNode): boolean {
|
||||
if (node.pointerEvents === "none") return false;
|
||||
return POSITIONED.has(node.position);
|
||||
}
|
||||
|
||||
function hasModalFeature(node: VomNode): boolean {
|
||||
return (
|
||||
node.modal === true ||
|
||||
node.tag === "dialog" ||
|
||||
node.role === "dialog" ||
|
||||
node.role === "alertdialog"
|
||||
);
|
||||
}
|
||||
|
||||
function classifyLayer(
|
||||
nodes: VomNode[],
|
||||
members: Set<number>,
|
||||
blockerCoverage: number,
|
||||
): BlockingLayer["kind"] {
|
||||
for (const node of nodes) {
|
||||
if (!members.has(node.id)) continue;
|
||||
if (hasModalFeature(node)) return "modal";
|
||||
if (FORM_TAGS.has(node.tag)) return "modal";
|
||||
if (node.tag === "iframe") return "modal";
|
||||
}
|
||||
return blockerCoverage >= MASK_COVERAGE_THRESHOLD ? "mask" : "modal";
|
||||
}
|
||||
|
||||
export function detectBlockingLayer(nodes: VomNode[], vp: Viewport): BlockingLayer | null {
|
||||
let blocker: { node: VomNode; coverage: number } | null = null;
|
||||
|
||||
for (const node of nodes) {
|
||||
if (!isBlockingCandidate(node)) continue;
|
||||
const cov = coverage(node.rect, vp);
|
||||
const qualifies = cov >= BLOCK_COVERAGE_THRESHOLD || (hasModalFeature(node) && cov >= 0.15);
|
||||
if (!qualifies) continue;
|
||||
if (
|
||||
blocker === null ||
|
||||
cov > blocker.coverage ||
|
||||
(cov === blocker.coverage && node.paintOrder > blocker.node.paintOrder)
|
||||
) {
|
||||
blocker = { node, coverage: cov };
|
||||
}
|
||||
}
|
||||
|
||||
if (!blocker) return null;
|
||||
|
||||
const threshold = blocker.node.paintOrder;
|
||||
const members = new Set<number>();
|
||||
for (const node of nodes) {
|
||||
if (node.paintOrder >= threshold) members.add(node.id);
|
||||
}
|
||||
|
||||
return {
|
||||
rootId: blocker.node.id,
|
||||
kind: classifyLayer(nodes, members, blocker.coverage),
|
||||
coverage: blocker.coverage,
|
||||
members,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,265 @@
|
||||
import { detectBlockingLayer } from "./layers";
|
||||
import type { BlockingLayer, VomNode, VomOptions, VomResult, VomScene } from "./types";
|
||||
|
||||
const SKIP_ROLES = new Set(["generic", "none", "presentation", "inlinetextbox"]);
|
||||
|
||||
const INTERACTIVE_ROLES = new Set([
|
||||
"button",
|
||||
"checkbox",
|
||||
"combobox",
|
||||
"listbox",
|
||||
"link",
|
||||
"menuitem",
|
||||
"menuitemcheckbox",
|
||||
"menuitemradio",
|
||||
"option",
|
||||
"radio",
|
||||
"scrollbar",
|
||||
"searchbox",
|
||||
"slider",
|
||||
"spinbutton",
|
||||
"switch",
|
||||
"tab",
|
||||
"textbox",
|
||||
"treeitem",
|
||||
]);
|
||||
|
||||
const STRUCTURAL_ROLES = new Set([
|
||||
"alert",
|
||||
"alertdialog",
|
||||
"article",
|
||||
"banner",
|
||||
"cell",
|
||||
"columnheader",
|
||||
"complementary",
|
||||
"contentinfo",
|
||||
"definition",
|
||||
"dialog",
|
||||
"directory",
|
||||
"document",
|
||||
"feed",
|
||||
"figure",
|
||||
"form",
|
||||
"grid",
|
||||
"group",
|
||||
"heading",
|
||||
"iframe",
|
||||
"img",
|
||||
"list",
|
||||
"listitem",
|
||||
"main",
|
||||
"navigation",
|
||||
"paragraph",
|
||||
"region",
|
||||
"rootwebarea",
|
||||
"row",
|
||||
"rowgroup",
|
||||
"rowheader",
|
||||
"search",
|
||||
"section",
|
||||
"status",
|
||||
"table",
|
||||
"term",
|
||||
"toolbar",
|
||||
"tree",
|
||||
"webarea",
|
||||
]);
|
||||
|
||||
const SENSITIVE_MASK = "•••";
|
||||
const DEFAULT_MAX_DEPTH = Number.POSITIVE_INFINITY;
|
||||
const DEFAULT_MAX_TOKENS = Number.POSITIVE_INFINITY;
|
||||
const MAX_VALUE_LENGTH = 200;
|
||||
|
||||
function estimateTokens(text: string): number {
|
||||
return Math.ceil(text.length / 4);
|
||||
}
|
||||
|
||||
function cleaned(value: string | undefined): string | undefined {
|
||||
const trimmed = value?.replace(/\s+/g, " ").trim();
|
||||
return trimmed ? trimmed : undefined;
|
||||
}
|
||||
|
||||
function normalizedRole(node: VomNode): string {
|
||||
return node.role?.toLowerCase() ?? "";
|
||||
}
|
||||
|
||||
function shouldRender(node: VomNode): boolean {
|
||||
if (!node.role) return false;
|
||||
|
||||
const role = normalizedRole(node);
|
||||
if (SKIP_ROLES.has(role)) return false;
|
||||
if (INTERACTIVE_ROLES.has(role)) return true;
|
||||
if (STRUCTURAL_ROLES.has(role)) return true;
|
||||
|
||||
return cleaned(node.name) !== undefined;
|
||||
}
|
||||
|
||||
function shouldReference(node: VomNode): boolean {
|
||||
return INTERACTIVE_ROLES.has(normalizedRole(node));
|
||||
}
|
||||
|
||||
function buildChildren(nodes: VomNode[]): Map<number | null, VomNode[]> {
|
||||
const children = new Map<number | null, VomNode[]>();
|
||||
const ids = new Set(nodes.map((node) => node.id));
|
||||
|
||||
for (const node of nodes) {
|
||||
const parentId = node.parentId !== null && ids.has(node.parentId) ? node.parentId : null;
|
||||
const siblings = children.get(parentId) ?? [];
|
||||
siblings.push(node);
|
||||
children.set(parentId, siblings);
|
||||
}
|
||||
|
||||
return children;
|
||||
}
|
||||
|
||||
function renderNodeLine(node: VomNode, depth: number, ref: string | undefined): string {
|
||||
let line = `${" ".repeat(depth)}${ref ? `@${ref} ` : ""}${node.role}`;
|
||||
|
||||
const name = cleaned(node.name);
|
||||
if (name) line += ` ${JSON.stringify(name)}`;
|
||||
|
||||
// For link nodes with an external href, annotate so the agent can
|
||||
// distinguish external navigation from same-origin links.
|
||||
if (node.role === "link" && node.href) {
|
||||
line += ` [→ ${node.href}]`;
|
||||
}
|
||||
|
||||
const value = node.sensitive ? SENSITIVE_MASK : cleaned(node.value)?.slice(0, MAX_VALUE_LENGTH);
|
||||
if (value !== undefined) line += ` =${JSON.stringify(value)}`;
|
||||
|
||||
return line;
|
||||
}
|
||||
|
||||
interface RenderState {
|
||||
lines: string[];
|
||||
refs: Array<{ ref: string; backendNodeId: number }>;
|
||||
nextRef: number;
|
||||
tokens: number;
|
||||
maxDepth: number;
|
||||
maxTokens: number;
|
||||
truncated: boolean;
|
||||
stopped: boolean;
|
||||
}
|
||||
|
||||
function renderTree(
|
||||
children: Map<number | null, VomNode[]>,
|
||||
parentId: number | null,
|
||||
depth: number,
|
||||
state: RenderState,
|
||||
): void {
|
||||
if (state.stopped) return;
|
||||
|
||||
const nodes = children.get(parentId) ?? [];
|
||||
for (const node of nodes) {
|
||||
if (state.stopped) return;
|
||||
|
||||
if (!shouldRender(node)) {
|
||||
renderTree(children, node.id, depth, state);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (depth > state.maxDepth) {
|
||||
state.truncated = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
const ref = shouldReference(node) ? `e${state.nextRef}` : undefined;
|
||||
const line = renderNodeLine(node, depth, ref);
|
||||
const nextTokens = state.tokens + estimateTokens(line);
|
||||
if (nextTokens > state.maxTokens) {
|
||||
state.truncated = true;
|
||||
state.stopped = true;
|
||||
return;
|
||||
}
|
||||
|
||||
state.lines.push(line);
|
||||
state.tokens = nextTokens;
|
||||
if (ref) {
|
||||
state.refs.push({ ref, backendNodeId: node.id });
|
||||
state.nextRef += 1;
|
||||
}
|
||||
|
||||
renderTree(children, node.id, depth + 1, state);
|
||||
}
|
||||
}
|
||||
|
||||
function renderNodes(nodes: VomNode[], options: VomOptions, initialLines: string[]): RenderState {
|
||||
const state: RenderState = {
|
||||
lines: [...initialLines],
|
||||
refs: [],
|
||||
nextRef: 1,
|
||||
tokens: estimateTokens(initialLines.join("\n")),
|
||||
maxDepth: options.maxDepth ?? DEFAULT_MAX_DEPTH,
|
||||
maxTokens: options.maxTokens ?? DEFAULT_MAX_TOKENS,
|
||||
truncated: false,
|
||||
stopped: false,
|
||||
};
|
||||
|
||||
renderTree(buildChildren(nodes), null, 1, state);
|
||||
|
||||
return state;
|
||||
}
|
||||
|
||||
function renderPageOcclusionLine(hiddenNodeCount: number): string {
|
||||
return `L2 page … occluded by L1 (~${hiddenNodeCount} nodes, not actionable)`;
|
||||
}
|
||||
|
||||
function collectDescendantsOfMembers(nodes: VomNode[], members: Set<number>): Set<number> {
|
||||
const included = new Set(members);
|
||||
let changed = true;
|
||||
|
||||
while (changed) {
|
||||
changed = false;
|
||||
for (const node of nodes) {
|
||||
if (included.has(node.id)) continue;
|
||||
if (node.parentId !== null && included.has(node.parentId)) {
|
||||
included.add(node.id);
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return included;
|
||||
}
|
||||
|
||||
function countRenderable(nodes: VomNode[]): number {
|
||||
return nodes.filter(shouldRender).length;
|
||||
}
|
||||
|
||||
function renderDoubleLayer(scene: VomScene, layer: BlockingLayer, options: VomOptions): VomResult {
|
||||
const included = collectDescendantsOfMembers(scene.nodes, layer.members);
|
||||
const visibleNodes = scene.nodes.filter((node) => included.has(node.id));
|
||||
const hiddenCount = countRenderable(scene.nodes.filter((node) => !included.has(node.id)));
|
||||
const header = [
|
||||
"@vom 1",
|
||||
`@view ${scene.viewport.width}x${scene.viewport.height}`,
|
||||
"@layers 2 focus=L1",
|
||||
`L1 ${layer.kind} cover=${Math.round(layer.coverage * 100)}%`,
|
||||
];
|
||||
const state = renderNodes(visibleNodes, options, header);
|
||||
state.lines.push(renderPageOcclusionLine(hiddenCount));
|
||||
|
||||
return {
|
||||
text: state.lines.join("\n"),
|
||||
refs: state.refs,
|
||||
truncated: state.truncated,
|
||||
};
|
||||
}
|
||||
|
||||
export function renderVom(scene: VomScene, options: VomOptions = {}): VomResult {
|
||||
const layer = detectBlockingLayer(scene.nodes, scene.viewport);
|
||||
if (layer) return renderDoubleLayer(scene, layer, options);
|
||||
|
||||
const state = renderNodes(scene.nodes, options, [
|
||||
"@vom 1",
|
||||
`@view ${scene.viewport.width}x${scene.viewport.height}`,
|
||||
"@layers 1 focus=L1",
|
||||
"L1 page",
|
||||
]);
|
||||
|
||||
return {
|
||||
text: state.lines.join("\n"),
|
||||
refs: state.refs,
|
||||
truncated: state.truncated,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
/** Viewport-relative layout box, CSS pixels. */
|
||||
export interface Rect {
|
||||
x: number;
|
||||
y: number;
|
||||
w: number;
|
||||
h: number;
|
||||
}
|
||||
|
||||
export interface Viewport {
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
export type LayerKind = "page" | "modal" | "mask";
|
||||
|
||||
/**
|
||||
* CDP-free data slice consumed by packages/vom.
|
||||
* The extension joins AX semantics and DOMSnapshot geometry by backendNodeId
|
||||
* before constructing these nodes.
|
||||
*/
|
||||
export interface VomNode {
|
||||
id: number;
|
||||
parentId: number | null;
|
||||
|
||||
role?: string;
|
||||
name?: string;
|
||||
value?: string;
|
||||
href?: string; // hostname of external link target; omitted for same-origin links
|
||||
|
||||
tag: string;
|
||||
rect: Rect | null;
|
||||
paintOrder: number;
|
||||
position: string;
|
||||
pointerEvents: string;
|
||||
|
||||
modal?: boolean;
|
||||
sensitive?: boolean;
|
||||
}
|
||||
|
||||
export interface VomScene {
|
||||
viewport: Viewport;
|
||||
nodes: VomNode[];
|
||||
}
|
||||
|
||||
export interface VomOptions {
|
||||
maxDepth?: number;
|
||||
maxTokens?: number;
|
||||
}
|
||||
|
||||
export interface VomResult {
|
||||
text: string;
|
||||
refs: Array<{ ref: string; backendNodeId: number }>;
|
||||
truncated: boolean;
|
||||
}
|
||||
|
||||
export interface BlockingLayer {
|
||||
rootId: number;
|
||||
kind: Exclude<LayerKind, "page">;
|
||||
coverage: number;
|
||||
members: Set<number>;
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ESNext",
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "Bundler",
|
||||
"noEmit": true,
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"resolveJsonModule": true,
|
||||
"strict": true,
|
||||
"skipLibCheck": true
|
||||
},
|
||||
"include": ["src/**/*"]
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
import { defineConfig } from "vitest/config";
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
environment: "node",
|
||||
include: ["src/**/*.test.ts"],
|
||||
},
|
||||
});
|
||||
Reference in New Issue
Block a user