mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat(vom): method update, solve context over exposed problem
This commit is contained in:
Ljy-0827
@@ -570,6 +570,82 @@ describe("buildVomScene", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("dedupes iframe fallback controls already exposed through AX", () => {
|
||||
const axNodes: CdpAxNode[] = [
|
||||
{
|
||||
nodeId: "1",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
backendDOMNodeId: 10,
|
||||
childIds: ["2"],
|
||||
},
|
||||
{
|
||||
nodeId: "2",
|
||||
parentId: "1",
|
||||
role: { type: "role", value: "Iframe" },
|
||||
backendDOMNodeId: 20,
|
||||
childIds: ["3"],
|
||||
},
|
||||
{
|
||||
nodeId: "3",
|
||||
parentId: "2",
|
||||
role: { type: "role", value: "textbox" },
|
||||
name: { type: "x", value: "输入密码" },
|
||||
backendDOMNodeId: 202,
|
||||
properties: [{ name: "inputType", value: { value: "password" } }],
|
||||
},
|
||||
];
|
||||
const captured: CapturedViewModel = {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "iframe",
|
||||
attrs: {},
|
||||
rect: { x: 100, y: 100, w: 400, h: 300 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
iframeNodes: new Map([
|
||||
[
|
||||
20,
|
||||
[
|
||||
{
|
||||
backendNodeId: 102,
|
||||
parentBackendNodeId: null,
|
||||
tag: "input",
|
||||
attrs: { type: "password", placeholder: "输入密码" },
|
||||
rect: { x: 0, y: 50, w: 300, h: 40 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
],
|
||||
]),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
|
||||
const passwordNodes = buildVomScene(axNodes, captured).nodes.filter(
|
||||
(node) => node.role === "textbox" && node.name === "输入密码",
|
||||
);
|
||||
|
||||
expect(passwordNodes).toHaveLength(1);
|
||||
expect(passwordNodes[0]).toEqual(expect.objectContaining({ id: 202, sensitive: true }));
|
||||
});
|
||||
|
||||
it("keeps unnamed iframe controls but skips unnamed iframe links", () => {
|
||||
const scene = buildVomScene(
|
||||
[
|
||||
@@ -843,6 +919,66 @@ describe("buildVomScene", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("uses nearby preceding text as a form control label", () => {
|
||||
const axNodes: CdpAxNode[] = [
|
||||
{
|
||||
nodeId: "1",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
backendDOMNodeId: 1,
|
||||
childIds: ["2"],
|
||||
},
|
||||
{
|
||||
nodeId: "2",
|
||||
parentId: "1",
|
||||
role: { type: "role", value: "textbox" },
|
||||
backendDOMNodeId: 20,
|
||||
},
|
||||
];
|
||||
const captured: CapturedViewModel = {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 1,
|
||||
parentBackendNodeId: null,
|
||||
tag: "div",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 500, h: 80 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: 1,
|
||||
tag: "span",
|
||||
attrs: {},
|
||||
textContent: "验证码:",
|
||||
rect: { x: 0, y: 0, w: 80, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 1,
|
||||
tag: "input",
|
||||
attrs: { type: "text" },
|
||||
rect: { x: 90, y: 0, w: 120, h: 30 },
|
||||
paintOrder: 2,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
cursor: "text",
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
expect(buildVomScene(axNodes, captured).nodes.find((node) => node.id === 20)).toEqual(
|
||||
expect.objectContaining({ role: "textbox", name: "验证码" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("promotes a cursor:pointer icon control to a button named from its <use> sprite", () => {
|
||||
// A close button built as <div class=close-button><svg><use href="#close"/></svg></div>
|
||||
// — the AX tree exposes it as `generic` with no name, so it would be dropped.
|
||||
@@ -1065,6 +1201,284 @@ describe("buildVomScene", () => {
|
||||
expect(rendered.refs).toEqual([{ ref: "e1", backendNodeId: 20 }]);
|
||||
});
|
||||
|
||||
it("builds active scope blocks from active aria-controls relationships", () => {
|
||||
const axNodes: CdpAxNode[] = [
|
||||
{
|
||||
nodeId: "1",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
backendDOMNodeId: 10,
|
||||
childIds: ["2"],
|
||||
},
|
||||
{
|
||||
nodeId: "2",
|
||||
parentId: "1",
|
||||
role: { type: "role", value: "tab" },
|
||||
name: { type: "computedString", value: "Reviews (12)" },
|
||||
backendDOMNodeId: 20,
|
||||
},
|
||||
];
|
||||
const captured: CapturedViewModel = {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "button",
|
||||
attrs: { role: "tab", "aria-selected": "true", "aria-controls": "reviews-panel" },
|
||||
rect: { x: 20, y: 20, w: 120, h: 40 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 30,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "div",
|
||||
attrs: { id: "reviews-panel" },
|
||||
rect: null,
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
textContent: "Jane - ear cups are small",
|
||||
},
|
||||
{
|
||||
backendNodeId: 31,
|
||||
parentBackendNodeId: 30,
|
||||
tag: "p",
|
||||
attrs: {},
|
||||
rect: null,
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
textContent: "Bob - great sound",
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const scene = buildVomScene(axNodes, captured);
|
||||
expect(scene.activeScopeBlocks).toEqual([
|
||||
{
|
||||
triggerId: 20,
|
||||
label: "Reviews (12)",
|
||||
lines: ["Jane - ear cups are small", "Bob - great sound"],
|
||||
},
|
||||
]);
|
||||
expect(renderVom(scene).text).toContain("[§ active: Reviews (12)]");
|
||||
});
|
||||
|
||||
it("maps hover surface probes to matching node labels", () => {
|
||||
const axNodes: CdpAxNode[] = [
|
||||
{
|
||||
nodeId: "1",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
backendDOMNodeId: 10,
|
||||
childIds: ["2"],
|
||||
},
|
||||
{
|
||||
nodeId: "2",
|
||||
parentId: "1",
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Products" },
|
||||
backendDOMNodeId: 20,
|
||||
},
|
||||
];
|
||||
const scene = buildVomScene(axNodes, {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
surfaceProbes: [
|
||||
{ triggerLabel: "Products", triggerAction: "hover", subItems: ["Shoes", "Bags"] },
|
||||
],
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "button",
|
||||
attrs: {},
|
||||
rect: { x: 20, y: 20, w: 120, h: 40 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(scene.surfaces).toEqual([
|
||||
{ triggerId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
|
||||
]);
|
||||
expect(renderVom(scene).text).toContain('@e1 button "Products" [→ Shoes | Bags]');
|
||||
});
|
||||
|
||||
it("enriches names and active scope signals from AX properties", () => {
|
||||
const axNodes: CdpAxNode[] = [
|
||||
{
|
||||
nodeId: "1",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
backendDOMNodeId: 10,
|
||||
childIds: ["2", "3"],
|
||||
},
|
||||
{
|
||||
nodeId: "2",
|
||||
parentId: "1",
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Products" },
|
||||
backendDOMNodeId: 20,
|
||||
properties: [
|
||||
{ name: "hasPopup", value: { value: "menu" } },
|
||||
{ name: "expanded", value: { value: "true" } },
|
||||
{ name: "controls", value: { value: "products-panel" } },
|
||||
],
|
||||
},
|
||||
{
|
||||
nodeId: "3",
|
||||
parentId: "1",
|
||||
role: { type: "role", value: "textbox" },
|
||||
name: { type: "computedString", value: "Card number" },
|
||||
backendDOMNodeId: 30,
|
||||
properties: [{ name: "inputType", value: { value: "credit-card" } }],
|
||||
},
|
||||
];
|
||||
const scene = buildVomScene(axNodes, {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "button",
|
||||
attrs: {},
|
||||
rect: { x: 20, y: 20, w: 120, h: 40 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 25,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "div",
|
||||
attrs: { id: "products-panel" },
|
||||
rect: null,
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
textContent: "Shoes Bags",
|
||||
},
|
||||
{
|
||||
backendNodeId: 30,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "input",
|
||||
attrs: {},
|
||||
rect: { x: 20, y: 80, w: 200, h: 40 },
|
||||
paintOrder: 2,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(scene.nodes.find((node) => node.id === 20)).toEqual(
|
||||
expect.objectContaining({
|
||||
name: "Products [expanded]",
|
||||
attrs: expect.objectContaining({
|
||||
"aria-expanded": "true",
|
||||
"aria-controls": "products-panel",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
expect(scene.activeScopeBlocks).toEqual([
|
||||
{ triggerId: 20, label: "Products [expanded]", lines: ["Shoes Bags"] },
|
||||
]);
|
||||
expect(scene.nodes.find((node) => node.id === 30)?.sensitive).toBe(true);
|
||||
});
|
||||
|
||||
it("aggregates AX virtual text into unnamed structural nodes", () => {
|
||||
const scene = buildVomScene(
|
||||
[
|
||||
{
|
||||
nodeId: "1",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
backendDOMNodeId: 10,
|
||||
childIds: ["2"],
|
||||
},
|
||||
{
|
||||
nodeId: "2",
|
||||
parentId: "1",
|
||||
role: { type: "role", value: "paragraph" },
|
||||
backendDOMNodeId: 20,
|
||||
childIds: ["3"],
|
||||
},
|
||||
{
|
||||
nodeId: "3",
|
||||
parentId: "2",
|
||||
role: { type: "role", value: "InlineTextBox" },
|
||||
name: { type: "computedString", value: "Inline only text" },
|
||||
},
|
||||
],
|
||||
{
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "p",
|
||||
attrs: {},
|
||||
rect: { x: 20, y: 20, w: 400, h: 40 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
},
|
||||
);
|
||||
|
||||
expect(scene.nodes.find((node) => node.id === 20)?.name).toBe("Inline only text");
|
||||
});
|
||||
|
||||
it("marks captured dialog elements as modal without AX role or aria-modal", () => {
|
||||
const scene = buildVomScene([], {
|
||||
viewport: { width: 640, height: 480 },
|
||||
@@ -1533,7 +1947,8 @@ describe("handleSnapshot", () => {
|
||||
expect(res.text).toContain("L1 modal cover=100%");
|
||||
// Iframe content must appear in L1 — both inputs rendered
|
||||
expect(res.text).toMatch(/textbox "请输入手机号"/);
|
||||
expect(res.text).toMatch(/textbox "密码" ="•••"/);
|
||||
expect(res.text).toMatch(/textbox "密码"/);
|
||||
expect(res.text).not.toMatch(/textbox "密码" ="•••"/);
|
||||
// Ref-store must include the iframe's input backendNodeIds
|
||||
const ctx = sm["sessions"].get("aa11")!;
|
||||
const phoneRef = res.text.match(/@(e\d+) textbox "请输入手机号"/)?.[1];
|
||||
|
||||
@@ -3,7 +3,13 @@
|
||||
// tab (defaulting to the Agent Window's active tab when omitted) and
|
||||
// returns a payload that mirrors the bsk-protocol Rust structs.
|
||||
|
||||
import { renderVom, type VomNode, type VomScene } from "@browser-skill/vom";
|
||||
import {
|
||||
type ActiveScopeBlock,
|
||||
type CondSurface,
|
||||
renderVom,
|
||||
type VomNode,
|
||||
type VomScene,
|
||||
} from "@browser-skill/vom";
|
||||
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
||||
import type { SessionManager } from "@/session-manager/manager";
|
||||
import type {
|
||||
@@ -234,6 +240,7 @@ export interface CdpAxNode {
|
||||
name?: { type: string; value?: string };
|
||||
description?: { value?: string };
|
||||
value?: { value?: string | number | boolean };
|
||||
properties?: Array<{ name?: string; value?: { value?: string | number | boolean } }>;
|
||||
childIds?: string[];
|
||||
}
|
||||
|
||||
@@ -268,6 +275,120 @@ function isSensitive(capturedNode: CapturedNode | undefined): boolean {
|
||||
return (attrs.type ?? "").toLowerCase() === "password";
|
||||
}
|
||||
|
||||
interface AxNodeSignals {
|
||||
hasPopup: boolean;
|
||||
expanded: boolean;
|
||||
selected: boolean;
|
||||
controls: string;
|
||||
sensitive: boolean;
|
||||
aggregatedText?: string;
|
||||
}
|
||||
|
||||
const AX_TEXT_AGGREGATE_ROLES = new Set([
|
||||
"paragraph",
|
||||
"listitem",
|
||||
"term",
|
||||
"definition",
|
||||
"cell",
|
||||
"gridcell",
|
||||
"caption",
|
||||
"figcaption",
|
||||
"blockquote",
|
||||
"note",
|
||||
"status",
|
||||
"log",
|
||||
"generic",
|
||||
"section",
|
||||
]);
|
||||
const AX_TEXT_LEAF_ROLES = new Set(["inlinetextbox", "statictext", "text"]);
|
||||
const AX_TEXT_STOP_ROLES = new Set([
|
||||
"button",
|
||||
"link",
|
||||
"combobox",
|
||||
"listbox",
|
||||
"menuitem",
|
||||
"menuitemcheckbox",
|
||||
"menuitemradio",
|
||||
"option",
|
||||
"radio",
|
||||
"checkbox",
|
||||
"textbox",
|
||||
"searchbox",
|
||||
"spinbutton",
|
||||
"slider",
|
||||
"switch",
|
||||
"tab",
|
||||
"treeitem",
|
||||
"columnheader",
|
||||
"rowheader",
|
||||
]);
|
||||
const SENSITIVE_AX_INPUT_TYPES = new Set(["password", "credit-card"]);
|
||||
|
||||
function axPropertyString(axNode: CdpAxNode, name: string): string | undefined {
|
||||
const prop = axNode.properties?.find((item) => item.name === name);
|
||||
return axString(prop?.value);
|
||||
}
|
||||
|
||||
function buildAxSignals(axNodes: CdpAxNode[]): Map<string, AxNodeSignals> {
|
||||
const axByNodeId = new Map(axNodes.map((node) => [node.nodeId, node]));
|
||||
const virtualText = new Map<string, string>();
|
||||
for (const node of axNodes) {
|
||||
if (typeof node.backendDOMNodeId === "number") continue;
|
||||
const role = axString(node.role)?.toLowerCase() ?? "";
|
||||
const name = axString(node.name);
|
||||
if (name && AX_TEXT_LEAF_ROLES.has(role)) virtualText.set(node.nodeId, name);
|
||||
}
|
||||
|
||||
const collectLeafText = (nodeId: string, depth: number): string[] => {
|
||||
if (depth > 8) return [];
|
||||
const node = axByNodeId.get(nodeId);
|
||||
if (!node) return [];
|
||||
const parts: string[] = [];
|
||||
for (const childId of node.childIds ?? []) {
|
||||
const vtext = virtualText.get(childId);
|
||||
if (vtext) {
|
||||
parts.push(vtext);
|
||||
continue;
|
||||
}
|
||||
const child = axByNodeId.get(childId);
|
||||
if (!child) continue;
|
||||
const childRole = axString(child.role)?.toLowerCase() ?? "";
|
||||
if (AX_TEXT_LEAF_ROLES.has(childRole)) continue;
|
||||
if (AX_TEXT_STOP_ROLES.has(childRole)) continue;
|
||||
const childName = axString(child.name);
|
||||
if (childName && !AX_TEXT_AGGREGATE_ROLES.has(childRole)) {
|
||||
parts.push(childName);
|
||||
} else {
|
||||
parts.push(...collectLeafText(childId, depth + 1));
|
||||
}
|
||||
}
|
||||
return parts;
|
||||
};
|
||||
|
||||
const signals = new Map<string, AxNodeSignals>();
|
||||
for (const node of axNodes) {
|
||||
const role = axString(node.role)?.toLowerCase() ?? "";
|
||||
const expanded = axPropertyString(node, "expanded") === "true";
|
||||
const selected = axPropertyString(node, "selected") === "true";
|
||||
const controls = axPropertyString(node, "controls") ?? "";
|
||||
const hasPopupValue = axPropertyString(node, "hasPopup") ?? "";
|
||||
const inputType = axPropertyString(node, "inputType") ?? "";
|
||||
let aggregatedText: string | undefined;
|
||||
if (AX_TEXT_AGGREGATE_ROLES.has(role) && !axString(node.name)) {
|
||||
aggregatedText = cleanAttr(collectLeafText(node.nodeId, 0).join(" "));
|
||||
}
|
||||
signals.set(node.nodeId, {
|
||||
hasPopup: hasPopupValue !== "" && hasPopupValue !== "false",
|
||||
expanded,
|
||||
selected,
|
||||
controls,
|
||||
sensitive: SENSITIVE_AX_INPUT_TYPES.has(inputType),
|
||||
aggregatedText,
|
||||
});
|
||||
}
|
||||
return signals;
|
||||
}
|
||||
|
||||
function findAxAncestor<T>(
|
||||
axNode: CdpAxNode,
|
||||
axById: Map<string, CdpAxNode>,
|
||||
@@ -331,6 +452,40 @@ function isRenderableIframeControl(node: CapturedNode): boolean {
|
||||
return tag !== "a" || name !== undefined;
|
||||
}
|
||||
|
||||
function normalizedControlType(node: VomNode | CapturedNode): string {
|
||||
const attrs = node.attrs ?? {};
|
||||
const tag = normalizeTag(node.tag);
|
||||
if (tag === "input") return (attrs.type ?? "text").toLowerCase();
|
||||
if ("sensitive" in node && node.sensitive) return "password";
|
||||
return tag;
|
||||
}
|
||||
|
||||
function sameLogicalIframeControl(existing: VomNode, candidate: CapturedNode, iframeId: number) {
|
||||
const role = iframeRoleFor(candidate);
|
||||
const name = cleanAttr(iframeNameFor(candidate));
|
||||
if (!role || !name) return false;
|
||||
if ((existing.role ?? "").toLowerCase() !== role) return false;
|
||||
if (cleanAttr(existing.name) !== name) return false;
|
||||
|
||||
const candidateType = normalizedControlType(candidate);
|
||||
const existingType = normalizedControlType(existing);
|
||||
if (candidateType !== existingType) return false;
|
||||
|
||||
return (
|
||||
existing.parentId === iframeId ||
|
||||
existing.domParentId === iframeId ||
|
||||
existing.domAncestorIds?.includes(iframeId) === true
|
||||
);
|
||||
}
|
||||
|
||||
function hasEquivalentIframeControl(
|
||||
nodes: VomNode[],
|
||||
candidate: CapturedNode,
|
||||
iframeId: number,
|
||||
): boolean {
|
||||
return nodes.some((node) => sameLogicalIframeControl(node, candidate, iframeId));
|
||||
}
|
||||
|
||||
function capturedIframeNameFor(node: CapturedNode): string | undefined {
|
||||
const ariaLabel = node.attrs["aria-label"]?.replace(/\s+/g, " ").trim();
|
||||
if (ariaLabel) return ariaLabel;
|
||||
@@ -350,6 +505,11 @@ function cleanAttr(value: string | undefined): string | undefined {
|
||||
const FORM_CONTROL_TAGS = new Set(["input", "textarea", "select"]);
|
||||
|
||||
const NATIVE_CONTROL_TAGS = new Set(["button", "input", "select", "textarea"]);
|
||||
const ACTIVE_SCOPE_MAX_BLOCKS = 8;
|
||||
const ACTIVE_SCOPE_MAX_LINES = 40;
|
||||
const ACTIVE_SCOPE_MAX_LINE_LENGTH = 160;
|
||||
const ACTIVE_SCOPE_MAX_TOTAL_CHARS = 8_000;
|
||||
const ACTIVE_SCOPE_SKIP_TAGS = new Set(["script", "style", "noscript", "template"]);
|
||||
|
||||
function buildCapturedChildren(capturedNodes: CapturedNode[]): Map<number, CapturedNode[]> {
|
||||
const children = new Map<number, CapturedNode[]>();
|
||||
@@ -427,6 +587,24 @@ function nearbyTextFor(
|
||||
return labels.length > 0 ? labels.join(" ") : undefined;
|
||||
}
|
||||
|
||||
function previousSiblingTextFor(
|
||||
node: CapturedNode,
|
||||
childrenByParentId: Map<number, CapturedNode[]>,
|
||||
): string | undefined {
|
||||
if (node.parentBackendNodeId === null) return undefined;
|
||||
const siblings = childrenByParentId.get(node.parentBackendNodeId) ?? [];
|
||||
const index = siblings.findIndex((sibling) => sibling.backendNodeId === node.backendNodeId);
|
||||
if (index <= 0) return undefined;
|
||||
|
||||
for (let i = index - 1; i >= Math.max(0, index - 4); i -= 1) {
|
||||
const sibling = siblings[i];
|
||||
if (["input", "textarea", "select", "button", "a"].includes(normalizeTag(sibling.tag))) break;
|
||||
const text = cleanAttr(sibling.textContent)?.replace(/[：:]\s*$/, "");
|
||||
if (text && text.length <= 40) return text;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
interface VomNodeDomSignals {
|
||||
capturedByBackendId: Map<number, CapturedNode>;
|
||||
childrenByParentId: Map<number, CapturedNode[]>;
|
||||
@@ -470,13 +648,18 @@ function formControlName(
|
||||
captured: CapturedNode | undefined,
|
||||
axName: string | undefined,
|
||||
axValue: string | undefined,
|
||||
signals: VomNodeDomSignals,
|
||||
): string | undefined {
|
||||
const placeholder = cleanAttr(captured?.attrs.placeholder);
|
||||
const ariaLabel = cleanAttr(captured?.attrs["aria-label"]);
|
||||
const title = cleanAttr(captured?.attrs.title);
|
||||
const nearbyLabel = captured
|
||||
? previousSiblingTextFor(captured, signals.childrenByParentId)
|
||||
: undefined;
|
||||
const hasValue = axValue !== undefined && axValue !== "";
|
||||
if (nearbyLabel) return nearbyLabel;
|
||||
if (!hasValue && placeholder) return placeholder;
|
||||
return axName ?? ariaLabel ?? placeholder ?? title;
|
||||
return axName ?? ariaLabel ?? placeholder ?? title ?? nearbyLabel;
|
||||
}
|
||||
|
||||
function vomNodeFromCaptured(
|
||||
@@ -516,12 +699,17 @@ function axVomNode(
|
||||
capturedNode: CapturedNode | undefined,
|
||||
axById: Map<string, CdpAxNode>,
|
||||
signals: VomNodeDomSignals,
|
||||
axSignals: Map<string, AxNodeSignals>,
|
||||
): VomNode {
|
||||
const role = axString(axNode.role);
|
||||
const value = axValue(axNode.value);
|
||||
const name = FORM_CONTROL_TAGS.has(normalizeTag(capturedNode?.tag))
|
||||
? formControlName(capturedNode, axString(axNode.name), value)
|
||||
: axString(axNode.name);
|
||||
const signalsForNode = axSignals.get(axNode.nodeId);
|
||||
let name = FORM_CONTROL_TAGS.has(normalizeTag(capturedNode?.tag))
|
||||
? formControlName(capturedNode, axString(axNode.name), value, signals)
|
||||
: (axString(axNode.name) ?? signalsForNode?.aggregatedText);
|
||||
if (name && signalsForNode?.hasPopup) {
|
||||
name = signalsForNode.expanded ? `${name} [expanded]` : `${name} [has-submenu]`;
|
||||
}
|
||||
const node: VomNode = {
|
||||
id: axNode.backendDOMNodeId as number,
|
||||
parentId: nearestBackendParent(axNode, axById),
|
||||
@@ -531,12 +719,22 @@ function axVomNode(
|
||||
position: capturedNode?.position || "static",
|
||||
pointerEvents: capturedNode?.pointerEvents || "auto",
|
||||
modal: isModalSignal(axNode, capturedNode),
|
||||
sensitive: isSensitive(capturedNode),
|
||||
sensitive: isSensitive(capturedNode) || signalsForNode?.sensitive === true,
|
||||
};
|
||||
if (role) node.role = role;
|
||||
if (name) node.name = name;
|
||||
if (value !== undefined) node.value = value;
|
||||
return applyCapturedSignals(node, capturedNode, signals);
|
||||
const enriched = applyCapturedSignals(node, capturedNode, signals);
|
||||
if (signalsForNode?.selected && !enriched.attrs?.["aria-selected"]) {
|
||||
enriched.attrs = { ...(enriched.attrs ?? {}), "aria-selected": "true" };
|
||||
}
|
||||
if (signalsForNode?.expanded && !enriched.attrs?.["aria-expanded"]) {
|
||||
enriched.attrs = { ...(enriched.attrs ?? {}), "aria-expanded": "true" };
|
||||
}
|
||||
if (signalsForNode?.controls && !enriched.attrs?.["aria-controls"]) {
|
||||
enriched.attrs = { ...(enriched.attrs ?? {}), "aria-controls": signalsForNode.controls };
|
||||
}
|
||||
return enriched;
|
||||
}
|
||||
|
||||
function iframeSignals(iframeNodes: CapturedNode[]): VomNodeDomSignals {
|
||||
@@ -561,6 +759,144 @@ function capturedOnlySignals(capturedNodes: CapturedNode[]): VomNodeDomSignals {
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeProbeKey(value: string | undefined): string {
|
||||
return cleanAttr(value)?.toLowerCase() ?? "";
|
||||
}
|
||||
|
||||
function controlledIds(attrs: Record<string, string>): string[] {
|
||||
return (attrs["aria-controls"] ?? "").split(/\s+/).filter(Boolean);
|
||||
}
|
||||
|
||||
function isActiveScopeTrigger(node: VomNode): boolean {
|
||||
const attrs = node.attrs ?? {};
|
||||
if (controlledIds(attrs).length === 0) return false;
|
||||
const role = normalizeTag(attrs.role) || normalizedVomRole(node);
|
||||
return (
|
||||
(role === "tab" && (attrs["aria-selected"] ?? "").toLowerCase() === "true") ||
|
||||
(attrs["aria-expanded"] ?? "").toLowerCase() === "true"
|
||||
);
|
||||
}
|
||||
|
||||
function normalizedVomRole(node: VomNode): string {
|
||||
return node.role?.toLowerCase() ?? "";
|
||||
}
|
||||
|
||||
function collectScopeLines(
|
||||
root: CapturedNode,
|
||||
triggerLabel: string,
|
||||
childrenByParentId: Map<number, CapturedNode[]>,
|
||||
): string[] {
|
||||
const lines: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
const triggerKey = normalizeProbeKey(triggerLabel);
|
||||
const stack = [root];
|
||||
while (stack.length > 0 && lines.length < ACTIVE_SCOPE_MAX_LINES) {
|
||||
const node = stack.shift() as CapturedNode;
|
||||
const tag = normalizeTag(node.tag);
|
||||
if (ACTIVE_SCOPE_SKIP_TAGS.has(tag)) continue;
|
||||
if ((node.attrs.type ?? "").toLowerCase() === "password") continue;
|
||||
|
||||
const text = cleanAttr(node.textContent);
|
||||
const key = normalizeProbeKey(text);
|
||||
if (text && key !== triggerKey && !seen.has(key)) {
|
||||
seen.add(key);
|
||||
lines.push(
|
||||
text.length > ACTIVE_SCOPE_MAX_LINE_LENGTH
|
||||
? text.slice(0, ACTIVE_SCOPE_MAX_LINE_LENGTH)
|
||||
: text,
|
||||
);
|
||||
}
|
||||
stack.push(...(childrenByParentId.get(node.backendNodeId) ?? []));
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
function buildActiveScopeBlocks(nodes: VomNode[], signals: VomNodeDomSignals): ActiveScopeBlock[] {
|
||||
const panelByDomId = new Map<string, CapturedNode>();
|
||||
for (const capturedNode of signals.capturedByBackendId.values()) {
|
||||
const domId = capturedNode.attrs.id;
|
||||
if (domId) panelByDomId.set(domId, capturedNode);
|
||||
}
|
||||
|
||||
const blocks: ActiveScopeBlock[] = [];
|
||||
let totalChars = 0;
|
||||
for (const node of nodes) {
|
||||
if (blocks.length >= ACTIVE_SCOPE_MAX_BLOCKS) break;
|
||||
if (!isActiveScopeTrigger(node)) continue;
|
||||
const label =
|
||||
cleanAttr(node.name) ?? cleanAttr(node.text) ?? cleanAttr(node.attrs?.["aria-label"]);
|
||||
if (!label) continue;
|
||||
|
||||
const lines: string[] = [];
|
||||
for (const controlId of controlledIds(node.attrs ?? {})) {
|
||||
const panel = panelByDomId.get(controlId);
|
||||
if (!panel) continue;
|
||||
lines.push(...collectScopeLines(panel, label, signals.childrenByParentId));
|
||||
if (lines.length >= ACTIVE_SCOPE_MAX_LINES) break;
|
||||
}
|
||||
|
||||
const uniqueLines: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const line of lines) {
|
||||
const key = normalizeProbeKey(line);
|
||||
if (!key || seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
uniqueLines.push(line);
|
||||
totalChars += line.length;
|
||||
if (
|
||||
uniqueLines.length >= ACTIVE_SCOPE_MAX_LINES ||
|
||||
totalChars >= ACTIVE_SCOPE_MAX_TOTAL_CHARS
|
||||
) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (uniqueLines.length === 0) continue;
|
||||
blocks.push({ triggerId: node.id, label, lines: uniqueLines });
|
||||
if (totalChars >= ACTIVE_SCOPE_MAX_TOTAL_CHARS) break;
|
||||
}
|
||||
return blocks;
|
||||
}
|
||||
|
||||
function labelForSurfaceMatch(node: VomNode): string {
|
||||
return (
|
||||
cleanAttr(node.name) ??
|
||||
cleanAttr(node.text) ??
|
||||
cleanAttr(node.attrs?.["aria-label"]) ??
|
||||
cleanAttr(node.attrs?.title) ??
|
||||
""
|
||||
);
|
||||
}
|
||||
|
||||
function buildConditionalSurfaces(nodes: VomNode[], captured: CapturedViewModel): CondSurface[] {
|
||||
const probes = captured.surfaceProbes ?? [];
|
||||
if (probes.length === 0) return [];
|
||||
|
||||
const surfaces: CondSurface[] = [];
|
||||
const nodesWithLabels = nodes
|
||||
.map((node) => ({ node, key: normalizeProbeKey(labelForSurfaceMatch(node)) }))
|
||||
.filter((entry) => entry.key.length > 0);
|
||||
const used = new Set<number>();
|
||||
for (const probe of probes) {
|
||||
const probeKey = normalizeProbeKey(probe.triggerLabel);
|
||||
if (!probeKey || probe.subItems.length === 0) continue;
|
||||
const match =
|
||||
nodesWithLabels.find((entry) => entry.key === probeKey && !used.has(entry.node.id)) ??
|
||||
nodesWithLabels.find(
|
||||
(entry) =>
|
||||
!used.has(entry.node.id) &&
|
||||
(probeKey.startsWith(entry.key) || entry.key.startsWith(probeKey)),
|
||||
);
|
||||
if (!match) continue;
|
||||
used.add(match.node.id);
|
||||
surfaces.push({
|
||||
triggerId: match.node.id,
|
||||
triggerAction: probe.triggerAction,
|
||||
subItems: probe.subItems,
|
||||
});
|
||||
}
|
||||
return surfaces;
|
||||
}
|
||||
|
||||
function axNodeInOverlaySubtree(
|
||||
axNode: CdpAxNode,
|
||||
axById: Map<string, CdpAxNode>,
|
||||
@@ -585,6 +921,7 @@ function axNodeInOverlaySubtree(
|
||||
export function buildVomScene(axNodes: CdpAxNode[], captured: CapturedViewModel): VomScene {
|
||||
const signals = capturedOnlySignals(captured.nodes);
|
||||
const { capturedByBackendId } = signals;
|
||||
const axSignals = buildAxSignals(axNodes);
|
||||
|
||||
const axById = new Map<string, CdpAxNode>();
|
||||
for (const node of axNodes) {
|
||||
@@ -601,7 +938,7 @@ export function buildVomScene(axNodes: CdpAxNode[], captured: CapturedViewModel)
|
||||
|
||||
const capturedNode = capturedByBackendId.get(axNode.backendDOMNodeId);
|
||||
const role = axString(axNode.role);
|
||||
const vomNode = axVomNode(axNode, capturedNode, axById, signals);
|
||||
const vomNode = axVomNode(axNode, capturedNode, axById, signals, axSignals);
|
||||
|
||||
// For link nodes, attach external hostname so the renderer can annotate it.
|
||||
// Relative hrefs and same-origin hrefs are omitted to avoid noise.
|
||||
@@ -633,7 +970,8 @@ export function buildVomScene(axNodes: CdpAxNode[], captured: CapturedViewModel)
|
||||
if (
|
||||
seenBackendIds.has(iframeNode.backendNodeId) ||
|
||||
excludedBackendNodeIds.has(iframeNode.backendNodeId) ||
|
||||
!isRenderableIframeControl(iframeNode)
|
||||
!isRenderableIframeControl(iframeNode) ||
|
||||
hasEquivalentIframeControl(nodes, iframeNode, iframeBackendId)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
@@ -651,7 +989,14 @@ export function buildVomScene(axNodes: CdpAxNode[], captured: CapturedViewModel)
|
||||
}
|
||||
}
|
||||
|
||||
return { viewport: captured.viewport, nodes };
|
||||
const activeScopeBlocks = buildActiveScopeBlocks(nodes, signals);
|
||||
const surfaces = buildConditionalSurfaces(nodes, captured);
|
||||
return {
|
||||
viewport: captured.viewport,
|
||||
nodes,
|
||||
...(surfaces.length > 0 ? { surfaces } : {}),
|
||||
...(activeScopeBlocks.length > 0 ? { activeScopeBlocks } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export interface SnapshotDeps {
|
||||
@@ -825,7 +1170,11 @@ export async function handleSnapshot(
|
||||
const axNodes = result.nodes ?? [];
|
||||
const captured = await captureForVom(deps.cdp, target.tabId);
|
||||
const scene = buildVomScene(axNodes, captured);
|
||||
const rendered = renderVom(scene, { maxDepth: params.max_depth, maxTokens: params.max_tokens });
|
||||
const rendered = renderVom(scene, {
|
||||
maxDepth: params.max_depth,
|
||||
maxTokens: params.max_tokens,
|
||||
activeRegionPolicy: true,
|
||||
});
|
||||
ctx.refStore.replace(
|
||||
rendered.refs.map(
|
||||
(r) => [r.ref, { backendNodeId: r.backendNodeId, tabId: target.tabId }] as const,
|
||||
|
||||
@@ -35,10 +35,17 @@ export interface CapturedNode {
|
||||
|
||||
export type CapturedIframeNodes = Map<number, CapturedNode[]>;
|
||||
|
||||
export interface CapturedSurfaceProbe {
|
||||
triggerLabel: string;
|
||||
triggerAction: "hover" | "focus" | string;
|
||||
subItems: string[];
|
||||
}
|
||||
|
||||
export interface CapturedViewModel {
|
||||
nodes: CapturedNode[];
|
||||
viewport: Viewport;
|
||||
iframeNodes: CapturedIframeNodes;
|
||||
surfaceProbes?: CapturedSurfaceProbe[];
|
||||
/** Backend node ids belonging to the agent overlay host + its shadow subtree. */
|
||||
excludedBackendNodeIds: Set<number>;
|
||||
}
|
||||
@@ -88,6 +95,20 @@ interface LayoutMetricsReply {
|
||||
layoutViewport?: { clientWidth?: number; clientHeight?: number; pageX?: number; pageY?: number };
|
||||
}
|
||||
|
||||
interface RuntimeEvaluateReply {
|
||||
result?: {
|
||||
value?: unknown;
|
||||
};
|
||||
}
|
||||
|
||||
interface HoverCandidate {
|
||||
triggerSel: string;
|
||||
affectedSub: string;
|
||||
label: string;
|
||||
x: number;
|
||||
y: number;
|
||||
}
|
||||
|
||||
interface CdpDomNode {
|
||||
backendNodeId?: number;
|
||||
children?: CdpDomNode[];
|
||||
@@ -126,6 +147,160 @@ function collectBackendIdsFromDomNode(node: CdpDomNode | undefined, out: Set<num
|
||||
}
|
||||
}
|
||||
|
||||
const MAX_HOVER_PROBE_MS = 2_000;
|
||||
const MAX_HOVER_TRIGGERS = 8;
|
||||
const HOVER_SETTLE_MS = 200;
|
||||
|
||||
function runtimeValue<T>(reply: RuntimeEvaluateReply): T | undefined {
|
||||
return reply.result?.value as T | undefined;
|
||||
}
|
||||
|
||||
function hoverCssScanExpression(): string {
|
||||
return `(() => {
|
||||
const visibilityProps = ["display", "visibility", "opacity", "maxHeight", "height", "overflow"];
|
||||
const pairs = [];
|
||||
const seenRules = new Set();
|
||||
for (const sheet of Array.from(document.styleSheets)) {
|
||||
let rules;
|
||||
try { rules = sheet.cssRules; } catch { continue; }
|
||||
if (!rules) continue;
|
||||
for (const rule of Array.from(rules)) {
|
||||
if (rule.type !== 1 || !rule.selectorText || !rule.style) continue;
|
||||
const selectorText = String(rule.selectorText);
|
||||
if (!selectorText.includes(":hover")) continue;
|
||||
if (!visibilityProps.some((prop) => rule.style[prop])) continue;
|
||||
for (const rawPart of selectorText.split(",")) {
|
||||
const part = rawPart.trim();
|
||||
const hoverIndex = part.indexOf(":hover");
|
||||
if (hoverIndex < 0) continue;
|
||||
const triggerSel = part.slice(0, hoverIndex).trim();
|
||||
const affectedSub = part.slice(hoverIndex + 6).trim();
|
||||
if (!triggerSel) continue;
|
||||
const key = triggerSel + "||" + affectedSub;
|
||||
if (seenRules.has(key)) continue;
|
||||
seenRules.add(key);
|
||||
pairs.push({ triggerSel, affectedSub });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const candidates = [];
|
||||
const seenLabels = new Set();
|
||||
for (const pair of pairs) {
|
||||
let elements;
|
||||
try { elements = Array.from(document.querySelectorAll(pair.triggerSel)); } catch { continue; }
|
||||
for (const el of elements) {
|
||||
if (!(el instanceof Element)) continue;
|
||||
const rect = el.getBoundingClientRect();
|
||||
const style = getComputedStyle(el);
|
||||
if (rect.width <= 0 || rect.height <= 0 || style.visibility === "hidden" || style.display === "none") continue;
|
||||
const label = (el.textContent || "").replace(/\\s+/g, " ").trim();
|
||||
if (!label || seenLabels.has(label)) continue;
|
||||
seenLabels.add(label);
|
||||
candidates.push({
|
||||
triggerSel: pair.triggerSel,
|
||||
affectedSub: pair.affectedSub,
|
||||
label,
|
||||
x: rect.left + rect.width / 2,
|
||||
y: rect.top + rect.height / 2,
|
||||
});
|
||||
if (candidates.length >= ${MAX_HOVER_TRIGGERS}) return candidates;
|
||||
}
|
||||
}
|
||||
return candidates;
|
||||
})()`;
|
||||
}
|
||||
|
||||
function hoverCollectExpression(candidate: HoverCandidate): string {
|
||||
return `(() => {
|
||||
const triggerSel = ${JSON.stringify(candidate.triggerSel)};
|
||||
const rawAffectedSub = ${JSON.stringify(candidate.affectedSub)};
|
||||
const x = ${JSON.stringify(candidate.x)};
|
||||
const y = ${JSON.stringify(candidate.y)};
|
||||
const hit = document.elementFromPoint(x, y);
|
||||
const trigger = hit instanceof Element ? hit.closest(triggerSel) : null;
|
||||
if (!trigger) return [];
|
||||
|
||||
const affectedSub = rawAffectedSub.replace(/^[>+~]\\s*/, "").trim();
|
||||
let targets = [];
|
||||
if (!affectedSub) {
|
||||
targets = [trigger];
|
||||
} else {
|
||||
try { targets = Array.from(trigger.querySelectorAll(affectedSub)); } catch { targets = []; }
|
||||
}
|
||||
const items = [];
|
||||
const seen = new Set();
|
||||
const push = (value) => {
|
||||
const text = String(value || "").replace(/\\s+/g, " ").trim();
|
||||
if (!text || seen.has(text)) return;
|
||||
seen.add(text);
|
||||
items.push(text);
|
||||
};
|
||||
for (const target of targets) {
|
||||
if (!(target instanceof Element)) continue;
|
||||
const style = getComputedStyle(target);
|
||||
if (style.display === "none" || style.visibility === "hidden" || style.opacity === "0") continue;
|
||||
const clickable = target.querySelectorAll('a, button, [role="menuitem"], [role="option"]');
|
||||
if (clickable.length > 0) {
|
||||
for (const child of Array.from(clickable)) push(child.textContent);
|
||||
} else {
|
||||
push(target.textContent);
|
||||
}
|
||||
}
|
||||
return items.slice(0, 12);
|
||||
})()`;
|
||||
}
|
||||
|
||||
async function wait(ms: number): Promise<void> {
|
||||
await new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function probeHoverSurfaces(cdp: CdpRunner, tabId: number): Promise<CapturedSurfaceProbe[]> {
|
||||
const started = Date.now();
|
||||
try {
|
||||
const scan = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: hoverCssScanExpression(),
|
||||
returnByValue: true,
|
||||
});
|
||||
const candidates = runtimeValue<HoverCandidate[]>(scan) ?? [];
|
||||
const results: CapturedSurfaceProbe[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const candidate of candidates.slice(0, MAX_HOVER_TRIGGERS)) {
|
||||
if (Date.now() - started > MAX_HOVER_PROBE_MS) break;
|
||||
if (!candidate.label || seen.has(candidate.label)) continue;
|
||||
try {
|
||||
await cdp.send(tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: candidate.x,
|
||||
y: candidate.y,
|
||||
});
|
||||
await wait(HOVER_SETTLE_MS);
|
||||
const collected = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: hoverCollectExpression(candidate),
|
||||
returnByValue: true,
|
||||
});
|
||||
await cdp.send(tabId, "Input.dispatchMouseEvent", { type: "mouseMoved", x: 0, y: 0 });
|
||||
const subItems = (runtimeValue<string[]>(collected) ?? [])
|
||||
.map((item) => item.replace(/\s+/g, " ").trim())
|
||||
.filter(Boolean);
|
||||
if (subItems.length === 0) continue;
|
||||
seen.add(candidate.label);
|
||||
results.push({
|
||||
triggerLabel: candidate.label,
|
||||
triggerAction: "hover",
|
||||
subItems,
|
||||
});
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
return results;
|
||||
} catch (err) {
|
||||
console.debug("[bsk capture] hover surface probe failed", err);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* When DOMSnapshot is unavailable, locate the marked overlay host via CDP and
|
||||
* collect every backendNodeId in its pierced subtree (open shadow included).
|
||||
@@ -319,7 +494,13 @@ export async function captureViewModel(cdp: CdpRunner, tabId: number): Promise<C
|
||||
const documents = snap.documents ?? [];
|
||||
const doc0 = documents[0];
|
||||
if (!doc0?.nodes?.backendNodeId) {
|
||||
return { nodes: [], viewport, iframeNodes: new Map(), excludedBackendNodeIds: new Set() };
|
||||
return {
|
||||
nodes: [],
|
||||
viewport,
|
||||
iframeNodes: new Map(),
|
||||
surfaceProbes: [],
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
}
|
||||
|
||||
const mainParsed = parseDocumentNodes(doc0, strings, dpr, scrollX, scrollY);
|
||||
@@ -352,5 +533,7 @@ export async function captureViewModel(cdp: CdpRunner, tabId: number): Promise<C
|
||||
}
|
||||
}
|
||||
|
||||
return { nodes, viewport, iframeNodes, excludedBackendNodeIds };
|
||||
const surfaceProbes = await probeHoverSurfaces(cdp, tabId);
|
||||
|
||||
return { nodes, viewport, iframeNodes, surfaceProbes, excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
@@ -135,11 +135,11 @@ describe("renderVom single-layer page", () => {
|
||||
]),
|
||||
);
|
||||
|
||||
expect(out.text).toContain('@e1 textbox "密码" ="•••"');
|
||||
expect(out.text).toContain('@e1 textbox "密码" [filled] ="•••"');
|
||||
expect(out.text).not.toContain("hunter2");
|
||||
});
|
||||
|
||||
it("masks sensitive textboxes even when the value is missing", () => {
|
||||
it("does not imply a sensitive textbox has a value when the value is missing", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
node({ id: 1, role: "RootWebArea", name: "Login" }),
|
||||
@@ -154,7 +154,8 @@ describe("renderVom single-layer page", () => {
|
||||
]),
|
||||
);
|
||||
|
||||
expect(out.text).toContain('@e1 textbox "密码" ="•••"');
|
||||
expect(out.text).toContain('@e1 textbox "密码" [empty]');
|
||||
expect(out.text).not.toContain('="•••"');
|
||||
});
|
||||
|
||||
it("continues rendering later siblings after a branch exceeds maxDepth", () => {
|
||||
@@ -278,6 +279,237 @@ describe("renderVom single-layer page", () => {
|
||||
expect(out.text).toContain('@e1 button "View" [ctx: Project Alpha]');
|
||||
expect(out.text).toContain('@e2 button "View" [ctx: Project Beta]');
|
||||
});
|
||||
|
||||
it("filters noisy template/layout text from weak label context", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
node({ id: 1, role: "RootWebArea", name: "Doc" }),
|
||||
node({
|
||||
id: 2,
|
||||
parentId: 1,
|
||||
role: "section",
|
||||
name: "top 开始 top 结束 bigpic、登录 开始 <![endif]",
|
||||
}),
|
||||
node({ id: 3, parentId: 2, role: "button", name: "更多", tag: "button" }),
|
||||
]),
|
||||
);
|
||||
|
||||
expect(out.text).toContain('@e1 button "更多"');
|
||||
expect(out.text).not.toContain("[ctx:");
|
||||
});
|
||||
|
||||
it("keeps active-region filtering disabled by default", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
node({ id: 1, role: "RootWebArea", name: "Doc" }),
|
||||
node({
|
||||
id: 2,
|
||||
parentId: 1,
|
||||
role: "button",
|
||||
name: "Background",
|
||||
tag: "button",
|
||||
rect: { x: 120, y: 120, w: 120, h: 40 },
|
||||
paintOrder: 1,
|
||||
}),
|
||||
node({
|
||||
id: 3,
|
||||
parentId: 1,
|
||||
role: "generic",
|
||||
tag: "div",
|
||||
rect: { x: 100, y: 100, w: 180, h: 120 },
|
||||
paintOrder: 10,
|
||||
position: "fixed",
|
||||
}),
|
||||
]),
|
||||
);
|
||||
|
||||
expect(out.text).toContain("@layers 1 focus=L1");
|
||||
expect(out.text).toContain('@e1 button "Background"');
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||
});
|
||||
|
||||
it("can filter refs blocked by active positioned regions without changing layer output", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
node({ id: 1, role: "RootWebArea", name: "Doc" }),
|
||||
node({
|
||||
id: 2,
|
||||
parentId: 1,
|
||||
role: "button",
|
||||
name: "Background",
|
||||
tag: "button",
|
||||
rect: { x: 120, y: 120, w: 120, h: 40 },
|
||||
paintOrder: 1,
|
||||
}),
|
||||
node({
|
||||
id: 3,
|
||||
parentId: 1,
|
||||
role: "generic",
|
||||
tag: "div",
|
||||
rect: { x: 100, y: 100, w: 180, h: 120 },
|
||||
paintOrder: 10,
|
||||
position: "fixed",
|
||||
}),
|
||||
node({
|
||||
id: 4,
|
||||
parentId: 1,
|
||||
role: "button",
|
||||
name: "Foreground",
|
||||
tag: "button",
|
||||
rect: { x: 320, y: 120, w: 120, h: 40 },
|
||||
paintOrder: 11,
|
||||
}),
|
||||
]),
|
||||
{ activeRegionPolicy: true },
|
||||
);
|
||||
|
||||
expect(out.text).toContain("@layers 1 focus=L1");
|
||||
expect(out.text).toContain("L1 page");
|
||||
expect(out.text).not.toContain("active-region");
|
||||
expect(out.text).not.toContain("Background");
|
||||
expect(out.text).toContain('@e1 button "Foreground"');
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]);
|
||||
});
|
||||
|
||||
it("renders conditional surface items inline on the trigger", () => {
|
||||
const out = renderVom({
|
||||
...scene([
|
||||
node({ id: 1, role: "RootWebArea", name: "Doc" }),
|
||||
node({ id: 2, parentId: 1, role: "button", name: "Products", tag: "button" }),
|
||||
]),
|
||||
surfaces: [
|
||||
{
|
||||
triggerId: 2,
|
||||
triggerAction: "hover",
|
||||
subItems: ["Shoes", "Bags", "Accessories"],
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(out.text).toContain('@e1 button "Products" [→ Shoes | Bags | Accessories]');
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||
});
|
||||
|
||||
it("injects active scope blocks immediately after their trigger without refs", () => {
|
||||
const out = renderVom({
|
||||
...scene([
|
||||
node({ id: 1, role: "RootWebArea", name: "Doc" }),
|
||||
node({ id: 2, parentId: 1, role: "tab", name: "Reviews (12)", tag: "button" }),
|
||||
]),
|
||||
activeScopeBlocks: [
|
||||
{
|
||||
triggerId: 2,
|
||||
label: "Reviews (12)",
|
||||
lines: ["Jane - ear cups are small", "Bob - great sound"],
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(out.text).toContain(
|
||||
[
|
||||
' @e1 tab "Reviews (12)"',
|
||||
" [§ active: Reviews (12)]",
|
||||
" Jane - ear cups are small",
|
||||
" Bob - great sound",
|
||||
].join("\n"),
|
||||
);
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||
});
|
||||
|
||||
it("does not render redundant children inside named ref controls", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
node({ id: 1, role: "RootWebArea", name: "Doc" }),
|
||||
node({ id: 2, parentId: 1, role: "button", name: "Save", tag: "button" }),
|
||||
node({ id: 3, parentId: 2, role: "img", name: "disk icon", tag: "svg" }),
|
||||
]),
|
||||
);
|
||||
|
||||
expect(out.text).toContain('@e1 button "Save"');
|
||||
expect(out.text).not.toContain("disk icon");
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||
});
|
||||
|
||||
it("does not render redundant children inside native text inputs", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
node({ id: 1, role: "RootWebArea", name: "Doc" }),
|
||||
node({
|
||||
id: 2,
|
||||
parentId: 1,
|
||||
role: "textbox",
|
||||
name: "手机号",
|
||||
value: "13311030827",
|
||||
tag: "input",
|
||||
}),
|
||||
node({ id: 3, parentId: 2, role: "StaticText", name: "13311030827" }),
|
||||
]),
|
||||
);
|
||||
|
||||
expect(out.text).toContain('@e1 textbox "手机号" [filled] ="13311030827"');
|
||||
expect(out.text).not.toContain('StaticText "13311030827"');
|
||||
});
|
||||
|
||||
it("adds semantic collection summaries for repeated item containers", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
node({ id: 1, role: "RootWebArea", name: "Forum" }),
|
||||
node({ id: 2, parentId: 1, role: "main", name: "Posts" }),
|
||||
node({ id: 3, parentId: 2, role: "searchbox", name: "Search posts", tag: "input" }),
|
||||
node({ id: 10, parentId: 2, role: "article" }),
|
||||
node({ id: 11, parentId: 10, role: "heading", name: "First post" }),
|
||||
node({ id: 12, parentId: 10, role: "link", name: "First post", tag: "a" }),
|
||||
node({ id: 13, parentId: 10, role: "button", name: "Upvote", tag: "button" }),
|
||||
node({ id: 20, parentId: 2, role: "article" }),
|
||||
node({ id: 21, parentId: 20, role: "heading", name: "Second post" }),
|
||||
node({ id: 22, parentId: 20, role: "link", name: "Second post", tag: "a" }),
|
||||
node({ id: 23, parentId: 20, role: "button", name: "Upvote", tag: "button" }),
|
||||
]),
|
||||
{ semanticCollections: true },
|
||||
);
|
||||
|
||||
expect(out.text).toContain("@collections");
|
||||
expect(out.text).toContain(
|
||||
'collection c1 role=main item_role=article label="First post" items=2',
|
||||
);
|
||||
expect(out.text).toContain('controls: search=@e1 "Search posts"');
|
||||
expect(out.text).toContain('title="First post" title_handle=@e2');
|
||||
expect(out.text).toContain('actions=[@e3 "Upvote"]');
|
||||
});
|
||||
|
||||
it("does not summarize login forms as semantic collections", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
node({ id: 1, role: "RootWebArea", name: "北京市小客车指标调控管理信息系统" }),
|
||||
node({ id: 2, parentId: 1, role: "list" }),
|
||||
node({ id: 3, parentId: 2, role: "listitem", name: "用户登录" }),
|
||||
node({ id: 4, parentId: 2, role: "listitem", name: "手机号：" }),
|
||||
node({
|
||||
id: 5,
|
||||
parentId: 4,
|
||||
role: "textbox",
|
||||
value: "13311030827",
|
||||
tag: "input",
|
||||
}),
|
||||
node({ id: 6, parentId: 2, role: "listitem", name: "密 码：" }),
|
||||
node({
|
||||
id: 7,
|
||||
parentId: 6,
|
||||
role: "textbox",
|
||||
value: "secret",
|
||||
tag: "input",
|
||||
sensitive: true,
|
||||
}),
|
||||
node({ id: 8, parentId: 2, role: "listitem", name: "验证码：" }),
|
||||
node({ id: 9, parentId: 8, role: "textbox", value: "h8Mh", tag: "input" }),
|
||||
node({ id: 10, parentId: 2, role: "listitem" }),
|
||||
node({ id: 11, parentId: 10, role: "button", name: "登录", tag: "button" }),
|
||||
]),
|
||||
);
|
||||
|
||||
expect(out.text).not.toContain("@collections");
|
||||
expect(out.text).toContain('@e2 textbox [filled] ="•••"');
|
||||
});
|
||||
});
|
||||
|
||||
describe("renderVom double-layer page", () => {
|
||||
@@ -331,7 +563,7 @@ describe("renderVom double-layer page", () => {
|
||||
"@layers 2 focus=L1",
|
||||
"L1 modal cover=100%",
|
||||
' dialog "提示"',
|
||||
' @e1 textbox "密码" ="•••"',
|
||||
' @e1 textbox "密码" [filled] ="•••"',
|
||||
' @e2 button "确认"',
|
||||
"L2 page … occluded by L1 (~2 nodes, not actionable)",
|
||||
].join("\n"),
|
||||
@@ -385,8 +617,8 @@ describe("renderVom double-layer page", () => {
|
||||
|
||||
expect(out.text).toContain("L1 modal cover=100%");
|
||||
expect(out.text).toContain(' Iframe "登录框"');
|
||||
expect(out.text).toContain(' @e1 textbox "请输入手机号"');
|
||||
expect(out.text).toContain(' @e2 textbox "密码" ="•••"');
|
||||
expect(out.text).toContain(' @e1 textbox "请输入手机号" [empty]');
|
||||
expect(out.text).toContain(' @e2 textbox "密码" [filled] ="•••"');
|
||||
expect(out.refs.map((r) => r.backendNodeId)).toEqual([101, 102]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
export { renderVom } from "./render";
|
||||
export type {
|
||||
ActiveScopeBlock,
|
||||
BlockingLayer,
|
||||
CondSurface,
|
||||
LayerKind,
|
||||
Rect,
|
||||
Viewport,
|
||||
|
||||
+668
-25
@@ -1,5 +1,14 @@
|
||||
import { detectBlockingLayer } from "./layers";
|
||||
import type { BlockingLayer, VomNode, VomOptions, VomResult, VomScene } from "./types";
|
||||
import { coverage, detectBlockingLayer } from "./layers";
|
||||
import type {
|
||||
ActiveScopeBlock,
|
||||
BlockingLayer,
|
||||
CondSurface,
|
||||
Rect,
|
||||
VomNode,
|
||||
VomOptions,
|
||||
VomResult,
|
||||
VomScene,
|
||||
} from "./types";
|
||||
|
||||
const SKIP_ROLES = new Set(["generic", "none", "presentation", "inlinetextbox"]);
|
||||
|
||||
@@ -72,6 +81,13 @@ const MAX_VALUE_LENGTH = 200;
|
||||
const MAX_CUSTOM_RECOVERY_AREA = 180_000;
|
||||
const MAX_RECOVERABLE_SINGLE_INTENT_AREA = 360_000;
|
||||
const MAX_HANDLE_CONTEXT_ITEMS = 3;
|
||||
const MAX_SURFACE_ITEMS = 12;
|
||||
const MAX_SCOPE_LINES = 40;
|
||||
const MAX_SEMANTIC_COLLECTIONS = 3;
|
||||
const MAX_SEMANTIC_ITEMS = 20;
|
||||
const MAX_SEMANTIC_CONTROLS = 8;
|
||||
const MAX_SEMANTIC_ACTIONS = 6;
|
||||
const MAX_SEMANTIC_FIELD_CHARS = 120;
|
||||
|
||||
const RECOVERY_HANDLER_ATTRS = ["onclick", "onmousedown", "onkeydown", "onkeyup", "onkeypress"];
|
||||
const NATIVE_TAGS = new Set(["button", "input", "select", "textarea"]);
|
||||
@@ -105,6 +121,8 @@ const WEAK_ACTION_LABELS = new Set([
|
||||
"continue",
|
||||
]);
|
||||
|
||||
const ACTIVE_REGION_POSITIONS = new Set(["fixed", "absolute", "sticky"]);
|
||||
|
||||
function estimateTokens(text: string): number {
|
||||
return Math.ceil(text.length / 4);
|
||||
}
|
||||
@@ -114,6 +132,31 @@ function cleaned(value: string | undefined): string | undefined {
|
||||
return trimmed ? trimmed : undefined;
|
||||
}
|
||||
|
||||
function isNoisyText(value: string): boolean {
|
||||
const clean = value.trim();
|
||||
const key = clean.toLowerCase();
|
||||
if (!clean) return true;
|
||||
if (clean.includes("<![endif]") || clean.includes("<!--") || clean.includes("-->")) return true;
|
||||
if (/<\/?[a-z][^>]*>/i.test(clean)) return true;
|
||||
if (/\$\(function|document\.|window\.|\.onclick\s*=|function\s*\(|var\s+\w+\s*=/.test(clean)) {
|
||||
return true;
|
||||
}
|
||||
if (/(https?:\/\/|encodeURIComponent|site_code|target="_blank")/.test(clean)) return true;
|
||||
if (/^\[?if\s+[^\]]+\]?>?$/i.test(clean)) return true;
|
||||
if (/^<!\[endif\]?>$/i.test(clean)) return true;
|
||||
if (/^(top|header|footer|nav|bigpic|login|登录|开始|结束)[、,\s]*(开始|结束)?$/i.test(clean)) {
|
||||
return true;
|
||||
}
|
||||
if (/(^|[、,\s])(top|bigpic|登录)\s*(开始|结束)($|[、,\s])/.test(key)) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
function cleanSemanticText(value: string | undefined): string | undefined {
|
||||
const clean = cleaned(value);
|
||||
if (!clean || isNoisyText(clean)) return undefined;
|
||||
return clean;
|
||||
}
|
||||
|
||||
function normalizedRole(node: VomNode): string {
|
||||
return node.role?.toLowerCase() ?? "";
|
||||
}
|
||||
@@ -275,6 +318,21 @@ function buildParentMap(nodes: VomNode[]): Map<number, number | null> {
|
||||
return parentMap;
|
||||
}
|
||||
|
||||
function isAncestorOf(
|
||||
parentMap: Map<number, number | null>,
|
||||
ancestorId: number,
|
||||
nodeId: number,
|
||||
): boolean {
|
||||
let current = parentMap.get(nodeId) ?? null;
|
||||
let guard = 0;
|
||||
while (current !== null && guard <= parentMap.size) {
|
||||
if (current === ancestorId) return true;
|
||||
current = parentMap.get(current) ?? null;
|
||||
guard += 1;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function applyRecovery(nodes: VomNode[]): VomNode[] {
|
||||
const parentMap = buildParentMap(nodes);
|
||||
const children = buildChildren(nodes);
|
||||
@@ -342,7 +400,7 @@ function duplicateReferenceNames(nodes: VomNode[]): Set<string> {
|
||||
function needsHandleContext(nodeName: string, duplicateRefNames: Set<string>): boolean {
|
||||
const key = normalizeContextKey(nodeName);
|
||||
if (!key) return false;
|
||||
return duplicateRefNames.has(key) || WEAK_ACTION_LABELS.has(key);
|
||||
return duplicateRefNames.has(key);
|
||||
}
|
||||
|
||||
function contextLabel(node: VomNode, targetName: string): string | undefined {
|
||||
@@ -368,7 +426,7 @@ function weakLabelContext(node: VomNode, targetName: string): string | undefined
|
||||
}
|
||||
|
||||
function weakContextText(label: string | undefined, targetName: string): string | undefined {
|
||||
const clean = cleaned(label);
|
||||
const clean = cleanSemanticText(label);
|
||||
if (!clean) return undefined;
|
||||
const key = normalizeContextKey(clean);
|
||||
if (
|
||||
@@ -416,7 +474,7 @@ function dedupeContexts(contexts: ContextCandidate[]): string[] {
|
||||
);
|
||||
const unique: string[] = [];
|
||||
for (const ctx of sorted) {
|
||||
const clean = cleaned(ctx.text);
|
||||
const clean = cleanSemanticText(ctx.text);
|
||||
if (!clean) continue;
|
||||
const key = normalizeContextKey(clean);
|
||||
const category = `${ctx.source}:${contextCategory(key)}`;
|
||||
@@ -537,6 +595,7 @@ function renderNodeLine(
|
||||
depth: number,
|
||||
ref: string | undefined,
|
||||
context: string[] = [],
|
||||
surface: CondSurface | undefined = undefined,
|
||||
): string {
|
||||
let line = `${" ".repeat(depth)}${ref ? `@${ref} ` : ""}${node.role}`;
|
||||
|
||||
@@ -550,12 +609,41 @@ function renderNodeLine(
|
||||
line += ` [→ ${node.href}]`;
|
||||
}
|
||||
|
||||
const value = node.sensitive ? SENSITIVE_MASK : cleaned(node.value)?.slice(0, MAX_VALUE_LENGTH);
|
||||
const rawValue = cleaned(node.value);
|
||||
const role = normalizedRole(node);
|
||||
if (["textbox", "searchbox"].includes(role)) {
|
||||
line += rawValue === undefined ? " [empty]" : " [filled]";
|
||||
}
|
||||
const value = node.sensitive
|
||||
? rawValue !== undefined
|
||||
? SENSITIVE_MASK
|
||||
: undefined
|
||||
: rawValue?.slice(0, MAX_VALUE_LENGTH);
|
||||
if (value !== undefined) line += ` =${JSON.stringify(value)}`;
|
||||
|
||||
if (surface && surface.subItems.length > 0) {
|
||||
const items = surface.subItems.slice(0, MAX_SURFACE_ITEMS).join(" | ");
|
||||
const suffix = surface.subItems.length > MAX_SURFACE_ITEMS ? " | …" : "";
|
||||
line += ` [→ ${items}${suffix}]`;
|
||||
}
|
||||
|
||||
return line;
|
||||
}
|
||||
|
||||
function shouldSkipRedundantRefChildren(node: VomNode): boolean {
|
||||
const role = normalizedRole(node);
|
||||
if (
|
||||
["textbox", "searchbox"].includes(role) &&
|
||||
["input", "textarea"].includes((node.tag ?? "").toLowerCase())
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
if (!cleaned(node.name)) return false;
|
||||
return ["button", "link", "menuitem", "tab", "switch", "checkbox", "radio", "combobox"].includes(
|
||||
role,
|
||||
);
|
||||
}
|
||||
|
||||
interface RenderState {
|
||||
lines: string[];
|
||||
refs: Array<{ ref: string; backendNodeId: number }>;
|
||||
@@ -569,6 +657,37 @@ interface RenderState {
|
||||
parentMap: Map<number, number | null>;
|
||||
nodesById: Map<number, VomNode>;
|
||||
duplicateRefNames: Set<string>;
|
||||
surfaceMap: Map<number, CondSurface>;
|
||||
scopeMap: Map<number, ActiveScopeBlock>;
|
||||
}
|
||||
|
||||
function emitActiveScopeBlock(scope: ActiveScopeBlock, depth: number, state: RenderState): void {
|
||||
if (state.stopped || scope.lines.length === 0) return;
|
||||
const indent = " ".repeat(depth);
|
||||
const header = `${indent}[§ active: ${scope.label}]`;
|
||||
const headerTokens = estimateTokens(header);
|
||||
if (state.tokens + headerTokens > state.maxTokens) {
|
||||
state.truncated = true;
|
||||
state.stopped = true;
|
||||
return;
|
||||
}
|
||||
state.lines.push(header);
|
||||
state.tokens += headerTokens;
|
||||
|
||||
for (const text of scope.lines.slice(0, MAX_SCOPE_LINES)) {
|
||||
if (state.stopped) return;
|
||||
const clean = cleaned(text);
|
||||
if (!clean) continue;
|
||||
const line = `${indent} ${clean}`;
|
||||
const nextTokens = state.tokens + estimateTokens(line);
|
||||
if (nextTokens > state.maxTokens) {
|
||||
state.truncated = true;
|
||||
state.stopped = true;
|
||||
return;
|
||||
}
|
||||
state.lines.push(line);
|
||||
state.tokens = nextTokens;
|
||||
}
|
||||
}
|
||||
|
||||
function renderTree(
|
||||
@@ -595,7 +714,8 @@ function renderTree(
|
||||
|
||||
const ref = shouldReference(node) ? `e${state.nextRef}` : undefined;
|
||||
const context = ref ? handleContext(node, state) : [];
|
||||
const line = renderNodeLine(node, depth, ref, context);
|
||||
const surface = state.surfaceMap.get(node.id);
|
||||
const line = renderNodeLine(node, depth, ref, context, surface);
|
||||
const nextTokens = state.tokens + estimateTokens(line);
|
||||
if (nextTokens > state.maxTokens) {
|
||||
state.truncated = true;
|
||||
@@ -610,11 +730,21 @@ function renderTree(
|
||||
state.nextRef += 1;
|
||||
}
|
||||
|
||||
const scope = state.scopeMap.get(node.id);
|
||||
if (scope) emitActiveScopeBlock(scope, depth + 1, state);
|
||||
|
||||
if (ref && shouldSkipRedundantRefChildren(node)) continue;
|
||||
renderTree(children, node.id, depth + 1, state);
|
||||
}
|
||||
}
|
||||
|
||||
function renderNodes(nodes: VomNode[], options: VomOptions, initialLines: string[]): RenderState {
|
||||
function renderNodes(
|
||||
nodes: VomNode[],
|
||||
options: VomOptions,
|
||||
initialLines: string[],
|
||||
surfaces: CondSurface[] = [],
|
||||
activeScopeBlocks: ActiveScopeBlock[] = [],
|
||||
): RenderState {
|
||||
const children = buildChildren(nodes);
|
||||
const state: RenderState = {
|
||||
lines: [...initialLines],
|
||||
@@ -629,6 +759,8 @@ function renderNodes(nodes: VomNode[], options: VomOptions, initialLines: string
|
||||
parentMap: buildParentMap(nodes),
|
||||
nodesById: new Map(nodes.map((node) => [node.id, node])),
|
||||
duplicateRefNames: duplicateReferenceNames(nodes),
|
||||
surfaceMap: new Map(surfaces.map((surface) => [surface.triggerId, surface])),
|
||||
scopeMap: new Map(activeScopeBlocks.map((scope) => [scope.triggerId, scope])),
|
||||
};
|
||||
|
||||
renderTree(children, null, 1, state);
|
||||
@@ -640,6 +772,122 @@ function renderPageOcclusionLine(hiddenNodeCount: number): string {
|
||||
return `L2 page … occluded by L1 (~${hiddenNodeCount} nodes, not actionable)`;
|
||||
}
|
||||
|
||||
function isPositionedRegionCandidate(node: VomNode): boolean {
|
||||
return (
|
||||
node.pointerEvents !== "none" &&
|
||||
node.rect !== null &&
|
||||
ACTIVE_REGION_POSITIONS.has(node.position)
|
||||
);
|
||||
}
|
||||
|
||||
function rectContains(rect: Rect, x: number, y: number): boolean {
|
||||
return x >= rect.x && x <= rect.x + rect.w && y >= rect.y && y <= rect.y + rect.h;
|
||||
}
|
||||
|
||||
function interactionPoints(rect: Rect): Array<[number, number]> {
|
||||
const epsX = Math.min(rect.w * 0.15, 8);
|
||||
const epsY = Math.min(rect.h * 0.15, 8);
|
||||
const midX = rect.x + rect.w / 2;
|
||||
const midY = rect.y + rect.h / 2;
|
||||
return [
|
||||
[midX, midY],
|
||||
[rect.x + epsX, rect.y + epsY],
|
||||
[rect.x + rect.w - epsX, rect.y + epsY],
|
||||
[rect.x + epsX, rect.y + rect.h - epsY],
|
||||
[rect.x + rect.w - epsX, rect.y + rect.h - epsY],
|
||||
[midX, rect.y + epsY],
|
||||
[midX, rect.y + rect.h - epsY],
|
||||
[rect.x + epsX, midY],
|
||||
[rect.x + rect.w - epsX, midY],
|
||||
];
|
||||
}
|
||||
|
||||
function isModalLike(node: VomNode): boolean {
|
||||
const role = normalizedRole(node);
|
||||
return (
|
||||
node.modal === true || node.tag === "dialog" || role === "dialog" || role === "alertdialog"
|
||||
);
|
||||
}
|
||||
|
||||
function activeRegionCandidatePriority(node: VomNode, viewportCoverage: number): number {
|
||||
if (isModalLike(node)) return 4;
|
||||
if (viewportCoverage >= 0.9) return 3;
|
||||
if (viewportCoverage >= 0.15) return 2;
|
||||
return 1;
|
||||
}
|
||||
|
||||
function isBlockedByRegion(
|
||||
target: VomNode,
|
||||
blocker: VomNode,
|
||||
parentMap: Map<number, number | null>,
|
||||
viewportCoverage: number,
|
||||
): boolean {
|
||||
if (target.id === blocker.id) return false;
|
||||
if (isAncestorOf(parentMap, blocker.id, target.id)) return false;
|
||||
if (isAncestorOf(parentMap, target.id, blocker.id)) return false;
|
||||
if (!target.rect || !blocker.rect) return false;
|
||||
|
||||
const points = interactionPoints(target.rect);
|
||||
|
||||
if (
|
||||
viewportCoverage < 0.9 &&
|
||||
target.paintOrder >= blocker.paintOrder &&
|
||||
points.some(([x, y]) => rectContains(blocker.rect as Rect, x, y))
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (blocker.paintOrder < target.paintOrder) return false;
|
||||
if (blocker.paintOrder === target.paintOrder && blocker.id <= target.id) return false;
|
||||
if (viewportCoverage >= 0.9) return true;
|
||||
|
||||
return points.some(([x, y]) => rectContains(blocker.rect as Rect, x, y));
|
||||
}
|
||||
|
||||
function collectDescendantsOfIds(nodes: VomNode[], roots: Set<number>): Set<number> {
|
||||
const included = new Set(roots);
|
||||
let changed = true;
|
||||
while (changed) {
|
||||
changed = false;
|
||||
for (const node of nodes) {
|
||||
if (included.has(node.id)) continue;
|
||||
if (node.parentId !== null && included.has(node.parentId)) {
|
||||
included.add(node.id);
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return included;
|
||||
}
|
||||
|
||||
function applyActiveRegionPolicy(nodes: VomNode[], scene: VomScene): VomNode[] {
|
||||
const parentMap = buildParentMap(nodes);
|
||||
const candidates = nodes
|
||||
.filter(isPositionedRegionCandidate)
|
||||
.map((node) => {
|
||||
const viewportCoverage = coverage(node.rect, scene.viewport);
|
||||
return {
|
||||
node,
|
||||
viewportCoverage,
|
||||
priority: activeRegionCandidatePriority(node, viewportCoverage),
|
||||
};
|
||||
})
|
||||
.filter((candidate) => candidate.priority > 1 || candidate.viewportCoverage > 0);
|
||||
|
||||
const blockedRoots = new Set<number>();
|
||||
for (const target of nodes) {
|
||||
if (!shouldReference(target)) continue;
|
||||
const blocked = candidates.some((candidate) =>
|
||||
isBlockedByRegion(target, candidate.node, parentMap, candidate.viewportCoverage),
|
||||
);
|
||||
if (blocked) blockedRoots.add(target.id);
|
||||
}
|
||||
|
||||
if (blockedRoots.size === 0) return nodes;
|
||||
const blocked = collectDescendantsOfIds(nodes, blockedRoots);
|
||||
return nodes.filter((node) => !blocked.has(node.id));
|
||||
}
|
||||
|
||||
function collectDescendantsOfMembers(nodes: VomNode[], members: Set<number>): Set<number> {
|
||||
const included = new Set(members);
|
||||
let changed = true;
|
||||
@@ -662,6 +910,386 @@ function countRenderable(nodes: VomNode[]): number {
|
||||
return nodes.filter(shouldRender).length;
|
||||
}
|
||||
|
||||
interface SemanticControl {
|
||||
kind: string;
|
||||
handle?: string;
|
||||
label: string;
|
||||
}
|
||||
|
||||
interface SemanticAction {
|
||||
handle: string;
|
||||
label: string;
|
||||
}
|
||||
|
||||
interface SemanticItem {
|
||||
title?: string;
|
||||
titleHandle?: string;
|
||||
author?: string;
|
||||
time?: string;
|
||||
score?: string;
|
||||
comments?: string;
|
||||
commentsHandle?: string;
|
||||
actions: SemanticAction[];
|
||||
}
|
||||
|
||||
interface SemanticCollection {
|
||||
role: string;
|
||||
itemRole: string;
|
||||
label?: string;
|
||||
controls: SemanticControl[];
|
||||
items: SemanticItem[];
|
||||
}
|
||||
|
||||
function truncateField(value: string): string {
|
||||
if ([...value].length <= MAX_SEMANTIC_FIELD_CHARS) return value;
|
||||
return `${[...value].slice(0, MAX_SEMANTIC_FIELD_CHARS - 1).join("")}…`;
|
||||
}
|
||||
|
||||
function quoteField(value: string): string {
|
||||
return JSON.stringify(truncateField(value));
|
||||
}
|
||||
|
||||
function roleIsCollectionItem(role: string): boolean {
|
||||
return role === "article" || role === "row" || role === "listitem";
|
||||
}
|
||||
|
||||
function roleIsCollectionContainer(role: string): boolean {
|
||||
return ["main", "feed", "list", "table", "grid", "rowgroup", "region", "section"].includes(role);
|
||||
}
|
||||
|
||||
function textForSemantics(node: VomNode): string | undefined {
|
||||
return (
|
||||
cleanSemanticText(node.name) ??
|
||||
cleanSemanticText(node.value) ??
|
||||
cleanSemanticText(node.text) ??
|
||||
cleanSemanticText(node.nearbyText)
|
||||
);
|
||||
}
|
||||
|
||||
function buildRefLookup(refs: Array<{ ref: string; backendNodeId: number }>): Map<number, string> {
|
||||
return new Map(
|
||||
refs.map((entry) => [
|
||||
entry.backendNodeId,
|
||||
entry.ref.startsWith("@") ? entry.ref : `@${entry.ref}`,
|
||||
]),
|
||||
);
|
||||
}
|
||||
|
||||
function collectDescendantNodes(
|
||||
children: Map<number | null, VomNode[]>,
|
||||
rootId: number,
|
||||
nodesById: Map<number, VomNode>,
|
||||
): VomNode[] {
|
||||
const out: VomNode[] = [];
|
||||
const stack = [rootId];
|
||||
const seen = new Set<number>();
|
||||
while (stack.length > 0) {
|
||||
const id = stack.pop() as number;
|
||||
if (!seen.add(id)) continue;
|
||||
const node = nodesById.get(id);
|
||||
if (node) out.push(node);
|
||||
for (const child of [...(children.get(id) ?? [])].reverse()) {
|
||||
stack.push(child.id);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function collectDescendantIdSet(
|
||||
children: Map<number | null, VomNode[]>,
|
||||
rootId: number,
|
||||
): Set<number> {
|
||||
const out = new Set<number>();
|
||||
const stack = [rootId];
|
||||
while (stack.length > 0) {
|
||||
const id = stack.pop() as number;
|
||||
if (!out.add(id)) continue;
|
||||
for (const child of [...(children.get(id) ?? [])].reverse()) stack.push(child.id);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function collectionLabel(
|
||||
node: VomNode,
|
||||
children: Map<number | null, VomNode[]>,
|
||||
nodesById: Map<number, VomNode>,
|
||||
): string | undefined {
|
||||
return (
|
||||
contextLabel(node, "") ??
|
||||
collectDescendantNodes(children, node.id, nodesById).find(
|
||||
(candidate) => normalizedRole(candidate) === "heading" && textForSemantics(candidate),
|
||||
)?.name
|
||||
);
|
||||
}
|
||||
|
||||
function classifyControl(label: string, role: string): string | undefined {
|
||||
const key = label.toLowerCase();
|
||||
if (role === "searchbox" || key.includes("search")) return "search";
|
||||
if (key.includes("sort") || key.includes("order")) return "sort";
|
||||
if (key.includes("filter")) return "filter";
|
||||
if (role === "tab") return "tab";
|
||||
if (key === "next" || key.includes("next page")) return "pagination";
|
||||
if (key === "previous" || key.includes("previous page")) return "pagination";
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function collectCollectionControls(
|
||||
container: VomNode,
|
||||
items: VomNode[],
|
||||
children: Map<number | null, VomNode[]>,
|
||||
nodesById: Map<number, VomNode>,
|
||||
refLookup: Map<number, string>,
|
||||
): SemanticControl[] {
|
||||
const itemIds = new Set(items.flatMap((item) => [...collectDescendantIdSet(children, item.id)]));
|
||||
const controls: SemanticControl[] = [];
|
||||
for (const node of collectDescendantNodes(children, container.id, nodesById)) {
|
||||
if (itemIds.has(node.id)) continue;
|
||||
const role = normalizedRole(node);
|
||||
if (!["button", "combobox", "searchbox", "tab", "link"].includes(role)) continue;
|
||||
const label = textForSemantics(node);
|
||||
if (!label) continue;
|
||||
const kind = classifyControl(label, role);
|
||||
if (!kind) continue;
|
||||
controls.push({ kind, handle: refLookup.get(node.id), label });
|
||||
if (controls.length >= MAX_SEMANTIC_CONTROLS) break;
|
||||
}
|
||||
return controls;
|
||||
}
|
||||
|
||||
function isFormLikeCollection(
|
||||
container: VomNode,
|
||||
items: VomNode[],
|
||||
children: Map<number | null, VomNode[]>,
|
||||
nodesById: Map<number, VomNode>,
|
||||
): boolean {
|
||||
const descendants = collectDescendantNodes(children, container.id, nodesById);
|
||||
const interactive = descendants.filter((node) => shouldReference(node));
|
||||
const textboxCount = interactive.filter((node) =>
|
||||
["textbox", "searchbox"].includes(normalizedRole(node)),
|
||||
).length;
|
||||
const sensitiveCount = descendants.filter((node) => node.sensitive).length;
|
||||
const labels = descendants
|
||||
.map((node) => textForSemantics(node))
|
||||
.filter((label): label is string => label !== undefined)
|
||||
.join(" ")
|
||||
.toLowerCase();
|
||||
const hasLoginSignals =
|
||||
/(用户登录|登录|验证码|密码|忘记密码|手机号|证书口令|captcha|password|sign in|log in)/i.test(
|
||||
labels,
|
||||
);
|
||||
|
||||
return (
|
||||
(textboxCount >= 2 && hasLoginSignals) ||
|
||||
(sensitiveCount > 0 && hasLoginSignals) ||
|
||||
(items.length <= 8 && textboxCount > 0 && interactive.length >= 3 && hasLoginSignals)
|
||||
);
|
||||
}
|
||||
|
||||
function isCommentLabel(label: string): boolean {
|
||||
const key = label.toLowerCase();
|
||||
return key.includes("comment") || key === "no comments";
|
||||
}
|
||||
|
||||
function parseScoreBetweenVoteButtons(
|
||||
item: VomNode,
|
||||
children: Map<number | null, VomNode[]>,
|
||||
): string | undefined {
|
||||
for (const descendantId of collectDescendantIdSet(children, item.id)) {
|
||||
const siblings = children.get(descendantId) ?? [];
|
||||
for (let index = 0; index + 2 < siblings.length; index += 1) {
|
||||
const prev = textForSemantics(siblings[index])?.toLowerCase() ?? "";
|
||||
const score = textForSemantics(siblings[index + 1]);
|
||||
const next = textForSemantics(siblings[index + 2])?.toLowerCase() ?? "";
|
||||
if (
|
||||
prev.includes("upvote") &&
|
||||
next.includes("downvote") &&
|
||||
score !== undefined &&
|
||||
/^-?\d+$/.test(score.replaceAll(",", ""))
|
||||
) {
|
||||
return score;
|
||||
}
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function extractAuthor(descendants: VomNode[]): string | undefined {
|
||||
for (let index = 0; index < descendants.length; index += 1) {
|
||||
const label = textForSemantics(descendants[index]);
|
||||
if (!label) continue;
|
||||
const key = label.toLowerCase();
|
||||
if (key === "submitted by" || key === "by" || key.endsWith(" by")) {
|
||||
for (const candidate of descendants.slice(index + 1, index + 9)) {
|
||||
if (normalizedRole(candidate) === "link") return textForSemantics(candidate);
|
||||
}
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function extractSemanticItem(
|
||||
item: VomNode,
|
||||
children: Map<number | null, VomNode[]>,
|
||||
nodesById: Map<number, VomNode>,
|
||||
refLookup: Map<number, string>,
|
||||
): SemanticItem {
|
||||
const descendants = collectDescendantNodes(children, item.id, nodesById);
|
||||
let title: string | undefined;
|
||||
let titleHandle: string | undefined;
|
||||
for (const role of ["heading", "link"]) {
|
||||
const node = descendants.find(
|
||||
(candidate) => normalizedRole(candidate) === role && textForSemantics(candidate),
|
||||
);
|
||||
if (!node) continue;
|
||||
title = textForSemantics(node);
|
||||
titleHandle = refLookup.get(node.id);
|
||||
if (!titleHandle && title) {
|
||||
const matchingLink = descendants.find(
|
||||
(candidate) =>
|
||||
normalizedRole(candidate) === "link" && textForSemantics(candidate) === title,
|
||||
);
|
||||
if (matchingLink) titleHandle = refLookup.get(matchingLink.id);
|
||||
}
|
||||
break;
|
||||
}
|
||||
if (!title) {
|
||||
title = textForSemantics(item);
|
||||
titleHandle = refLookup.get(item.id);
|
||||
}
|
||||
|
||||
let time: string | undefined;
|
||||
let comments: string | undefined;
|
||||
let commentsHandle: string | undefined;
|
||||
const actions: SemanticAction[] = [];
|
||||
for (const node of descendants) {
|
||||
const role = normalizedRole(node);
|
||||
const label = textForSemantics(node);
|
||||
if (!time && role === "time") time = label;
|
||||
if (!comments && label && isCommentLabel(label)) {
|
||||
comments = label;
|
||||
commentsHandle = refLookup.get(node.id);
|
||||
}
|
||||
if (
|
||||
actions.length < MAX_SEMANTIC_ACTIONS &&
|
||||
shouldReference(node) &&
|
||||
!["link", "textbox", "searchbox"].includes(role)
|
||||
) {
|
||||
const handle = refLookup.get(node.id);
|
||||
if (handle && label) actions.push({ handle, label });
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
title,
|
||||
titleHandle,
|
||||
author: extractAuthor(descendants),
|
||||
time,
|
||||
score: parseScoreBetweenVoteButtons(item, children),
|
||||
comments,
|
||||
commentsHandle,
|
||||
actions,
|
||||
};
|
||||
}
|
||||
|
||||
function detectSemanticCollections(
|
||||
nodes: VomNode[],
|
||||
refs: Array<{ ref: string; backendNodeId: number }>,
|
||||
): SemanticCollection[] {
|
||||
const children = buildChildren(nodes);
|
||||
const nodesById = new Map(nodes.map((node) => [node.id, node]));
|
||||
const refLookup = buildRefLookup(refs);
|
||||
const collections: SemanticCollection[] = [];
|
||||
const coveredItems = new Set<number>();
|
||||
|
||||
for (const container of nodes) {
|
||||
if (collections.length >= MAX_SEMANTIC_COLLECTIONS) break;
|
||||
const role = normalizedRole(container);
|
||||
if (!roleIsCollectionContainer(role)) continue;
|
||||
const childList = children.get(container.id) ?? [];
|
||||
const counts = new Map<string, number>();
|
||||
for (const child of childList) {
|
||||
const childRole = normalizedRole(child);
|
||||
if (roleIsCollectionItem(childRole)) counts.set(childRole, (counts.get(childRole) ?? 0) + 1);
|
||||
}
|
||||
const best = [...counts].filter(([, count]) => count >= 2).sort((a, b) => b[1] - a[1])[0];
|
||||
if (!best) continue;
|
||||
const [itemRole] = best;
|
||||
const items = childList.filter(
|
||||
(child) => normalizedRole(child) === itemRole && !coveredItems.has(child.id),
|
||||
);
|
||||
if (items.length < 2) continue;
|
||||
if (isFormLikeCollection(container, items, children, nodesById)) continue;
|
||||
for (const item of items) coveredItems.add(item.id);
|
||||
collections.push({
|
||||
role,
|
||||
itemRole,
|
||||
label: collectionLabel(container, children, nodesById),
|
||||
controls: collectCollectionControls(container, items, children, nodesById, refLookup),
|
||||
items: items
|
||||
.slice(0, MAX_SEMANTIC_ITEMS)
|
||||
.map((item) => extractSemanticItem(item, children, nodesById, refLookup)),
|
||||
});
|
||||
}
|
||||
return collections;
|
||||
}
|
||||
|
||||
function renderSemanticCollections(collections: SemanticCollection[]): string[] {
|
||||
if (collections.length === 0) return [];
|
||||
const lines = ["@collections"];
|
||||
for (const [index, collection] of collections.entries()) {
|
||||
const label = collection.label ? ` label=${quoteField(collection.label)}` : "";
|
||||
lines.push(
|
||||
`collection c${index + 1} role=${collection.role} item_role=${collection.itemRole}${label} items=${collection.items.length}`,
|
||||
);
|
||||
if (collection.controls.length > 0) {
|
||||
const rendered = collection.controls
|
||||
.map(
|
||||
(control) =>
|
||||
`${control.kind}=${control.handle ? `${control.handle} ` : ""}${quoteField(control.label)}`,
|
||||
)
|
||||
.join("; ");
|
||||
lines.push(` controls: ${rendered}`);
|
||||
}
|
||||
lines.push(
|
||||
" columns: index title title_handle author time score comments comments_handle actions",
|
||||
);
|
||||
collection.items.forEach((item, itemIndex) => {
|
||||
const parts = [`${itemIndex + 1}. `];
|
||||
if (item.title) parts.push(`title=${quoteField(item.title)}`);
|
||||
if (item.titleHandle) parts.push(`title_handle=${item.titleHandle}`);
|
||||
if (item.author) parts.push(`author=${quoteField(item.author)}`);
|
||||
if (item.time) parts.push(`time=${quoteField(item.time)}`);
|
||||
if (item.score) parts.push(`score=${quoteField(item.score)}`);
|
||||
if (item.comments) parts.push(`comments=${quoteField(item.comments)}`);
|
||||
if (item.commentsHandle) parts.push(`comments_handle=${item.commentsHandle}`);
|
||||
if (item.actions.length > 0) {
|
||||
parts.push(
|
||||
`actions=[${item.actions.map((action) => `${action.handle} ${quoteField(action.label)}`).join(", ")}]`,
|
||||
);
|
||||
}
|
||||
lines.push(` ${parts.join(" ")}`);
|
||||
});
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
function insertAfterHeader(text: string, insertLines: string[]): string {
|
||||
if (insertLines.length === 0) return text;
|
||||
const lines = text.split("\n");
|
||||
const l1Index = lines.findIndex((line) => line.startsWith("L1 "));
|
||||
const insertAt = l1Index >= 0 ? l1Index + 1 : Math.min(lines.length, 3);
|
||||
lines.splice(insertAt, 0, ...insertLines);
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
function augmentVomResult(result: VomResult, nodes: VomNode[], options: VomOptions): VomResult {
|
||||
if (!options.semanticCollections) return result;
|
||||
const semanticLines = renderSemanticCollections(detectSemanticCollections(nodes, result.refs));
|
||||
return semanticLines.length === 0
|
||||
? result
|
||||
: { ...result, text: insertAfterHeader(result.text, semanticLines) };
|
||||
}
|
||||
|
||||
function renderDoubleLayer(scene: VomScene, layer: BlockingLayer, options: VomOptions): VomResult {
|
||||
const included = collectDescendantsOfMembers(scene.nodes, layer.members);
|
||||
const visibleNodes = scene.nodes.filter((node) => included.has(node.id));
|
||||
@@ -672,14 +1300,18 @@ function renderDoubleLayer(scene: VomScene, layer: BlockingLayer, options: VomOp
|
||||
"@layers 2 focus=L1",
|
||||
`L1 ${layer.kind} cover=${Math.round(layer.coverage * 100)}%`,
|
||||
];
|
||||
const state = renderNodes(visibleNodes, options, header);
|
||||
const state = renderNodes(visibleNodes, options, header, scene.surfaces, scene.activeScopeBlocks);
|
||||
state.lines.push(renderPageOcclusionLine(hiddenCount));
|
||||
|
||||
return {
|
||||
text: state.lines.join("\n"),
|
||||
refs: state.refs,
|
||||
truncated: state.truncated,
|
||||
};
|
||||
return augmentVomResult(
|
||||
{
|
||||
text: state.lines.join("\n"),
|
||||
refs: state.refs,
|
||||
truncated: state.truncated,
|
||||
},
|
||||
visibleNodes,
|
||||
options,
|
||||
);
|
||||
}
|
||||
|
||||
export function renderVom(scene: VomScene, options: VomOptions = {}): VomResult {
|
||||
@@ -687,17 +1319,28 @@ export function renderVom(scene: VomScene, options: VomOptions = {}): VomResult
|
||||
const renderScene = { ...scene, nodes };
|
||||
const layer = detectBlockingLayer(nodes, scene.viewport);
|
||||
if (layer) return renderDoubleLayer(renderScene, layer, options);
|
||||
const visibleNodes = options.activeRegionPolicy ? applyActiveRegionPolicy(nodes, scene) : nodes;
|
||||
|
||||
const state = renderNodes(nodes, options, [
|
||||
"@vom 1",
|
||||
`@view ${scene.viewport.width}x${scene.viewport.height}`,
|
||||
"@layers 1 focus=L1",
|
||||
"L1 page",
|
||||
]);
|
||||
const state = renderNodes(
|
||||
visibleNodes,
|
||||
options,
|
||||
[
|
||||
"@vom 1",
|
||||
`@view ${scene.viewport.width}x${scene.viewport.height}`,
|
||||
"@layers 1 focus=L1",
|
||||
"L1 page",
|
||||
],
|
||||
scene.surfaces,
|
||||
scene.activeScopeBlocks,
|
||||
);
|
||||
|
||||
return {
|
||||
text: state.lines.join("\n"),
|
||||
refs: state.refs,
|
||||
truncated: state.truncated,
|
||||
};
|
||||
return augmentVomResult(
|
||||
{
|
||||
text: state.lines.join("\n"),
|
||||
refs: state.refs,
|
||||
truncated: state.truncated,
|
||||
},
|
||||
visibleNodes,
|
||||
options,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -50,11 +50,37 @@ export interface VomNode {
|
||||
export interface VomScene {
|
||||
viewport: Viewport;
|
||||
nodes: VomNode[];
|
||||
surfaces?: CondSurface[];
|
||||
activeScopeBlocks?: ActiveScopeBlock[];
|
||||
}
|
||||
|
||||
export interface VomOptions {
|
||||
maxDepth?: number;
|
||||
maxTokens?: number;
|
||||
/**
|
||||
* Experimental: filter refs that are geometrically blocked by foreground
|
||||
* fixed/absolute/sticky regions. Disabled by default and does not alter the
|
||||
* public layer header format.
|
||||
*/
|
||||
activeRegionPolicy?: boolean;
|
||||
/**
|
||||
* Experimental: emit semantic summaries for repeated collection-like
|
||||
* containers. Disabled by default because false positives in menus/forms are
|
||||
* still noisy.
|
||||
*/
|
||||
semanticCollections?: boolean;
|
||||
}
|
||||
|
||||
export interface CondSurface {
|
||||
triggerId: number;
|
||||
triggerAction: "hover" | "focus" | string;
|
||||
subItems: string[];
|
||||
}
|
||||
|
||||
export interface ActiveScopeBlock {
|
||||
triggerId: number;
|
||||
label: string;
|
||||
lines: string[];
|
||||
}
|
||||
|
||||
export interface VomResult {
|
||||
|
||||
Reference in New Issue
Block a user