figmar/BrowserSkill
1
0
Fork 0
mirror of https://github.com/Tencent/BrowserSkill.git synced 2026-09-10 13:01:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(vom): method update, solve context over exposed problem

Browse Source
This commit is contained in:
Ljy-0827
2026-07-24 17:25:57 +08:00
parent 5e0d26ccb2
commit f319672721
7 changed files with 1895 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/extension/src/tools/__tests__/observation.test.ts
+416 -1
View File
@@ -570,6 +570,82 @@ describe("buildVomScene", () => {
]);
});
it("dedupes iframe fallback controls already exposed through AX", () => {
const axNodes: CdpAxNode[] = [
{
nodeId: "1",
role: { type: "role", value: "RootWebArea" },
backendDOMNodeId: 10,
childIds: ["2"],
},
{
nodeId: "2",
parentId: "1",
role: { type: "role", value: "Iframe" },
backendDOMNodeId: 20,
childIds: ["3"],
},
{
nodeId: "3",
parentId: "2",
role: { type: "role", value: "textbox" },
name: { type: "x", value: "输入密码" },
backendDOMNodeId: 202,
properties: [{ name: "inputType", value: { value: "password" } }],
},
];
const captured: CapturedViewModel = {
viewport: { width: 1000, height: 800 },
nodes: [
{
backendNodeId: 10,
parentBackendNodeId: null,
tag: "body",
attrs: {},
rect: { x: 0, y: 0, w: 1000, h: 800 },
paintOrder: 0,
position: "static",
pointerEvents: "auto",
},
{
backendNodeId: 20,
parentBackendNodeId: 10,
tag: "iframe",
attrs: {},
rect: { x: 100, y: 100, w: 400, h: 300 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
},
],
iframeNodes: new Map([
[
20,
[
{
backendNodeId: 102,
parentBackendNodeId: null,
tag: "input",
attrs: { type: "password", placeholder: "输入密码" },
rect: { x: 0, y: 50, w: 300, h: 40 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
},
],
],
]),
excludedBackendNodeIds: new Set(),
};
const passwordNodes = buildVomScene(axNodes, captured).nodes.filter(
(node) => node.role === "textbox" && node.name === "输入密码",
);
expect(passwordNodes).toHaveLength(1);
expect(passwordNodes[0]).toEqual(expect.objectContaining({ id: 202, sensitive: true }));
});
it("keeps unnamed iframe controls but skips unnamed iframe links", () => {
const scene = buildVomScene(
[
@@ -843,6 +919,66 @@ describe("buildVomScene", () => {
);
});
it("uses nearby preceding text as a form control label", () => {
const axNodes: CdpAxNode[] = [
{
nodeId: "1",
role: { type: "role", value: "RootWebArea" },
backendDOMNodeId: 1,
childIds: ["2"],
},
{
nodeId: "2",
parentId: "1",
role: { type: "role", value: "textbox" },
backendDOMNodeId: 20,
},
];
const captured: CapturedViewModel = {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
backendNodeId: 1,
parentBackendNodeId: null,
tag: "div",
attrs: {},
rect: { x: 0, y: 0, w: 500, h: 80 },
paintOrder: 0,
position: "static",
pointerEvents: "auto",
},
{
backendNodeId: 10,
parentBackendNodeId: 1,
tag: "span",
attrs: {},
textContent: "验证码:",
rect: { x: 0, y: 0, w: 80, h: 30 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
},
{
backendNodeId: 20,
parentBackendNodeId: 1,
tag: "input",
attrs: { type: "text" },
rect: { x: 90, y: 0, w: 120, h: 30 },
paintOrder: 2,
position: "static",
pointerEvents: "auto",
cursor: "text",
},
],
};
expect(buildVomScene(axNodes, captured).nodes.find((node) => node.id === 20)).toEqual(
expect.objectContaining({ role: "textbox", name: "验证码" }),
);
});
it("promotes a cursor:pointer icon control to a button named from its <use> sprite", () => {
// A close button built as <div class=close-button><svg><use href="#close"/></svg></div>
// — the AX tree exposes it as `generic` with no name, so it would be dropped.
@@ -1065,6 +1201,284 @@ describe("buildVomScene", () => {
expect(rendered.refs).toEqual([{ ref: "e1", backendNodeId: 20 }]);
});
it("builds active scope blocks from active aria-controls relationships", () => {
const axNodes: CdpAxNode[] = [
{
nodeId: "1",
role: { type: "role", value: "RootWebArea" },
backendDOMNodeId: 10,
childIds: ["2"],
},
{
nodeId: "2",
parentId: "1",
role: { type: "role", value: "tab" },
name: { type: "computedString", value: "Reviews (12)" },
backendDOMNodeId: 20,
},
];
const captured: CapturedViewModel = {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
backendNodeId: 10,
parentBackendNodeId: null,
tag: "body",
attrs: {},
rect: { x: 0, y: 0, w: 1000, h: 800 },
paintOrder: 0,
position: "static",
pointerEvents: "auto",
},
{
backendNodeId: 20,
parentBackendNodeId: 10,
tag: "button",
attrs: { role: "tab", "aria-selected": "true", "aria-controls": "reviews-panel" },
rect: { x: 20, y: 20, w: 120, h: 40 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
},
{
backendNodeId: 30,
parentBackendNodeId: 10,
tag: "div",
attrs: { id: "reviews-panel" },
rect: null,
paintOrder: 0,
position: "static",
pointerEvents: "auto",
textContent: "Jane - ear cups are small",
},
{
backendNodeId: 31,
parentBackendNodeId: 30,
tag: "p",
attrs: {},
rect: null,
paintOrder: 0,
position: "static",
pointerEvents: "auto",
textContent: "Bob - great sound",
},
],
};
const scene = buildVomScene(axNodes, captured);
expect(scene.activeScopeBlocks).toEqual([
{
triggerId: 20,
label: "Reviews (12)",
lines: ["Jane - ear cups are small", "Bob - great sound"],
},
]);
expect(renderVom(scene).text).toContain("[§ active: Reviews (12)]");
});
it("maps hover surface probes to matching node labels", () => {
const axNodes: CdpAxNode[] = [
{
nodeId: "1",
role: { type: "role", value: "RootWebArea" },
backendDOMNodeId: 10,
childIds: ["2"],
},
{
nodeId: "2",
parentId: "1",
role: { type: "role", value: "button" },
name: { type: "computedString", value: "Products" },
backendDOMNodeId: 20,
},
];
const scene = buildVomScene(axNodes, {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
surfaceProbes: [
{ triggerLabel: "Products", triggerAction: "hover", subItems: ["Shoes", "Bags"] },
],
nodes: [
{
backendNodeId: 10,
parentBackendNodeId: null,
tag: "body",
attrs: {},
rect: { x: 0, y: 0, w: 1000, h: 800 },
paintOrder: 0,
position: "static",
pointerEvents: "auto",
},
{
backendNodeId: 20,
parentBackendNodeId: 10,
tag: "button",
attrs: {},
rect: { x: 20, y: 20, w: 120, h: 40 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
},
],
});
expect(scene.surfaces).toEqual([
{ triggerId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
]);
expect(renderVom(scene).text).toContain('@e1 button "Products" [→ Shoes | Bags]');
});
it("enriches names and active scope signals from AX properties", () => {
const axNodes: CdpAxNode[] = [
{
nodeId: "1",
role: { type: "role", value: "RootWebArea" },
backendDOMNodeId: 10,
childIds: ["2", "3"],
},
{
nodeId: "2",
parentId: "1",
role: { type: "role", value: "button" },
name: { type: "computedString", value: "Products" },
backendDOMNodeId: 20,
properties: [
{ name: "hasPopup", value: { value: "menu" } },
{ name: "expanded", value: { value: "true" } },
{ name: "controls", value: { value: "products-panel" } },
],
},
{
nodeId: "3",
parentId: "1",
role: { type: "role", value: "textbox" },
name: { type: "computedString", value: "Card number" },
backendDOMNodeId: 30,
properties: [{ name: "inputType", value: { value: "credit-card" } }],
},
];
const scene = buildVomScene(axNodes, {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
backendNodeId: 10,
parentBackendNodeId: null,
tag: "body",
attrs: {},
rect: { x: 0, y: 0, w: 1000, h: 800 },
paintOrder: 0,
position: "static",
pointerEvents: "auto",
},
{
backendNodeId: 20,
parentBackendNodeId: 10,
tag: "button",
attrs: {},
rect: { x: 20, y: 20, w: 120, h: 40 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
},
{
backendNodeId: 25,
parentBackendNodeId: 10,
tag: "div",
attrs: { id: "products-panel" },
rect: null,
paintOrder: 0,
position: "static",
pointerEvents: "auto",
textContent: "Shoes Bags",
},
{
backendNodeId: 30,
parentBackendNodeId: 10,
tag: "input",
attrs: {},
rect: { x: 20, y: 80, w: 200, h: 40 },
paintOrder: 2,
position: "static",
pointerEvents: "auto",
},
],
});
expect(scene.nodes.find((node) => node.id === 20)).toEqual(
expect.objectContaining({
name: "Products [expanded]",
attrs: expect.objectContaining({
"aria-expanded": "true",
"aria-controls": "products-panel",
}),
}),
);
expect(scene.activeScopeBlocks).toEqual([
{ triggerId: 20, label: "Products [expanded]", lines: ["Shoes Bags"] },
]);
expect(scene.nodes.find((node) => node.id === 30)?.sensitive).toBe(true);
});
it("aggregates AX virtual text into unnamed structural nodes", () => {
const scene = buildVomScene(
[
{
nodeId: "1",
role: { type: "role", value: "RootWebArea" },
backendDOMNodeId: 10,
childIds: ["2"],
},
{
nodeId: "2",
parentId: "1",
role: { type: "role", value: "paragraph" },
backendDOMNodeId: 20,
childIds: ["3"],
},
{
nodeId: "3",
parentId: "2",
role: { type: "role", value: "InlineTextBox" },
name: { type: "computedString", value: "Inline only text" },
},
],
{
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
backendNodeId: 10,
parentBackendNodeId: null,
tag: "body",
attrs: {},
rect: { x: 0, y: 0, w: 1000, h: 800 },
paintOrder: 0,
position: "static",
pointerEvents: "auto",
},
{
backendNodeId: 20,
parentBackendNodeId: 10,
tag: "p",
attrs: {},
rect: { x: 20, y: 20, w: 400, h: 40 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
},
],
},
);
expect(scene.nodes.find((node) => node.id === 20)?.name).toBe("Inline only text");
});
it("marks captured dialog elements as modal without AX role or aria-modal", () => {
const scene = buildVomScene([], {
viewport: { width: 640, height: 480 },
@@ -1533,7 +1947,8 @@ describe("handleSnapshot", () => {
expect(res.text).toContain("L1 modal cover=100%");
// Iframe content must appear in L1 — both inputs rendered
expect(res.text).toMatch(/textbox "请输入手机号"/);
expect(res.text).toMatch(/textbox "密码" ="•••"/);
expect(res.text).toMatch(/textbox "密码"/);
expect(res.text).not.toMatch(/textbox "密码" ="•••"/);
// Ref-store must include the iframe's input backendNodeIds
const ctx = sm["sessions"].get("aa11")!;
const phoneRef = res.text.match(/@(e\d+) textbox "请输入手机号"/)?.[1];
apps/extension/src/tools/observation.ts
+360 -11
View File
@@ -3,7 +3,13 @@
// tab (defaulting to the Agent Window's active tab when omitted) and
// returns a payload that mirrors the bsk-protocol Rust structs.
import { renderVom, type VomNode, type VomScene } from "@browser-skill/vom";
import {
type ActiveScopeBlock,
type CondSurface,
renderVom,
type VomNode,
type VomScene,
} from "@browser-skill/vom";
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
import type { SessionManager } from "@/session-manager/manager";
import type {
@@ -234,6 +240,7 @@ export interface CdpAxNode {
name?: { type: string; value?: string };
description?: { value?: string };
value?: { value?: string | number | boolean };
properties?: Array<{ name?: string; value?: { value?: string | number | boolean } }>;
childIds?: string[];
}
@@ -268,6 +275,120 @@ function isSensitive(capturedNode: CapturedNode | undefined): boolean {
return (attrs.type ?? "").toLowerCase() === "password";
}
interface AxNodeSignals {
hasPopup: boolean;
expanded: boolean;
selected: boolean;
controls: string;
sensitive: boolean;
aggregatedText?: string;
}
const AX_TEXT_AGGREGATE_ROLES = new Set([
"paragraph",
"listitem",
"term",
"definition",
"cell",
"gridcell",
"caption",
"figcaption",
"blockquote",
"note",
"status",
"log",
"generic",
"section",
]);
const AX_TEXT_LEAF_ROLES = new Set(["inlinetextbox", "statictext", "text"]);
const AX_TEXT_STOP_ROLES = new Set([
"button",
"link",
"combobox",
"listbox",
"menuitem",
"menuitemcheckbox",
"menuitemradio",
"option",
"radio",
"checkbox",
"textbox",
"searchbox",
"spinbutton",
"slider",
"switch",
"tab",
"treeitem",
"columnheader",
"rowheader",
]);
const SENSITIVE_AX_INPUT_TYPES = new Set(["password", "credit-card"]);
function axPropertyString(axNode: CdpAxNode, name: string): string | undefined {
const prop = axNode.properties?.find((item) => item.name === name);
return axString(prop?.value);
}
function buildAxSignals(axNodes: CdpAxNode[]): Map<string, AxNodeSignals> {
const axByNodeId = new Map(axNodes.map((node) => [node.nodeId, node]));
const virtualText = new Map<string, string>();
for (const node of axNodes) {
if (typeof node.backendDOMNodeId === "number") continue;
const role = axString(node.role)?.toLowerCase() ?? "";
const name = axString(node.name);
if (name && AX_TEXT_LEAF_ROLES.has(role)) virtualText.set(node.nodeId, name);
}
const collectLeafText = (nodeId: string, depth: number): string[] => {
if (depth > 8) return [];
const node = axByNodeId.get(nodeId);
if (!node) return [];
const parts: string[] = [];
for (const childId of node.childIds ?? []) {
const vtext = virtualText.get(childId);
if (vtext) {
parts.push(vtext);
continue;
}
const child = axByNodeId.get(childId);
if (!child) continue;
const childRole = axString(child.role)?.toLowerCase() ?? "";
if (AX_TEXT_LEAF_ROLES.has(childRole)) continue;
if (AX_TEXT_STOP_ROLES.has(childRole)) continue;
const childName = axString(child.name);
if (childName && !AX_TEXT_AGGREGATE_ROLES.has(childRole)) {
parts.push(childName);
} else {
parts.push(...collectLeafText(childId, depth + 1));
}
}
return parts;
};
const signals = new Map<string, AxNodeSignals>();
for (const node of axNodes) {
const role = axString(node.role)?.toLowerCase() ?? "";
const expanded = axPropertyString(node, "expanded") === "true";
const selected = axPropertyString(node, "selected") === "true";
const controls = axPropertyString(node, "controls") ?? "";
const hasPopupValue = axPropertyString(node, "hasPopup") ?? "";
const inputType = axPropertyString(node, "inputType") ?? "";
let aggregatedText: string | undefined;
if (AX_TEXT_AGGREGATE_ROLES.has(role) && !axString(node.name)) {
aggregatedText = cleanAttr(collectLeafText(node.nodeId, 0).join(" "));
}
signals.set(node.nodeId, {
hasPopup: hasPopupValue !== "" && hasPopupValue !== "false",
expanded,
selected,
controls,
sensitive: SENSITIVE_AX_INPUT_TYPES.has(inputType),
aggregatedText,
});
}
return signals;
}
function findAxAncestor<T>(
axNode: CdpAxNode,
axById: Map<string, CdpAxNode>,
@@ -331,6 +452,40 @@ function isRenderableIframeControl(node: CapturedNode): boolean {
return tag !== "a" || name !== undefined;
}
function normalizedControlType(node: VomNode | CapturedNode): string {
const attrs = node.attrs ?? {};
const tag = normalizeTag(node.tag);
if (tag === "input") return (attrs.type ?? "text").toLowerCase();
if ("sensitive" in node && node.sensitive) return "password";
return tag;
}
function sameLogicalIframeControl(existing: VomNode, candidate: CapturedNode, iframeId: number) {
const role = iframeRoleFor(candidate);
const name = cleanAttr(iframeNameFor(candidate));
if (!role || !name) return false;
if ((existing.role ?? "").toLowerCase() !== role) return false;
if (cleanAttr(existing.name) !== name) return false;
const candidateType = normalizedControlType(candidate);
const existingType = normalizedControlType(existing);
if (candidateType !== existingType) return false;
return (
existing.parentId === iframeId ||
existing.domParentId === iframeId ||
existing.domAncestorIds?.includes(iframeId) === true
);
}
function hasEquivalentIframeControl(
nodes: VomNode[],
candidate: CapturedNode,
iframeId: number,
): boolean {
return nodes.some((node) => sameLogicalIframeControl(node, candidate, iframeId));
}
function capturedIframeNameFor(node: CapturedNode): string | undefined {
const ariaLabel = node.attrs["aria-label"]?.replace(/\s+/g, " ").trim();
if (ariaLabel) return ariaLabel;
@@ -350,6 +505,11 @@ function cleanAttr(value: string | undefined): string | undefined {
const FORM_CONTROL_TAGS = new Set(["input", "textarea", "select"]);
const NATIVE_CONTROL_TAGS = new Set(["button", "input", "select", "textarea"]);
const ACTIVE_SCOPE_MAX_BLOCKS = 8;
const ACTIVE_SCOPE_MAX_LINES = 40;
const ACTIVE_SCOPE_MAX_LINE_LENGTH = 160;
const ACTIVE_SCOPE_MAX_TOTAL_CHARS = 8_000;
const ACTIVE_SCOPE_SKIP_TAGS = new Set(["script", "style", "noscript", "template"]);
function buildCapturedChildren(capturedNodes: CapturedNode[]): Map<number, CapturedNode[]> {
const children = new Map<number, CapturedNode[]>();
@@ -427,6 +587,24 @@ function nearbyTextFor(
return labels.length > 0 ? labels.join(" ") : undefined;
}
function previousSiblingTextFor(
node: CapturedNode,
childrenByParentId: Map<number, CapturedNode[]>,
): string | undefined {
if (node.parentBackendNodeId === null) return undefined;
const siblings = childrenByParentId.get(node.parentBackendNodeId) ?? [];
const index = siblings.findIndex((sibling) => sibling.backendNodeId === node.backendNodeId);
if (index <= 0) return undefined;
for (let i = index - 1; i >= Math.max(0, index - 4); i -= 1) {
const sibling = siblings[i];
if (["input", "textarea", "select", "button", "a"].includes(normalizeTag(sibling.tag))) break;
const text = cleanAttr(sibling.textContent)?.replace(/[：:]\s*$/, "");
if (text && text.length <= 40) return text;
}
return undefined;
}
interface VomNodeDomSignals {
capturedByBackendId: Map<number, CapturedNode>;
childrenByParentId: Map<number, CapturedNode[]>;
@@ -470,13 +648,18 @@ function formControlName(
captured: CapturedNode | undefined,
axName: string | undefined,
axValue: string | undefined,
signals: VomNodeDomSignals,
): string | undefined {
const placeholder = cleanAttr(captured?.attrs.placeholder);
const ariaLabel = cleanAttr(captured?.attrs["aria-label"]);
const title = cleanAttr(captured?.attrs.title);
const nearbyLabel = captured
? previousSiblingTextFor(captured, signals.childrenByParentId)
: undefined;
const hasValue = axValue !== undefined && axValue !== "";
if (nearbyLabel) return nearbyLabel;
if (!hasValue && placeholder) return placeholder;
return axName ?? ariaLabel ?? placeholder ?? title;
return axName ?? ariaLabel ?? placeholder ?? title ?? nearbyLabel;
}
function vomNodeFromCaptured(
@@ -516,12 +699,17 @@ function axVomNode(
capturedNode: CapturedNode | undefined,
axById: Map<string, CdpAxNode>,
signals: VomNodeDomSignals,
axSignals: Map<string, AxNodeSignals>,
): VomNode {
const role = axString(axNode.role);
const value = axValue(axNode.value);
const name = FORM_CONTROL_TAGS.has(normalizeTag(capturedNode?.tag))
? formControlName(capturedNode, axString(axNode.name), value)
: axString(axNode.name);
const signalsForNode = axSignals.get(axNode.nodeId);
let name = FORM_CONTROL_TAGS.has(normalizeTag(capturedNode?.tag))
? formControlName(capturedNode, axString(axNode.name), value, signals)
: (axString(axNode.name) ?? signalsForNode?.aggregatedText);
if (name && signalsForNode?.hasPopup) {
name = signalsForNode.expanded ? `${name} [expanded]` : `${name} [has-submenu]`;
}
const node: VomNode = {
id: axNode.backendDOMNodeId as number,
parentId: nearestBackendParent(axNode, axById),
@@ -531,12 +719,22 @@ function axVomNode(
position: capturedNode?.position || "static",
pointerEvents: capturedNode?.pointerEvents || "auto",
modal: isModalSignal(axNode, capturedNode),
sensitive: isSensitive(capturedNode),
sensitive: isSensitive(capturedNode) || signalsForNode?.sensitive === true,
};
if (role) node.role = role;
if (name) node.name = name;
if (value !== undefined) node.value = value;
return applyCapturedSignals(node, capturedNode, signals);
const enriched = applyCapturedSignals(node, capturedNode, signals);
if (signalsForNode?.selected && !enriched.attrs?.["aria-selected"]) {
enriched.attrs = { ...(enriched.attrs ?? {}), "aria-selected": "true" };
}
if (signalsForNode?.expanded && !enriched.attrs?.["aria-expanded"]) {
enriched.attrs = { ...(enriched.attrs ?? {}), "aria-expanded": "true" };
}
if (signalsForNode?.controls && !enriched.attrs?.["aria-controls"]) {
enriched.attrs = { ...(enriched.attrs ?? {}), "aria-controls": signalsForNode.controls };
}
return enriched;
}
function iframeSignals(iframeNodes: CapturedNode[]): VomNodeDomSignals {
@@ -561,6 +759,144 @@ function capturedOnlySignals(capturedNodes: CapturedNode[]): VomNodeDomSignals {
};
}
function normalizeProbeKey(value: string | undefined): string {
return cleanAttr(value)?.toLowerCase() ?? "";
}
function controlledIds(attrs: Record<string, string>): string[] {
return (attrs["aria-controls"] ?? "").split(/\s+/).filter(Boolean);
}
function isActiveScopeTrigger(node: VomNode): boolean {
const attrs = node.attrs ?? {};
if (controlledIds(attrs).length === 0) return false;
const role = normalizeTag(attrs.role) || normalizedVomRole(node);
return (
(role === "tab" && (attrs["aria-selected"] ?? "").toLowerCase() === "true") ||
(attrs["aria-expanded"] ?? "").toLowerCase() === "true"
);
}
function normalizedVomRole(node: VomNode): string {
return node.role?.toLowerCase() ?? "";
}
function collectScopeLines(
root: CapturedNode,
triggerLabel: string,
childrenByParentId: Map<number, CapturedNode[]>,
): string[] {
const lines: string[] = [];
const seen = new Set<string>();
const triggerKey = normalizeProbeKey(triggerLabel);
const stack = [root];
while (stack.length > 0 && lines.length < ACTIVE_SCOPE_MAX_LINES) {
const node = stack.shift() as CapturedNode;
const tag = normalizeTag(node.tag);
if (ACTIVE_SCOPE_SKIP_TAGS.has(tag)) continue;
if ((node.attrs.type ?? "").toLowerCase() === "password") continue;
const text = cleanAttr(node.textContent);
const key = normalizeProbeKey(text);
if (text && key !== triggerKey && !seen.has(key)) {
seen.add(key);
lines.push(
text.length > ACTIVE_SCOPE_MAX_LINE_LENGTH
? text.slice(0, ACTIVE_SCOPE_MAX_LINE_LENGTH)
: text,
);
}
stack.push(...(childrenByParentId.get(node.backendNodeId) ?? []));
}
return lines;
}
function buildActiveScopeBlocks(nodes: VomNode[], signals: VomNodeDomSignals): ActiveScopeBlock[] {
const panelByDomId = new Map<string, CapturedNode>();
for (const capturedNode of signals.capturedByBackendId.values()) {
const domId = capturedNode.attrs.id;
if (domId) panelByDomId.set(domId, capturedNode);
}
const blocks: ActiveScopeBlock[] = [];
let totalChars = 0;
for (const node of nodes) {
if (blocks.length >= ACTIVE_SCOPE_MAX_BLOCKS) break;
if (!isActiveScopeTrigger(node)) continue;
const label =
cleanAttr(node.name) ?? cleanAttr(node.text) ?? cleanAttr(node.attrs?.["aria-label"]);
if (!label) continue;
const lines: string[] = [];
for (const controlId of controlledIds(node.attrs ?? {})) {
const panel = panelByDomId.get(controlId);
if (!panel) continue;
lines.push(...collectScopeLines(panel, label, signals.childrenByParentId));
if (lines.length >= ACTIVE_SCOPE_MAX_LINES) break;
}
const uniqueLines: string[] = [];
const seen = new Set<string>();
for (const line of lines) {
const key = normalizeProbeKey(line);
if (!key || seen.has(key)) continue;
seen.add(key);
uniqueLines.push(line);
totalChars += line.length;
if (
uniqueLines.length >= ACTIVE_SCOPE_MAX_LINES ||
totalChars >= ACTIVE_SCOPE_MAX_TOTAL_CHARS
) {
break;
}
}
if (uniqueLines.length === 0) continue;
blocks.push({ triggerId: node.id, label, lines: uniqueLines });
if (totalChars >= ACTIVE_SCOPE_MAX_TOTAL_CHARS) break;
}
return blocks;
}
function labelForSurfaceMatch(node: VomNode): string {
return (
cleanAttr(node.name) ??
cleanAttr(node.text) ??
cleanAttr(node.attrs?.["aria-label"]) ??
cleanAttr(node.attrs?.title) ??
""
);
}
function buildConditionalSurfaces(nodes: VomNode[], captured: CapturedViewModel): CondSurface[] {
const probes = captured.surfaceProbes ?? [];
if (probes.length === 0) return [];
const surfaces: CondSurface[] = [];
const nodesWithLabels = nodes
.map((node) => ({ node, key: normalizeProbeKey(labelForSurfaceMatch(node)) }))
.filter((entry) => entry.key.length > 0);
const used = new Set<number>();
for (const probe of probes) {
const probeKey = normalizeProbeKey(probe.triggerLabel);
if (!probeKey || probe.subItems.length === 0) continue;
const match =
nodesWithLabels.find((entry) => entry.key === probeKey && !used.has(entry.node.id)) ??
nodesWithLabels.find(
(entry) =>
!used.has(entry.node.id) &&
(probeKey.startsWith(entry.key) || entry.key.startsWith(probeKey)),
);
if (!match) continue;
used.add(match.node.id);
surfaces.push({
triggerId: match.node.id,
triggerAction: probe.triggerAction,
subItems: probe.subItems,
});
}
return surfaces;
}
function axNodeInOverlaySubtree(
axNode: CdpAxNode,
axById: Map<string, CdpAxNode>,
@@ -585,6 +921,7 @@ function axNodeInOverlaySubtree(
export function buildVomScene(axNodes: CdpAxNode[], captured: CapturedViewModel): VomScene {
const signals = capturedOnlySignals(captured.nodes);
const { capturedByBackendId } = signals;
const axSignals = buildAxSignals(axNodes);
const axById = new Map<string, CdpAxNode>();
for (const node of axNodes) {
@@ -601,7 +938,7 @@ export function buildVomScene(axNodes: CdpAxNode[], captured: CapturedViewModel)
const capturedNode = capturedByBackendId.get(axNode.backendDOMNodeId);
const role = axString(axNode.role);
const vomNode = axVomNode(axNode, capturedNode, axById, signals);
const vomNode = axVomNode(axNode, capturedNode, axById, signals, axSignals);
// For link nodes, attach external hostname so the renderer can annotate it.
// Relative hrefs and same-origin hrefs are omitted to avoid noise.
@@ -633,7 +970,8 @@ export function buildVomScene(axNodes: CdpAxNode[], captured: CapturedViewModel)
if (
seenBackendIds.has(iframeNode.backendNodeId) ||
excludedBackendNodeIds.has(iframeNode.backendNodeId) ||
!isRenderableIframeControl(iframeNode)
!isRenderableIframeControl(iframeNode) ||
hasEquivalentIframeControl(nodes, iframeNode, iframeBackendId)
) {
continue;
}
@@ -651,7 +989,14 @@ export function buildVomScene(axNodes: CdpAxNode[], captured: CapturedViewModel)
}
}
return { viewport: captured.viewport, nodes };
const activeScopeBlocks = buildActiveScopeBlocks(nodes, signals);
const surfaces = buildConditionalSurfaces(nodes, captured);
return {
viewport: captured.viewport,
nodes,
...(surfaces.length > 0 ? { surfaces } : {}),
...(activeScopeBlocks.length > 0 ? { activeScopeBlocks } : {}),
};
}
export interface SnapshotDeps {
@@ -825,7 +1170,11 @@ export async function handleSnapshot(
const axNodes = result.nodes ?? [];
const captured = await captureForVom(deps.cdp, target.tabId);
const scene = buildVomScene(axNodes, captured);
const rendered = renderVom(scene, { maxDepth: params.max_depth, maxTokens: params.max_tokens });
const rendered = renderVom(scene, {
maxDepth: params.max_depth,
maxTokens: params.max_tokens,
activeRegionPolicy: true,
});
ctx.refStore.replace(
rendered.refs.map(
(r) => [r.ref, { backendNodeId: r.backendNodeId, tabId: target.tabId }] as const,
apps/extension/src/tools/vom/capture.ts
+185 -2
View File
@@ -35,10 +35,17 @@ export interface CapturedNode {
export type CapturedIframeNodes = Map<number, CapturedNode[]>;
export interface CapturedSurfaceProbe {
triggerLabel: string;
triggerAction: "hover" | "focus" | string;
subItems: string[];
}
export interface CapturedViewModel {
nodes: CapturedNode[];
viewport: Viewport;
iframeNodes: CapturedIframeNodes;
surfaceProbes?: CapturedSurfaceProbe[];
/** Backend node ids belonging to the agent overlay host + its shadow subtree. */
excludedBackendNodeIds: Set<number>;
}
@@ -88,6 +95,20 @@ interface LayoutMetricsReply {
layoutViewport?: { clientWidth?: number; clientHeight?: number; pageX?: number; pageY?: number };
}
interface RuntimeEvaluateReply {
result?: {
value?: unknown;
};
}
interface HoverCandidate {
triggerSel: string;
affectedSub: string;
label: string;
x: number;
y: number;
}
interface CdpDomNode {
backendNodeId?: number;
children?: CdpDomNode[];
@@ -126,6 +147,160 @@ function collectBackendIdsFromDomNode(node: CdpDomNode | undefined, out: Set<num
}
}
const MAX_HOVER_PROBE_MS = 2_000;
const MAX_HOVER_TRIGGERS = 8;
const HOVER_SETTLE_MS = 200;
function runtimeValue<T>(reply: RuntimeEvaluateReply): T | undefined {
return reply.result?.value as T | undefined;
}
function hoverCssScanExpression(): string {
return `(() => {
const visibilityProps = ["display", "visibility", "opacity", "maxHeight", "height", "overflow"];
const pairs = [];
const seenRules = new Set();
for (const sheet of Array.from(document.styleSheets)) {
let rules;
try { rules = sheet.cssRules; } catch { continue; }
if (!rules) continue;
for (const rule of Array.from(rules)) {
if (rule.type !== 1 || !rule.selectorText || !rule.style) continue;
const selectorText = String(rule.selectorText);
if (!selectorText.includes(":hover")) continue;
if (!visibilityProps.some((prop) => rule.style[prop])) continue;
for (const rawPart of selectorText.split(",")) {
const part = rawPart.trim();
const hoverIndex = part.indexOf(":hover");
if (hoverIndex < 0) continue;
const triggerSel = part.slice(0, hoverIndex).trim();
const affectedSub = part.slice(hoverIndex + 6).trim();
if (!triggerSel) continue;
const key = triggerSel + "||" + affectedSub;
if (seenRules.has(key)) continue;
seenRules.add(key);
pairs.push({ triggerSel, affectedSub });
}
}
}
const candidates = [];
const seenLabels = new Set();
for (const pair of pairs) {
let elements;
try { elements = Array.from(document.querySelectorAll(pair.triggerSel)); } catch { continue; }
for (const el of elements) {
if (!(el instanceof Element)) continue;
const rect = el.getBoundingClientRect();
const style = getComputedStyle(el);
if (rect.width <= 0 || rect.height <= 0 || style.visibility === "hidden" || style.display === "none") continue;
const label = (el.textContent || "").replace(/\\s+/g, " ").trim();
if (!label || seenLabels.has(label)) continue;
seenLabels.add(label);
candidates.push({
triggerSel: pair.triggerSel,
affectedSub: pair.affectedSub,
label,
x: rect.left + rect.width / 2,
y: rect.top + rect.height / 2,
});
if (candidates.length >= ${MAX_HOVER_TRIGGERS}) return candidates;
}
}
return candidates;
})()`;
}
function hoverCollectExpression(candidate: HoverCandidate): string {
return `(() => {
const triggerSel = ${JSON.stringify(candidate.triggerSel)};
const rawAffectedSub = ${JSON.stringify(candidate.affectedSub)};
const x = ${JSON.stringify(candidate.x)};
const y = ${JSON.stringify(candidate.y)};
const hit = document.elementFromPoint(x, y);
const trigger = hit instanceof Element ? hit.closest(triggerSel) : null;
if (!trigger) return [];
const affectedSub = rawAffectedSub.replace(/^[>+~]\\s*/, "").trim();
let targets = [];
if (!affectedSub) {
targets = [trigger];
} else {
try { targets = Array.from(trigger.querySelectorAll(affectedSub)); } catch { targets = []; }
}
const items = [];
const seen = new Set();
const push = (value) => {
const text = String(value || "").replace(/\\s+/g, " ").trim();
if (!text || seen.has(text)) return;
seen.add(text);
items.push(text);
};
for (const target of targets) {
if (!(target instanceof Element)) continue;
const style = getComputedStyle(target);
if (style.display === "none" || style.visibility === "hidden" || style.opacity === "0") continue;
const clickable = target.querySelectorAll('a, button, [role="menuitem"], [role="option"]');
if (clickable.length > 0) {
for (const child of Array.from(clickable)) push(child.textContent);
} else {
push(target.textContent);
}
}
return items.slice(0, 12);
})()`;
}
async function wait(ms: number): Promise<void> {
await new Promise((resolve) => setTimeout(resolve, ms));
}
async function probeHoverSurfaces(cdp: CdpRunner, tabId: number): Promise<CapturedSurfaceProbe[]> {
const started = Date.now();
try {
const scan = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
expression: hoverCssScanExpression(),
returnByValue: true,
});
const candidates = runtimeValue<HoverCandidate[]>(scan) ?? [];
const results: CapturedSurfaceProbe[] = [];
const seen = new Set<string>();
for (const candidate of candidates.slice(0, MAX_HOVER_TRIGGERS)) {
if (Date.now() - started > MAX_HOVER_PROBE_MS) break;
if (!candidate.label || seen.has(candidate.label)) continue;
try {
await cdp.send(tabId, "Input.dispatchMouseEvent", {
type: "mouseMoved",
x: candidate.x,
y: candidate.y,
});
await wait(HOVER_SETTLE_MS);
const collected = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
expression: hoverCollectExpression(candidate),
returnByValue: true,
});
await cdp.send(tabId, "Input.dispatchMouseEvent", { type: "mouseMoved", x: 0, y: 0 });
const subItems = (runtimeValue<string[]>(collected) ?? [])
.map((item) => item.replace(/\s+/g, " ").trim())
.filter(Boolean);
if (subItems.length === 0) continue;
seen.add(candidate.label);
results.push({
triggerLabel: candidate.label,
triggerAction: "hover",
subItems,
});
} catch {
continue;
}
}
return results;
} catch (err) {
console.debug("[bsk capture] hover surface probe failed", err);
return [];
}
}
/**
* When DOMSnapshot is unavailable, locate the marked overlay host via CDP and
* collect every backendNodeId in its pierced subtree (open shadow included).
@@ -319,7 +494,13 @@ export async function captureViewModel(cdp: CdpRunner, tabId: number): Promise<C
const documents = snap.documents ?? [];
const doc0 = documents[0];
if (!doc0?.nodes?.backendNodeId) {
return { nodes: [], viewport, iframeNodes: new Map(), excludedBackendNodeIds: new Set() };
return {
nodes: [],
viewport,
iframeNodes: new Map(),
surfaceProbes: [],
excludedBackendNodeIds: new Set(),
};
}
const mainParsed = parseDocumentNodes(doc0, strings, dpr, scrollX, scrollY);
@@ -352,5 +533,7 @@ export async function captureViewModel(cdp: CdpRunner, tabId: number): Promise<C
}
}
return { nodes, viewport, iframeNodes, excludedBackendNodeIds };
const surfaceProbes = await probeHoverSurfaces(cdp, tabId);
return { nodes, viewport, iframeNodes, surfaceProbes, excludedBackendNodeIds };
}