feat(extension): vom hover description bug fix

This commit is contained in:
Ljy-0827 committed 2026-08-06 15:30:11 +08:00
1 parent 91225e5fdb
commit dbc4845c0d
13 files changed
+1081 -175

No files matched your search

@@ -1430,7 +1430,7 @@ describe("buildVomScene", () => {
expect(renderVom(scene).text).toContain("[§ active: Reviews (12)]"); expect(renderVom(scene).text).toContain("[§ active: Reviews (12)]");
}); });
it("maps hover surface probes to matching node labels", () => { it("maps hover surface probes to matching backend node ids", () => {
const axNodes: CdpAxNode[] = [ const axNodes: CdpAxNode[] = [
{ {
nodeId: "1", nodeId: "1",
@@ -1451,7 +1451,7 @@ describe("buildVomScene", () => {
iframeNodes: new Map(), iframeNodes: new Map(),
excludedBackendNodeIds: new Set(), excludedBackendNodeIds: new Set(),
surfaceProbes: [ surfaceProbes: [
{ triggerLabel: "Products", triggerAction: "hover", subItems: ["Shoes", "Bags"] }, { triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
], ],
nodes: [ nodes: [
{ {
@@ -1483,6 +1483,195 @@ describe("buildVomScene", () => {
expect(renderVom(scene).text).toContain('@e1 button "Products" [hover first: Shoes | Bags]'); expect(renderVom(scene).text).toContain('@e1 button "Products" [hover first: Shoes | Bags]');
}); });
it("maps hover surface probes to rendered descendants in the same trigger subtree", () => {
const axNodes: CdpAxNode[] = [
{
nodeId: "1",
role: { type: "role", value: "RootWebArea" },
backendDOMNodeId: 10,
childIds: ["2"],
},
{
nodeId: "2",
parentId: "1",
role: { type: "role", value: "button" },
name: { type: "computedString", value: "image" },
backendDOMNodeId: 21,
},
];
const scene = buildVomScene(axNodes, {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
surfaceProbes: [
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
],
nodes: [
{
backendNodeId: 10,
parentBackendNodeId: null,
tag: "body",
attrs: {},
rect: { x: 0, y: 0, w: 1000, h: 800 },
paintOrder: 0,
position: "static",
pointerEvents: "auto",
},
{
backendNodeId: 20,
parentBackendNodeId: 10,
tag: "div",
attrs: { class: "tg-avatar" },
rect: { x: 900, y: 10, w: 30, h: 30 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
},
{
backendNodeId: 21,
parentBackendNodeId: 20,
tag: "div",
attrs: { class: "tg-avatar__inner" },
rect: { x: 902, y: 12, w: 26, h: 26 },
paintOrder: 2,
position: "static",
pointerEvents: "auto",
},
],
});
expect(renderVom(scene).text).toContain('@e1 button "image" [hover first: My profile]');
});
it("maps hover surface probes to DOM-recovered custom controls", () => {
const axNodes: CdpAxNode[] = [
{
nodeId: "1",
role: { type: "role", value: "RootWebArea" },
backendDOMNodeId: 10,
childIds: ["2"],
},
{
nodeId: "2",
parentId: "1",
role: { type: "role", value: "img" },
name: { type: "computedString", value: "image" },
backendDOMNodeId: 21,
},
];
const scene = buildVomScene(axNodes, {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
surfaceProbes: [
{
triggerBackendNodeId: 20,
triggerPoint: { x: 915, y: 25 },
triggerAction: "hover",
subItems: ["My profile", "Sign out"],
},
],
nodes: [
{
backendNodeId: 10,
parentBackendNodeId: null,
tag: "body",
attrs: {},
rect: { x: 0, y: 0, w: 1000, h: 800 },
paintOrder: 0,
position: "static",
pointerEvents: "auto",
cursor: "auto",
},
{
backendNodeId: 20,
parentBackendNodeId: 10,
tag: "div",
attrs: { class: "tg-avatar" },
rect: { x: 900, y: 10, w: 30, h: 30 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
cursor: "pointer",
},
{
backendNodeId: 21,
parentBackendNodeId: 20,
tag: "div",
attrs: { class: "tg-avatar__inner" },
rect: { x: 902, y: 12, w: 26, h: 26 },
paintOrder: 2,
position: "static",
pointerEvents: "auto",
cursor: "pointer",
},
],
});
expect(scene.surfaces).toEqual([
{ triggerId: 21, triggerAction: "hover", subItems: ["My profile", "Sign out"] },
]);
expect(renderVom(scene).text).toContain(
'@e1 button "image" [hover first: My profile | Sign out]',
);
});
it("does not attach hover probes to distant geometry matches", () => {
const axNodes: CdpAxNode[] = [
{
nodeId: "1",
role: { type: "role", value: "RootWebArea" },
backendDOMNodeId: 10,
childIds: ["2"],
},
{
nodeId: "2",
parentId: "1",
role: { type: "role", value: "link" },
name: { type: "computedString", value: "0" },
backendDOMNodeId: 42,
},
];
const scene = buildVomScene(axNodes, {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
surfaceProbes: [
{
triggerBackendNodeId: 999,
triggerPoint: { x: 900, y: 20 },
triggerAction: "hover",
subItems: ["My profile"],
},
],
nodes: [
{
backendNodeId: 10,
parentBackendNodeId: null,
tag: "body",
attrs: {},
rect: { x: 0, y: 0, w: 1000, h: 800 },
paintOrder: 0,
position: "static",
pointerEvents: "auto",
},
{
backendNodeId: 42,
parentBackendNodeId: 10,
tag: "a",
attrs: {},
rect: { x: 860, y: 120, w: 40, h: 30 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
},
],
});
expect(scene.surfaces).toBeUndefined();
expect(renderVom(scene).text).not.toContain("[hover first:");
});
it("enriches names and active scope signals from AX properties", () => { it("enriches names and active scope signals from AX properties", () => {
const axNodes: CdpAxNode[] = [ const axNodes: CdpAxNode[] = [
{ {
@@ -1953,7 +2142,7 @@ describe("handleSnapshot", () => {
}); });
const res = await handleObserve( const res = await handleObserve(
sm, sm,
{ session_id: "aa11" }, { session_id: "aa11", debug_surfaces: true },
{ {
cdp: { cdp: {
send: send as unknown as <T = unknown>( send: send as unknown as <T = unknown>(
@@ -1974,6 +2163,7 @@ describe("handleSnapshot", () => {
); );
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(res.debug).toEqual({ surface_probes: [] });
expect(send).toHaveBeenCalledWith( expect(send).toHaveBeenCalledWith(
4, 4,
"Runtime.evaluate", "Runtime.evaluate",
+1
View File
@@ -342,6 +342,7 @@ export class ToolDispatcher {
cdp: this.cdp, cdp: this.cdp,
tabsApi: chromeTabsCaptureApi, tabsApi: chromeTabsCaptureApi,
conditionalSurfaceProbe: !this.hasHoverLatch(), conditionalSurfaceProbe: !this.hasHoverLatch(),
hoverProbeBypassOverlay: bypassOverlay,
} }
: undefined, : undefined,
), ),
+109 -26
View File
@@ -5,7 +5,9 @@
import { import {
type ActiveScopeBlock, type ActiveScopeBlock,
applyVomInteractionRecovery,
type CondSurface, type CondSurface,
isVomReferenceNode,
renderVom, renderVom,
type VomNode, type VomNode,
type VomScene, type VomScene,
@@ -908,39 +910,26 @@ function buildActiveScopeBlocks(nodes: VomNode[], signals: VomNodeDomSignals): A
return blocks; return blocks;
} }
function labelForSurfaceMatch(node: VomNode): string {
return (
cleanAttr(node.name) ??
cleanAttr(node.text) ??
cleanAttr(node.attrs?.["aria-label"]) ??
cleanAttr(node.attrs?.title) ??
""
);
}
function buildConditionalSurfaces(nodes: VomNode[], captured: CapturedViewModel): CondSurface[] { function buildConditionalSurfaces(nodes: VomNode[], captured: CapturedViewModel): CondSurface[] {
const probes = captured.surfaceProbes ?? []; const probes = captured.surfaceProbes ?? [];
if (probes.length === 0) return []; if (probes.length === 0) return [];
const surfaces: CondSurface[] = []; const surfaces: CondSurface[] = [];
const nodesWithLabels = nodes const recoveredNodes = applyVomInteractionRecovery(nodes);
.map((node) => ({ node, key: normalizeProbeKey(labelForSurfaceMatch(node)) })) const signals = capturedOnlySignals(captured.nodes);
.filter((entry) => entry.key.length > 0);
const used = new Set<number>(); const used = new Set<number>();
for (const probe of probes) { for (const probe of probes) {
const probeKey = normalizeProbeKey(probe.triggerLabel); if (probe.subItems.length === 0 || used.has(probe.triggerBackendNodeId)) continue;
if (!probeKey || probe.subItems.length === 0) continue; const match = findSurfaceTriggerNode(
const match = probe.triggerBackendNodeId,
nodesWithLabels.find((entry) => entry.key === probeKey && !used.has(entry.node.id)) ?? recoveredNodes,
nodesWithLabels.find( signals,
(entry) => probe.triggerPoint,
!used.has(entry.node.id) && );
(probeKey.startsWith(entry.key) || entry.key.startsWith(probeKey)),
);
if (!match) continue; if (!match) continue;
used.add(match.node.id); used.add(match.id);
surfaces.push({ surfaces.push({
triggerId: match.node.id, triggerId: match.id,
triggerAction: probe.triggerAction, triggerAction: probe.triggerAction,
subItems: probe.subItems, subItems: probe.subItems,
}); });
@@ -948,6 +937,77 @@ function buildConditionalSurfaces(nodes: VomNode[], captured: CapturedViewModel)
return surfaces; return surfaces;
} }
function findSurfaceTriggerNode(
triggerBackendNodeId: number,
nodes: VomNode[],
signals: VomNodeDomSignals,
triggerPoint?: { x: number; y: number },
): VomNode | undefined {
const renderedById = new Map(nodes.map((node) => [node.id, node]));
const exact = renderedById.get(triggerBackendNodeId);
if (exact && isSurfaceAttachableNode(exact)) return exact;
const domDescendant = nodes.find(
(node) =>
isSurfaceAttachableNode(node) &&
(node.domParentId === triggerBackendNodeId ||
node.domAncestorIds?.includes(triggerBackendNodeId) === true),
);
if (domDescendant) return domDescendant;
const queue = [...(signals.childrenByParentId.get(triggerBackendNodeId) ?? [])];
while (queue.length > 0) {
const child = queue.shift() as CapturedNode;
const rendered = renderedById.get(child.backendNodeId);
if (rendered && isSurfaceAttachableNode(rendered)) return rendered;
queue.push(...(signals.childrenByParentId.get(child.backendNodeId) ?? []));
}
let current = signals.capturedByBackendId.get(triggerBackendNodeId);
while (current?.parentBackendNodeId !== null && current?.parentBackendNodeId !== undefined) {
const parent = renderedById.get(current.parentBackendNodeId);
if (parent && isSurfaceAttachableNode(parent)) return parent;
current = signals.capturedByBackendId.get(current.parentBackendNodeId);
}
if (triggerPoint) {
return findSurfaceNodeByPoint(triggerPoint, nodes, signals);
}
return undefined;
}
function isSurfaceAttachableNode(node: VomNode): boolean {
return isVomReferenceNode(node);
}
function findSurfaceNodeByPoint(
point: { x: number; y: number },
nodes: VomNode[],
signals: VomNodeDomSignals,
): VomNode | undefined {
let best: { node: VomNode; score: number } | undefined;
for (const node of nodes) {
if (!isSurfaceAttachableNode(node)) continue;
const rect = node.rect ?? signals.capturedByBackendId.get(node.id)?.rect;
if (!rect) continue;
const contains =
point.x >= rect.x &&
point.x <= rect.x + rect.w &&
point.y >= rect.y &&
point.y <= rect.y + rect.h;
const centerX = rect.x + rect.w / 2;
const centerY = rect.y + rect.h / 2;
const distance = Math.hypot(point.x - centerX, point.y - centerY);
if (!contains && distance > 40) continue;
const score = contains ? distance : distance + 1_000;
if (!best || score < best.score) {
best = { node, score };
}
}
return best?.node;
}
function axNodeInOverlaySubtree( function axNodeInOverlaySubtree(
axNode: CdpAxNode, axNode: CdpAxNode,
axById: Map<string, CdpAxNode>, axById: Map<string, CdpAxNode>,
@@ -1073,6 +1133,7 @@ export interface SnapshotDeps {
query(q: chrome.tabs.QueryInfo): Promise<chrome.tabs.Tab[]>; query(q: chrome.tabs.QueryInfo): Promise<chrome.tabs.Tab[]>;
}; };
conditionalSurfaceProbe?: boolean; conditionalSurfaceProbe?: boolean;
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
} }
let defaultDeps: SnapshotDeps | null = null; let defaultDeps: SnapshotDeps | null = null;
@@ -1218,9 +1279,13 @@ async function captureForVom(
cdp: CdpRunner, cdp: CdpRunner,
tabId: number, tabId: number,
conditionalSurfaceProbe: boolean, conditionalSurfaceProbe: boolean,
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>,
): Promise<CapturedViewModel> { ): Promise<CapturedViewModel> {
try { try {
return await captureViewModel(cdp, tabId, { conditionalSurfaceProbe }); return await captureViewModel(cdp, tabId, {
conditionalSurfaceProbe,
hoverProbeBypassOverlay,
});
} catch { } catch {
return fallbackCapturedViewModel(cdp, tabId); return fallbackCapturedViewModel(cdp, tabId);
} }
@@ -1261,7 +1326,12 @@ async function handleVomObservation(
const axNodes = result.nodes ?? []; const axNodes = result.nodes ?? [];
const effectiveConditionalSurfaceProbe = const effectiveConditionalSurfaceProbe =
deps.conditionalSurfaceProbe ?? conditionalSurfaceProbe; deps.conditionalSurfaceProbe ?? conditionalSurfaceProbe;
const captured = await captureForVom(deps.cdp, target.tabId, effectiveConditionalSurfaceProbe); const captured = await captureForVom(
deps.cdp,
target.tabId,
effectiveConditionalSurfaceProbe,
deps.hoverProbeBypassOverlay,
);
const scene = buildVomScene(axNodes, captured, { pageUrl: target.url }); const scene = buildVomScene(axNodes, captured, { pageUrl: target.url });
const rendered = renderVom(scene, { const rendered = renderVom(scene, {
maxDepth: params.max_depth, maxDepth: params.max_depth,
@@ -1278,6 +1348,19 @@ async function handleVomObservation(
ref_count: rendered.refs.length, ref_count: rendered.refs.length,
tab_id: target.tabId, tab_id: target.tabId,
truncated: rendered.truncated, truncated: rendered.truncated,
...(toolName === "observe" && (params as ObserveParams).debug_surfaces
? {
debug: {
surface_probes: (captured.surfaceProbes ?? []).map((probe) => ({
trigger_backend_node_id: probe.triggerBackendNodeId,
...(probe.triggerPoint ? { trigger_point: probe.triggerPoint } : {}),
trigger_action: probe.triggerAction,
sub_items: probe.subItems,
...(probe.confidence ? { confidence: probe.confidence } : {}),
})),
},
}
: {}),
}); });
} catch (err) { } catch (err) {
return { return {
@@ -51,6 +51,156 @@ function fakeSnapshotReply() {
}; };
} }
function hoverTriggerSnapshotReply() {
const S = [
"html",
"body",
"button",
"class",
"user-avatar",
"position",
"static",
"pointer-events",
"auto",
"cursor",
"pointer",
];
const i = (s: string) => S.indexOf(s);
return {
strings: S,
documents: [
{
nodes: {
parentIndex: [-1, 0, 1],
nodeType: [1, 1, 1],
nodeName: [i("html"), i("body"), i("button")],
backendNodeId: [10, 11, 12],
attributes: [[], [], [i("class"), i("user-avatar")]],
},
layout: {
nodeIndex: [1, 2],
styles: [
[i("static"), i("auto"), i("pointer")],
[i("static"), i("auto"), i("pointer")],
],
bounds: [
[0, 0, 1000, 800],
[940, 16, 32, 32],
],
paintOrders: [0, 1],
},
},
],
};
}
function twoHoverTriggerSnapshotReply() {
const S = [
"html",
"body",
"button",
"class",
"user-avatar",
"secondary-trigger",
"position",
"static",
"pointer-events",
"auto",
"cursor",
"pointer",
];
const i = (s: string) => S.indexOf(s);
return {
strings: S,
documents: [
{
nodes: {
parentIndex: [-1, 0, 1, 1],
nodeType: [1, 1, 1, 1],
nodeName: [i("html"), i("body"), i("button"), i("button")],
backendNodeId: [10, 11, 12, 13],
attributes: [
[],
[],
[i("class"), i("user-avatar")],
[i("class"), i("secondary-trigger")],
],
},
layout: {
nodeIndex: [1, 2, 3],
styles: [
[i("static"), i("auto"), i("pointer")],
[i("static"), i("auto"), i("pointer")],
[i("static"), i("auto"), i("pointer")],
],
bounds: [
[0, 0, 1000, 800],
[940, 16, 32, 32],
[100, 120, 32, 32],
],
paintOrders: [0, 1, 2],
},
},
],
};
}
function nestedHoverTriggerSnapshotReply() {
const S = [
"html",
"body",
"div",
"img",
"class",
"tg-avatar",
"tg-avatar__inner",
"tg-avatar__image",
"position",
"static",
"pointer-events",
"auto",
"cursor",
"pointer",
];
const i = (s: string) => S.indexOf(s);
return {
strings: S,
documents: [
{
nodes: {
parentIndex: [-1, 0, 1, 2, 3],
nodeType: [1, 1, 1, 1, 1],
nodeName: [i("html"), i("body"), i("div"), i("div"), i("img")],
backendNodeId: [10, 11, 12, 13, 14],
attributes: [
[],
[],
[i("class"), i("tg-avatar")],
[i("class"), i("tg-avatar__inner")],
[i("class"), i("tg-avatar__image")],
],
},
layout: {
nodeIndex: [1, 2, 3, 4],
styles: [
[i("static"), i("auto"), i("pointer")],
[i("static"), i("auto"), i("pointer")],
[i("static"), i("auto"), i("pointer")],
[i("static"), i("auto"), i("pointer")],
],
bounds: [
[0, 0, 1000, 800],
[940, 16, 32, 32],
[942, 18, 28, 28],
[944, 20, 24, 24],
],
paintOrders: [0, 1, 2, 3],
},
},
],
};
}
function makeCdp(snapshot: unknown) { function makeCdp(snapshot: unknown) {
return { return {
send: vi.fn(async (_tab: number, method: string) => { send: vi.fn(async (_tab: number, method: string) => {
@@ -126,11 +276,11 @@ describe("captureViewModel", () => {
}); });
it("runs hover surface probes when explicitly enabled", async () => { it("runs hover surface probes when explicitly enabled", async () => {
let runtimeCalls = 0; let hoverStateCalls = 0;
const cdp = { const cdp = {
send: vi.fn(async (_tabId: number, method: string, params?: object) => { send: vi.fn(async (_tabId: number, method: string, params?: object) => {
if (method === "DOMSnapshot.enable") return {}; if (method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") return fakeSnapshotReply(); if (method === "DOMSnapshot.captureSnapshot") return hoverTriggerSnapshotReply();
if (method === "Page.getLayoutMetrics") { if (method === "Page.getLayoutMetrics") {
return { return {
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
@@ -141,22 +291,23 @@ describe("captureViewModel", () => {
if (expression.includes("input,textarea,select")) { if (expression.includes("input,textarea,select")) {
return { result: { value: { controls: [], childFrames: [] } } }; return { result: { value: { controls: [], childFrames: [] } } };
} }
runtimeCalls += 1; if (expression.includes("document.styleSheets")) {
return runtimeCalls === 1 return { result: { value: [] } };
? { }
result: { if (expression.includes("querySelectorAll(selectors)")) {
value: [ hoverStateCalls += 1;
{ return hoverStateCalls === 1
triggerSel: ".menu", ? { result: { value: [] } }
affectedSub: " .items", : {
label: "Products", result: {
x: 50, value: [
y: 20, { text: "My profile", role: "", tag: "a", x: 900, y: 60 },
}, { text: "Sign out", role: "", tag: "a", x: 900, y: 90 },
], ],
}, },
} };
: { result: { value: ["Shoes", "Bags"] } }; }
throw new Error(`unexpected Runtime.evaluate: ${expression.slice(0, 80)}`);
} }
if (method === "Input.dispatchMouseEvent") return {}; if (method === "Input.dispatchMouseEvent") return {};
throw new Error(`unexpected ${method}`); throw new Error(`unexpected ${method}`);
@@ -166,12 +317,18 @@ describe("captureViewModel", () => {
const result = await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true }); const result = await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
expect(result.surfaceProbes).toEqual([ expect(result.surfaceProbes).toEqual([
{ triggerLabel: "Products", triggerAction: "hover", subItems: ["Shoes", "Bags"] }, {
triggerBackendNodeId: 12,
triggerPoint: { x: 956, y: 32 },
triggerAction: "hover",
subItems: ["My profile", "Sign out"],
confidence: "high",
},
]); ]);
expect(cdp.send).toHaveBeenCalledWith( expect(cdp.send).toHaveBeenCalledWith(
4, 4,
"Input.dispatchMouseEvent", "Input.dispatchMouseEvent",
expect.objectContaining({ type: "mouseMoved", x: 50, y: 20 }), expect.objectContaining({ type: "mouseMoved", x: 956, y: 32 }),
); );
expect(cdp.send).toHaveBeenCalledWith( expect(cdp.send).toHaveBeenCalledWith(
4, 4,
@@ -180,6 +337,91 @@ describe("captureViewModel", () => {
); );
}); });
it("uses a fresh baseline for each hover candidate", async () => {
let hoverStateCalls = 0;
const cdp = {
send: vi.fn(async (_tabId: number, method: string, params?: object) => {
if (method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") return twoHoverTriggerSnapshotReply();
if (method === "Page.getLayoutMetrics") {
return {
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
if (method === "Runtime.evaluate") {
const expression = (params as { expression?: string } | undefined)?.expression ?? "";
if (expression.includes("input,textarea,select")) {
return { result: { value: { controls: [], childFrames: [] } } };
}
if (expression.includes("document.styleSheets")) {
return { result: { value: [] } };
}
if (expression.includes("querySelectorAll(selectors)")) {
hoverStateCalls += 1;
const visibleMenu = [
{ text: "My profile", role: "", tag: "a", x: 900, y: 60 },
{ text: "Sign out", role: "", tag: "a", x: 900, y: 90 },
];
return hoverStateCalls === 1
? { result: { value: [] } }
: { result: { value: visibleMenu } };
}
throw new Error(`unexpected Runtime.evaluate: ${expression.slice(0, 80)}`);
}
if (method === "Input.dispatchMouseEvent") return {};
throw new Error(`unexpected ${method}`);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
const result = await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
expect(result.surfaceProbes).toEqual([
expect.objectContaining({
triggerBackendNodeId: 12,
subItems: ["My profile", "Sign out"],
}),
]);
expect(hoverStateCalls).toBeGreaterThanOrEqual(4);
});
it("deduplicates nested hover candidates for one visual trigger", async () => {
let hoverMoves = 0;
const cdp = {
send: vi.fn(async (_tabId: number, method: string, params?: object) => {
if (method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") return nestedHoverTriggerSnapshotReply();
if (method === "Page.getLayoutMetrics") {
return {
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
if (method === "Runtime.evaluate") {
const expression = (params as { expression?: string } | undefined)?.expression ?? "";
if (expression.includes("input,textarea,select")) {
return { result: { value: { controls: [], childFrames: [] } } };
}
if (expression.includes("document.styleSheets")) {
return { result: { value: [] } };
}
if (expression.includes("querySelectorAll(selectors)")) {
return { result: { value: [] } };
}
throw new Error(`unexpected Runtime.evaluate: ${expression.slice(0, 80)}`);
}
if (method === "Input.dispatchMouseEvent") {
const point = params as { x?: number; y?: number };
if (point.x !== -10 && point.x !== 0) hoverMoves += 1;
return {};
}
throw new Error(`unexpected ${method}`);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
expect(hoverMoves).toBe(1);
});
it("excludes the agent's own overlay shadow host and its inlined shadow subtree", async () => { it("excludes the agent's own overlay shadow host and its inlined shadow subtree", async () => {
// DOMSnapshot inlines an open shadow root's content as descendants of the // DOMSnapshot inlines an open shadow root's content as descendants of the
// host. The agent's WXT overlay host carries a fixed full-viewport // host. The agent's WXT overlay host carries a fixed full-viewport
+372 -113
View File
@@ -45,9 +45,11 @@ export interface CapturedNode {
export type CapturedIframeNodes = Map<number, CapturedNode[]>; export type CapturedIframeNodes = Map<number, CapturedNode[]>;
export interface CapturedSurfaceProbe { export interface CapturedSurfaceProbe {
triggerLabel: string; triggerBackendNodeId: number;
triggerPoint?: { x: number; y: number };
triggerAction: "hover" | "focus" | string; triggerAction: "hover" | "focus" | string;
subItems: string[]; subItems: string[];
confidence?: "high" | "medium" | "low";
} }
export interface CapturedViewModel { export interface CapturedViewModel {
@@ -61,6 +63,7 @@ export interface CapturedViewModel {
export interface CaptureViewModelOptions { export interface CaptureViewModelOptions {
conditionalSurfaceProbe?: boolean; conditionalSurfaceProbe?: boolean;
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
} }
/** Sparse array format Chrome uses for infrequently-set per-node fields. */ /** Sparse array format Chrome uses for infrequently-set per-node fields. */
@@ -225,11 +228,12 @@ interface RuntimeEvaluateReply {
} }
interface HoverCandidate { interface HoverCandidate {
triggerSel: string; backendNodeId: number;
affectedSub: string; label?: string;
label: string;
x: number; x: number;
y: number; y: number;
score: number;
reasons: string[];
} }
interface CdpDomNode { interface CdpDomNode {
@@ -322,17 +326,19 @@ function collectBackendIdsFromDomNode(node: CdpDomNode | undefined, out: Set<num
} }
const MAX_HOVER_PROBE_MS = 2_000; const MAX_HOVER_PROBE_MS = 2_000;
const MAX_HOVER_TRIGGERS = 8; const MAX_HOVER_TRIGGERS = 6;
const HOVER_SETTLE_MS = 200; const MAX_HOVER_SURFACES = 3;
const HOVER_SETTLE_MS = 300;
const MAX_HOVER_SUB_ITEMS = 12;
function runtimeValue<T>(reply: RuntimeEvaluateReply): T | undefined { function runtimeValue<T>(reply: RuntimeEvaluateReply): T | undefined {
return reply.result?.value as T | undefined; return reply.result?.value as T | undefined;
} }
function hoverCssScanExpression(): string { function hoverCssTriggerScanExpression(): string {
return `(() => { return `(() => {
const visibilityProps = ["display", "visibility", "opacity", "maxHeight", "height", "overflow"]; const visibilityProps = ["display", "visibility", "opacity", "maxHeight", "height", "overflow"];
const pairs = []; const centres = [];
const seenRules = new Set(); const seenRules = new Set();
for (const sheet of Array.from(document.styleSheets)) { for (const sheet of Array.from(document.styleSheets)) {
let rules; let rules;
@@ -348,90 +354,73 @@ function hoverCssScanExpression(): string {
const hoverIndex = part.indexOf(":hover"); const hoverIndex = part.indexOf(":hover");
if (hoverIndex < 0) continue; if (hoverIndex < 0) continue;
const triggerSel = part.slice(0, hoverIndex).trim(); const triggerSel = part.slice(0, hoverIndex).trim();
const affectedSub = part.slice(hoverIndex + 6).trim();
if (!triggerSel) continue; if (!triggerSel) continue;
const key = triggerSel + "||" + affectedSub; if (seenRules.has(triggerSel)) continue;
if (seenRules.has(key)) continue; seenRules.add(triggerSel);
seenRules.add(key); let elements;
pairs.push({ triggerSel, affectedSub }); try { elements = Array.from(document.querySelectorAll(triggerSel)); } catch { continue; }
for (const el of elements) {
if (!(el instanceof HTMLElement)) continue;
const rect = el.getBoundingClientRect();
const style = getComputedStyle(el);
if (rect.width <= 0 || rect.height <= 0 || style.visibility === "hidden" || style.display === "none" || style.pointerEvents === "none") continue;
centres.push({ x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 });
if (centres.length >= 24) return centres;
}
} }
} }
} }
return centres;
const candidates = [];
const seenLabels = new Set();
const isUnsafeTrigger = (el) => {
if (!(el instanceof HTMLElement)) return true;
const tag = el.tagName.toLowerCase();
if (["input", "textarea", "select", "option"].includes(tag)) return true;
if (el.isContentEditable) return true;
if (el.hasAttribute("disabled") || el.hasAttribute("inert")) return true;
if ((el.getAttribute("aria-disabled") || "").toLowerCase() === "true") return true;
return false;
};
for (const pair of pairs) {
let elements;
try { elements = Array.from(document.querySelectorAll(pair.triggerSel)); } catch { continue; }
for (const el of elements) {
if (!(el instanceof Element)) continue;
if (isUnsafeTrigger(el)) continue;
const rect = el.getBoundingClientRect();
const style = getComputedStyle(el);
if (rect.width <= 0 || rect.height <= 0 || style.visibility === "hidden" || style.display === "none" || style.pointerEvents === "none") continue;
const label = (el.textContent || "").replace(/\\s+/g, " ").trim();
if (!label || seenLabels.has(label)) continue;
seenLabels.add(label);
candidates.push({
triggerSel: pair.triggerSel,
affectedSub: pair.affectedSub,
label,
x: rect.left + rect.width / 2,
y: rect.top + rect.height / 2,
});
if (candidates.length >= ${MAX_HOVER_TRIGGERS}) return candidates;
}
}
return candidates;
})()`; })()`;
} }
function hoverCollectExpression(candidate: HoverCandidate): string { interface HoverRuntimeItem {
return `(() => { text: string;
const triggerSel = ${JSON.stringify(candidate.triggerSel)}; role: string;
const rawAffectedSub = ${JSON.stringify(candidate.affectedSub)}; tag: string;
const x = ${JSON.stringify(candidate.x)}; x: number;
const y = ${JSON.stringify(candidate.y)}; y: number;
const hit = document.elementFromPoint(x, y); }
const trigger = hit instanceof Element ? hit.closest(triggerSel) : null;
if (!trigger) return [];
const affectedSub = rawAffectedSub.replace(/^[>+~]\\s*/, "").trim(); function hoverStateExpression(): string {
let targets = []; return `(() => {
if (!affectedSub) {
targets = [trigger];
} else {
try { targets = Array.from(trigger.querySelectorAll(affectedSub)); } catch { targets = []; }
}
const items = []; const items = [];
const seen = new Set(); const seen = new Set();
const push = (value) => { const selectors = [
const text = String(value || "").replace(/\\s+/g, " ").trim(); "a",
"button",
"[role='menuitem']",
"[role='menuitemcheckbox']",
"[role='menuitemradio']",
"[role='option']",
"[role='tab']",
"[role='link']",
"[role='button']"
].join(",");
const push = (el) => {
if (!(el instanceof HTMLElement)) return;
const style = getComputedStyle(el);
const rect = el.getBoundingClientRect();
if (rect.width <= 0 || rect.height <= 0) return;
if (style.display === "none" || style.visibility === "hidden" || style.opacity === "0") return;
const text = String(
el.getAttribute("aria-label") ||
el.getAttribute("title") ||
el.textContent ||
""
).replace(/\\s+/g, " ").trim();
if (!text || seen.has(text)) return; if (!text || seen.has(text)) return;
seen.add(text); seen.add(text);
items.push(text); items.push({
text,
role: (el.getAttribute("role") || "").toLowerCase(),
tag: el.tagName.toLowerCase(),
x: rect.left,
y: rect.top,
});
}; };
for (const target of targets) { for (const el of Array.from(document.querySelectorAll(selectors))) push(el);
if (!(target instanceof Element)) continue; return items.slice(0, 400);
const style = getComputedStyle(target);
if (style.display === "none" || style.visibility === "hidden" || style.opacity === "0") continue;
const clickable = target.querySelectorAll('a, button, [role="menuitem"], [role="option"]');
if (clickable.length > 0) {
for (const child of Array.from(clickable)) push(child.textContent);
} else {
push(target.textContent);
}
}
return items.slice(0, 12);
})()`; })()`;
} }
@@ -453,45 +442,313 @@ async function clearHover(cdp: CdpRunner, tabId: number): Promise<void> {
); );
} }
async function probeHoverSurfaces(cdp: CdpRunner, tabId: number): Promise<CapturedSurfaceProbe[]> { function isVisibleProbeNode(node: CapturedNode): boolean {
const rect = node.rect;
return (
!!rect &&
rect.w > 0 &&
rect.h > 0 &&
node.pointerEvents !== "none" &&
node.attrs.hidden === undefined &&
node.attrs.inert === undefined &&
(node.attrs["aria-hidden"] ?? "").toLowerCase() !== "true"
);
}
function isUnsafeHoverTrigger(node: CapturedNode): boolean {
const tag = node.tag.toLowerCase();
if (["input", "textarea", "select", "option"].includes(tag)) return true;
if ((node.attrs.contenteditable ?? "").toLowerCase() === "true") return true;
if (node.attrs.disabled !== undefined || node.attrs.inert !== undefined) return true;
return (node.attrs["aria-disabled"] ?? "").toLowerCase() === "true";
}
function capturedText(node: CapturedNode): string | undefined {
const value =
node.attrs["aria-label"] ?? node.attrs.title ?? node.attrs.alt ?? node.textContent ?? "";
const clean = value.replace(/\s+/g, " ").trim();
return clean || undefined;
}
function hasGraphicDescendant(
node: CapturedNode,
childrenByParentId: Map<number, CapturedNode[]>,
depth = 0,
): boolean {
if (depth > 3) return false;
for (const child of childrenByParentId.get(node.backendNodeId) ?? []) {
const tag = child.tag.toLowerCase();
if (["img", "svg", "use", "path", "i"].includes(tag)) return true;
if (hasGraphicDescendant(child, childrenByParentId, depth + 1)) return true;
}
return false;
}
function hasHoverPopupSignal(node: CapturedNode): boolean {
const attrs = node.attrs;
const haystack = [
attrs.id,
attrs.class,
attrs["data-testid"],
attrs["data-test"],
attrs["data-cy"],
attrs["aria-haspopup"],
attrs["aria-controls"],
]
.filter(Boolean)
.join(" ")
.toLowerCase();
return /(menu|dropdown|popover|popup|avatar|profile|account|user)/.test(haystack);
}
function roleOf(node: CapturedNode): string {
return (node.attrs.role ?? "").toLowerCase();
}
function hasDirectInteractiveSignal(node: CapturedNode): boolean {
const tag = node.tag.toLowerCase();
const role = roleOf(node);
return (
["button", "a", "summary"].includes(tag) ||
["button", "link", "menuitem", "tab", "combobox"].includes(role) ||
node.attrs.tabindex !== undefined ||
node.cursor === "pointer" ||
node.attrs.onclick !== undefined ||
node.attrs.onmouseenter !== undefined ||
node.attrs.onmouseover !== undefined ||
node.attrs["aria-haspopup"] !== undefined ||
node.attrs["aria-expanded"] === "false"
);
}
function scoreHoverCandidate(
node: CapturedNode,
childrenByParentId: Map<number, CapturedNode[]>,
cssHoverPoints: Array<{ x: number; y: number }>,
): HoverCandidate | null {
if (!isVisibleProbeNode(node) || isUnsafeHoverTrigger(node)) {
return null;
}
const rect = node.rect;
if (!rect) return null;
const area = rect.w * rect.h;
if (area <= 0 || area > 160_000) return null;
const reasons: string[] = [];
let score = 0;
const tag = node.tag.toLowerCase();
const role = roleOf(node);
const label = capturedText(node);
const graphic = hasGraphicDescendant(node, childrenByParentId);
const directInteractive = hasDirectInteractiveSignal(node);
const popupSignal = hasHoverPopupSignal(node);
const compactTopbarIcon = !label && graphic && rect.y < 120 && rect.w <= 96 && rect.h <= 96;
if (!directInteractive && !popupSignal && !compactTopbarIcon) return null;
if (node.attrs["aria-haspopup"] !== undefined) {
score += 80;
reasons.push("aria-haspopup");
}
if ((node.attrs["aria-expanded"] ?? "").toLowerCase() === "false") {
score += 55;
reasons.push("collapsed");
}
if (popupSignal) {
score += 45;
reasons.push("popup-signal");
}
if (tag === "button" || role === "button") {
score += 30;
reasons.push("button");
}
if (node.cursor === "pointer") {
score += 25;
reasons.push("pointer");
}
if (!label && graphic) {
score += 40;
reasons.push("icon-only");
}
if (rect.y < 120) {
score += 25;
reasons.push("topbar");
}
if (rect.w <= 80 && rect.h <= 80) {
score += 20;
reasons.push("compact");
}
if (
cssHoverPoints.some(
(point) =>
point.x >= rect.x &&
point.x <= rect.x + rect.w &&
point.y >= rect.y &&
point.y <= rect.y + rect.h,
)
) {
score += 40;
reasons.push("css-hover");
}
if (score < 45) return null;
return {
backendNodeId: node.backendNodeId,
label,
x: rect.x + rect.w / 2,
y: rect.y + rect.h / 2,
score,
reasons,
};
}
function buildHoverCandidates(
nodes: CapturedNode[],
cssHoverPoints: Array<{ x: number; y: number }>,
): HoverCandidate[] {
const childrenByParentId = new Map<number, CapturedNode[]>();
const parentByBackendId = new Map<number, number | null>();
for (const node of nodes) {
parentByBackendId.set(node.backendNodeId, node.parentBackendNodeId);
if (node.parentBackendNodeId === null) continue;
const children = childrenByParentId.get(node.parentBackendNodeId) ?? [];
children.push(node);
childrenByParentId.set(node.parentBackendNodeId, children);
}
const candidates = nodes
.map((node) => scoreHoverCandidate(node, childrenByParentId, cssHoverPoints))
.filter((candidate): candidate is HoverCandidate => candidate !== null)
.sort((a, b) => b.score - a.score);
const deduped: HoverCandidate[] = [];
const seen = new Set<number>();
for (const candidate of candidates) {
if (seen.has(candidate.backendNodeId)) continue;
if (deduped.some((existing) => sameHoverCluster(existing, candidate, parentByBackendId))) {
continue;
}
seen.add(candidate.backendNodeId);
deduped.push(candidate);
if (deduped.length >= MAX_HOVER_TRIGGERS) break;
}
return deduped;
}
function sameHoverCluster(
a: HoverCandidate,
b: HoverCandidate,
parentByBackendId: Map<number, number | null>,
): boolean {
if (Math.hypot(a.x - b.x, a.y - b.y) <= 8) return true;
return (
isBackendAncestor(a.backendNodeId, b.backendNodeId, parentByBackendId) ||
isBackendAncestor(b.backendNodeId, a.backendNodeId, parentByBackendId)
);
}
function isBackendAncestor(
ancestorId: number,
nodeId: number,
parentByBackendId: Map<number, number | null>,
): boolean {
let parentId = parentByBackendId.get(nodeId);
let guard = 0;
while (parentId !== null && parentId !== undefined && guard < parentByBackendId.size) {
if (parentId === ancestorId) return true;
parentId = parentByBackendId.get(parentId);
guard += 1;
}
return false;
}
function diffHoverItems(before: HoverRuntimeItem[], after: HoverRuntimeItem[]): string[] {
const beforeKeys = new Set(before.map((item) => item.text.toLowerCase()));
const out: string[] = [];
const seen = new Set<string>();
for (const item of after) {
const text = item.text.replace(/\s+/g, " ").trim();
const key = text.toLowerCase();
if (!text || beforeKeys.has(key) || seen.has(key)) continue;
seen.add(key);
out.push(text);
if (out.length >= MAX_HOVER_SUB_ITEMS) break;
}
return out;
}
function confidenceForHover(
candidate: HoverCandidate,
subItems: string[],
): "high" | "medium" | "low" {
if (subItems.length >= 2 && candidate.score >= 80) return "high";
if (subItems.length >= 2 || candidate.score >= 80) return "medium";
return "low";
}
async function probeHoverSurfaces(
cdp: CdpRunner,
tabId: number,
nodes: CapturedNode[],
options: CaptureViewModelOptions,
): Promise<CapturedSurfaceProbe[]> {
const started = Date.now(); const started = Date.now();
try { try {
const scan = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", { const cssScan = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
expression: hoverCssScanExpression(), expression: hoverCssTriggerScanExpression(),
returnByValue: true, returnByValue: true,
}); });
const candidates = runtimeValue<HoverCandidate[]>(scan) ?? []; const cssHoverPoints = runtimeValue<Array<{ x: number; y: number }>>(cssScan) ?? [];
const candidates = buildHoverCandidates(nodes, cssHoverPoints);
if (candidates.length === 0) return [];
const results: CapturedSurfaceProbe[] = []; const results: CapturedSurfaceProbe[] = [];
const seen = new Set<string>(); const seen = new Set<number>();
for (const candidate of candidates.slice(0, MAX_HOVER_TRIGGERS)) { await options.hoverProbeBypassOverlay?.(tabId, true).catch(() => undefined);
if (Date.now() - started > MAX_HOVER_PROBE_MS) break; try {
if (!candidate.label || seen.has(candidate.label)) continue; for (const candidate of candidates.slice(0, MAX_HOVER_TRIGGERS)) {
try { if (Date.now() - started > MAX_HOVER_PROBE_MS) break;
await cdp.send(tabId, "Input.dispatchMouseEvent", { if (results.length >= MAX_HOVER_SURFACES) break;
type: "mouseMoved", if (seen.has(candidate.backendNodeId)) continue;
x: candidate.x, try {
y: candidate.y, await clearHover(cdp, tabId);
}); await wait(HOVER_SETTLE_MS);
await wait(HOVER_SETTLE_MS); const baselineReply = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
const collected = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", { expression: hoverStateExpression(),
expression: hoverCollectExpression(candidate), returnByValue: true,
returnByValue: true, });
}); const baselineItems = runtimeValue<HoverRuntimeItem[]>(baselineReply) ?? [];
const subItems = (runtimeValue<string[]>(collected) ?? [])
.map((item) => item.replace(/\s+/g, " ").trim()) await cdp.send(tabId, "Input.dispatchMouseEvent", {
.filter(Boolean); type: "mouseMoved",
if (subItems.length === 0) continue; x: candidate.x,
seen.add(candidate.label); y: candidate.y,
results.push({ });
triggerLabel: candidate.label, await wait(HOVER_SETTLE_MS);
triggerAction: "hover", const collected = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
subItems, expression: hoverStateExpression(),
}); returnByValue: true,
} catch { });
continue; const subItems = diffHoverItems(
} finally { baselineItems,
await clearHover(cdp, tabId); runtimeValue<HoverRuntimeItem[]>(collected) ?? [],
);
if (subItems.length === 0) continue;
seen.add(candidate.backendNodeId);
results.push({
triggerBackendNodeId: candidate.backendNodeId,
triggerPoint: { x: candidate.x, y: candidate.y },
triggerAction: "hover",
subItems,
confidence: confidenceForHover(candidate, subItems),
});
} catch {
continue;
} finally {
await clearHover(cdp, tabId);
}
} }
} finally {
await options.hoverProbeBypassOverlay?.(tabId, false).catch(() => undefined);
} }
return results; return results;
} catch (err) { } catch (err) {
@@ -797,7 +1054,9 @@ export async function captureViewModel(
await enrichFormControlStates(cdp, tabId, [nodes, ...iframeNodes.values()]); await enrichFormControlStates(cdp, tabId, [nodes, ...iframeNodes.values()]);
const surfaceProbes = options.conditionalSurfaceProbe ? await probeHoverSurfaces(cdp, tabId) : []; const surfaceProbes = options.conditionalSurfaceProbe
? await probeHoverSurfaces(cdp, tabId, nodes, options)
: [];
return { nodes, viewport, iframeNodes, surfaceProbes, excludedBackendNodeIds }; return { nodes, viewport, iframeNodes, surfaceProbes, excludedBackendNodeIds };
} }
+15 -2
View File
@@ -314,8 +314,21 @@ export interface SnapshotResult {
dialogs?: JavaScriptDialogInfo[]; dialogs?: JavaScriptDialogInfo[];
} }
export type ObserveParams = SnapshotParams; export interface ObserveParams extends SnapshotParams {
export type ObserveResult = SnapshotResult; debug_surfaces?: boolean;
}
export interface ObserveResult extends SnapshotResult {
debug?: {
surface_probes?: Array<{
trigger_backend_node_id: number;
trigger_point?: { x: number; y: number };
trigger_action: string;
sub_items: string[];
confidence?: string;
}>;
};
}
export interface GetHtmlParams { export interface GetHtmlParams {
session_id: string; session_id: string;
+5
View File
@@ -29,6 +29,10 @@ pub struct ObserveArgs {
/// Soft cap on rendered tokens (~4 chars/token). /// Soft cap on rendered tokens (~4 chars/token).
#[arg(long = "max-tokens")] #[arg(long = "max-tokens")]
pub max_tokens: Option<u32>, pub max_tokens: Option<u32>,
/// Include conditional surface probe diagnostics in JSON output.
#[arg(long = "debug-surfaces")]
pub debug_surfaces: bool,
} }
pub fn dispatch(args: ObserveArgs, format: Format) -> Result<(), CliError> { pub fn dispatch(args: ObserveArgs, format: Format) -> Result<(), CliError> {
@@ -42,6 +46,7 @@ fn run(sock: PathBuf, args: ObserveArgs, format: Format) -> Result<(), CliError>
tab_id: args.tab_id, tab_id: args.tab_id,
max_depth: args.max_depth, max_depth: args.max_depth,
max_tokens: args.max_tokens, max_tokens: args.max_tokens,
debug_surfaces: args.debug_surfaces,
}; };
let reply: ObserveResult = call(sock, params)?; let reply: ObserveResult = call(sock, params)?;
match format { match format {
+2
View File
@@ -478,6 +478,7 @@ async fn observe_returns_semantic_text_and_ref_count() {
tab_id: 13, tab_id: 13,
truncated: false, truncated: false,
dialogs: vec![], dialogs: vec![],
debug: None,
}) })
.unwrap(), .unwrap(),
) )
@@ -492,6 +493,7 @@ async fn observe_returns_semantic_text_and_ref_count() {
tab_id: None, tab_id: None,
max_depth: None, max_depth: None,
max_tokens: None, max_tokens: None,
debug_surfaces: false,
}, },
) )
.await .await
@@ -7,6 +7,11 @@
"session_id" "session_id"
], ],
"properties": { "properties": {
"debug_surfaces": {
"description": "Include implementation diagnostics for conditional surface probes.",
"default": false,
"type": "boolean"
},
"max_depth": { "max_depth": {
"description": "Cap the depth of the rendered VOM tree.", "description": "Cap the depth of the rendered VOM tree.",
"type": [ "type": [
@@ -8,6 +8,16 @@
"text" "text"
], ],
"properties": { "properties": {
"debug": {
"anyOf": [
{
"$ref": "#/definitions/ObserveDebug"
},
{
"type": "null"
}
]
},
"dialogs": { "dialogs": {
"type": "array", "type": "array",
"items": { "items": {
@@ -103,6 +113,73 @@
"prompt", "prompt",
"beforeunload" "beforeunload"
] ]
},
"ObserveDebug": {
"type": "object",
"properties": {
"surface_probes": {
"type": "array",
"items": {
"$ref": "#/definitions/SurfaceProbeDebug"
}
}
}
},
"SurfaceProbeDebug": {
"type": "object",
"required": [
"sub_items",
"trigger_action",
"trigger_backend_node_id"
],
"properties": {
"confidence": {
"type": [
"string",
"null"
]
},
"sub_items": {
"type": "array",
"items": {
"type": "string"
}
},
"trigger_action": {
"type": "string"
},
"trigger_backend_node_id": {
"type": "integer",
"format": "int64"
},
"trigger_point": {
"anyOf": [
{
"$ref": "#/definitions/SurfaceProbePoint"
},
{
"type": "null"
}
]
}
}
},
"SurfaceProbePoint": {
"type": "object",
"required": [
"x",
"y"
],
"properties": {
"x": {
"type": "number",
"format": "double"
},
"y": {
"type": "number",
"format": "double"
}
}
} }
} }
} }
@@ -74,6 +74,32 @@ pub struct ObserveParams {
/// best-effort heuristic based on character count). /// best-effort heuristic based on character count).
#[serde(default, skip_serializing_if = "Option::is_none")] #[serde(default, skip_serializing_if = "Option::is_none")]
pub max_tokens: Option<u32>, pub max_tokens: Option<u32>,
/// Include implementation diagnostics for conditional surface probes.
#[serde(default)]
pub debug_surfaces: bool,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct SurfaceProbePoint {
pub x: f64,
pub y: f64,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct SurfaceProbeDebug {
pub trigger_backend_node_id: i64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub trigger_point: Option<SurfaceProbePoint>,
pub trigger_action: String,
pub sub_items: Vec<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub confidence: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct ObserveDebug {
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub surface_probes: Vec<SurfaceProbeDebug>,
} }
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
@@ -92,6 +118,8 @@ pub struct ObserveResult {
pub truncated: bool, pub truncated: bool,
#[serde(default, skip_serializing_if = "Vec::is_empty")] #[serde(default, skip_serializing_if = "Vec::is_empty")]
pub dialogs: Vec<JavaScriptDialogInfo>, pub dialogs: Vec<JavaScriptDialogInfo>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub debug: Option<ObserveDebug>,
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@@ -280,6 +308,7 @@ mod tests {
tab_id: 42, tab_id: 42,
truncated: false, truncated: false,
dialogs: Vec::new(), dialogs: Vec::new(),
debug: None,
}; };
let v = serde_json::to_value(&r).unwrap(); let v = serde_json::to_value(&r).unwrap();
assert_eq!(v.get("ref_count").and_then(|v| v.as_u64()), Some(1)); assert_eq!(v.get("ref_count").and_then(|v| v.as_u64()), Some(1));
+1 -1
View File
@@ -1,4 +1,4 @@
export { renderVom } from "./render"; export { applyVomInteractionRecovery, isVomReferenceNode, renderVom } from "./render";
export type { export type {
ActiveScopeBlock, ActiveScopeBlock,
BlockingLayer, BlockingLayer,
+10 -10
View File
@@ -197,7 +197,7 @@ function shouldRender(node: VomNode): boolean {
return cleaned(node.name) !== undefined; return cleaned(node.name) !== undefined;
} }
function shouldReference(node: VomNode): boolean { export function isVomReferenceNode(node: VomNode): boolean {
return INTERACTIVE_ROLES.has(normalizedRole(node)) || isUrlBearingStructuralAction(node); return INTERACTIVE_ROLES.has(normalizedRole(node)) || isUrlBearingStructuralAction(node);
} }
@@ -342,7 +342,7 @@ function isAncestorOf(
return false; return false;
} }
function applyRecovery(nodes: VomNode[]): VomNode[] { export function applyVomInteractionRecovery(nodes: VomNode[]): VomNode[] {
const parentMap = buildParentMap(nodes); const parentMap = buildParentMap(nodes);
const children = buildChildren(nodes); const children = buildChildren(nodes);
const recoveredIds = new Set<number>(); const recoveredIds = new Set<number>();
@@ -350,7 +350,7 @@ function applyRecovery(nodes: VomNode[]): VomNode[] {
const recoveredNameById = new Map<number, string>(); const recoveredNameById = new Map<number, string>();
for (const node of nodes) { for (const node of nodes) {
if (shouldReference(node)) continue; if (isVomReferenceNode(node)) continue;
const role = recoveredRole(node, children); const role = recoveredRole(node, children);
if (!role || !passesRecoveryGuards(node, children)) continue; if (!role || !passesRecoveryGuards(node, children)) continue;
recoveredIds.add(node.id); recoveredIds.add(node.id);
@@ -397,7 +397,7 @@ function buildChildren(nodes: VomNode[]): Map<number | null, VomNode[]> {
function duplicateReferenceNames(nodes: VomNode[]): Set<string> { function duplicateReferenceNames(nodes: VomNode[]): Set<string> {
const counts = new Map<string, number>(); const counts = new Map<string, number>();
for (const node of nodes) { for (const node of nodes) {
if (!shouldReference(node)) continue; if (!isVomReferenceNode(node)) continue;
const name = cleaned(node.name); const name = cleaned(node.name);
if (!name) continue; if (!name) continue;
const key = normalizeContextKey(name); const key = normalizeContextKey(name);
@@ -427,7 +427,7 @@ function contextLabel(node: VomNode, targetName: string): string | undefined {
} }
function weakLabelContext(node: VomNode, targetName: string): string | undefined { function weakLabelContext(node: VomNode, targetName: string): string | undefined {
if (shouldReference(node)) return undefined; if (isVomReferenceNode(node)) return undefined;
return weakContextText( return weakContextText(
cleaned(node.name) ?? cleaned(node.text) ?? cleaned(node.nearbyText), cleaned(node.name) ?? cleaned(node.text) ?? cleaned(node.nearbyText),
targetName, targetName,
@@ -509,7 +509,7 @@ function collectWeakLabelsFromSubtree(
state: RenderState, state: RenderState,
labels: string[], labels: string[],
): void { ): void {
if (shouldReference(node)) return; if (isVomReferenceNode(node)) return;
const label = weakLabelContext(node, nodeName); const label = weakLabelContext(node, nodeName);
if (label) { if (label) {
labels.push(label); labels.push(label);
@@ -666,7 +666,7 @@ function descendantTextCoveredByName(node: VomNode, state: RenderState): boolean
const texts: string[] = []; const texts: string[] = [];
while (stack.length > 0) { while (stack.length > 0) {
const child = stack.pop() as VomNode; const child = stack.pop() as VomNode;
if (shouldReference(child)) return false; if (isVomReferenceNode(child)) return false;
const text = cleaned(child.name) ?? cleaned(child.text) ?? cleaned(child.value); const text = cleaned(child.name) ?? cleaned(child.text) ?? cleaned(child.value);
if (text) texts.push(text); if (text) texts.push(text);
stack.push(...(state.children.get(child.id) ?? [])); stack.push(...(state.children.get(child.id) ?? []));
@@ -750,7 +750,7 @@ function renderTree(
continue; continue;
} }
const ref = shouldReference(node) ? `e${state.nextRef}` : undefined; const ref = isVomReferenceNode(node) ? `e${state.nextRef}` : undefined;
const context = ref ? handleContext(node, state) : []; const context = ref ? handleContext(node, state) : [];
const surface = state.surfaceMap.get(node.id); const surface = state.surfaceMap.get(node.id);
const line = renderNodeLine(node, depth, ref, context, surface); const line = renderNodeLine(node, depth, ref, context, surface);
@@ -915,7 +915,7 @@ function applyActiveRegionPolicy(nodes: VomNode[], scene: VomScene): VomNode[] {
const blockedRoots = new Set<number>(); const blockedRoots = new Set<number>();
for (const target of nodes) { for (const target of nodes) {
if (!shouldReference(target)) continue; if (!isVomReferenceNode(target)) continue;
const blocked = candidates.some((candidate) => const blocked = candidates.some((candidate) =>
isBlockedByRegion(target, candidate.node, parentMap, candidate.viewportCoverage), isBlockedByRegion(target, candidate.node, parentMap, candidate.viewportCoverage),
); );
@@ -970,7 +970,7 @@ function renderDoubleLayer(scene: VomScene, layer: BlockingLayer, options: VomOp
} }
export function renderVom(scene: VomScene, options: VomOptions = {}): VomResult { export function renderVom(scene: VomScene, options: VomOptions = {}): VomResult {
const nodes = applyRecovery(scene.nodes); const nodes = applyVomInteractionRecovery(scene.nodes);
const renderScene = { ...scene, nodes }; const renderScene = { ...scene, nodes };
const layer = detectBlockingLayer(nodes, scene.viewport); const layer = detectBlockingLayer(nodes, scene.viewport);
if (layer) return renderDoubleLayer(renderScene, layer, options); if (layer) return renderDoubleLayer(renderScene, layer, options);