figmar/BrowserSkill
1
0
Fork 0
mirror of https://github.com/Tencent/BrowserSkill.git synced 2026-09-10 13:01:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

perf(vom): make active hover probing opt-in on observe

Browse Source 
Every tool.observe moved the real cursor across the page to look for
hover-only menus and tooltips. Measured against real Chrome, that cost a
median of 2.5s and up to 4.7s per observation — 73% to 99% of total
observe wall clock — and returned 3 hits across 12 pages. A positive
control confirms the probes work; the yield on real pages simply does not
justify putting them on the default path.

observe now probes only when asked, via `probe_hover` (`--probe-hover`).
Agents that know which element hides a menu should keep using
`bsk hover <ref>`, which is both cheaper and precise. A held hover latch
still suppresses probing, since probing would move the cursor off the
element the caller is deliberately holding.

Correctness fixes that apply whether or not probing is requested:

- Hover probing ran between the DOM snapshot and the accessibility tree
  fetch, so a hover that opened a menu left one half of an observation
  describing the page before the change and the other half after it. Both
  chains now run after all capture is complete.
- The overlay bypass toggled once per chain and swallowed restore
  failures, which could leave the agent overlay hidden for the rest of
  the tab's life. It is now reference counted across both chains, and a
  failed restore is reported instead of discarded.
- Observations report whether they hovered the page and whether that
  revealed content the static tree lacks, so callers can judge how far
  the live page may have drifted from the returned snapshot.

Probe budgets were checked at the top of the loop, so a candidate could
start with 1ms remaining and still run a full settle window, overshooting
by up to 634ms. Candidates are now admitted only if they fit. Raising
MAX_HOVER_PROBE_MS to 2600 is part of that fix rather than a relaxation:
2000 was never the real ceiling, and enforcing it literally would have
silently dropped a candidate the positive control depends on.

Tooltip misses are cached too, with a shorter TTL than hits, so repeat
observations of the same page no longer re-probe every unlabelled control
from scratch.
This commit is contained in:
drakezhang
2026-09-02 10:59:18 +08:00
parent f06a514bd2
commit 05a36075b8
18 changed files with 917 additions and 338 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/extension/src/tools/__tests__/observation.test.ts
+356 -216
View File
@@ -1904,36 +1904,41 @@ describe("buildVomScene", () => {
backendDOMNodeId: 20,
},
];
const scene = buildVomScene(axNodes, {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
surfaceProbes: [
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
],
nodes: [
{
backendNodeId: 10,
parentBackendNodeId: null,
tag: "body",
attrs: {},
rect: { x: 0, y: 0, w: 1000, h: 800 },
paintOrder: 0,
position: "static",
pointerEvents: "auto",
},
{
backendNodeId: 20,
parentBackendNodeId: 10,
tag: "button",
attrs: {},
rect: { x: 20, y: 20, w: 120, h: 40 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
},
],
});
const scene = buildVomScene(
axNodes,
{
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
backendNodeId: 10,
parentBackendNodeId: null,
tag: "body",
attrs: {},
rect: { x: 0, y: 0, w: 1000, h: 800 },
paintOrder: 0,
position: "static",
pointerEvents: "auto",
},
{
backendNodeId: 20,
parentBackendNodeId: 10,
tag: "button",
attrs: {},
rect: { x: 20, y: 20, w: 120, h: 40 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
},
],
},
{
surfaceProbes: [
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
],
},
);
expect(scene.surfaces).toEqual([
{ triggerId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
@@ -1957,46 +1962,51 @@ describe("buildVomScene", () => {
backendDOMNodeId: 21,
},
];
const scene = buildVomScene(axNodes, {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
surfaceProbes: [
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
],
nodes: [
{
backendNodeId: 10,
parentBackendNodeId: null,
tag: "body",
attrs: {},
rect: { x: 0, y: 0, w: 1000, h: 800 },
paintOrder: 0,
position: "static",
pointerEvents: "auto",
},
{
backendNodeId: 20,
parentBackendNodeId: 10,
tag: "div",
attrs: { class: "tg-avatar" },
rect: { x: 900, y: 10, w: 30, h: 30 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
},
{
backendNodeId: 21,
parentBackendNodeId: 20,
tag: "div",
attrs: { class: "tg-avatar__inner" },
rect: { x: 902, y: 12, w: 26, h: 26 },
paintOrder: 2,
position: "static",
pointerEvents: "auto",
},
],
});
const scene = buildVomScene(
axNodes,
{
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
backendNodeId: 10,
parentBackendNodeId: null,
tag: "body",
attrs: {},
rect: { x: 0, y: 0, w: 1000, h: 800 },
paintOrder: 0,
position: "static",
pointerEvents: "auto",
},
{
backendNodeId: 20,
parentBackendNodeId: 10,
tag: "div",
attrs: { class: "tg-avatar" },
rect: { x: 900, y: 10, w: 30, h: 30 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
},
{
backendNodeId: 21,
parentBackendNodeId: 20,
tag: "div",
attrs: { class: "tg-avatar__inner" },
rect: { x: 902, y: 12, w: 26, h: 26 },
paintOrder: 2,
position: "static",
pointerEvents: "auto",
},
],
},
{
surfaceProbes: [
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
],
},
);
expect(renderVom(scene).text).toContain('@e1 button "image" [hover first: My profile]');
});
@@ -2017,48 +2027,53 @@ describe("buildVomScene", () => {
backendDOMNodeId: 21,
},
];
const scene = buildVomScene(axNodes, {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
surfaceProbes: [
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Sign out"] },
{ triggerBackendNodeId: 21, triggerAction: "hover", subItems: ["Settings"] },
],
nodes: [
{
backendNodeId: 10,
parentBackendNodeId: null,
tag: "body",
attrs: {},
rect: { x: 0, y: 0, w: 1000, h: 800 },
paintOrder: 0,
position: "static",
pointerEvents: "auto",
},
{
backendNodeId: 20,
parentBackendNodeId: 10,
tag: "div",
attrs: { class: "tg-avatar" },
rect: { x: 900, y: 10, w: 30, h: 30 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
},
{
backendNodeId: 21,
parentBackendNodeId: 20,
tag: "div",
attrs: { class: "tg-avatar__inner" },
rect: { x: 902, y: 12, w: 26, h: 26 },
paintOrder: 2,
position: "static",
pointerEvents: "auto",
},
],
});
const scene = buildVomScene(
axNodes,
{
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
backendNodeId: 10,
parentBackendNodeId: null,
tag: "body",
attrs: {},
rect: { x: 0, y: 0, w: 1000, h: 800 },
paintOrder: 0,
position: "static",
pointerEvents: "auto",
},
{
backendNodeId: 20,
parentBackendNodeId: 10,
tag: "div",
attrs: { class: "tg-avatar" },
rect: { x: 900, y: 10, w: 30, h: 30 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
},
{
backendNodeId: 21,
parentBackendNodeId: 20,
tag: "div",
attrs: { class: "tg-avatar__inner" },
rect: { x: 902, y: 12, w: 26, h: 26 },
paintOrder: 2,
position: "static",
pointerEvents: "auto",
},
],
},
{
surfaceProbes: [
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Sign out"] },
{ triggerBackendNodeId: 21, triggerAction: "hover", subItems: ["Settings"] },
],
},
);
expect(scene.surfaces).toEqual([
{ triggerId: 21, triggerAction: "hover", subItems: ["My profile"] },
@@ -2081,54 +2096,59 @@ describe("buildVomScene", () => {
backendDOMNodeId: 21,
},
];
const scene = buildVomScene(axNodes, {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
surfaceProbes: [
{
triggerBackendNodeId: 20,
triggerPoint: { x: 915, y: 25 },
triggerAction: "hover",
subItems: ["My profile", "Sign out"],
},
],
nodes: [
{
backendNodeId: 10,
parentBackendNodeId: null,
tag: "body",
attrs: {},
rect: { x: 0, y: 0, w: 1000, h: 800 },
paintOrder: 0,
position: "static",
pointerEvents: "auto",
cursor: "auto",
},
{
backendNodeId: 20,
parentBackendNodeId: 10,
tag: "div",
attrs: { class: "tg-avatar" },
rect: { x: 900, y: 10, w: 30, h: 30 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
cursor: "pointer",
},
{
backendNodeId: 21,
parentBackendNodeId: 20,
tag: "div",
attrs: { class: "tg-avatar__inner" },
rect: { x: 902, y: 12, w: 26, h: 26 },
paintOrder: 2,
position: "static",
pointerEvents: "auto",
cursor: "pointer",
},
],
});
const scene = buildVomScene(
axNodes,
{
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
backendNodeId: 10,
parentBackendNodeId: null,
tag: "body",
attrs: {},
rect: { x: 0, y: 0, w: 1000, h: 800 },
paintOrder: 0,
position: "static",
pointerEvents: "auto",
cursor: "auto",
},
{
backendNodeId: 20,
parentBackendNodeId: 10,
tag: "div",
attrs: { class: "tg-avatar" },
rect: { x: 900, y: 10, w: 30, h: 30 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
cursor: "pointer",
},
{
backendNodeId: 21,
parentBackendNodeId: 20,
tag: "div",
attrs: { class: "tg-avatar__inner" },
rect: { x: 902, y: 12, w: 26, h: 26 },
paintOrder: 2,
position: "static",
pointerEvents: "auto",
cursor: "pointer",
},
],
},
{
surfaceProbes: [
{
triggerBackendNodeId: 20,
triggerPoint: { x: 915, y: 25 },
triggerAction: "hover",
subItems: ["My profile", "Sign out"],
},
],
},
);
expect(scene.surfaces).toEqual([
{ triggerId: 20, triggerAction: "hover", subItems: ["My profile", "Sign out"] },
@@ -2154,41 +2174,46 @@ describe("buildVomScene", () => {
backendDOMNodeId: 42,
},
];
const scene = buildVomScene(axNodes, {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
surfaceProbes: [
{
triggerBackendNodeId: 999,
triggerPoint: { x: 900, y: 20 },
triggerAction: "hover",
subItems: ["My profile"],
},
],
nodes: [
{
backendNodeId: 10,
parentBackendNodeId: null,
tag: "body",
attrs: {},
rect: { x: 0, y: 0, w: 1000, h: 800 },
paintOrder: 0,
position: "static",
pointerEvents: "auto",
},
{
backendNodeId: 42,
parentBackendNodeId: 10,
tag: "a",
attrs: {},
rect: { x: 860, y: 120, w: 40, h: 30 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
},
],
});
const scene = buildVomScene(
axNodes,
{
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
backendNodeId: 10,
parentBackendNodeId: null,
tag: "body",
attrs: {},
rect: { x: 0, y: 0, w: 1000, h: 800 },
paintOrder: 0,
position: "static",
pointerEvents: "auto",
},
{
backendNodeId: 42,
parentBackendNodeId: 10,
tag: "a",
attrs: {},
rect: { x: 860, y: 120, w: 40, h: 30 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
},
],
},
{
surfaceProbes: [
{
triggerBackendNodeId: 999,
triggerPoint: { x: 900, y: 20 },
triggerAction: "hover",
subItems: ["My profile"],
},
],
},
);
expect(scene.surfaces).toBeUndefined();
expect(renderVom(scene).text).not.toContain("[hover first:");
@@ -2606,7 +2631,106 @@ describe("handleSnapshot", () => {
);
});
it("enables conditional surface probing for semantic observe", async () => {
it("hovers the page only after the accessibility tree has been captured", async () => {
// Hovering can open menus and reflow the page. Probing between the DOM and
// AX captures would leave the two halves of one observation describing the
// page on either side of that change.
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
await sm.start("aa11");
const root: CdpAxNode = {
nodeId: "1",
role: { type: "role", value: "RootWebArea" },
name: { type: "computedString", value: "Example" },
backendDOMNodeId: 100,
childIds: ["2"],
};
const button: CdpAxNode = {
nodeId: "2",
parentId: "1",
role: { type: "role", value: "button" },
name: { type: "computedString", value: "Products" },
backendDOMNodeId: 200,
};
const strings = ["body", "button", "position", "static", "pointer-events", "auto", "cursor"];
const i = (s: string) => strings.indexOf(s);
const methodOrder: string[] = [];
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
methodOrder.push(method);
if (method === "Accessibility.enable") return {};
if (method === "Accessibility.getFullAXTree") return { nodes: [root, button] };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
}
if (method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") {
return {
strings,
documents: [
{
nodes: {
parentIndex: [-1, 0],
nodeName: [i("body"), i("button")],
backendNodeId: [100, 200],
attributes: [[], []],
},
layout: {
nodeIndex: [0, 1],
styles: [
[i("static"), i("auto"), i("auto")],
[i("static"), i("auto"), i("auto")],
],
bounds: [
[0, 0, 1000, 800],
[20, 20, 120, 40],
],
paintOrders: [0, 1],
},
},
],
};
}
if (method === "Input.dispatchMouseEvent") return {};
if (method === "Runtime.evaluate") {
const expression = (params as { expression?: string } | undefined)?.expression ?? "";
// Report the button as a `:hover` rule target so a real hover fires.
if (expression.includes("document.styleSheets")) {
return { result: { value: [{ x: 80, y: 40 }] } };
}
return { result: { value: [] } };
}
throw new Error(`unexpected CDP method ${method}`);
});
const res = await handleObserve(
sm,
{ session_id: "aa11", probe_hover: true },
{
cdp: {
send: send as unknown as <T = unknown>(
tabId: number,
method: string,
params?: object,
) => Promise<T>,
trackSessionTab: vi.fn(),
},
tabsApi: {
get: vi.fn(
async (tabId: number) =>
({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
),
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
},
},
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
const firstHover = methodOrder.indexOf("Input.dispatchMouseEvent");
expect(firstHover).toBeGreaterThan(-1);
expect(methodOrder.lastIndexOf("DOMSnapshot.captureSnapshot")).toBeLessThan(firstHover);
expect(methodOrder.lastIndexOf("Accessibility.getFullAXTree")).toBeLessThan(firstHover);
});
it("runs conditional surface probing only when observe opts in", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
await sm.start("aa11");
const root: CdpAxNode = {
@@ -2662,30 +2786,46 @@ describe("handleSnapshot", () => {
if (method === "Runtime.evaluate") return { result: { value: [] } };
throw new Error(`unexpected CDP method ${method}`);
});
const res = await handleObserve(
const deps = {
cdp: {
send: send as unknown as <T = unknown>(
tabId: number,
method: string,
params?: object,
) => Promise<T>,
trackSessionTab: vi.fn(),
},
tabsApi: {
get: vi.fn(
async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
),
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
},
};
const withoutProbe = await handleObserve(
sm,
{ session_id: "aa11", debug_surfaces: true },
{
cdp: {
send: send as unknown as <T = unknown>(
tabId: number,
method: string,
params?: object,
) => Promise<T>,
trackSessionTab: vi.fn(),
},
tabsApi: {
get: vi.fn(
async (tabId: number) =>
({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
),
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
},
},
deps,
);
if ("code" in withoutProbe) {
throw new Error(`unexpected error: ${JSON.stringify(withoutProbe)}`);
}
expect(withoutProbe.hover_probe).toBeUndefined();
expect(send).not.toHaveBeenCalledWith(
4,
"Runtime.evaluate",
expect.objectContaining({ returnByValue: true }),
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(res.debug).toEqual({ surface_probes: [] });
const withProbe = await handleObserve(
sm,
{ session_id: "aa11", debug_surfaces: true, probe_hover: true },
deps,
);
if ("code" in withProbe) throw new Error(`unexpected error: ${JSON.stringify(withProbe)}`);
expect(withProbe.debug).toEqual({ surface_probes: [] });
expect(withProbe.hover_probe).toEqual({ performed: true, revealed_content: false });
expect(send).toHaveBeenCalledWith(
4,
"Runtime.evaluate",
apps/extension/src/tools/dispatcher.ts
+5 -1
View File
@@ -383,7 +383,11 @@ export class ToolDispatcher {
? {
cdp: this.cdp,
tabsApi: chromeTabsCaptureApi,
conditionalSurfaceProbe: !this.hasHoverLatchForScope(hoverScope),
// Active hover probing is opt-in. A held hover latch still
// suppresses it, because probing would move the cursor off
// the element the caller is deliberately holding.
conditionalSurfaceProbe:
params.probe_hover === true && !this.hasHoverLatchForScope(hoverScope),
hoverProbeBypassOverlay: bypassOverlay,
}
: undefined,
apps/extension/src/tools/observation.ts
+106 -19
View File
@@ -49,11 +49,14 @@ import {
import { resolveSnapshotRef } from "./snapshot-ref";
import {
type CapturedNode,
type CapturedSurfaceProbe,
type CapturedViewModel,
captureViewModel,
collectOverlayExcludedBackendIds,
probeHoverSurfaces,
} from "./vom/capture";
import { type CapturedFrameDocument, captureFrameData } from "./vom/frame-capture";
import { withOverlayBypass } from "./vom/hover-perception";
import { probeTooltipNames } from "./vom/name-enrichment";
import {
type CaptureVomObservationResult,
@@ -525,8 +528,11 @@ function buildActiveScopeBlocks(nodes: VomNode[], signals: VomNodeDomSignals): A
return blocks;
}
function buildConditionalSurfaces(nodes: VomNode[], captured: CapturedViewModel): CondSurface[] {
const probes = captured.surfaceProbes ?? [];
function buildConditionalSurfaces(
nodes: VomNode[],
captured: CapturedViewModel,
probes: CapturedSurfaceProbe[],
): CondSurface[] {
if (probes.length === 0) return [];
const surfaces: CondSurface[] = [];
@@ -649,6 +655,7 @@ function findSurfaceNodeByPoint(
export interface BuildVomSceneOptions {
pageUrl?: string;
supplementalNames?: ReadonlyMap<string, string>;
surfaceProbes?: CapturedSurfaceProbe[];
}
export type VomFrameDocument = CapturedFrameDocument<CdpAxNode>;
@@ -817,18 +824,19 @@ export function buildFrameVomScene(
excludedBackendNodeIds: captured.excludedBackendNodeIds,
supplementalNames: options.supplementalNames,
});
return attachCapturedSceneAnnotations(scene, documents, captured);
return attachCapturedSceneAnnotations(scene, documents, captured, options.surfaceProbes ?? []);
}
function attachCapturedSceneAnnotations(
scene: VomScene,
documents: VomFrameDocument[],
captured: CapturedViewModel,
surfaceProbes: CapturedSurfaceProbe[],
): VomScene {
const rootDocument = documents.find((document) => document.frameId === scene.rootFrameId);
const signals = capturedOnlySignals(rootDocument?.domNodes ?? captured.nodes);
const activeScopeBlocks = buildActiveScopeBlocks(scene.nodes, signals);
const surfaces = buildConditionalSurfaces(scene.nodes, captured);
const surfaces = buildConditionalSurfaces(scene.nodes, captured, surfaceProbes);
return {
...scene,
...(surfaces.length > 0 ? { surfaces } : {}),
@@ -1010,17 +1018,69 @@ async function captureForVom(
options: CaptureVomObservationOptions,
): Promise<CapturedViewModel> {
try {
return await captureViewModel(cdp, tabId, {
conditionalSurfaceProbe: options.conditionalSurfaceProbe ?? false,
hoverProbeBypassOverlay: options.hoverProbeBypassOverlay,
signal: options.signal,
});
return await captureViewModel(cdp, tabId, { signal: options.signal });
} catch (error) {
if (isAbortError(error)) throw error;
return fallbackCapturedViewModel(cdp, tabId, options.signal);
}
}
export interface HoverProbeOutcome {
/** Whether any active hover was dispatched during this observation. */
performed: boolean;
/**
* Whether hovering surfaced content absent from the static snapshot. Cached
* tooltip names count, so this may over-report on repeat observations — it
* errs towards telling the caller the snapshot is less fresh than it looks.
*/
revealedContent: boolean;
surfaceProbes: CapturedSurfaceProbe[];
tooltipNames: Map<string, string>;
}
const NO_HOVER_PROBES: HoverProbeOutcome = {
performed: false,
revealedContent: false,
surfaceProbes: [],
tooltipNames: new Map(),
};
/**
* Runs both active-hover chains back to back.
*
* Ordering matters: this happens after DOM *and* accessibility capture so a
* hover that opens a menu cannot leave one half of the observation describing
* the page before the change and the other half after it. Sharing one overlay
* bypass span also means the agent overlay toggles once per observation
* instead of once per chain.
*/
async function runHoverProbes(
cdp: CdpRunner,
tabId: number,
captured: CapturedViewModel,
documents: VomFrameDocument[],
staticSemantics: ReturnType<typeof resolveSemanticGraph>,
options: CaptureVomObservationOptions,
): Promise<HoverProbeOutcome> {
if (!options.conditionalSurfaceProbe) return NO_HOVER_PROBES;
return withOverlayBypass(options.hoverProbeBypassOverlay, tabId, async () => {
const surfaceProbes = await probeHoverSurfaces(cdp, tabId, captured.nodes, {
signal: options.signal,
});
throwIfAborted(options.signal, "observation");
const tooltipNames = await probeTooltipNames(cdp, tabId, documents, staticSemantics, {
signal: options.signal,
});
return {
performed: true,
revealedContent: surfaceProbes.length > 0 || tooltipNames.size > 0,
surfaceProbes,
tooltipNames,
};
});
}
export interface CaptureVomObservationOptions extends VomOptions {
conditionalSurfaceProbe?: boolean;
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
@@ -1051,19 +1111,26 @@ export async function captureVomObservation(
excludedBackendNodeIds: captured.excludedBackendNodeIds,
});
const staticSemantics = resolveSemanticGraph(semanticGraph, { identifierFallback: false });
const tooltipNames = options.conditionalSurfaceProbe
? await probeTooltipNames(cdp, tabId, normalizedDocuments, staticSemantics, {
signal: options.signal,
hoverProbeBypassOverlay: options.hoverProbeBypassOverlay,
})
: new Map<string, string>();
const hoverProbes = await runHoverProbes(
cdp,
tabId,
captured,
normalizedDocuments,
staticSemantics,
options,
);
throwIfAborted(options.signal, "observation");
const scene = projectSemanticGraph(
normalizeSemanticStructure(
resolveSemanticGraph(semanticGraph, { supplementalNames: tooltipNames }),
resolveSemanticGraph(semanticGraph, { supplementalNames: hoverProbes.tooltipNames }),
),
);
const decoratedScene = attachCapturedSceneAnnotations(scene, normalizedDocuments, captured);
const decoratedScene = attachCapturedSceneAnnotations(
scene,
normalizedDocuments,
captured,
hoverProbes.surfaceProbes,
);
const rendered = renderVom(decoratedScene, {
maxDepth: options.maxDepth,
maxTokens: options.maxTokens,
@@ -1075,7 +1142,11 @@ export async function captureVomObservation(
rootFrameId: captured.rootFrameId ?? normalizedDocuments[0]?.frameId ?? "root",
frameDocuments: normalizedDocuments,
rendered,
surfaceProbes: captured.surfaceProbes,
surfaceProbes: hoverProbes.surfaceProbes,
hoverProbe: {
performed: hoverProbes.performed,
revealedContent: hoverProbes.revealedContent,
},
});
}
@@ -1141,6 +1212,14 @@ async function handleVomObservation(
ref_count: observation.refs.length,
tab_id: target.tabId,
truncated: observation.truncated,
...(toolName === "observe" && observation.hoverProbe?.performed
? {
hover_probe: {
performed: true,
revealed_content: observation.hoverProbe.revealedContent,
},
}
: {}),
...(toolName === "observe" && (params as ObserveParams).debug_surfaces
? {
debug: {
@@ -1179,5 +1258,13 @@ export async function handleObserve(
deps: SnapshotDeps = getDefaultDeps(),
signal?: AbortSignal,
): Promise<ObserveResult | RpcError> {
return handleVomObservation(manager, params, "observe", "transient_input", true, deps, signal);
return handleVomObservation(
manager,
params,
"observe",
"transient_input",
params.probe_hover === true,
deps,
signal,
);
}
apps/extension/src/tools/vom/__tests__/capture.test.ts
+11 -9
View File
@@ -1,6 +1,6 @@
import { describe, expect, it, vi } from "vitest";
import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "../../../lib/overlay-bridge";
import { captureViewModel, collectOverlayExcludedBackendIds } from "../capture";
import { captureViewModel, collectOverlayExcludedBackendIds, probeHoverSurfaces } from "../capture";
// Minimal but format-accurate captureSnapshot reply: a body with one
// fixed full-screen overlay div carrying a password input.
@@ -251,11 +251,10 @@ describe("captureViewModel", () => {
expect(input?.attrs).toEqual({ type: "password" });
});
it("does not run conditional surface probes by default", async () => {
it("never hovers the page while capturing", async () => {
const cdp = makeCdp(fakeSnapshotReply());
const result = await captureViewModel(cdp, 4);
await captureViewModel(cdp, 4);
expect(result.surfaceProbes).toEqual([]);
expect(cdp.send).not.toHaveBeenCalledWith(4, "Input.dispatchMouseEvent", expect.anything());
});
@@ -319,9 +318,10 @@ describe("captureViewModel", () => {
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
const result = await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
const captured = await captureViewModel(cdp, 4);
const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes);
expect(result.surfaceProbes).toEqual([
expect(surfaceProbes).toEqual([
{
triggerBackendNodeId: 12,
triggerPoint: { x: 956, y: 32 },
@@ -378,9 +378,10 @@ describe("captureViewModel", () => {
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
const result = await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
const captured = await captureViewModel(cdp, 4);
const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes);
expect(result.surfaceProbes).toEqual([
expect(surfaceProbes).toEqual([
expect.objectContaining({
triggerBackendNodeId: 12,
subItems: ["My profile", "Sign out"],
@@ -422,7 +423,8 @@ describe("captureViewModel", () => {
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
const captured = await captureViewModel(cdp, 4);
await probeHoverSurfaces(cdp, 4, captured.nodes);
expect(hoverMoves).toBe(1);
});
apps/extension/src/tools/vom/__tests__/hover-perception.test.ts
+84
View File
@@ -0,0 +1,84 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { ProbeBudget, resetOverlayBypassDepth, withOverlayBypass } from "../hover-perception";
afterEach(() => {
resetOverlayBypassDepth();
});
describe("ProbeBudget", () => {
it("refuses a candidate that cannot finish inside the budget", () => {
const budget = new ProbeBudget(500);
expect(budget.canAfford(400)).toBe(true);
expect(budget.canAfford(600)).toBe(false);
});
it("stops admitting candidates once the remaining budget is smaller than one", async () => {
const budget = new ProbeBudget(60);
await new Promise((resolve) => setTimeout(resolve, 40));
// A top-of-loop `elapsed > total` check would still admit this candidate
// and then overshoot by its full cost.
expect(budget.canAfford(50)).toBe(false);
});
});
describe("withOverlayBypass", () => {
it("toggles the overlay once around nested probe phases", async () => {
const bypass = vi.fn(async () => {});
await withOverlayBypass(bypass, 4, async () => {
await withOverlayBypass(bypass, 4, async () => {
expect(bypass).toHaveBeenCalledTimes(1);
});
expect(bypass).toHaveBeenCalledTimes(1);
});
expect(bypass.mock.calls).toEqual([
[4, true],
[4, false],
]);
});
it("restores the overlay when the probe throws", async () => {
const bypass = vi.fn(async () => {});
await expect(
withOverlayBypass(bypass, 4, async () => {
throw new Error("probe exploded");
}),
).rejects.toThrow("probe exploded");
expect(bypass).toHaveBeenLastCalledWith(4, false);
});
it("reports a failed restore instead of swallowing it", async () => {
const onRestoreFailure = vi.fn();
const bypass = vi.fn(async (_tabId: number, enabled: boolean) => {
if (!enabled) throw new Error("tab is gone");
});
await withOverlayBypass(bypass, 4, async () => undefined, { onRestoreFailure });
expect(onRestoreFailure).toHaveBeenCalledTimes(1);
});
it("does not pin the overlay off for the tab when a restore fails", async () => {
const failing = vi.fn(async (_tabId: number, enabled: boolean) => {
if (!enabled) throw new Error("tab is gone");
});
await withOverlayBypass(failing, 4, async () => undefined, { onRestoreFailure: () => {} });
const bypass = vi.fn(async () => {});
await withOverlayBypass(bypass, 4, async () => undefined);
expect(bypass.mock.calls).toEqual([
[4, true],
[4, false],
]);
});
it("runs the probe untouched when no bypass is wired up", async () => {
await expect(withOverlayBypass(undefined, 4, async () => "done")).resolves.toBe("done");
});
});
apps/extension/src/tools/vom/__tests__/name-enrichment.test.ts
+31
View File
@@ -132,6 +132,37 @@ describe("tooltip name enrichment", () => {
expect(cdp.sendToTarget).not.toHaveBeenCalled();
});
it("remembers controls that revealed no tooltip so the miss is paid once", async () => {
const cdp: CdpRunner = {
send: vi.fn(async () => ({})) as CdpRunner["send"],
sendToTarget: vi.fn(async (_target, method) => {
if (method === "Page.createIsolatedWorld") return { executionContextId: 31 };
if (method === "DOM.resolveNode") return { object: { objectId: "button-801" } };
if (method === "Runtime.callFunctionOn") return { result: { value: [] } };
return {};
}) as CdpRunner["sendToTarget"],
};
const documents = [
document({ tabId: 11, sessionId: "miss-session" }, 801, undefined, "miss-frame"),
];
const graph = semantics(documents);
const first = await probeTooltipNames(cdp, 11, documents, graph);
expect(first.size).toBe(0);
const hoversAfterFirst = vi
.mocked(cdp.send)
.mock.calls.filter(([, method]) => method === "Input.dispatchMouseEvent").length;
expect(hoversAfterFirst).toBeGreaterThan(0);
const second = await probeTooltipNames(cdp, 11, documents, graph);
expect(second.size).toBe(0);
expect(
vi.mocked(cdp.send).mock.calls.filter(([, method]) => method === "Input.dispatchMouseEvent")
.length,
).toBe(hoversAfterFirst);
});
it("keeps tooltip evidence isolated across sibling frame sessions", async () => {
let hoveredX = -10;
const cdp: CdpRunner = {
apps/extension/src/tools/vom/capture.ts
+80 -65
View File
@@ -9,7 +9,7 @@ import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
import { childFrameProjection, type GeometryProjection, projectRectToViewport } from "../geometry";
import type { CdpRunner } from "../shared";
import { clearHover, waitForHover } from "./hover-perception";
import { clearHover, ProbeBudget, waitForHover } from "./hover-perception";
const REQUESTED_STYLES = ["position", "pointer-events", "cursor", "visibility", "opacity"] as const;
const STYLE_COL = Object.fromEntries(
@@ -71,14 +71,11 @@ export interface CapturedViewModel {
/** Explicit DOMSnapshot frame ancestry; never inferred from backend node ids. */
frameParentIds?: Map<string, string>;
rootFrameId?: string;
surfaceProbes?: CapturedSurfaceProbe[];
/** Backend node ids belonging to the agent overlay host + its shadow subtree. */
excludedBackendNodeIds: Set<number>;
}
export interface CaptureViewModelOptions {
conditionalSurfaceProbe?: boolean;
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
signal?: AbortSignal;
}
@@ -368,7 +365,16 @@ function collectBackendIdsFromDomNode(node: CdpDomNode | undefined, out: Set<num
}
}
const MAX_HOVER_PROBE_MS = 2_000;
/**
* Ceiling for the whole hover-surface phase.
*
* The previous 2000 was only a floor: the loop checked elapsed time at the top,
* so a candidate could start with 1ms left and still run its two settle
* windows, pushing real cost to ~2.6s. This is that true ceiling, now actually
* enforced by an up-front affordability check, so the same number of candidates
* get probed under a bound that no longer lies.
*/
const MAX_HOVER_PROBE_MS = 2_600;
const MAX_HOVER_TRIGGERS = 6;
const MAX_HOVER_SURFACES = 3;
const HOVER_SETTLE_MS = 300;
@@ -614,13 +620,35 @@ function confidenceForHover(
return "low";
}
async function probeHoverSurfaces(
/**
* One candidate costs two settle windows (baseline + post-hover) plus a few
* CDP round trips. Used to decide whether the next candidate still fits the
* budget before paying for it.
*/
const HOVER_CANDIDATE_COST_MS = HOVER_SETTLE_MS * 2;
export interface HoverSurfaceProbeOptions {
signal?: AbortSignal;
}
/**
* Hovers a bounded set of likely menu triggers and reports the sub-items each
* one reveals.
*
* Runs after DOM *and* accessibility capture. Hovering can open menus and
* change layout, so probing between the two captures would leave the DOM half
* of an observation describing the page before the change and the AX half
* describing it after.
*
* The caller owns the overlay bypass span (see `withOverlayBypass`).
*/
export async function probeHoverSurfaces(
cdp: CdpRunner,
tabId: number,
nodes: CapturedNode[],
options: CaptureViewModelOptions,
options: HoverSurfaceProbeOptions = {},
): Promise<CapturedSurfaceProbe[]> {
const started = Date.now();
const budget = new ProbeBudget(MAX_HOVER_PROBE_MS);
try {
throwIfAborted(options.signal);
const cssScan = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
@@ -635,60 +663,54 @@ async function probeHoverSurfaces(
const results: CapturedSurfaceProbe[] = [];
const seen = new Set<number>();
throwIfAborted(options.signal);
await options.hoverProbeBypassOverlay?.(tabId, true).catch(() => undefined);
try {
for (const candidate of candidates.slice(0, MAX_HOVER_TRIGGERS)) {
throwIfAborted(options.signal);
for (const candidate of candidates.slice(0, MAX_HOVER_TRIGGERS)) {
if (!budget.canAfford(HOVER_CANDIDATE_COST_MS)) break;
if (results.length >= MAX_HOVER_SURFACES) break;
if (seen.has(candidate.backendNodeId)) continue;
try {
await clearHover(cdp, tabId);
throwIfAborted(options.signal);
if (Date.now() - started > MAX_HOVER_PROBE_MS) break;
if (results.length >= MAX_HOVER_SURFACES) break;
if (seen.has(candidate.backendNodeId)) continue;
try {
await clearHover(cdp, tabId);
throwIfAborted(options.signal);
await waitForHover(HOVER_SETTLE_MS, options.signal);
const baselineReply = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
expression: hoverStateExpression(),
returnByValue: true,
});
throwIfAborted(options.signal);
const baselineItems = runtimeValue<HoverRuntimeItem[]>(baselineReply) ?? [];
await waitForHover(HOVER_SETTLE_MS, options.signal);
const baselineReply = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
expression: hoverStateExpression(),
returnByValue: true,
});
throwIfAborted(options.signal);
const baselineItems = runtimeValue<HoverRuntimeItem[]>(baselineReply) ?? [];
throwIfAborted(options.signal);
await cdp.send(tabId, "Input.dispatchMouseEvent", {
type: "mouseMoved",
x: candidate.x,
y: candidate.y,
});
throwIfAborted(options.signal);
await waitForHover(HOVER_SETTLE_MS, options.signal);
const collected = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
expression: hoverStateExpression(),
returnByValue: true,
});
throwIfAborted(options.signal);
const subItems = diffHoverItems(
baselineItems,
runtimeValue<HoverRuntimeItem[]>(collected) ?? [],
);
if (subItems.length === 0) continue;
seen.add(candidate.backendNodeId);
results.push({
triggerBackendNodeId: candidate.backendNodeId,
triggerPoint: { x: candidate.x, y: candidate.y },
triggerAction: "hover",
subItems,
confidence: confidenceForHover(candidate, subItems),
});
} catch (error) {
if (isAbortError(error)) throw error;
continue;
} finally {
await clearHover(cdp, tabId);
}
throwIfAborted(options.signal);
await cdp.send(tabId, "Input.dispatchMouseEvent", {
type: "mouseMoved",
x: candidate.x,
y: candidate.y,
});
throwIfAborted(options.signal);
await waitForHover(HOVER_SETTLE_MS, options.signal);
const collected = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
expression: hoverStateExpression(),
returnByValue: true,
});
throwIfAborted(options.signal);
const subItems = diffHoverItems(
baselineItems,
runtimeValue<HoverRuntimeItem[]>(collected) ?? [],
);
if (subItems.length === 0) continue;
seen.add(candidate.backendNodeId);
results.push({
triggerBackendNodeId: candidate.backendNodeId,
triggerPoint: { x: candidate.x, y: candidate.y },
triggerAction: "hover",
subItems,
confidence: confidenceForHover(candidate, subItems),
});
} catch (error) {
if (isAbortError(error)) throw error;
continue;
} finally {
await clearHover(cdp, tabId);
}
} finally {
await options.hoverProbeBypassOverlay?.(tabId, false).catch(() => undefined);
}
return results;
} catch (err) {
@@ -1042,7 +1064,6 @@ export async function captureViewModel(
nodes: [],
viewport,
iframeNodes: new Map(),
surfaceProbes: [],
excludedBackendNodeIds: new Set(),
};
}
@@ -1071,11 +1092,6 @@ export async function captureViewModel(
await enrichFormControlStates(cdp, tabId, [nodes, ...iframeNodes.values()], options.signal);
throwIfAborted(options.signal);
const surfaceProbes = options.conditionalSurfaceProbe
? await probeHoverSurfaces(cdp, tabId, nodes, options)
: [];
throwIfAborted(options.signal);
const frameNodes = new Map<string, CapturedNode[]>();
if (topContext.frameId) frameNodes.set(topContext.frameId, nodes);
for (const iframeFrameNodes of iframeNodes.values()) {
@@ -1096,7 +1112,6 @@ export async function captureViewModel(
frameOwnerBackendNodeIds: frameParsed.frameOwnerBackendNodeIds,
frameParentIds: frameParsed.frameParentIds,
...(topContext.frameId ? { rootFrameId: topContext.frameId } : {}),
surfaceProbes,
excludedBackendNodeIds,
};
}
apps/extension/src/tools/vom/hover-perception.ts
+88
View File
@@ -35,3 +35,91 @@ export async function clearHover(cdp: CdpRunner, tabId: number): Promise<void> {
}),
);
}
/**
* Wall-clock budget for a probe phase.
*
* Callers ask whether the *next* candidate still fits before starting it,
* rather than checking elapsed time at the top of the loop. A top-of-loop
* check lets a candidate start with 1ms of budget left and then run to
* completion, overshooting by a full settle window.
*/
export class ProbeBudget {
private readonly startedAt = Date.now();
constructor(private readonly totalMs: number) {}
get elapsedMs(): number {
return Date.now() - this.startedAt;
}
/** True when a candidate costing `estimatedMs` still fits in the budget. */
canAfford(estimatedMs: number): boolean {
return this.elapsedMs + estimatedMs <= this.totalMs;
}
}
export type OverlayBypass = (tabId: number, enabled: boolean) => Promise<void>;
export interface OverlayBypassScopeOptions {
/**
* Called when the overlay could not be restored. Defaults to `console.error`
* so a stuck-bypassed overlay is never silent; tests inject a spy.
*/
onRestoreFailure?: (error: unknown) => void;
}
const overlayBypassDepth = new Map<number, number>();
/**
* Runs `body` with the agent overlay bypassed, restoring it afterwards.
*
* Reference counted per tab so nested probe phases share a single bypass span
* instead of toggling the overlay once per phase. Restore failures are
* reported rather than swallowed, and the depth is released even when restore
* throws so a failure cannot pin the overlay off for the tab's lifetime.
*/
export async function withOverlayBypass<T>(
bypass: OverlayBypass | undefined,
tabId: number,
body: () => Promise<T>,
options: OverlayBypassScopeOptions = {},
): Promise<T> {
if (!bypass) return body();
const depth = overlayBypassDepth.get(tabId) ?? 0;
overlayBypassDepth.set(tabId, depth + 1);
if (depth === 0) {
try {
await bypass(tabId, true);
} catch (error) {
// Failing to hide the overlay only risks probing our own UI, so continue.
console.debug("[bsk hover] overlay bypass could not be enabled", error);
}
}
try {
return await body();
} finally {
const current = overlayBypassDepth.get(tabId) ?? 1;
if (current <= 1) {
overlayBypassDepth.delete(tabId);
try {
await bypass(tabId, false);
} catch (error) {
(options.onRestoreFailure ?? defaultRestoreFailure)(error);
}
} else {
overlayBypassDepth.set(tabId, current - 1);
}
}
}
function defaultRestoreFailure(error: unknown): void {
console.error("[bsk hover] agent overlay could not be restored after probing", error);
}
/** Test seam: drops any bypass depth left behind by a crashed probe. */
export function resetOverlayBypassDepth(): void {
overlayBypassDepth.clear();
}
apps/extension/src/tools/vom/name-enrichment.ts
+26 -15
View File
@@ -1,7 +1,7 @@
import { isVomReferenceNode } from "@browser-skill/vom";
import { type CdpRunner, sendToCdpTarget } from "../shared";
import type { FrameDocument, FrameOwnedAxNode } from "./frame-document";
import { clearHover, waitForHover } from "./hover-perception";
import { clearHover, ProbeBudget, waitForHover } from "./hover-perception";
import { stableIdentifierName } from "./semantic-graph/name-evidence";
import { frameBackendKey, type ResolvedSemanticGraph } from "./semantic-graph/types";
@@ -9,11 +9,18 @@ const MAX_TOOLTIP_CANDIDATES = 8;
const MAX_TOOLTIP_PROBE_MS = 2_400;
const TOOLTIP_SETTLE_MS = 250;
const CACHE_TTL_MS = 5 * 60_000;
/**
* Misses expire sooner than hits: a control that gains a tooltip once the app
* finishes booting should still be discoverable, but not at the cost of
* re-probing every unlabelled control on every observation.
*/
const MISS_CACHE_TTL_MS = 60_000;
const MAX_CACHE_ENTRIES = 500;
const FORM_CONTROL_TAGS = new Set(["input", "select", "textarea"]);
interface TooltipCacheEntry {
name: string;
/** `null` records "probed, revealed no tooltip" so the miss is not re-paid. */
name: string | null;
expiresAt: number;
}
@@ -27,7 +34,6 @@ interface TooltipCandidate<T extends FrameOwnedAxNode> {
export interface TooltipNameProbeOptions {
signal?: AbortSignal;
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
}
const tooltipCache = new Map<string, TooltipCacheEntry>();
@@ -39,10 +45,11 @@ function cacheKey<T extends FrameOwnedAxNode>(
return `${document.target.tabId}\u0000${document.frameId}\u0000${document.url ?? ""}\u0000${backendNodeId}`;
}
function cachedName<T extends FrameOwnedAxNode>(
/** Returns the cache entry, or `undefined` when this control was never probed. */
function cachedEntry<T extends FrameOwnedAxNode>(
document: FrameDocument<T>,
backendNodeId: number,
): string | undefined {
): TooltipCacheEntry | undefined {
const key = cacheKey(document, backendNodeId);
const entry = tooltipCache.get(key);
if (!entry) return undefined;
@@ -50,13 +57,13 @@ function cachedName<T extends FrameOwnedAxNode>(
tooltipCache.delete(key);
return undefined;
}
return entry.name;
return entry;
}
function storeName<T extends FrameOwnedAxNode>(
document: FrameDocument<T>,
backendNodeId: number,
name: string,
name: string | null,
): void {
if (tooltipCache.size >= MAX_CACHE_ENTRIES) {
const oldest = tooltipCache.keys().next().value;
@@ -64,7 +71,7 @@ function storeName<T extends FrameOwnedAxNode>(
}
tooltipCache.set(cacheKey(document, backendNodeId), {
name,
expiresAt: Date.now() + CACHE_TTL_MS,
expiresAt: Date.now() + (name === null ? MISS_CACHE_TTL_MS : CACHE_TTL_MS),
});
}
@@ -237,22 +244,27 @@ export async function probeTooltipNames<T extends FrameOwnedAxNode>(
const names = new Map<string, string>();
const pending: TooltipCandidate<T>[] = [];
for (const candidate of candidates(documents, graph)) {
const cached = cachedName(candidate.document, candidate.backendNodeId);
const cached = cachedEntry(candidate.document, candidate.backendNodeId);
if (cached) {
names.set(frameBackendKey(candidate.document.frameId, candidate.backendNodeId), cached);
// A cached miss is still a decision: skip the control instead of re-probing.
if (cached.name !== null) {
names.set(
frameBackendKey(candidate.document.frameId, candidate.backendNodeId),
cached.name,
);
}
} else if (pending.length < MAX_TOOLTIP_CANDIDATES) {
pending.push(candidate);
}
}
if (pending.length === 0) return names;
const started = Date.now();
const budget = new ProbeBudget(MAX_TOOLTIP_PROBE_MS);
const contexts = new Map<string, number>();
await options.hoverProbeBypassOverlay?.(tabId, true).catch(() => undefined);
try {
for (const candidate of pending) {
if (options.signal?.aborted) throw new DOMException("observation aborted", "AbortError");
if (Date.now() - started >= MAX_TOOLTIP_PROBE_MS) break;
if (!budget.canAfford(TOOLTIP_SETTLE_MS)) break;
let objectId: string | undefined;
try {
let contextId = contexts.get(candidate.document.frameId);
@@ -278,9 +290,9 @@ export async function probeTooltipNames<T extends FrameOwnedAxNode>(
await waitForHover(TOOLTIP_SETTLE_MS, options.signal);
const after = await tooltipTexts(cdp, candidate.document, objectId);
const name = cleanTooltipDiff(before, after);
storeName(candidate.document, candidate.backendNodeId, name ?? null);
if (!name) continue;
names.set(frameBackendKey(candidate.document.frameId, candidate.backendNodeId), name);
storeName(candidate.document, candidate.backendNodeId, name);
} catch (error) {
if (error instanceof Error && error.name === "AbortError") throw error;
} finally {
@@ -290,7 +302,6 @@ export async function probeTooltipNames<T extends FrameOwnedAxNode>(
}
} finally {
await clearHover(cdp, tabId);
await options.hoverProbeBypassOverlay?.(tabId, false).catch(() => undefined);
}
return names;
}
apps/extension/src/tools/vom/record-safe-observation.ts
+13
View File
@@ -37,6 +37,17 @@ export interface CaptureVomObservationResult {
frames: CaptureVomFrame[];
matchNodes: CaptureVomMatchNode[];
surfaceProbes?: CapturedSurfaceProbe[];
/**
* Whether this observation actively hovered the page, and whether that
* revealed content the static tree does not contain. Lets callers judge how
* far the page may have drifted from the returned snapshot.
*/
hoverProbe?: HoverProbeReport;
}
export interface HoverProbeReport {
performed: boolean;
revealedContent: boolean;
}
function projectFrames(documents: CapturedFrameDocument<FrameAxNode>[]): CaptureVomFrame[] {
@@ -75,6 +86,7 @@ export function projectRecordSafeObservation(input: {
frameDocuments: CapturedFrameDocument<FrameAxNode>[];
rendered: VomResult;
surfaceProbes?: CapturedSurfaceProbe[];
hoverProbe?: HoverProbeReport;
}): CaptureVomObservationResult {
return {
text: input.rendered.text,
@@ -84,5 +96,6 @@ export function projectRecordSafeObservation(input: {
frames: projectFrames(input.frameDocuments),
matchNodes: projectMatchNodes(input.frameDocuments),
surfaceProbes: input.surfaceProbes,
...(input.hoverProbe ? { hoverProbe: input.hoverProbe } : {}),
};
}
apps/extension/src/transport/types.ts
+5
View File
@@ -318,9 +318,14 @@ export interface SnapshotResult {
export interface ObserveParams extends SnapshotParams {
debug_surfaces?: boolean;
probe_hover?: boolean;
}
export interface ObserveResult extends SnapshotResult {
hover_probe?: {
performed: boolean;
revealed_content: boolean;
};
debug?: {
surface_probes?: Array<{
trigger_backend_node_id: number;