figmar/BrowserSkill
1
0
Fork 0
mirror of https://github.com/Tencent/BrowserSkill.git synced 2026-09-10 13:01:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

merge: resolve SKILL.md conflicts with main

Browse Source 
Keep the expanded command-reference tables from feat/upload-download
(including the new File Transfer section) over main's compact two-line
format.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
drakezhang
2026-09-02 16:42:15 +08:00
co-authored by Cursor
parent 1bd13d5c3c 66ae555e40
commit d70df753f6
32 changed files with 1524 additions and 831 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/extension/src/content/__tests__/BorrowConfirmationOverlay.test.tsx
+4 -4
View File
@@ -23,7 +23,7 @@ describe("BorrowConfirmationOverlay", () => {
id: "req-1",
isActiveTab: true,
tabTitle: "Example",
timeoutMs: 5000,
timeoutMs: 60_000,
onAllow,
onDeny,
},
@@ -32,7 +32,7 @@ describe("BorrowConfirmationOverlay", () => {
);
expect(screen.getByText("允许借用标签页？")).toBeTruthy();
for (let i = 0; i < 5; i++) {
for (let i = 0; i < 60; i++) {
await act(async () => {
await vi.advanceTimersByTimeAsync(1000);
});
@@ -66,7 +66,7 @@ describe("BorrowConfirmationOverlay", () => {
id: "req-fallback",
isActiveTab: true,
tabTitle: "Example",
timeoutMs: 5000,
timeoutMs: 60_000,
onAllow,
onDeny,
},
@@ -74,7 +74,7 @@ describe("BorrowConfirmationOverlay", () => {
/>,
);
for (let i = 0; i < 5; i++) {
for (let i = 0; i < 60; i++) {
await act(async () => {
await vi.advanceTimersByTimeAsync(1000);
});
apps/extension/src/tools/__tests__/borrow-confirmation.test.ts
+32
View File
@@ -127,6 +127,38 @@ describe("requestBorrowConfirmation", () => {
expect(message.requestId.length).toBeGreaterThan(0);
});
it("proactively focuses the user window before sending the overlay message", async () => {
tabs.get.mockResolvedValueOnce({ id: 42, title: "Tab" });
windows.getLastFocused.mockResolvedValueOnce(
userWindowWithActiveTab({ windowId: 77, tabId: 7, url: "https://app.example/" }),
);
tabs.sendMessage.mockResolvedValueOnce({ type: "borrow-response", allowed: true });
const pending = requestBorrowConfirmation(42, {
deps: { tabs, windows, notifications },
});
await vi.runAllTimersAsync();
await pending;
expect(windows.update).toHaveBeenCalledWith(77, { focused: true });
});
it("still proceeds when proactive window focus fails", async () => {
tabs.get.mockResolvedValueOnce({ id: 42, title: "Tab" });
windows.getLastFocused.mockResolvedValueOnce(
userWindowWithActiveTab({ windowId: 77, tabId: 7, url: "https://app.example/" }),
);
windows.update.mockRejectedValueOnce(new Error("window gone"));
tabs.sendMessage.mockResolvedValueOnce({ type: "borrow-response", allowed: true });
const pending = requestBorrowConfirmation(42, {
deps: { tabs, windows, notifications },
});
await vi.runAllTimersAsync();
expect(await pending).toBe(true);
});
it("sends confirmation to the borrow target when it is the active tab", async () => {
tabs.get.mockResolvedValueOnce({ id: 42, title: "Active Tab" });
windows.getLastFocused.mockResolvedValueOnce(
apps/extension/src/tools/__tests__/observation.test.ts
+356 -216
View File
@@ -1904,36 +1904,41 @@ describe("buildVomScene", () => {
backendDOMNodeId: 20,
},
];
const scene = buildVomScene(axNodes, {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
surfaceProbes: [
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
],
nodes: [
{
backendNodeId: 10,
parentBackendNodeId: null,
tag: "body",
attrs: {},
rect: { x: 0, y: 0, w: 1000, h: 800 },
paintOrder: 0,
position: "static",
pointerEvents: "auto",
},
{
backendNodeId: 20,
parentBackendNodeId: 10,
tag: "button",
attrs: {},
rect: { x: 20, y: 20, w: 120, h: 40 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
},
],
});
const scene = buildVomScene(
axNodes,
{
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
backendNodeId: 10,
parentBackendNodeId: null,
tag: "body",
attrs: {},
rect: { x: 0, y: 0, w: 1000, h: 800 },
paintOrder: 0,
position: "static",
pointerEvents: "auto",
},
{
backendNodeId: 20,
parentBackendNodeId: 10,
tag: "button",
attrs: {},
rect: { x: 20, y: 20, w: 120, h: 40 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
},
],
},
{
surfaceProbes: [
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
],
},
);
expect(scene.surfaces).toEqual([
{ triggerId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
@@ -1957,46 +1962,51 @@ describe("buildVomScene", () => {
backendDOMNodeId: 21,
},
];
const scene = buildVomScene(axNodes, {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
surfaceProbes: [
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
],
nodes: [
{
backendNodeId: 10,
parentBackendNodeId: null,
tag: "body",
attrs: {},
rect: { x: 0, y: 0, w: 1000, h: 800 },
paintOrder: 0,
position: "static",
pointerEvents: "auto",
},
{
backendNodeId: 20,
parentBackendNodeId: 10,
tag: "div",
attrs: { class: "tg-avatar" },
rect: { x: 900, y: 10, w: 30, h: 30 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
},
{
backendNodeId: 21,
parentBackendNodeId: 20,
tag: "div",
attrs: { class: "tg-avatar__inner" },
rect: { x: 902, y: 12, w: 26, h: 26 },
paintOrder: 2,
position: "static",
pointerEvents: "auto",
},
],
});
const scene = buildVomScene(
axNodes,
{
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
backendNodeId: 10,
parentBackendNodeId: null,
tag: "body",
attrs: {},
rect: { x: 0, y: 0, w: 1000, h: 800 },
paintOrder: 0,
position: "static",
pointerEvents: "auto",
},
{
backendNodeId: 20,
parentBackendNodeId: 10,
tag: "div",
attrs: { class: "tg-avatar" },
rect: { x: 900, y: 10, w: 30, h: 30 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
},
{
backendNodeId: 21,
parentBackendNodeId: 20,
tag: "div",
attrs: { class: "tg-avatar__inner" },
rect: { x: 902, y: 12, w: 26, h: 26 },
paintOrder: 2,
position: "static",
pointerEvents: "auto",
},
],
},
{
surfaceProbes: [
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
],
},
);
expect(renderVom(scene).text).toContain('@e1 button "image" [hover first: My profile]');
});
@@ -2017,48 +2027,53 @@ describe("buildVomScene", () => {
backendDOMNodeId: 21,
},
];
const scene = buildVomScene(axNodes, {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
surfaceProbes: [
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Sign out"] },
{ triggerBackendNodeId: 21, triggerAction: "hover", subItems: ["Settings"] },
],
nodes: [
{
backendNodeId: 10,
parentBackendNodeId: null,
tag: "body",
attrs: {},
rect: { x: 0, y: 0, w: 1000, h: 800 },
paintOrder: 0,
position: "static",
pointerEvents: "auto",
},
{
backendNodeId: 20,
parentBackendNodeId: 10,
tag: "div",
attrs: { class: "tg-avatar" },
rect: { x: 900, y: 10, w: 30, h: 30 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
},
{
backendNodeId: 21,
parentBackendNodeId: 20,
tag: "div",
attrs: { class: "tg-avatar__inner" },
rect: { x: 902, y: 12, w: 26, h: 26 },
paintOrder: 2,
position: "static",
pointerEvents: "auto",
},
],
});
const scene = buildVomScene(
axNodes,
{
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
backendNodeId: 10,
parentBackendNodeId: null,
tag: "body",
attrs: {},
rect: { x: 0, y: 0, w: 1000, h: 800 },
paintOrder: 0,
position: "static",
pointerEvents: "auto",
},
{
backendNodeId: 20,
parentBackendNodeId: 10,
tag: "div",
attrs: { class: "tg-avatar" },
rect: { x: 900, y: 10, w: 30, h: 30 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
},
{
backendNodeId: 21,
parentBackendNodeId: 20,
tag: "div",
attrs: { class: "tg-avatar__inner" },
rect: { x: 902, y: 12, w: 26, h: 26 },
paintOrder: 2,
position: "static",
pointerEvents: "auto",
},
],
},
{
surfaceProbes: [
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Sign out"] },
{ triggerBackendNodeId: 21, triggerAction: "hover", subItems: ["Settings"] },
],
},
);
expect(scene.surfaces).toEqual([
{ triggerId: 21, triggerAction: "hover", subItems: ["My profile"] },
@@ -2081,54 +2096,59 @@ describe("buildVomScene", () => {
backendDOMNodeId: 21,
},
];
const scene = buildVomScene(axNodes, {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
surfaceProbes: [
{
triggerBackendNodeId: 20,
triggerPoint: { x: 915, y: 25 },
triggerAction: "hover",
subItems: ["My profile", "Sign out"],
},
],
nodes: [
{
backendNodeId: 10,
parentBackendNodeId: null,
tag: "body",
attrs: {},
rect: { x: 0, y: 0, w: 1000, h: 800 },
paintOrder: 0,
position: "static",
pointerEvents: "auto",
cursor: "auto",
},
{
backendNodeId: 20,
parentBackendNodeId: 10,
tag: "div",
attrs: { class: "tg-avatar" },
rect: { x: 900, y: 10, w: 30, h: 30 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
cursor: "pointer",
},
{
backendNodeId: 21,
parentBackendNodeId: 20,
tag: "div",
attrs: { class: "tg-avatar__inner" },
rect: { x: 902, y: 12, w: 26, h: 26 },
paintOrder: 2,
position: "static",
pointerEvents: "auto",
cursor: "pointer",
},
],
});
const scene = buildVomScene(
axNodes,
{
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
backendNodeId: 10,
parentBackendNodeId: null,
tag: "body",
attrs: {},
rect: { x: 0, y: 0, w: 1000, h: 800 },
paintOrder: 0,
position: "static",
pointerEvents: "auto",
cursor: "auto",
},
{
backendNodeId: 20,
parentBackendNodeId: 10,
tag: "div",
attrs: { class: "tg-avatar" },
rect: { x: 900, y: 10, w: 30, h: 30 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
cursor: "pointer",
},
{
backendNodeId: 21,
parentBackendNodeId: 20,
tag: "div",
attrs: { class: "tg-avatar__inner" },
rect: { x: 902, y: 12, w: 26, h: 26 },
paintOrder: 2,
position: "static",
pointerEvents: "auto",
cursor: "pointer",
},
],
},
{
surfaceProbes: [
{
triggerBackendNodeId: 20,
triggerPoint: { x: 915, y: 25 },
triggerAction: "hover",
subItems: ["My profile", "Sign out"],
},
],
},
);
expect(scene.surfaces).toEqual([
{ triggerId: 20, triggerAction: "hover", subItems: ["My profile", "Sign out"] },
@@ -2154,41 +2174,46 @@ describe("buildVomScene", () => {
backendDOMNodeId: 42,
},
];
const scene = buildVomScene(axNodes, {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
surfaceProbes: [
{
triggerBackendNodeId: 999,
triggerPoint: { x: 900, y: 20 },
triggerAction: "hover",
subItems: ["My profile"],
},
],
nodes: [
{
backendNodeId: 10,
parentBackendNodeId: null,
tag: "body",
attrs: {},
rect: { x: 0, y: 0, w: 1000, h: 800 },
paintOrder: 0,
position: "static",
pointerEvents: "auto",
},
{
backendNodeId: 42,
parentBackendNodeId: 10,
tag: "a",
attrs: {},
rect: { x: 860, y: 120, w: 40, h: 30 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
},
],
});
const scene = buildVomScene(
axNodes,
{
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
backendNodeId: 10,
parentBackendNodeId: null,
tag: "body",
attrs: {},
rect: { x: 0, y: 0, w: 1000, h: 800 },
paintOrder: 0,
position: "static",
pointerEvents: "auto",
},
{
backendNodeId: 42,
parentBackendNodeId: 10,
tag: "a",
attrs: {},
rect: { x: 860, y: 120, w: 40, h: 30 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
},
],
},
{
surfaceProbes: [
{
triggerBackendNodeId: 999,
triggerPoint: { x: 900, y: 20 },
triggerAction: "hover",
subItems: ["My profile"],
},
],
},
);
expect(scene.surfaces).toBeUndefined();
expect(renderVom(scene).text).not.toContain("[hover first:");
@@ -2606,7 +2631,106 @@ describe("handleSnapshot", () => {
);
});
it("enables conditional surface probing for semantic observe", async () => {
it("hovers the page only after the accessibility tree has been captured", async () => {
// Hovering can open menus and reflow the page. Probing between the DOM and
// AX captures would leave the two halves of one observation describing the
// page on either side of that change.
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
await sm.start("aa11");
const root: CdpAxNode = {
nodeId: "1",
role: { type: "role", value: "RootWebArea" },
name: { type: "computedString", value: "Example" },
backendDOMNodeId: 100,
childIds: ["2"],
};
const button: CdpAxNode = {
nodeId: "2",
parentId: "1",
role: { type: "role", value: "button" },
name: { type: "computedString", value: "Products" },
backendDOMNodeId: 200,
};
const strings = ["body", "button", "position", "static", "pointer-events", "auto", "cursor"];
const i = (s: string) => strings.indexOf(s);
const methodOrder: string[] = [];
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
methodOrder.push(method);
if (method === "Accessibility.enable") return {};
if (method === "Accessibility.getFullAXTree") return { nodes: [root, button] };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
}
if (method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") {
return {
strings,
documents: [
{
nodes: {
parentIndex: [-1, 0],
nodeName: [i("body"), i("button")],
backendNodeId: [100, 200],
attributes: [[], []],
},
layout: {
nodeIndex: [0, 1],
styles: [
[i("static"), i("auto"), i("auto")],
[i("static"), i("auto"), i("auto")],
],
bounds: [
[0, 0, 1000, 800],
[20, 20, 120, 40],
],
paintOrders: [0, 1],
},
},
],
};
}
if (method === "Input.dispatchMouseEvent") return {};
if (method === "Runtime.evaluate") {
const expression = (params as { expression?: string } | undefined)?.expression ?? "";
// Report the button as a `:hover` rule target so a real hover fires.
if (expression.includes("document.styleSheets")) {
return { result: { value: [{ x: 80, y: 40 }] } };
}
return { result: { value: [] } };
}
throw new Error(`unexpected CDP method ${method}`);
});
const res = await handleObserve(
sm,
{ session_id: "aa11", probe_hover: true },
{
cdp: {
send: send as unknown as <T = unknown>(
tabId: number,
method: string,
params?: object,
) => Promise<T>,
trackSessionTab: vi.fn(),
},
tabsApi: {
get: vi.fn(
async (tabId: number) =>
({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
),
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
},
},
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
const firstHover = methodOrder.indexOf("Input.dispatchMouseEvent");
expect(firstHover).toBeGreaterThan(-1);
expect(methodOrder.lastIndexOf("DOMSnapshot.captureSnapshot")).toBeLessThan(firstHover);
expect(methodOrder.lastIndexOf("Accessibility.getFullAXTree")).toBeLessThan(firstHover);
});
it("runs conditional surface probing only when observe opts in", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
await sm.start("aa11");
const root: CdpAxNode = {
@@ -2662,30 +2786,46 @@ describe("handleSnapshot", () => {
if (method === "Runtime.evaluate") return { result: { value: [] } };
throw new Error(`unexpected CDP method ${method}`);
});
const res = await handleObserve(
const deps = {
cdp: {
send: send as unknown as <T = unknown>(
tabId: number,
method: string,
params?: object,
) => Promise<T>,
trackSessionTab: vi.fn(),
},
tabsApi: {
get: vi.fn(
async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
),
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
},
};
const withoutProbe = await handleObserve(
sm,
{ session_id: "aa11", debug_surfaces: true },
{
cdp: {
send: send as unknown as <T = unknown>(
tabId: number,
method: string,
params?: object,
) => Promise<T>,
trackSessionTab: vi.fn(),
},
tabsApi: {
get: vi.fn(
async (tabId: number) =>
({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
),
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
},
},
deps,
);
if ("code" in withoutProbe) {
throw new Error(`unexpected error: ${JSON.stringify(withoutProbe)}`);
}
expect(withoutProbe.hover_probe).toBeUndefined();
expect(send).not.toHaveBeenCalledWith(
4,
"Runtime.evaluate",
expect.objectContaining({ returnByValue: true }),
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(res.debug).toEqual({ surface_probes: [] });
const withProbe = await handleObserve(
sm,
{ session_id: "aa11", debug_surfaces: true, probe_hover: true },
deps,
);
if ("code" in withProbe) throw new Error(`unexpected error: ${JSON.stringify(withProbe)}`);
expect(withProbe.debug).toEqual({ surface_probes: [] });
expect(withProbe.hover_probe).toEqual({ performed: true, revealed_content: false });
expect(send).toHaveBeenCalledWith(
4,
"Runtime.evaluate",
apps/extension/src/tools/borrow-confirmation.ts
+8 -1
View File
@@ -24,7 +24,7 @@ export interface BorrowResponseMessage {
allowed: boolean;
}
export const CONFIRMATION_TIMEOUT_MS = 5000;
export const CONFIRMATION_TIMEOUT_MS = 60_000;
const EXIT_ANIMATION_MS = 150;
/** Matches BorrowConfirmationOverlay progress ring/bar transition (duration-1000). */
export const PROGRESS_TRANSITION_MS = 1000;
@@ -480,6 +480,13 @@ export async function requestBorrowConfirmation(
settle(false);
}, BACKGROUND_TIMEOUT_MS);
// Bring the user window to the front so the overlay is visible even
// when the Agent Window has stolen focus. This is fire-and-forget: if
// it fails the overlay / OS notification still give the user a path.
void windowsApi.update(notificationAnchor.windowId, { focused: true }).catch((err) => {
console.debug("[bsk borrow] proactive focus of user window failed", err);
});
// Surface the OS notification *before* messaging any candidate so the
// user has a parallel signal even if every content script is missing.
// The Allow / Deny buttons let the user authorize explicitly when the
apps/extension/src/tools/dispatcher.ts
+5 -1
View File
@@ -387,7 +387,11 @@ export class ToolDispatcher {
? {
cdp: this.cdp,
tabsApi: chromeTabsCaptureApi,
conditionalSurfaceProbe: !this.hasHoverLatchForScope(hoverScope),
// Active hover probing is opt-in. A held hover latch still
// suppresses it, because probing would move the cursor off
// the element the caller is deliberately holding.
conditionalSurfaceProbe:
params.probe_hover === true && !this.hasHoverLatchForScope(hoverScope),
hoverProbeBypassOverlay: bypassOverlay,
}
: undefined,
apps/extension/src/tools/observation.ts
+106 -19
View File
@@ -49,11 +49,14 @@ import {
import { resolveSnapshotRef } from "./snapshot-ref";
import {
type CapturedNode,
type CapturedSurfaceProbe,
type CapturedViewModel,
captureViewModel,
collectOverlayExcludedBackendIds,
probeHoverSurfaces,
} from "./vom/capture";
import { type CapturedFrameDocument, captureFrameData } from "./vom/frame-capture";
import { withOverlayBypass } from "./vom/hover-perception";
import { probeTooltipNames } from "./vom/name-enrichment";
import {
type CaptureVomObservationResult,
@@ -525,8 +528,11 @@ function buildActiveScopeBlocks(nodes: VomNode[], signals: VomNodeDomSignals): A
return blocks;
}
function buildConditionalSurfaces(nodes: VomNode[], captured: CapturedViewModel): CondSurface[] {
const probes = captured.surfaceProbes ?? [];
function buildConditionalSurfaces(
nodes: VomNode[],
captured: CapturedViewModel,
probes: CapturedSurfaceProbe[],
): CondSurface[] {
if (probes.length === 0) return [];
const surfaces: CondSurface[] = [];
@@ -649,6 +655,7 @@ function findSurfaceNodeByPoint(
export interface BuildVomSceneOptions {
pageUrl?: string;
supplementalNames?: ReadonlyMap<string, string>;
surfaceProbes?: CapturedSurfaceProbe[];
}
export type VomFrameDocument = CapturedFrameDocument<CdpAxNode>;
@@ -817,18 +824,19 @@ export function buildFrameVomScene(
excludedBackendNodeIds: captured.excludedBackendNodeIds,
supplementalNames: options.supplementalNames,
});
return attachCapturedSceneAnnotations(scene, documents, captured);
return attachCapturedSceneAnnotations(scene, documents, captured, options.surfaceProbes ?? []);
}
function attachCapturedSceneAnnotations(
scene: VomScene,
documents: VomFrameDocument[],
captured: CapturedViewModel,
surfaceProbes: CapturedSurfaceProbe[],
): VomScene {
const rootDocument = documents.find((document) => document.frameId === scene.rootFrameId);
const signals = capturedOnlySignals(rootDocument?.domNodes ?? captured.nodes);
const activeScopeBlocks = buildActiveScopeBlocks(scene.nodes, signals);
const surfaces = buildConditionalSurfaces(scene.nodes, captured);
const surfaces = buildConditionalSurfaces(scene.nodes, captured, surfaceProbes);
return {
...scene,
...(surfaces.length > 0 ? { surfaces } : {}),
@@ -1010,17 +1018,69 @@ async function captureForVom(
options: CaptureVomObservationOptions,
): Promise<CapturedViewModel> {
try {
return await captureViewModel(cdp, tabId, {
conditionalSurfaceProbe: options.conditionalSurfaceProbe ?? false,
hoverProbeBypassOverlay: options.hoverProbeBypassOverlay,
signal: options.signal,
});
return await captureViewModel(cdp, tabId, { signal: options.signal });
} catch (error) {
if (isAbortError(error)) throw error;
return fallbackCapturedViewModel(cdp, tabId, options.signal);
}
}
export interface HoverProbeOutcome {
/** Whether any active hover was dispatched during this observation. */
performed: boolean;
/**
* Whether hovering surfaced content absent from the static snapshot. Cached
* tooltip names count, so this may over-report on repeat observations — it
* errs towards telling the caller the snapshot is less fresh than it looks.
*/
revealedContent: boolean;
surfaceProbes: CapturedSurfaceProbe[];
tooltipNames: Map<string, string>;
}
const NO_HOVER_PROBES: HoverProbeOutcome = {
performed: false,
revealedContent: false,
surfaceProbes: [],
tooltipNames: new Map(),
};
/**
* Runs both active-hover chains back to back.
*
* Ordering matters: this happens after DOM *and* accessibility capture so a
* hover that opens a menu cannot leave one half of the observation describing
* the page before the change and the other half after it. Sharing one overlay
* bypass span also means the agent overlay toggles once per observation
* instead of once per chain.
*/
async function runHoverProbes(
cdp: CdpRunner,
tabId: number,
captured: CapturedViewModel,
documents: VomFrameDocument[],
staticSemantics: ReturnType<typeof resolveSemanticGraph>,
options: CaptureVomObservationOptions,
): Promise<HoverProbeOutcome> {
if (!options.conditionalSurfaceProbe) return NO_HOVER_PROBES;
return withOverlayBypass(options.hoverProbeBypassOverlay, tabId, async () => {
const surfaceProbes = await probeHoverSurfaces(cdp, tabId, captured.nodes, {
signal: options.signal,
});
throwIfAborted(options.signal, "observation");
const tooltipNames = await probeTooltipNames(cdp, tabId, documents, staticSemantics, {
signal: options.signal,
});
return {
performed: true,
revealedContent: surfaceProbes.length > 0 || tooltipNames.size > 0,
surfaceProbes,
tooltipNames,
};
});
}
export interface CaptureVomObservationOptions extends VomOptions {
conditionalSurfaceProbe?: boolean;
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
@@ -1051,19 +1111,26 @@ export async function captureVomObservation(
excludedBackendNodeIds: captured.excludedBackendNodeIds,
});
const staticSemantics = resolveSemanticGraph(semanticGraph, { identifierFallback: false });
const tooltipNames = options.conditionalSurfaceProbe
? await probeTooltipNames(cdp, tabId, normalizedDocuments, staticSemantics, {
signal: options.signal,
hoverProbeBypassOverlay: options.hoverProbeBypassOverlay,
})
: new Map<string, string>();
const hoverProbes = await runHoverProbes(
cdp,
tabId,
captured,
normalizedDocuments,
staticSemantics,
options,
);
throwIfAborted(options.signal, "observation");
const scene = projectSemanticGraph(
normalizeSemanticStructure(
resolveSemanticGraph(semanticGraph, { supplementalNames: tooltipNames }),
resolveSemanticGraph(semanticGraph, { supplementalNames: hoverProbes.tooltipNames }),
),
);
const decoratedScene = attachCapturedSceneAnnotations(scene, normalizedDocuments, captured);
const decoratedScene = attachCapturedSceneAnnotations(
scene,
normalizedDocuments,
captured,
hoverProbes.surfaceProbes,
);
const rendered = renderVom(decoratedScene, {
maxDepth: options.maxDepth,
maxTokens: options.maxTokens,
@@ -1075,7 +1142,11 @@ export async function captureVomObservation(
rootFrameId: captured.rootFrameId ?? normalizedDocuments[0]?.frameId ?? "root",
frameDocuments: normalizedDocuments,
rendered,
surfaceProbes: captured.surfaceProbes,
surfaceProbes: hoverProbes.surfaceProbes,
hoverProbe: {
performed: hoverProbes.performed,
revealedContent: hoverProbes.revealedContent,
},
});
}
@@ -1141,6 +1212,14 @@ async function handleVomObservation(
ref_count: observation.refs.length,
tab_id: target.tabId,
truncated: observation.truncated,
...(toolName === "observe" && observation.hoverProbe?.performed
? {
hover_probe: {
performed: true,
revealed_content: observation.hoverProbe.revealedContent,
},
}
: {}),
...(toolName === "observe" && (params as ObserveParams).debug_surfaces
? {
debug: {
@@ -1179,5 +1258,13 @@ export async function handleObserve(
deps: SnapshotDeps = getDefaultDeps(),
signal?: AbortSignal,
): Promise<ObserveResult | RpcError> {
return handleVomObservation(manager, params, "observe", "transient_input", true, deps, signal);
return handleVomObservation(
manager,
params,
"observe",
"transient_input",
params.probe_hover === true,
deps,
signal,
);
}
apps/extension/src/tools/vom/__tests__/capture.test.ts
+11 -9
View File
@@ -1,6 +1,6 @@
import { describe, expect, it, vi } from "vitest";
import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "../../../lib/overlay-bridge";
import { captureViewModel, collectOverlayExcludedBackendIds } from "../capture";
import { captureViewModel, collectOverlayExcludedBackendIds, probeHoverSurfaces } from "../capture";
// Minimal but format-accurate captureSnapshot reply: a body with one
// fixed full-screen overlay div carrying a password input.
@@ -251,11 +251,10 @@ describe("captureViewModel", () => {
expect(input?.attrs).toEqual({ type: "password" });
});
it("does not run conditional surface probes by default", async () => {
it("never hovers the page while capturing", async () => {
const cdp = makeCdp(fakeSnapshotReply());
const result = await captureViewModel(cdp, 4);
await captureViewModel(cdp, 4);
expect(result.surfaceProbes).toEqual([]);
expect(cdp.send).not.toHaveBeenCalledWith(4, "Input.dispatchMouseEvent", expect.anything());
});
@@ -319,9 +318,10 @@ describe("captureViewModel", () => {
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
const result = await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
const captured = await captureViewModel(cdp, 4);
const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes);
expect(result.surfaceProbes).toEqual([
expect(surfaceProbes).toEqual([
{
triggerBackendNodeId: 12,
triggerPoint: { x: 956, y: 32 },
@@ -378,9 +378,10 @@ describe("captureViewModel", () => {
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
const result = await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
const captured = await captureViewModel(cdp, 4);
const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes);
expect(result.surfaceProbes).toEqual([
expect(surfaceProbes).toEqual([
expect.objectContaining({
triggerBackendNodeId: 12,
subItems: ["My profile", "Sign out"],
@@ -422,7 +423,8 @@ describe("captureViewModel", () => {
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
const captured = await captureViewModel(cdp, 4);
await probeHoverSurfaces(cdp, 4, captured.nodes);
expect(hoverMoves).toBe(1);
});
apps/extension/src/tools/vom/__tests__/hover-perception.test.ts
+84
View File
@@ -0,0 +1,84 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { ProbeBudget, resetOverlayBypassDepth, withOverlayBypass } from "../hover-perception";
afterEach(() => {
resetOverlayBypassDepth();
});
describe("ProbeBudget", () => {
it("refuses a candidate that cannot finish inside the budget", () => {
const budget = new ProbeBudget(500);
expect(budget.canAfford(400)).toBe(true);
expect(budget.canAfford(600)).toBe(false);
});
it("stops admitting candidates once the remaining budget is smaller than one", async () => {
const budget = new ProbeBudget(60);
await new Promise((resolve) => setTimeout(resolve, 40));
// A top-of-loop `elapsed > total` check would still admit this candidate
// and then overshoot by its full cost.
expect(budget.canAfford(50)).toBe(false);
});
});
describe("withOverlayBypass", () => {
it("toggles the overlay once around nested probe phases", async () => {
const bypass = vi.fn(async () => {});
await withOverlayBypass(bypass, 4, async () => {
await withOverlayBypass(bypass, 4, async () => {
expect(bypass).toHaveBeenCalledTimes(1);
});
expect(bypass).toHaveBeenCalledTimes(1);
});
expect(bypass.mock.calls).toEqual([
[4, true],
[4, false],
]);
});
it("restores the overlay when the probe throws", async () => {
const bypass = vi.fn(async () => {});
await expect(
withOverlayBypass(bypass, 4, async () => {
throw new Error("probe exploded");
}),
).rejects.toThrow("probe exploded");
expect(bypass).toHaveBeenLastCalledWith(4, false);
});
it("reports a failed restore instead of swallowing it", async () => {
const onRestoreFailure = vi.fn();
const bypass = vi.fn(async (_tabId: number, enabled: boolean) => {
if (!enabled) throw new Error("tab is gone");
});
await withOverlayBypass(bypass, 4, async () => undefined, { onRestoreFailure });
expect(onRestoreFailure).toHaveBeenCalledTimes(1);
});
it("does not pin the overlay off for the tab when a restore fails", async () => {
const failing = vi.fn(async (_tabId: number, enabled: boolean) => {
if (!enabled) throw new Error("tab is gone");
});
await withOverlayBypass(failing, 4, async () => undefined, { onRestoreFailure: () => {} });
const bypass = vi.fn(async () => {});
await withOverlayBypass(bypass, 4, async () => undefined);
expect(bypass.mock.calls).toEqual([
[4, true],
[4, false],
]);
});
it("runs the probe untouched when no bypass is wired up", async () => {
await expect(withOverlayBypass(undefined, 4, async () => "done")).resolves.toBe("done");
});
});
apps/extension/src/tools/vom/__tests__/name-enrichment.test.ts
+31
View File
@@ -132,6 +132,37 @@ describe("tooltip name enrichment", () => {
expect(cdp.sendToTarget).not.toHaveBeenCalled();
});
it("remembers controls that revealed no tooltip so the miss is paid once", async () => {
const cdp: CdpRunner = {
send: vi.fn(async () => ({})) as CdpRunner["send"],
sendToTarget: vi.fn(async (_target, method) => {
if (method === "Page.createIsolatedWorld") return { executionContextId: 31 };
if (method === "DOM.resolveNode") return { object: { objectId: "button-801" } };
if (method === "Runtime.callFunctionOn") return { result: { value: [] } };
return {};
}) as CdpRunner["sendToTarget"],
};
const documents = [
document({ tabId: 11, sessionId: "miss-session" }, 801, undefined, "miss-frame"),
];
const graph = semantics(documents);
const first = await probeTooltipNames(cdp, 11, documents, graph);
expect(first.size).toBe(0);
const hoversAfterFirst = vi
.mocked(cdp.send)
.mock.calls.filter(([, method]) => method === "Input.dispatchMouseEvent").length;
expect(hoversAfterFirst).toBeGreaterThan(0);
const second = await probeTooltipNames(cdp, 11, documents, graph);
expect(second.size).toBe(0);
expect(
vi.mocked(cdp.send).mock.calls.filter(([, method]) => method === "Input.dispatchMouseEvent")
.length,
).toBe(hoversAfterFirst);
});
it("keeps tooltip evidence isolated across sibling frame sessions", async () => {
let hoveredX = -10;
const cdp: CdpRunner = {
apps/extension/src/tools/vom/capture.ts
+80 -65
View File
@@ -9,7 +9,7 @@ import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
import { childFrameProjection, type GeometryProjection, projectRectToViewport } from "../geometry";
import type { CdpRunner } from "../shared";
import { clearHover, waitForHover } from "./hover-perception";
import { clearHover, ProbeBudget, waitForHover } from "./hover-perception";
const REQUESTED_STYLES = ["position", "pointer-events", "cursor", "visibility", "opacity"] as const;
const STYLE_COL = Object.fromEntries(
@@ -71,14 +71,11 @@ export interface CapturedViewModel {
/** Explicit DOMSnapshot frame ancestry; never inferred from backend node ids. */
frameParentIds?: Map<string, string>;
rootFrameId?: string;
surfaceProbes?: CapturedSurfaceProbe[];
/** Backend node ids belonging to the agent overlay host + its shadow subtree. */
excludedBackendNodeIds: Set<number>;
}
export interface CaptureViewModelOptions {
conditionalSurfaceProbe?: boolean;
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
signal?: AbortSignal;
}
@@ -368,7 +365,16 @@ function collectBackendIdsFromDomNode(node: CdpDomNode | undefined, out: Set<num
}
}
const MAX_HOVER_PROBE_MS = 2_000;
/**
* Ceiling for the whole hover-surface phase.
*
* The previous 2000 was only a floor: the loop checked elapsed time at the top,
* so a candidate could start with 1ms left and still run its two settle
* windows, pushing real cost to ~2.6s. This is that true ceiling, now actually
* enforced by an up-front affordability check, so the same number of candidates
* get probed under a bound that no longer lies.
*/
const MAX_HOVER_PROBE_MS = 2_600;
const MAX_HOVER_TRIGGERS = 6;
const MAX_HOVER_SURFACES = 3;
const HOVER_SETTLE_MS = 300;
@@ -614,13 +620,35 @@ function confidenceForHover(
return "low";
}
async function probeHoverSurfaces(
/**
* One candidate costs two settle windows (baseline + post-hover) plus a few
* CDP round trips. Used to decide whether the next candidate still fits the
* budget before paying for it.
*/
const HOVER_CANDIDATE_COST_MS = HOVER_SETTLE_MS * 2;
export interface HoverSurfaceProbeOptions {
signal?: AbortSignal;
}
/**
* Hovers a bounded set of likely menu triggers and reports the sub-items each
* one reveals.
*
* Runs after DOM *and* accessibility capture. Hovering can open menus and
* change layout, so probing between the two captures would leave the DOM half
* of an observation describing the page before the change and the AX half
* describing it after.
*
* The caller owns the overlay bypass span (see `withOverlayBypass`).
*/
export async function probeHoverSurfaces(
cdp: CdpRunner,
tabId: number,
nodes: CapturedNode[],
options: CaptureViewModelOptions,
options: HoverSurfaceProbeOptions = {},
): Promise<CapturedSurfaceProbe[]> {
const started = Date.now();
const budget = new ProbeBudget(MAX_HOVER_PROBE_MS);
try {
throwIfAborted(options.signal);
const cssScan = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
@@ -635,60 +663,54 @@ async function probeHoverSurfaces(
const results: CapturedSurfaceProbe[] = [];
const seen = new Set<number>();
throwIfAborted(options.signal);
await options.hoverProbeBypassOverlay?.(tabId, true).catch(() => undefined);
try {
for (const candidate of candidates.slice(0, MAX_HOVER_TRIGGERS)) {
throwIfAborted(options.signal);
for (const candidate of candidates.slice(0, MAX_HOVER_TRIGGERS)) {
if (!budget.canAfford(HOVER_CANDIDATE_COST_MS)) break;
if (results.length >= MAX_HOVER_SURFACES) break;
if (seen.has(candidate.backendNodeId)) continue;
try {
await clearHover(cdp, tabId);
throwIfAborted(options.signal);
if (Date.now() - started > MAX_HOVER_PROBE_MS) break;
if (results.length >= MAX_HOVER_SURFACES) break;
if (seen.has(candidate.backendNodeId)) continue;
try {
await clearHover(cdp, tabId);
throwIfAborted(options.signal);
await waitForHover(HOVER_SETTLE_MS, options.signal);
const baselineReply = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
expression: hoverStateExpression(),
returnByValue: true,
});
throwIfAborted(options.signal);
const baselineItems = runtimeValue<HoverRuntimeItem[]>(baselineReply) ?? [];
await waitForHover(HOVER_SETTLE_MS, options.signal);
const baselineReply = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
expression: hoverStateExpression(),
returnByValue: true,
});
throwIfAborted(options.signal);
const baselineItems = runtimeValue<HoverRuntimeItem[]>(baselineReply) ?? [];
throwIfAborted(options.signal);
await cdp.send(tabId, "Input.dispatchMouseEvent", {
type: "mouseMoved",
x: candidate.x,
y: candidate.y,
});
throwIfAborted(options.signal);
await waitForHover(HOVER_SETTLE_MS, options.signal);
const collected = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
expression: hoverStateExpression(),
returnByValue: true,
});
throwIfAborted(options.signal);
const subItems = diffHoverItems(
baselineItems,
runtimeValue<HoverRuntimeItem[]>(collected) ?? [],
);
if (subItems.length === 0) continue;
seen.add(candidate.backendNodeId);
results.push({
triggerBackendNodeId: candidate.backendNodeId,
triggerPoint: { x: candidate.x, y: candidate.y },
triggerAction: "hover",
subItems,
confidence: confidenceForHover(candidate, subItems),
});
} catch (error) {
if (isAbortError(error)) throw error;
continue;
} finally {
await clearHover(cdp, tabId);
}
throwIfAborted(options.signal);
await cdp.send(tabId, "Input.dispatchMouseEvent", {
type: "mouseMoved",
x: candidate.x,
y: candidate.y,
});
throwIfAborted(options.signal);
await waitForHover(HOVER_SETTLE_MS, options.signal);
const collected = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
expression: hoverStateExpression(),
returnByValue: true,
});
throwIfAborted(options.signal);
const subItems = diffHoverItems(
baselineItems,
runtimeValue<HoverRuntimeItem[]>(collected) ?? [],
);
if (subItems.length === 0) continue;
seen.add(candidate.backendNodeId);
results.push({
triggerBackendNodeId: candidate.backendNodeId,
triggerPoint: { x: candidate.x, y: candidate.y },
triggerAction: "hover",
subItems,
confidence: confidenceForHover(candidate, subItems),
});
} catch (error) {
if (isAbortError(error)) throw error;
continue;
} finally {
await clearHover(cdp, tabId);
}
} finally {
await options.hoverProbeBypassOverlay?.(tabId, false).catch(() => undefined);
}
return results;
} catch (err) {
@@ -1042,7 +1064,6 @@ export async function captureViewModel(
nodes: [],
viewport,
iframeNodes: new Map(),
surfaceProbes: [],
excludedBackendNodeIds: new Set(),
};
}
@@ -1071,11 +1092,6 @@ export async function captureViewModel(
await enrichFormControlStates(cdp, tabId, [nodes, ...iframeNodes.values()], options.signal);
throwIfAborted(options.signal);
const surfaceProbes = options.conditionalSurfaceProbe
? await probeHoverSurfaces(cdp, tabId, nodes, options)
: [];
throwIfAborted(options.signal);
const frameNodes = new Map<string, CapturedNode[]>();
if (topContext.frameId) frameNodes.set(topContext.frameId, nodes);
for (const iframeFrameNodes of iframeNodes.values()) {
@@ -1096,7 +1112,6 @@ export async function captureViewModel(
frameOwnerBackendNodeIds: frameParsed.frameOwnerBackendNodeIds,
frameParentIds: frameParsed.frameParentIds,
...(topContext.frameId ? { rootFrameId: topContext.frameId } : {}),
surfaceProbes,
excludedBackendNodeIds,
};
}
apps/extension/src/tools/vom/hover-perception.ts
+88
View File
@@ -35,3 +35,91 @@ export async function clearHover(cdp: CdpRunner, tabId: number): Promise<void> {
}),
);
}
/**
* Wall-clock budget for a probe phase.
*
* Callers ask whether the *next* candidate still fits before starting it,
* rather than checking elapsed time at the top of the loop. A top-of-loop
* check lets a candidate start with 1ms of budget left and then run to
* completion, overshooting by a full settle window.
*/
export class ProbeBudget {
private readonly startedAt = Date.now();
constructor(private readonly totalMs: number) {}
get elapsedMs(): number {
return Date.now() - this.startedAt;
}
/** True when a candidate costing `estimatedMs` still fits in the budget. */
canAfford(estimatedMs: number): boolean {
return this.elapsedMs + estimatedMs <= this.totalMs;
}
}
export type OverlayBypass = (tabId: number, enabled: boolean) => Promise<void>;
export interface OverlayBypassScopeOptions {
/**
* Called when the overlay could not be restored. Defaults to `console.error`
* so a stuck-bypassed overlay is never silent; tests inject a spy.
*/
onRestoreFailure?: (error: unknown) => void;
}
const overlayBypassDepth = new Map<number, number>();
/**
* Runs `body` with the agent overlay bypassed, restoring it afterwards.
*
* Reference counted per tab so nested probe phases share a single bypass span
* instead of toggling the overlay once per phase. Restore failures are
* reported rather than swallowed, and the depth is released even when restore
* throws so a failure cannot pin the overlay off for the tab's lifetime.
*/
export async function withOverlayBypass<T>(
bypass: OverlayBypass | undefined,
tabId: number,
body: () => Promise<T>,
options: OverlayBypassScopeOptions = {},
): Promise<T> {
if (!bypass) return body();
const depth = overlayBypassDepth.get(tabId) ?? 0;
overlayBypassDepth.set(tabId, depth + 1);
if (depth === 0) {
try {
await bypass(tabId, true);
} catch (error) {
// Failing to hide the overlay only risks probing our own UI, so continue.
console.debug("[bsk hover] overlay bypass could not be enabled", error);
}
}
try {
return await body();
} finally {
const current = overlayBypassDepth.get(tabId) ?? 1;
if (current <= 1) {
overlayBypassDepth.delete(tabId);
try {
await bypass(tabId, false);
} catch (error) {
(options.onRestoreFailure ?? defaultRestoreFailure)(error);
}
} else {
overlayBypassDepth.set(tabId, current - 1);
}
}
}
function defaultRestoreFailure(error: unknown): void {
console.error("[bsk hover] agent overlay could not be restored after probing", error);
}
/** Test seam: drops any bypass depth left behind by a crashed probe. */
export function resetOverlayBypassDepth(): void {
overlayBypassDepth.clear();
}
apps/extension/src/tools/vom/name-enrichment.ts
+26 -15
View File
@@ -1,7 +1,7 @@
import { isVomReferenceNode } from "@browser-skill/vom";
import { type CdpRunner, sendToCdpTarget } from "../shared";
import type { FrameDocument, FrameOwnedAxNode } from "./frame-document";
import { clearHover, waitForHover } from "./hover-perception";
import { clearHover, ProbeBudget, waitForHover } from "./hover-perception";
import { stableIdentifierName } from "./semantic-graph/name-evidence";
import { frameBackendKey, type ResolvedSemanticGraph } from "./semantic-graph/types";
@@ -9,11 +9,18 @@ const MAX_TOOLTIP_CANDIDATES = 8;
const MAX_TOOLTIP_PROBE_MS = 2_400;
const TOOLTIP_SETTLE_MS = 250;
const CACHE_TTL_MS = 5 * 60_000;
/**
* Misses expire sooner than hits: a control that gains a tooltip once the app
* finishes booting should still be discoverable, but not at the cost of
* re-probing every unlabelled control on every observation.
*/
const MISS_CACHE_TTL_MS = 60_000;
const MAX_CACHE_ENTRIES = 500;
const FORM_CONTROL_TAGS = new Set(["input", "select", "textarea"]);
interface TooltipCacheEntry {
name: string;
/** `null` records "probed, revealed no tooltip" so the miss is not re-paid. */
name: string | null;
expiresAt: number;
}
@@ -27,7 +34,6 @@ interface TooltipCandidate<T extends FrameOwnedAxNode> {
export interface TooltipNameProbeOptions {
signal?: AbortSignal;
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
}
const tooltipCache = new Map<string, TooltipCacheEntry>();
@@ -39,10 +45,11 @@ function cacheKey<T extends FrameOwnedAxNode>(
return `${document.target.tabId}\u0000${document.frameId}\u0000${document.url ?? ""}\u0000${backendNodeId}`;
}
function cachedName<T extends FrameOwnedAxNode>(
/** Returns the cache entry, or `undefined` when this control was never probed. */
function cachedEntry<T extends FrameOwnedAxNode>(
document: FrameDocument<T>,
backendNodeId: number,
): string | undefined {
): TooltipCacheEntry | undefined {
const key = cacheKey(document, backendNodeId);
const entry = tooltipCache.get(key);
if (!entry) return undefined;
@@ -50,13 +57,13 @@ function cachedName<T extends FrameOwnedAxNode>(
tooltipCache.delete(key);
return undefined;
}
return entry.name;
return entry;
}
function storeName<T extends FrameOwnedAxNode>(
document: FrameDocument<T>,
backendNodeId: number,
name: string,
name: string | null,
): void {
if (tooltipCache.size >= MAX_CACHE_ENTRIES) {
const oldest = tooltipCache.keys().next().value;
@@ -64,7 +71,7 @@ function storeName<T extends FrameOwnedAxNode>(
}
tooltipCache.set(cacheKey(document, backendNodeId), {
name,
expiresAt: Date.now() + CACHE_TTL_MS,
expiresAt: Date.now() + (name === null ? MISS_CACHE_TTL_MS : CACHE_TTL_MS),
});
}
@@ -237,22 +244,27 @@ export async function probeTooltipNames<T extends FrameOwnedAxNode>(
const names = new Map<string, string>();
const pending: TooltipCandidate<T>[] = [];
for (const candidate of candidates(documents, graph)) {
const cached = cachedName(candidate.document, candidate.backendNodeId);
const cached = cachedEntry(candidate.document, candidate.backendNodeId);
if (cached) {
names.set(frameBackendKey(candidate.document.frameId, candidate.backendNodeId), cached);
// A cached miss is still a decision: skip the control instead of re-probing.
if (cached.name !== null) {
names.set(
frameBackendKey(candidate.document.frameId, candidate.backendNodeId),
cached.name,
);
}
} else if (pending.length < MAX_TOOLTIP_CANDIDATES) {
pending.push(candidate);
}
}
if (pending.length === 0) return names;
const started = Date.now();
const budget = new ProbeBudget(MAX_TOOLTIP_PROBE_MS);
const contexts = new Map<string, number>();
await options.hoverProbeBypassOverlay?.(tabId, true).catch(() => undefined);
try {
for (const candidate of pending) {
if (options.signal?.aborted) throw new DOMException("observation aborted", "AbortError");
if (Date.now() - started >= MAX_TOOLTIP_PROBE_MS) break;
if (!budget.canAfford(TOOLTIP_SETTLE_MS)) break;
let objectId: string | undefined;
try {
let contextId = contexts.get(candidate.document.frameId);
@@ -278,9 +290,9 @@ export async function probeTooltipNames<T extends FrameOwnedAxNode>(
await waitForHover(TOOLTIP_SETTLE_MS, options.signal);
const after = await tooltipTexts(cdp, candidate.document, objectId);
const name = cleanTooltipDiff(before, after);
storeName(candidate.document, candidate.backendNodeId, name ?? null);
if (!name) continue;
names.set(frameBackendKey(candidate.document.frameId, candidate.backendNodeId), name);
storeName(candidate.document, candidate.backendNodeId, name);
} catch (error) {
if (error instanceof Error && error.name === "AbortError") throw error;
} finally {
@@ -290,7 +302,6 @@ export async function probeTooltipNames<T extends FrameOwnedAxNode>(
}
} finally {
await clearHover(cdp, tabId);
await options.hoverProbeBypassOverlay?.(tabId, false).catch(() => undefined);
}
return names;
}
apps/extension/src/tools/vom/record-safe-observation.ts
+13
View File
@@ -37,6 +37,17 @@ export interface CaptureVomObservationResult {
frames: CaptureVomFrame[];
matchNodes: CaptureVomMatchNode[];
surfaceProbes?: CapturedSurfaceProbe[];
/**
* Whether this observation actively hovered the page, and whether that
* revealed content the static tree does not contain. Lets callers judge how
* far the page may have drifted from the returned snapshot.
*/
hoverProbe?: HoverProbeReport;
}
export interface HoverProbeReport {
performed: boolean;
revealedContent: boolean;
}
function projectFrames(documents: CapturedFrameDocument<FrameAxNode>[]): CaptureVomFrame[] {
@@ -75,6 +86,7 @@ export function projectRecordSafeObservation(input: {
frameDocuments: CapturedFrameDocument<FrameAxNode>[];
rendered: VomResult;
surfaceProbes?: CapturedSurfaceProbe[];
hoverProbe?: HoverProbeReport;
}): CaptureVomObservationResult {
return {
text: input.rendered.text,
@@ -84,5 +96,6 @@ export function projectRecordSafeObservation(input: {
frames: projectFrames(input.frameDocuments),
matchNodes: projectMatchNodes(input.frameDocuments),
surfaceProbes: input.surfaceProbes,
...(input.hoverProbe ? { hoverProbe: input.hoverProbe } : {}),
};
}
apps/extension/src/transport/types.ts
+5
View File
@@ -330,9 +330,14 @@ export interface SnapshotResult {
export interface ObserveParams extends SnapshotParams {
debug_surfaces?: boolean;
probe_hover?: boolean;
}
export interface ObserveResult extends SnapshotResult {
hover_probe?: {
performed: boolean;
revealed_content: boolean;
};
debug?: {
surface_probes?: Array<{
trigger_backend_node_id: number;