mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
Merge pull request #68 from Tencent/feat/more-browser-actions
feat(extension): hover end to end capability support
This commit is contained in:
Zhang GH
@@ -9,6 +9,8 @@ import type {
|
||||
} from "@/transport/types";
|
||||
import { ToolDispatcher } from "../dispatcher";
|
||||
|
||||
type TestDispatcherCdp = NonNullable<ConstructorParameters<typeof ToolDispatcher>[0]["cdp"]>;
|
||||
|
||||
function fakeTransport() {
|
||||
const handlers = new Set<FrameHandler>();
|
||||
const stateHandlers = new Set<ConnectionStateHandler>();
|
||||
@@ -129,7 +131,7 @@ describe("ToolDispatcher", () => {
|
||||
setUserAgentOverride: vi.fn(async () => {}),
|
||||
setTouchEmulationEnabled: vi.fn(async () => {}),
|
||||
};
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions, cdp });
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
|
||||
dispatcher.start();
|
||||
|
||||
deliver(makeRequest("tool.console", { session_id: "aa11" }));
|
||||
@@ -172,7 +174,7 @@ describe("ToolDispatcher", () => {
|
||||
setUserAgentOverride: vi.fn(async () => {}),
|
||||
setTouchEmulationEnabled: vi.fn(async () => {}),
|
||||
};
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions, cdp });
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
|
||||
dispatcher.start();
|
||||
|
||||
deliver(makeRequest("tool.session_stop", { session_id: "aa11" }));
|
||||
@@ -311,6 +313,277 @@ describe("ToolDispatcher", () => {
|
||||
expect(onBrowserControlResumed).toHaveBeenCalledWith("aa11");
|
||||
});
|
||||
|
||||
it("reasserts remembered hover before follow-up work and releases only after actions", async () => {
|
||||
vi.stubGlobal("chrome", {
|
||||
tabs: {
|
||||
sendMessage: vi.fn(async () => undefined),
|
||||
},
|
||||
});
|
||||
const { transport } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
const order: string[] = [];
|
||||
const cdp = {
|
||||
send: vi.fn(async <T>() => {
|
||||
order.push("hover");
|
||||
return {} as T;
|
||||
}),
|
||||
detachSession: vi.fn(async () => {}),
|
||||
ensureNetworkCapture: vi.fn(async () => {}),
|
||||
networkEntriesSince: vi.fn(() => ({
|
||||
tab_id: 7,
|
||||
entries: [],
|
||||
next_since: 0,
|
||||
truncated: false,
|
||||
})),
|
||||
setDeviceMetricsOverride: vi.fn(async () => {}),
|
||||
clearDeviceMetricsOverride: vi.fn(async () => {}),
|
||||
setUserAgentOverride: vi.fn(async () => {}),
|
||||
setTouchEmulationEnabled: vi.fn(async () => {}),
|
||||
};
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
|
||||
|
||||
(
|
||||
dispatcher as unknown as { rememberHover: (sessionId: string, result: unknown) => unknown }
|
||||
).rememberHover("aa11", {
|
||||
tab_id: 7,
|
||||
x: 10,
|
||||
y: 20,
|
||||
});
|
||||
const helpers = dispatcher as unknown as {
|
||||
withHoverReassert: <T>(
|
||||
params: { session_id: string; tab_id?: number },
|
||||
work: () => Promise<T>,
|
||||
options?: { releaseAfter?: boolean },
|
||||
) => Promise<T>;
|
||||
setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
|
||||
};
|
||||
|
||||
await helpers.withHoverReassert({ session_id: "aa11", tab_id: 7 }, async () => {
|
||||
order.push("observe");
|
||||
return {};
|
||||
});
|
||||
expect(order).toEqual(["hover", "observe"]);
|
||||
expect(chrome.tabs.sendMessage).not.toHaveBeenCalledWith(
|
||||
7,
|
||||
expect.objectContaining({ enabled: false }),
|
||||
);
|
||||
|
||||
await helpers.setHoverBypass("aa11", 7, true);
|
||||
await helpers.withHoverReassert(
|
||||
{ session_id: "aa11", tab_id: 7 },
|
||||
async () => {
|
||||
order.push("click");
|
||||
return {};
|
||||
},
|
||||
{ releaseAfter: true },
|
||||
);
|
||||
|
||||
expect(cdp.send).toHaveBeenLastCalledWith(7, "Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: 10,
|
||||
y: 20,
|
||||
});
|
||||
expect(chrome.tabs.sendMessage).toHaveBeenCalledWith(
|
||||
7,
|
||||
expect.objectContaining({ enabled: false }),
|
||||
);
|
||||
});
|
||||
|
||||
it("does not reassert or disable hover latches from another session", async () => {
|
||||
vi.stubGlobal("chrome", {
|
||||
tabs: {
|
||||
sendMessage: vi.fn(async () => undefined),
|
||||
},
|
||||
});
|
||||
const { transport } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
const cdp = {
|
||||
send: vi.fn(async <T>() => ({}) as T),
|
||||
detachSession: vi.fn(async () => {}),
|
||||
ensureNetworkCapture: vi.fn(async () => {}),
|
||||
networkEntriesSince: vi.fn(() => ({
|
||||
tab_id: 7,
|
||||
entries: [],
|
||||
next_since: 0,
|
||||
truncated: false,
|
||||
})),
|
||||
setDeviceMetricsOverride: vi.fn(async () => {}),
|
||||
clearDeviceMetricsOverride: vi.fn(async () => {}),
|
||||
setUserAgentOverride: vi.fn(async () => {}),
|
||||
setTouchEmulationEnabled: vi.fn(async () => {}),
|
||||
};
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
|
||||
const helpers = dispatcher as unknown as {
|
||||
rememberHover: (sessionId: string, result: unknown) => unknown;
|
||||
setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
|
||||
withHoverReassert: <T>(
|
||||
params: { session_id: string; tab_id?: number },
|
||||
work: () => Promise<T>,
|
||||
options?: { releaseAfter?: boolean },
|
||||
) => Promise<T>;
|
||||
releaseHoverLatch: (sessionId?: string, tabId?: number) => Promise<void>;
|
||||
};
|
||||
|
||||
helpers.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 });
|
||||
await helpers.setHoverBypass("aa11", 7, true);
|
||||
|
||||
await helpers.withHoverReassert({ session_id: "bb22", tab_id: 7 }, async () => ({}), {
|
||||
releaseAfter: true,
|
||||
});
|
||||
|
||||
expect(cdp.send).not.toHaveBeenCalled();
|
||||
expect(chrome.tabs.sendMessage).not.toHaveBeenCalledWith(
|
||||
7,
|
||||
expect.objectContaining({ enabled: false }),
|
||||
);
|
||||
|
||||
await helpers.releaseHoverLatch("aa11", 7);
|
||||
expect(chrome.tabs.sendMessage).toHaveBeenCalledWith(
|
||||
7,
|
||||
expect.objectContaining({ enabled: false }),
|
||||
);
|
||||
});
|
||||
|
||||
it("transfers hover overlay bypass ownership between sessions", async () => {
|
||||
vi.stubGlobal("chrome", {
|
||||
tabs: {
|
||||
sendMessage: vi.fn(async () => undefined),
|
||||
},
|
||||
});
|
||||
const { transport } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
const helpers = dispatcher as unknown as {
|
||||
setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
|
||||
};
|
||||
|
||||
await helpers.setHoverBypass("aa11", 7, true);
|
||||
await helpers.setHoverBypass("bb22", 7, true);
|
||||
await helpers.setHoverBypass("aa11", 7, false);
|
||||
expect(chrome.tabs.sendMessage).not.toHaveBeenCalledWith(
|
||||
7,
|
||||
expect.objectContaining({ enabled: false }),
|
||||
);
|
||||
|
||||
await helpers.setHoverBypass("bb22", 7, false);
|
||||
expect(chrome.tabs.sendMessage).toHaveBeenCalledWith(
|
||||
7,
|
||||
expect.objectContaining({ enabled: false }),
|
||||
);
|
||||
expect(chrome.tabs.sendMessage).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("limits default-target hover reassertion to the active tab", async () => {
|
||||
vi.stubGlobal("chrome", {
|
||||
tabs: {
|
||||
query: vi.fn(async () => [{ id: 9, windowId: 4242, active: true }]),
|
||||
get: vi.fn(async (tabId: number) => ({ id: tabId, windowId: 4242, active: tabId === 9 })),
|
||||
sendMessage: vi.fn(async () => undefined),
|
||||
},
|
||||
});
|
||||
const { transport } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
await sessions.start("aa11");
|
||||
const cdp = {
|
||||
send: vi.fn(async <T>() => ({}) as T),
|
||||
detachSession: vi.fn(async () => {}),
|
||||
ensureNetworkCapture: vi.fn(async () => {}),
|
||||
networkEntriesSince: vi.fn(() => ({
|
||||
tab_id: 9,
|
||||
entries: [],
|
||||
next_since: 0,
|
||||
truncated: false,
|
||||
})),
|
||||
setDeviceMetricsOverride: vi.fn(async () => {}),
|
||||
clearDeviceMetricsOverride: vi.fn(async () => {}),
|
||||
setUserAgentOverride: vi.fn(async () => {}),
|
||||
setTouchEmulationEnabled: vi.fn(async () => {}),
|
||||
};
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
|
||||
const helpers = dispatcher as unknown as {
|
||||
rememberHover: (sessionId: string, result: unknown) => unknown;
|
||||
withHoverReassert: <T>(
|
||||
params: { session_id: string; tab_id?: number },
|
||||
work: () => Promise<T>,
|
||||
) => Promise<T>;
|
||||
};
|
||||
|
||||
helpers.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 });
|
||||
helpers.rememberHover("aa11", { tab_id: 9, x: 30, y: 40 });
|
||||
|
||||
await helpers.withHoverReassert({ session_id: "aa11" }, async () => ({}));
|
||||
|
||||
expect(cdp.send).toHaveBeenCalledTimes(1);
|
||||
expect(cdp.send).toHaveBeenCalledWith(9, "Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: 30,
|
||||
y: 40,
|
||||
});
|
||||
});
|
||||
|
||||
it("releases the current session hover latch before navigation-style work", async () => {
|
||||
vi.stubGlobal("chrome", {
|
||||
tabs: {
|
||||
sendMessage: vi.fn(async () => undefined),
|
||||
},
|
||||
});
|
||||
const { transport } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
const helpers = dispatcher as unknown as {
|
||||
rememberHover: (sessionId: string, result: unknown) => unknown;
|
||||
setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
|
||||
withHoverReleaseForRequest: <T>(
|
||||
params: { session_id: string; tab_id?: number },
|
||||
work: () => Promise<T>,
|
||||
) => Promise<T>;
|
||||
withHoverReassert: <T>(
|
||||
params: { session_id: string; tab_id?: number },
|
||||
work: () => Promise<T>,
|
||||
) => Promise<T>;
|
||||
};
|
||||
helpers.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 });
|
||||
await helpers.setHoverBypass("aa11", 7, true);
|
||||
|
||||
await helpers.withHoverReleaseForRequest({ session_id: "aa11", tab_id: 7 }, async () => ({}));
|
||||
await helpers.withHoverReassert({ session_id: "aa11", tab_id: 7 }, async () => ({}));
|
||||
|
||||
expect(chrome.tabs.sendMessage).toHaveBeenCalledWith(
|
||||
7,
|
||||
expect.objectContaining({ enabled: false }),
|
||||
);
|
||||
});
|
||||
|
||||
it("disconnects the transport when send() fails so keepalive can rebuild", async () => {
|
||||
const { transport, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
|
||||
@@ -4,6 +4,7 @@ import type { CdpRunner } from "@/tools/shared";
|
||||
import {
|
||||
handleClick,
|
||||
handleFill,
|
||||
handleHover,
|
||||
handlePress,
|
||||
handleSelect,
|
||||
modifiersBitfield,
|
||||
@@ -435,6 +436,50 @@ describe("handleClick", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("handleHover", () => {
|
||||
it("keeps overlay bypass enabled after a successful hover when requested", async () => {
|
||||
const bypassOverlay = vi.fn().mockResolvedValue(undefined);
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e3", 1234, { tabId: 4 });
|
||||
const fake = makeFakeCdp({
|
||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||
"DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }),
|
||||
"Runtime.evaluate": (params: unknown) => {
|
||||
const expr = String((params as { expression?: string })?.expression ?? "");
|
||||
if (expr.includes("overlayHostPresent") && !expr.includes("hitIndex")) {
|
||||
return { result: { value: { overlayHostPresent: true, overlayHostConnected: true } } };
|
||||
}
|
||||
if (expr.includes("hitIndex")) {
|
||||
return {
|
||||
result: {
|
||||
value: { overlayHostPresent: true, overlayHostConnected: true, hitIndex: 0 },
|
||||
},
|
||||
};
|
||||
}
|
||||
throw new Error(`unexpected Runtime.evaluate: ${expr.slice(0, 80)}`);
|
||||
},
|
||||
"Input.dispatchMouseEvent": () => ({}),
|
||||
});
|
||||
|
||||
const res = await handleHover(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e3", settle_ms: 0 },
|
||||
{
|
||||
cdp: fake.cdp,
|
||||
tabsApi: fake.tabsApi,
|
||||
bypassOverlay,
|
||||
keepOverlayBypassAfterHover: true,
|
||||
},
|
||||
);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(res).toMatchObject({ tab_id: 4, used_ref: "e3", x: 60, y: 40 });
|
||||
expect(bypassOverlay).toHaveBeenCalledTimes(1);
|
||||
expect(bypassOverlay).toHaveBeenCalledWith(4, true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("handleFill", () => {
|
||||
it("returns not_found for unknown ref", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
|
||||
@@ -1523,7 +1523,7 @@ describe("buildVomScene", () => {
|
||||
expect(renderVom(scene).text).toContain("[§ active: Reviews (12)]");
|
||||
});
|
||||
|
||||
it("maps hover surface probes to matching node labels", () => {
|
||||
it("maps hover surface probes to matching backend node ids", () => {
|
||||
const axNodes: CdpAxNode[] = [
|
||||
{
|
||||
nodeId: "1",
|
||||
@@ -1544,7 +1544,7 @@ describe("buildVomScene", () => {
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
surfaceProbes: [
|
||||
{ triggerLabel: "Products", triggerAction: "hover", subItems: ["Shoes", "Bags"] },
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
|
||||
],
|
||||
nodes: [
|
||||
{
|
||||
@@ -1573,7 +1573,260 @@ describe("buildVomScene", () => {
|
||||
expect(scene.surfaces).toEqual([
|
||||
{ triggerId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
|
||||
]);
|
||||
expect(renderVom(scene).text).toContain('@e1 button "Products" [hover: Shoes | Bags]');
|
||||
expect(renderVom(scene).text).toContain('@e1 button "Products" [hover first: Shoes | Bags]');
|
||||
});
|
||||
|
||||
it("maps hover surface probes to rendered descendants in the same trigger subtree", () => {
|
||||
const axNodes: CdpAxNode[] = [
|
||||
{
|
||||
nodeId: "1",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
backendDOMNodeId: 10,
|
||||
childIds: ["2"],
|
||||
},
|
||||
{
|
||||
nodeId: "2",
|
||||
parentId: "1",
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "image" },
|
||||
backendDOMNodeId: 21,
|
||||
},
|
||||
];
|
||||
const scene = buildVomScene(axNodes, {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
surfaceProbes: [
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
|
||||
],
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar" },
|
||||
rect: { x: 900, y: 10, w: 30, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 21,
|
||||
parentBackendNodeId: 20,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar__inner" },
|
||||
rect: { x: 902, y: 12, w: 26, h: 26 },
|
||||
paintOrder: 2,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(renderVom(scene).text).toContain('@e1 button "image" [hover first: My profile]');
|
||||
});
|
||||
|
||||
it("deduplicates hover surface probes by original trigger backend id", () => {
|
||||
const axNodes: CdpAxNode[] = [
|
||||
{
|
||||
nodeId: "1",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
backendDOMNodeId: 10,
|
||||
childIds: ["2"],
|
||||
},
|
||||
{
|
||||
nodeId: "2",
|
||||
parentId: "1",
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "image" },
|
||||
backendDOMNodeId: 21,
|
||||
},
|
||||
];
|
||||
const scene = buildVomScene(axNodes, {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
surfaceProbes: [
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Sign out"] },
|
||||
{ triggerBackendNodeId: 21, triggerAction: "hover", subItems: ["Settings"] },
|
||||
],
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar" },
|
||||
rect: { x: 900, y: 10, w: 30, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 21,
|
||||
parentBackendNodeId: 20,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar__inner" },
|
||||
rect: { x: 902, y: 12, w: 26, h: 26 },
|
||||
paintOrder: 2,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(scene.surfaces).toEqual([
|
||||
{ triggerId: 21, triggerAction: "hover", subItems: ["My profile"] },
|
||||
]);
|
||||
});
|
||||
|
||||
it("maps hover surface probes to DOM-recovered custom controls", () => {
|
||||
const axNodes: CdpAxNode[] = [
|
||||
{
|
||||
nodeId: "1",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
backendDOMNodeId: 10,
|
||||
childIds: ["2"],
|
||||
},
|
||||
{
|
||||
nodeId: "2",
|
||||
parentId: "1",
|
||||
role: { type: "role", value: "img" },
|
||||
name: { type: "computedString", value: "image" },
|
||||
backendDOMNodeId: 21,
|
||||
},
|
||||
];
|
||||
const scene = buildVomScene(axNodes, {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
surfaceProbes: [
|
||||
{
|
||||
triggerBackendNodeId: 20,
|
||||
triggerPoint: { x: 915, y: 25 },
|
||||
triggerAction: "hover",
|
||||
subItems: ["My profile", "Sign out"],
|
||||
},
|
||||
],
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
cursor: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar" },
|
||||
rect: { x: 900, y: 10, w: 30, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
cursor: "pointer",
|
||||
},
|
||||
{
|
||||
backendNodeId: 21,
|
||||
parentBackendNodeId: 20,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar__inner" },
|
||||
rect: { x: 902, y: 12, w: 26, h: 26 },
|
||||
paintOrder: 2,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
cursor: "pointer",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(scene.surfaces).toEqual([
|
||||
{ triggerId: 21, triggerAction: "hover", subItems: ["My profile", "Sign out"] },
|
||||
]);
|
||||
expect(renderVom(scene).text).toContain(
|
||||
'@e1 button "image" [hover first: My profile | Sign out]',
|
||||
);
|
||||
});
|
||||
|
||||
it("does not attach hover probes to distant geometry matches", () => {
|
||||
const axNodes: CdpAxNode[] = [
|
||||
{
|
||||
nodeId: "1",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
backendDOMNodeId: 10,
|
||||
childIds: ["2"],
|
||||
},
|
||||
{
|
||||
nodeId: "2",
|
||||
parentId: "1",
|
||||
role: { type: "role", value: "link" },
|
||||
name: { type: "computedString", value: "0" },
|
||||
backendDOMNodeId: 42,
|
||||
},
|
||||
];
|
||||
const scene = buildVomScene(axNodes, {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
surfaceProbes: [
|
||||
{
|
||||
triggerBackendNodeId: 999,
|
||||
triggerPoint: { x: 900, y: 20 },
|
||||
triggerAction: "hover",
|
||||
subItems: ["My profile"],
|
||||
},
|
||||
],
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 42,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "a",
|
||||
attrs: {},
|
||||
rect: { x: 860, y: 120, w: 40, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(scene.surfaces).toBeUndefined();
|
||||
expect(renderVom(scene).text).not.toContain("[hover first:");
|
||||
});
|
||||
|
||||
it("enriches names and active scope signals from AX properties", () => {
|
||||
@@ -2046,7 +2299,7 @@ describe("handleSnapshot", () => {
|
||||
});
|
||||
const res = await handleObserve(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{ session_id: "aa11", debug_surfaces: true },
|
||||
{
|
||||
cdp: {
|
||||
send: send as unknown as <T = unknown>(
|
||||
@@ -2067,6 +2320,7 @@ describe("handleSnapshot", () => {
|
||||
);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(res.debug).toEqual({ surface_probes: [] });
|
||||
expect(send).toHaveBeenCalledWith(
|
||||
4,
|
||||
"Runtime.evaluate",
|
||||
|
||||
@@ -8,6 +8,8 @@ import type {
|
||||
EvaluateParams,
|
||||
FillParams,
|
||||
GetHtmlParams,
|
||||
HoverParams,
|
||||
HoverResult,
|
||||
NavigateBackParams,
|
||||
NavigateForwardParams,
|
||||
NavigateParams,
|
||||
@@ -33,7 +35,7 @@ import { handleConsole } from "./console";
|
||||
import { type EmulateCdpRunner, handleEmulate } from "./emulate";
|
||||
import { handleEvaluate } from "./evaluate";
|
||||
import { handleRequestHelp } from "./human-loop";
|
||||
import { handleClick, handleFill, handlePress, handleSelect } from "./interaction";
|
||||
import { handleClick, handleFill, handleHover, handlePress, handleSelect } from "./interaction";
|
||||
import {
|
||||
handleNavigate,
|
||||
handleNavigateBack,
|
||||
@@ -56,7 +58,7 @@ import {
|
||||
type SessionStartParams,
|
||||
type SessionStopParams,
|
||||
} from "./session";
|
||||
import { chromeTabsApi } from "./shared";
|
||||
import { chromeTabsApi, lookupSession, resolveTargetTab } from "./shared";
|
||||
import {
|
||||
type BorrowConfirmationApprover,
|
||||
handleTabBorrow,
|
||||
@@ -81,6 +83,18 @@ type DispatcherCdpRunner = CdpRunner &
|
||||
detachSession(sessionId: string): Promise<void>;
|
||||
};
|
||||
|
||||
interface HoverLatch {
|
||||
sessionId: string;
|
||||
tabId: number;
|
||||
x: number;
|
||||
y: number;
|
||||
}
|
||||
|
||||
interface HoverLatchScope {
|
||||
session_id: string;
|
||||
tab_id?: number;
|
||||
}
|
||||
|
||||
export interface DispatcherDeps {
|
||||
transport: Transport;
|
||||
sessions: SessionManager;
|
||||
@@ -126,6 +140,8 @@ export class ToolDispatcher {
|
||||
private readonly approveBorrow?: BorrowConfirmationApprover;
|
||||
private readonly helpNotificationCopy?: () => { title: string; body: string };
|
||||
private subscription: { dispose(): void } | null = null;
|
||||
private readonly hoverBypassTabs = new Map<number, string>();
|
||||
private readonly hoverLatches = new Map<number, HoverLatch>();
|
||||
/**
|
||||
* Per-rpc-id `AbortController` registry. Populated inside
|
||||
* [`dispatch`] before we await the tool handler and torn down in
|
||||
@@ -271,16 +287,20 @@ export class ToolDispatcher {
|
||||
switch (req.method) {
|
||||
case "tool.session_start":
|
||||
return handleSessionStart(this.sessions, req.params as SessionStartParams);
|
||||
case "tool.session_stop":
|
||||
case "tool.session_stop": {
|
||||
await this.releaseHoverLatch((req.params as SessionStopParams).session_id);
|
||||
return handleSessionStop(this.sessions, req.params as SessionStopParams, {
|
||||
cdp: this.cdp,
|
||||
});
|
||||
}
|
||||
case "tool.tab_list":
|
||||
return handleTabList(this.sessions, req.params as TabListParams);
|
||||
case "tool.tab_create":
|
||||
return handleTabCreate(this.sessions, req.params as TabCreateParams);
|
||||
case "tool.tab_close":
|
||||
return handleTabClose(this.sessions, req.params as TabCloseParams);
|
||||
return this.withHoverReleaseForRequest(req.params as TabCloseParams, () =>
|
||||
handleTabClose(this.sessions, req.params as TabCloseParams),
|
||||
);
|
||||
case "tool.tab_select":
|
||||
return handleTabSelect(this.sessions, req.params as TabSelectParams);
|
||||
case "tool.tab_borrow":
|
||||
@@ -319,17 +339,31 @@ export class ToolDispatcher {
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi } : undefined,
|
||||
);
|
||||
case "tool.snapshot":
|
||||
return handleSnapshot(
|
||||
this.sessions,
|
||||
req.params as SnapshotParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
|
||||
return this.withHoverReassert(req.params as SnapshotParams, () =>
|
||||
handleSnapshot(
|
||||
this.sessions,
|
||||
req.params as SnapshotParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
|
||||
),
|
||||
);
|
||||
case "tool.observe":
|
||||
return handleObserve(
|
||||
this.sessions,
|
||||
req.params as ObserveParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
|
||||
case "tool.observe": {
|
||||
const params = req.params as ObserveParams;
|
||||
const hoverScope = await this.resolveHoverLatchScope(params);
|
||||
return this.withHoverReassert(params, () =>
|
||||
handleObserve(
|
||||
this.sessions,
|
||||
params,
|
||||
this.cdp
|
||||
? {
|
||||
cdp: this.cdp,
|
||||
tabsApi: chromeTabsCaptureApi,
|
||||
conditionalSurfaceProbe: !this.hasHoverLatchForScope(hoverScope),
|
||||
hoverProbeBypassOverlay: bypassOverlay,
|
||||
}
|
||||
: undefined,
|
||||
),
|
||||
);
|
||||
}
|
||||
case "tool.get_html":
|
||||
return handleGetHtml(
|
||||
this.sessions,
|
||||
@@ -337,68 +371,95 @@ export class ToolDispatcher {
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
|
||||
);
|
||||
case "tool.navigate":
|
||||
return handleNavigate(
|
||||
this.sessions,
|
||||
req.params as NavigateParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
return this.withHoverReleaseForRequest(req.params as NavigateParams, () =>
|
||||
handleNavigate(
|
||||
this.sessions,
|
||||
req.params as NavigateParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
),
|
||||
);
|
||||
case "tool.navigate_back":
|
||||
return handleNavigateBack(
|
||||
this.sessions,
|
||||
req.params as NavigateBackParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
return this.withHoverReleaseForRequest(req.params as NavigateBackParams, () =>
|
||||
handleNavigateBack(
|
||||
this.sessions,
|
||||
req.params as NavigateBackParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
),
|
||||
);
|
||||
case "tool.navigate_forward":
|
||||
return handleNavigateForward(
|
||||
this.sessions,
|
||||
req.params as NavigateForwardParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
return this.withHoverReleaseForRequest(req.params as NavigateForwardParams, () =>
|
||||
handleNavigateForward(
|
||||
this.sessions,
|
||||
req.params as NavigateForwardParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
),
|
||||
);
|
||||
case "tool.reload":
|
||||
return handleReload(
|
||||
this.sessions,
|
||||
req.params as ReloadParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
return this.withHoverReleaseForRequest(req.params as ReloadParams, () =>
|
||||
handleReload(
|
||||
this.sessions,
|
||||
req.params as ReloadParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
),
|
||||
);
|
||||
case "tool.click":
|
||||
return handleClick(
|
||||
this.sessions,
|
||||
return this.withHoverReassert(
|
||||
req.params as ClickParams,
|
||||
() =>
|
||||
handleClick(
|
||||
this.sessions,
|
||||
req.params as ClickParams,
|
||||
this.cdp
|
||||
? {
|
||||
cdp: this.cdp,
|
||||
tabsApi: chromeTabsApi,
|
||||
signal,
|
||||
bypassOverlay,
|
||||
}
|
||||
: undefined,
|
||||
),
|
||||
{ releaseAfter: true },
|
||||
);
|
||||
case "tool.hover": {
|
||||
const result = await handleHover(
|
||||
this.sessions,
|
||||
req.params as HoverParams,
|
||||
this.cdp
|
||||
? {
|
||||
cdp: this.cdp,
|
||||
tabsApi: chromeTabsApi,
|
||||
signal,
|
||||
bypassOverlay: async (tabId, enabled) => {
|
||||
try {
|
||||
await chrome.tabs.sendMessage(tabId, {
|
||||
type: OVERLAY_AUTOMATION_BYPASS,
|
||||
enabled,
|
||||
});
|
||||
} catch {
|
||||
// Content script may be unavailable on restricted pages.
|
||||
}
|
||||
},
|
||||
bypassOverlay: (tabId, enabled) =>
|
||||
this.setHoverBypass((req.params as HoverParams).session_id, tabId, enabled),
|
||||
keepOverlayBypassAfterHover: true,
|
||||
}
|
||||
: undefined,
|
||||
);
|
||||
return this.rememberHover((req.params as HoverParams).session_id, result);
|
||||
}
|
||||
case "tool.fill":
|
||||
return handleFill(
|
||||
this.sessions,
|
||||
req.params as FillParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
return this.withHoverReleaseForRequest(req.params as FillParams, () =>
|
||||
handleFill(
|
||||
this.sessions,
|
||||
req.params as FillParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
),
|
||||
);
|
||||
case "tool.press":
|
||||
return handlePress(
|
||||
this.sessions,
|
||||
req.params as PressParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
return this.withHoverReleaseForRequest(req.params as PressParams, () =>
|
||||
handlePress(
|
||||
this.sessions,
|
||||
req.params as PressParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
),
|
||||
);
|
||||
case "tool.select":
|
||||
return handleSelect(
|
||||
this.sessions,
|
||||
req.params as SelectParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
return this.withHoverReleaseForRequest(req.params as SelectParams, () =>
|
||||
handleSelect(
|
||||
this.sessions,
|
||||
req.params as SelectParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
),
|
||||
);
|
||||
case "tool.evaluate":
|
||||
return handleEvaluate(
|
||||
@@ -471,6 +532,114 @@ export class ToolDispatcher {
|
||||
} satisfies RpcError;
|
||||
}
|
||||
}
|
||||
|
||||
private async setHoverBypass(sessionId: string, tabId: number, enabled: boolean): Promise<void> {
|
||||
const owner = this.hoverBypassTabs.get(tabId);
|
||||
if (enabled) {
|
||||
if (owner === sessionId) return;
|
||||
if (owner === undefined) await bypassOverlay(tabId, true);
|
||||
this.hoverBypassTabs.set(tabId, sessionId);
|
||||
} else {
|
||||
if (owner !== sessionId) return;
|
||||
await bypassOverlay(tabId, false);
|
||||
this.hoverBypassTabs.delete(tabId);
|
||||
}
|
||||
}
|
||||
|
||||
private rememberHover(sessionId: string, result: HoverResult | RpcError): HoverResult | RpcError {
|
||||
if (!isRpcError(result)) {
|
||||
this.hoverLatches.set(result.tab_id, {
|
||||
sessionId,
|
||||
tabId: result.tab_id,
|
||||
x: result.x,
|
||||
y: result.y,
|
||||
});
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private hasHoverLatchForScope(scope: HoverLatchScope): boolean {
|
||||
return this.hoverLatchesForRequest(scope).length > 0;
|
||||
}
|
||||
|
||||
private async withHoverReassert<T>(
|
||||
params: { session_id: string; tab_id?: number },
|
||||
work: () => Promise<T>,
|
||||
options: { releaseAfter?: boolean } = {},
|
||||
): Promise<T> {
|
||||
const scope = await this.resolveHoverLatchScope(params);
|
||||
await this.reassertHover(scope);
|
||||
try {
|
||||
return await work();
|
||||
} finally {
|
||||
if (options.releaseAfter) {
|
||||
await this.releaseHoverLatch(scope.session_id, scope.tab_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async withHoverReleaseForRequest<T>(
|
||||
params: { session_id: string; tab_id?: number },
|
||||
work: () => Promise<T>,
|
||||
): Promise<T> {
|
||||
const scope = await this.resolveHoverLatchScope(params);
|
||||
await this.releaseHoverLatch(scope.session_id, scope.tab_id);
|
||||
return work();
|
||||
}
|
||||
|
||||
private async resolveHoverLatchScope(params: {
|
||||
session_id: string;
|
||||
tab_id?: number;
|
||||
}): Promise<HoverLatchScope> {
|
||||
if (params.tab_id !== undefined) return params;
|
||||
const ctx = lookupSession(this.sessions, params, "hover latch");
|
||||
if (isRpcError(ctx)) return params;
|
||||
const target = await resolveTargetTab(this.sessions, ctx, undefined, chromeTabsApi);
|
||||
if (isRpcError(target)) return params;
|
||||
return { session_id: params.session_id, tab_id: target.tabId };
|
||||
}
|
||||
|
||||
private hoverLatchesForRequest(params: { session_id: string; tab_id?: number }): HoverLatch[] {
|
||||
return [...this.hoverLatches.values()].filter((latch) => {
|
||||
if (latch.sessionId !== params.session_id) return false;
|
||||
return params.tab_id === undefined || latch.tabId === params.tab_id;
|
||||
});
|
||||
}
|
||||
|
||||
private async reassertHover(params: { session_id: string; tab_id?: number }): Promise<void> {
|
||||
if (!this.cdp) return;
|
||||
await Promise.all(
|
||||
this.hoverLatchesForRequest(params).map((latch) =>
|
||||
this.cdp!.send(latch.tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: latch.x,
|
||||
y: latch.y,
|
||||
}).catch((err) => {
|
||||
console.debug("[bsk dispatcher] hover reassert failed", err);
|
||||
this.hoverLatches.delete(latch.tabId);
|
||||
}),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
private async releaseHoverLatch(sessionId?: string, tabId?: number): Promise<void> {
|
||||
const matchesScope = (entrySessionId: string, entryTabId: number): boolean => {
|
||||
if (sessionId !== undefined && entrySessionId !== sessionId) return false;
|
||||
return tabId === undefined || entryTabId === tabId;
|
||||
};
|
||||
const tabs = new Set<number>();
|
||||
for (const [bypassTabId, bypassSessionId] of this.hoverBypassTabs) {
|
||||
if (!matchesScope(bypassSessionId, bypassTabId)) continue;
|
||||
tabs.add(bypassTabId);
|
||||
this.hoverBypassTabs.delete(bypassTabId);
|
||||
}
|
||||
for (const latch of this.hoverLatches.values()) {
|
||||
if (!matchesScope(latch.sessionId, latch.tabId)) continue;
|
||||
tabs.add(latch.tabId);
|
||||
this.hoverLatches.delete(latch.tabId);
|
||||
}
|
||||
await Promise.all([...tabs].map((tabId) => bypassOverlay(tabId, false)));
|
||||
}
|
||||
}
|
||||
|
||||
function isRpcError(v: unknown): v is RpcError {
|
||||
@@ -497,6 +666,7 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
|
||||
case "tool.navigate_forward":
|
||||
case "tool.reload":
|
||||
case "tool.click":
|
||||
case "tool.hover":
|
||||
case "tool.fill":
|
||||
case "tool.press":
|
||||
case "tool.select":
|
||||
@@ -512,6 +682,17 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
|
||||
}
|
||||
}
|
||||
|
||||
async function bypassOverlay(tabId: number, enabled: boolean): Promise<void> {
|
||||
try {
|
||||
await chrome.tabs.sendMessage(tabId, {
|
||||
type: OVERLAY_AUTOMATION_BYPASS,
|
||||
enabled,
|
||||
});
|
||||
} catch {
|
||||
// Content script may be unavailable on restricted pages.
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves never; rejects with `AbortLikeError` as soon as the signal
|
||||
* fires (or immediately if it is already aborted). Used by the
|
||||
|
||||
@@ -17,6 +17,8 @@ import type {
|
||||
ClickResult,
|
||||
FillParams,
|
||||
FillResult,
|
||||
HoverParams,
|
||||
HoverResult,
|
||||
KeyModifier,
|
||||
MouseButton,
|
||||
PressParams,
|
||||
@@ -53,9 +55,12 @@ export interface InteractionDeps {
|
||||
defaultTimeoutMs?: number;
|
||||
/** Temporarily disable overlay click blocker during CDP automation. */
|
||||
bypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||
/** Keep hover hit-testing active for the caller's next observation/action. */
|
||||
keepOverlayBypassAfterHover?: boolean;
|
||||
}
|
||||
|
||||
const DEFAULT_TIMEOUT_MS = 30_000;
|
||||
const DEFAULT_HOVER_SETTLE_MS = 200;
|
||||
|
||||
let defaultDeps: { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } | null = null;
|
||||
function getDefaultDeps(): { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } {
|
||||
@@ -100,6 +105,27 @@ function throwIfAborted(signal: AbortSignal | undefined): RpcError | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
function isAbortLikeError(err: unknown): boolean {
|
||||
return err instanceof DOMException && err.name === "AbortError";
|
||||
}
|
||||
|
||||
async function wait(ms: number, signal?: AbortSignal): Promise<void> {
|
||||
if (signal?.aborted) throw new DOMException("aborted", "AbortError");
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const cleanup = () => signal?.removeEventListener("abort", onAbort);
|
||||
const timer = setTimeout(() => {
|
||||
cleanup();
|
||||
resolve();
|
||||
}, ms);
|
||||
const onAbort = () => {
|
||||
clearTimeout(timer);
|
||||
cleanup();
|
||||
reject(new DOMException("aborted", "AbortError"));
|
||||
};
|
||||
signal?.addEventListener("abort", onAbort, { once: true });
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve `{ref?, selector?}` into a `backendNodeId`. Returns an
|
||||
* `RpcError` if the caller supplied neither (or both), or if neither
|
||||
@@ -286,7 +312,104 @@ export async function handleClick(
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
} finally {
|
||||
if (automationBypassEnabled && deps.bypassOverlay) {
|
||||
if (automationBypassEnabled && deps.bypassOverlay && !deps.keepOverlayBypassAfterHover) {
|
||||
try {
|
||||
await deps.bypassOverlay(target.tabId, false);
|
||||
} catch (err) {
|
||||
console.debug("[bsk interaction] overlay bypass disable failed", err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||
tab_id: target.tabId,
|
||||
used_ref: node.usedRef,
|
||||
used_selector: node.usedSelector,
|
||||
x: centre.x,
|
||||
y: centre.y,
|
||||
});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// tool.hover
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export async function handleHover(
|
||||
manager: SessionManager,
|
||||
params: HoverParams,
|
||||
deps: InteractionDeps = getDefaultDeps(),
|
||||
): Promise<HoverResult | RpcError> {
|
||||
const ctxOrErr = lookupSession(manager, params, "hover");
|
||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||
const ctx = ctxOrErr;
|
||||
const aborted = throwIfAborted(deps.signal);
|
||||
if (aborted) return aborted;
|
||||
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
|
||||
if (isRpcError(target)) return target;
|
||||
const denied = enforceAgentWindow(ctx, target, "hover");
|
||||
if (denied) return denied;
|
||||
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
||||
|
||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "hover");
|
||||
if (isRpcError(node)) return node;
|
||||
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
|
||||
if (scrollErr) return scrollErr;
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
}
|
||||
|
||||
const centre = await nodeCentre(deps.cdp, target.tabId, node.backendNodeId);
|
||||
if (isRpcError(centre)) return centre;
|
||||
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "hover aborted" };
|
||||
}
|
||||
|
||||
const modifiers = modifiersBitfield(params.modifiers);
|
||||
const overlayBlocking = await checkOverlayAtPoint(deps.cdp, target.tabId, centre.x, centre.y);
|
||||
let automationBypassEnabled = false;
|
||||
let hoverCompleted = false;
|
||||
if (overlayBlocking && deps.bypassOverlay) {
|
||||
try {
|
||||
await deps.bypassOverlay(target.tabId, true);
|
||||
automationBypassEnabled = true;
|
||||
} catch (err) {
|
||||
console.debug("[bsk interaction] overlay bypass enable failed", err);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: centre.x,
|
||||
y: centre.y,
|
||||
modifiers,
|
||||
});
|
||||
const settleMs = params.settle_ms ?? DEFAULT_HOVER_SETTLE_MS;
|
||||
if (settleMs > 0) {
|
||||
await wait(settleMs, deps.signal);
|
||||
}
|
||||
hoverCompleted = true;
|
||||
} catch (err) {
|
||||
if (isAbortLikeError(err)) {
|
||||
return { code: "cancelled", message: "hover aborted" };
|
||||
}
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
} finally {
|
||||
if (
|
||||
automationBypassEnabled &&
|
||||
deps.bypassOverlay &&
|
||||
(!deps.keepOverlayBypassAfterHover || !hoverCompleted)
|
||||
) {
|
||||
try {
|
||||
await deps.bypassOverlay(target.tabId, false);
|
||||
} catch (err) {
|
||||
|
||||
@@ -5,7 +5,9 @@
|
||||
|
||||
import {
|
||||
type ActiveScopeBlock,
|
||||
applyVomInteractionRecovery,
|
||||
type CondSurface,
|
||||
isVomReferenceNode,
|
||||
renderVom,
|
||||
type VomNode,
|
||||
type VomScene,
|
||||
@@ -927,39 +929,28 @@ function buildActiveScopeBlocks(nodes: VomNode[], signals: VomNodeDomSignals): A
|
||||
return blocks;
|
||||
}
|
||||
|
||||
function labelForSurfaceMatch(node: VomNode): string {
|
||||
return (
|
||||
cleanAttr(node.name) ??
|
||||
cleanAttr(node.text) ??
|
||||
cleanAttr(node.attrs?.["aria-label"]) ??
|
||||
cleanAttr(node.attrs?.title) ??
|
||||
""
|
||||
);
|
||||
}
|
||||
|
||||
function buildConditionalSurfaces(nodes: VomNode[], captured: CapturedViewModel): CondSurface[] {
|
||||
const probes = captured.surfaceProbes ?? [];
|
||||
if (probes.length === 0) return [];
|
||||
|
||||
const surfaces: CondSurface[] = [];
|
||||
const nodesWithLabels = nodes
|
||||
.map((node) => ({ node, key: normalizeProbeKey(labelForSurfaceMatch(node)) }))
|
||||
.filter((entry) => entry.key.length > 0);
|
||||
const recoveredNodes = applyVomInteractionRecovery(nodes);
|
||||
const signals = capturedOnlySignals(captured.nodes);
|
||||
const used = new Set<number>();
|
||||
for (const probe of probes) {
|
||||
const probeKey = normalizeProbeKey(probe.triggerLabel);
|
||||
if (!probeKey || probe.subItems.length === 0) continue;
|
||||
const match =
|
||||
nodesWithLabels.find((entry) => entry.key === probeKey && !used.has(entry.node.id)) ??
|
||||
nodesWithLabels.find(
|
||||
(entry) =>
|
||||
!used.has(entry.node.id) &&
|
||||
(probeKey.startsWith(entry.key) || entry.key.startsWith(probeKey)),
|
||||
);
|
||||
if (probe.subItems.length === 0 || used.has(probe.triggerBackendNodeId)) continue;
|
||||
const match = findSurfaceTriggerNode(
|
||||
probe.triggerBackendNodeId,
|
||||
recoveredNodes,
|
||||
signals,
|
||||
probe.triggerPoint,
|
||||
);
|
||||
if (!match) continue;
|
||||
used.add(match.node.id);
|
||||
if (used.has(match.id)) continue;
|
||||
used.add(probe.triggerBackendNodeId);
|
||||
used.add(match.id);
|
||||
surfaces.push({
|
||||
triggerId: match.node.id,
|
||||
triggerId: match.id,
|
||||
triggerAction: probe.triggerAction,
|
||||
subItems: probe.subItems,
|
||||
});
|
||||
@@ -967,6 +958,77 @@ function buildConditionalSurfaces(nodes: VomNode[], captured: CapturedViewModel)
|
||||
return surfaces;
|
||||
}
|
||||
|
||||
function findSurfaceTriggerNode(
|
||||
triggerBackendNodeId: number,
|
||||
nodes: VomNode[],
|
||||
signals: VomNodeDomSignals,
|
||||
triggerPoint?: { x: number; y: number },
|
||||
): VomNode | undefined {
|
||||
const renderedById = new Map(nodes.map((node) => [node.id, node]));
|
||||
const exact = renderedById.get(triggerBackendNodeId);
|
||||
if (exact && isSurfaceAttachableNode(exact)) return exact;
|
||||
|
||||
const domDescendant = nodes.find(
|
||||
(node) =>
|
||||
isSurfaceAttachableNode(node) &&
|
||||
(node.domParentId === triggerBackendNodeId ||
|
||||
node.domAncestorIds?.includes(triggerBackendNodeId) === true),
|
||||
);
|
||||
if (domDescendant) return domDescendant;
|
||||
|
||||
const queue = [...(signals.childrenByParentId.get(triggerBackendNodeId) ?? [])];
|
||||
while (queue.length > 0) {
|
||||
const child = queue.shift() as CapturedNode;
|
||||
const rendered = renderedById.get(child.backendNodeId);
|
||||
if (rendered && isSurfaceAttachableNode(rendered)) return rendered;
|
||||
queue.push(...(signals.childrenByParentId.get(child.backendNodeId) ?? []));
|
||||
}
|
||||
|
||||
let current = signals.capturedByBackendId.get(triggerBackendNodeId);
|
||||
while (current?.parentBackendNodeId !== null && current?.parentBackendNodeId !== undefined) {
|
||||
const parent = renderedById.get(current.parentBackendNodeId);
|
||||
if (parent && isSurfaceAttachableNode(parent)) return parent;
|
||||
current = signals.capturedByBackendId.get(current.parentBackendNodeId);
|
||||
}
|
||||
|
||||
if (triggerPoint) {
|
||||
return findSurfaceNodeByPoint(triggerPoint, nodes, signals);
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function isSurfaceAttachableNode(node: VomNode): boolean {
|
||||
return isVomReferenceNode(node);
|
||||
}
|
||||
|
||||
function findSurfaceNodeByPoint(
|
||||
point: { x: number; y: number },
|
||||
nodes: VomNode[],
|
||||
signals: VomNodeDomSignals,
|
||||
): VomNode | undefined {
|
||||
let best: { node: VomNode; score: number } | undefined;
|
||||
for (const node of nodes) {
|
||||
if (!isSurfaceAttachableNode(node)) continue;
|
||||
const rect = node.rect ?? signals.capturedByBackendId.get(node.id)?.rect;
|
||||
if (!rect) continue;
|
||||
const contains =
|
||||
point.x >= rect.x &&
|
||||
point.x <= rect.x + rect.w &&
|
||||
point.y >= rect.y &&
|
||||
point.y <= rect.y + rect.h;
|
||||
const centerX = rect.x + rect.w / 2;
|
||||
const centerY = rect.y + rect.h / 2;
|
||||
const distance = Math.hypot(point.x - centerX, point.y - centerY);
|
||||
if (!contains && distance > 40) continue;
|
||||
const score = contains ? distance : distance + 1_000;
|
||||
if (!best || score < best.score) {
|
||||
best = { node, score };
|
||||
}
|
||||
}
|
||||
return best?.node;
|
||||
}
|
||||
|
||||
function axNodeInOverlaySubtree(
|
||||
axNode: CdpAxNode,
|
||||
axById: Map<string, CdpAxNode>,
|
||||
@@ -1091,6 +1153,8 @@ export interface SnapshotDeps {
|
||||
get(tabId: number): Promise<chrome.tabs.Tab>;
|
||||
query(q: chrome.tabs.QueryInfo): Promise<chrome.tabs.Tab[]>;
|
||||
};
|
||||
conditionalSurfaceProbe?: boolean;
|
||||
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||
}
|
||||
|
||||
let defaultDeps: SnapshotDeps | null = null;
|
||||
@@ -1236,9 +1300,13 @@ async function captureForVom(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
conditionalSurfaceProbe: boolean,
|
||||
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>,
|
||||
): Promise<CapturedViewModel> {
|
||||
try {
|
||||
return await captureViewModel(cdp, tabId, { conditionalSurfaceProbe });
|
||||
return await captureViewModel(cdp, tabId, {
|
||||
conditionalSurfaceProbe,
|
||||
hoverProbeBypassOverlay,
|
||||
});
|
||||
} catch {
|
||||
return fallbackCapturedViewModel(cdp, tabId);
|
||||
}
|
||||
@@ -1277,7 +1345,14 @@ async function handleVomObservation(
|
||||
{},
|
||||
);
|
||||
const axNodes = result.nodes ?? [];
|
||||
const captured = await captureForVom(deps.cdp, target.tabId, conditionalSurfaceProbe);
|
||||
const effectiveConditionalSurfaceProbe =
|
||||
deps.conditionalSurfaceProbe ?? conditionalSurfaceProbe;
|
||||
const captured = await captureForVom(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
effectiveConditionalSurfaceProbe,
|
||||
deps.hoverProbeBypassOverlay,
|
||||
);
|
||||
const scene = buildVomScene(axNodes, captured, { pageUrl: target.url });
|
||||
const rendered = renderVom(scene, {
|
||||
maxDepth: params.max_depth,
|
||||
@@ -1294,6 +1369,19 @@ async function handleVomObservation(
|
||||
ref_count: rendered.refs.length,
|
||||
tab_id: target.tabId,
|
||||
truncated: rendered.truncated,
|
||||
...(toolName === "observe" && (params as ObserveParams).debug_surfaces
|
||||
? {
|
||||
debug: {
|
||||
surface_probes: (captured.surfaceProbes ?? []).map((probe) => ({
|
||||
trigger_backend_node_id: probe.triggerBackendNodeId,
|
||||
...(probe.triggerPoint ? { trigger_point: probe.triggerPoint } : {}),
|
||||
trigger_action: probe.triggerAction,
|
||||
sub_items: probe.subItems,
|
||||
...(probe.confidence ? { confidence: probe.confidence } : {}),
|
||||
})),
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
} catch (err) {
|
||||
return {
|
||||
|
||||
@@ -51,6 +51,156 @@ function fakeSnapshotReply() {
|
||||
};
|
||||
}
|
||||
|
||||
function hoverTriggerSnapshotReply() {
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
"button",
|
||||
"class",
|
||||
"user-avatar",
|
||||
"position",
|
||||
"static",
|
||||
"pointer-events",
|
||||
"auto",
|
||||
"cursor",
|
||||
"pointer",
|
||||
];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
return {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeType: [1, 1, 1],
|
||||
nodeName: [i("html"), i("body"), i("button")],
|
||||
backendNodeId: [10, 11, 12],
|
||||
attributes: [[], [], [i("class"), i("user-avatar")]],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2],
|
||||
styles: [
|
||||
[i("static"), i("auto"), i("pointer")],
|
||||
[i("static"), i("auto"), i("pointer")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[940, 16, 32, 32],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function twoHoverTriggerSnapshotReply() {
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
"button",
|
||||
"class",
|
||||
"user-avatar",
|
||||
"secondary-dropdown-trigger",
|
||||
"position",
|
||||
"static",
|
||||
"pointer-events",
|
||||
"auto",
|
||||
"cursor",
|
||||
"pointer",
|
||||
];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
return {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 1],
|
||||
nodeType: [1, 1, 1, 1],
|
||||
nodeName: [i("html"), i("body"), i("button"), i("button")],
|
||||
backendNodeId: [10, 11, 12, 13],
|
||||
attributes: [
|
||||
[],
|
||||
[],
|
||||
[i("class"), i("user-avatar")],
|
||||
[i("class"), i("secondary-dropdown-trigger")],
|
||||
],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2, 3],
|
||||
styles: [
|
||||
[i("static"), i("auto"), i("pointer")],
|
||||
[i("static"), i("auto"), i("pointer")],
|
||||
[i("static"), i("auto"), i("pointer")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[940, 16, 32, 32],
|
||||
[100, 120, 32, 32],
|
||||
],
|
||||
paintOrders: [0, 1, 2],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function nestedHoverTriggerSnapshotReply() {
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
"div",
|
||||
"img",
|
||||
"class",
|
||||
"tg-avatar",
|
||||
"tg-avatar__inner",
|
||||
"tg-avatar__image",
|
||||
"position",
|
||||
"static",
|
||||
"pointer-events",
|
||||
"auto",
|
||||
"cursor",
|
||||
"pointer",
|
||||
];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
return {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 2, 3],
|
||||
nodeType: [1, 1, 1, 1, 1],
|
||||
nodeName: [i("html"), i("body"), i("div"), i("div"), i("img")],
|
||||
backendNodeId: [10, 11, 12, 13, 14],
|
||||
attributes: [
|
||||
[],
|
||||
[],
|
||||
[i("class"), i("tg-avatar")],
|
||||
[i("class"), i("tg-avatar__inner")],
|
||||
[i("class"), i("tg-avatar__image")],
|
||||
],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2, 3, 4],
|
||||
styles: [
|
||||
[i("static"), i("auto"), i("pointer")],
|
||||
[i("static"), i("auto"), i("pointer")],
|
||||
[i("static"), i("auto"), i("pointer")],
|
||||
[i("static"), i("auto"), i("pointer")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[940, 16, 32, 32],
|
||||
[942, 18, 28, 28],
|
||||
[944, 20, 24, 24],
|
||||
],
|
||||
paintOrders: [0, 1, 2, 3],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function makeCdp(snapshot: unknown) {
|
||||
return {
|
||||
send: vi.fn(async (_tab: number, method: string) => {
|
||||
@@ -126,11 +276,11 @@ describe("captureViewModel", () => {
|
||||
});
|
||||
|
||||
it("runs hover surface probes when explicitly enabled", async () => {
|
||||
let runtimeCalls = 0;
|
||||
let hoverStateCalls = 0;
|
||||
const cdp = {
|
||||
send: vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return fakeSnapshotReply();
|
||||
if (method === "DOMSnapshot.captureSnapshot") return hoverTriggerSnapshotReply();
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
@@ -141,22 +291,23 @@ describe("captureViewModel", () => {
|
||||
if (expression.includes("input,textarea,select")) {
|
||||
return { result: { value: { controls: [], childFrames: [] } } };
|
||||
}
|
||||
runtimeCalls += 1;
|
||||
return runtimeCalls === 1
|
||||
? {
|
||||
result: {
|
||||
value: [
|
||||
{
|
||||
triggerSel: ".menu",
|
||||
affectedSub: " .items",
|
||||
label: "Products",
|
||||
x: 50,
|
||||
y: 20,
|
||||
},
|
||||
],
|
||||
},
|
||||
}
|
||||
: { result: { value: ["Shoes", "Bags"] } };
|
||||
if (expression.includes("document.styleSheets")) {
|
||||
return { result: { value: [] } };
|
||||
}
|
||||
if (expression.includes("querySelectorAll(selectors)")) {
|
||||
hoverStateCalls += 1;
|
||||
return hoverStateCalls === 1
|
||||
? { result: { value: [] } }
|
||||
: {
|
||||
result: {
|
||||
value: [
|
||||
{ text: "My profile", role: "", tag: "a", x: 900, y: 60 },
|
||||
{ text: "Sign out", role: "", tag: "a", x: 900, y: 90 },
|
||||
],
|
||||
},
|
||||
};
|
||||
}
|
||||
throw new Error(`unexpected Runtime.evaluate: ${expression.slice(0, 80)}`);
|
||||
}
|
||||
if (method === "Input.dispatchMouseEvent") return {};
|
||||
throw new Error(`unexpected ${method}`);
|
||||
@@ -166,12 +317,18 @@ describe("captureViewModel", () => {
|
||||
const result = await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
|
||||
|
||||
expect(result.surfaceProbes).toEqual([
|
||||
{ triggerLabel: "Products", triggerAction: "hover", subItems: ["Shoes", "Bags"] },
|
||||
{
|
||||
triggerBackendNodeId: 12,
|
||||
triggerPoint: { x: 956, y: 32 },
|
||||
triggerAction: "hover",
|
||||
subItems: ["My profile", "Sign out"],
|
||||
confidence: "high",
|
||||
},
|
||||
]);
|
||||
expect(cdp.send).toHaveBeenCalledWith(
|
||||
4,
|
||||
"Input.dispatchMouseEvent",
|
||||
expect.objectContaining({ type: "mouseMoved", x: 50, y: 20 }),
|
||||
expect.objectContaining({ type: "mouseMoved", x: 956, y: 32 }),
|
||||
);
|
||||
expect(cdp.send).toHaveBeenCalledWith(
|
||||
4,
|
||||
@@ -180,6 +337,91 @@ describe("captureViewModel", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("uses a fresh baseline for each hover candidate", async () => {
|
||||
let hoverStateCalls = 0;
|
||||
const cdp = {
|
||||
send: vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return twoHoverTriggerSnapshotReply();
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
};
|
||||
}
|
||||
if (method === "Runtime.evaluate") {
|
||||
const expression = (params as { expression?: string } | undefined)?.expression ?? "";
|
||||
if (expression.includes("input,textarea,select")) {
|
||||
return { result: { value: { controls: [], childFrames: [] } } };
|
||||
}
|
||||
if (expression.includes("document.styleSheets")) {
|
||||
return { result: { value: [] } };
|
||||
}
|
||||
if (expression.includes("querySelectorAll(selectors)")) {
|
||||
hoverStateCalls += 1;
|
||||
const visibleMenu = [
|
||||
{ text: "My profile", role: "", tag: "a", x: 900, y: 60 },
|
||||
{ text: "Sign out", role: "", tag: "a", x: 900, y: 90 },
|
||||
];
|
||||
return hoverStateCalls === 1
|
||||
? { result: { value: [] } }
|
||||
: { result: { value: visibleMenu } };
|
||||
}
|
||||
throw new Error(`unexpected Runtime.evaluate: ${expression.slice(0, 80)}`);
|
||||
}
|
||||
if (method === "Input.dispatchMouseEvent") return {};
|
||||
throw new Error(`unexpected ${method}`);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
|
||||
const result = await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
|
||||
|
||||
expect(result.surfaceProbes).toEqual([
|
||||
expect.objectContaining({
|
||||
triggerBackendNodeId: 12,
|
||||
subItems: ["My profile", "Sign out"],
|
||||
}),
|
||||
]);
|
||||
expect(hoverStateCalls).toBeGreaterThanOrEqual(4);
|
||||
});
|
||||
|
||||
it("deduplicates nested hover candidates for one visual trigger", async () => {
|
||||
let hoverMoves = 0;
|
||||
const cdp = {
|
||||
send: vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return nestedHoverTriggerSnapshotReply();
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
};
|
||||
}
|
||||
if (method === "Runtime.evaluate") {
|
||||
const expression = (params as { expression?: string } | undefined)?.expression ?? "";
|
||||
if (expression.includes("input,textarea,select")) {
|
||||
return { result: { value: { controls: [], childFrames: [] } } };
|
||||
}
|
||||
if (expression.includes("document.styleSheets")) {
|
||||
return { result: { value: [] } };
|
||||
}
|
||||
if (expression.includes("querySelectorAll(selectors)")) {
|
||||
return { result: { value: [] } };
|
||||
}
|
||||
throw new Error(`unexpected Runtime.evaluate: ${expression.slice(0, 80)}`);
|
||||
}
|
||||
if (method === "Input.dispatchMouseEvent") {
|
||||
const point = params as { x?: number; y?: number };
|
||||
if (point.x !== -10 && point.x !== 0) hoverMoves += 1;
|
||||
return {};
|
||||
}
|
||||
throw new Error(`unexpected ${method}`);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
|
||||
await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
|
||||
|
||||
expect(hoverMoves).toBe(1);
|
||||
});
|
||||
|
||||
it("excludes the agent's own overlay shadow host and its inlined shadow subtree", async () => {
|
||||
// DOMSnapshot inlines an open shadow root's content as descendants of the
|
||||
// host. The agent's WXT overlay host carries a fixed full-viewport
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
// `styles` columns are [position, pointer-events, cursor] in that order.
|
||||
|
||||
import type { Rect, Viewport } from "@browser-skill/vom";
|
||||
import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
|
||||
import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
|
||||
import type { CdpRunner } from "../shared";
|
||||
|
||||
@@ -45,9 +46,11 @@ export interface CapturedNode {
|
||||
export type CapturedIframeNodes = Map<number, CapturedNode[]>;
|
||||
|
||||
export interface CapturedSurfaceProbe {
|
||||
triggerLabel: string;
|
||||
triggerBackendNodeId: number;
|
||||
triggerPoint?: { x: number; y: number };
|
||||
triggerAction: "hover" | "focus" | string;
|
||||
subItems: string[];
|
||||
confidence?: "high" | "medium" | "low";
|
||||
}
|
||||
|
||||
export interface CapturedViewModel {
|
||||
@@ -61,6 +64,7 @@ export interface CapturedViewModel {
|
||||
|
||||
export interface CaptureViewModelOptions {
|
||||
conditionalSurfaceProbe?: boolean;
|
||||
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||
}
|
||||
|
||||
/** Sparse array format Chrome uses for infrequently-set per-node fields. */
|
||||
@@ -225,11 +229,12 @@ interface RuntimeEvaluateReply {
|
||||
}
|
||||
|
||||
interface HoverCandidate {
|
||||
triggerSel: string;
|
||||
affectedSub: string;
|
||||
label: string;
|
||||
backendNodeId: number;
|
||||
label?: string;
|
||||
x: number;
|
||||
y: number;
|
||||
score: number;
|
||||
reasons: string[];
|
||||
}
|
||||
|
||||
interface CdpDomNode {
|
||||
@@ -322,17 +327,19 @@ function collectBackendIdsFromDomNode(node: CdpDomNode | undefined, out: Set<num
|
||||
}
|
||||
|
||||
const MAX_HOVER_PROBE_MS = 2_000;
|
||||
const MAX_HOVER_TRIGGERS = 8;
|
||||
const HOVER_SETTLE_MS = 200;
|
||||
const MAX_HOVER_TRIGGERS = 6;
|
||||
const MAX_HOVER_SURFACES = 3;
|
||||
const HOVER_SETTLE_MS = 300;
|
||||
const MAX_HOVER_SUB_ITEMS = 12;
|
||||
|
||||
function runtimeValue<T>(reply: RuntimeEvaluateReply): T | undefined {
|
||||
return reply.result?.value as T | undefined;
|
||||
}
|
||||
|
||||
function hoverCssScanExpression(): string {
|
||||
function hoverCssTriggerScanExpression(): string {
|
||||
return `(() => {
|
||||
const visibilityProps = ["display", "visibility", "opacity", "maxHeight", "height", "overflow"];
|
||||
const pairs = [];
|
||||
const centres = [];
|
||||
const seenRules = new Set();
|
||||
for (const sheet of Array.from(document.styleSheets)) {
|
||||
let rules;
|
||||
@@ -348,90 +355,73 @@ function hoverCssScanExpression(): string {
|
||||
const hoverIndex = part.indexOf(":hover");
|
||||
if (hoverIndex < 0) continue;
|
||||
const triggerSel = part.slice(0, hoverIndex).trim();
|
||||
const affectedSub = part.slice(hoverIndex + 6).trim();
|
||||
if (!triggerSel) continue;
|
||||
const key = triggerSel + "||" + affectedSub;
|
||||
if (seenRules.has(key)) continue;
|
||||
seenRules.add(key);
|
||||
pairs.push({ triggerSel, affectedSub });
|
||||
if (seenRules.has(triggerSel)) continue;
|
||||
seenRules.add(triggerSel);
|
||||
let elements;
|
||||
try { elements = Array.from(document.querySelectorAll(triggerSel)); } catch { continue; }
|
||||
for (const el of elements) {
|
||||
if (!(el instanceof HTMLElement)) continue;
|
||||
const rect = el.getBoundingClientRect();
|
||||
const style = getComputedStyle(el);
|
||||
if (rect.width <= 0 || rect.height <= 0 || style.visibility === "hidden" || style.display === "none" || style.pointerEvents === "none") continue;
|
||||
centres.push({ x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 });
|
||||
if (centres.length >= 24) return centres;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const candidates = [];
|
||||
const seenLabels = new Set();
|
||||
const isUnsafeTrigger = (el) => {
|
||||
if (!(el instanceof HTMLElement)) return true;
|
||||
const tag = el.tagName.toLowerCase();
|
||||
if (["input", "textarea", "select", "option"].includes(tag)) return true;
|
||||
if (el.isContentEditable) return true;
|
||||
if (el.hasAttribute("disabled") || el.hasAttribute("inert")) return true;
|
||||
if ((el.getAttribute("aria-disabled") || "").toLowerCase() === "true") return true;
|
||||
return false;
|
||||
};
|
||||
for (const pair of pairs) {
|
||||
let elements;
|
||||
try { elements = Array.from(document.querySelectorAll(pair.triggerSel)); } catch { continue; }
|
||||
for (const el of elements) {
|
||||
if (!(el instanceof Element)) continue;
|
||||
if (isUnsafeTrigger(el)) continue;
|
||||
const rect = el.getBoundingClientRect();
|
||||
const style = getComputedStyle(el);
|
||||
if (rect.width <= 0 || rect.height <= 0 || style.visibility === "hidden" || style.display === "none" || style.pointerEvents === "none") continue;
|
||||
const label = (el.textContent || "").replace(/\\s+/g, " ").trim();
|
||||
if (!label || seenLabels.has(label)) continue;
|
||||
seenLabels.add(label);
|
||||
candidates.push({
|
||||
triggerSel: pair.triggerSel,
|
||||
affectedSub: pair.affectedSub,
|
||||
label,
|
||||
x: rect.left + rect.width / 2,
|
||||
y: rect.top + rect.height / 2,
|
||||
});
|
||||
if (candidates.length >= ${MAX_HOVER_TRIGGERS}) return candidates;
|
||||
}
|
||||
}
|
||||
return candidates;
|
||||
return centres;
|
||||
})()`;
|
||||
}
|
||||
|
||||
function hoverCollectExpression(candidate: HoverCandidate): string {
|
||||
return `(() => {
|
||||
const triggerSel = ${JSON.stringify(candidate.triggerSel)};
|
||||
const rawAffectedSub = ${JSON.stringify(candidate.affectedSub)};
|
||||
const x = ${JSON.stringify(candidate.x)};
|
||||
const y = ${JSON.stringify(candidate.y)};
|
||||
const hit = document.elementFromPoint(x, y);
|
||||
const trigger = hit instanceof Element ? hit.closest(triggerSel) : null;
|
||||
if (!trigger) return [];
|
||||
interface HoverRuntimeItem {
|
||||
text: string;
|
||||
role: string;
|
||||
tag: string;
|
||||
x: number;
|
||||
y: number;
|
||||
}
|
||||
|
||||
const affectedSub = rawAffectedSub.replace(/^[>+~]\\s*/, "").trim();
|
||||
let targets = [];
|
||||
if (!affectedSub) {
|
||||
targets = [trigger];
|
||||
} else {
|
||||
try { targets = Array.from(trigger.querySelectorAll(affectedSub)); } catch { targets = []; }
|
||||
}
|
||||
function hoverStateExpression(): string {
|
||||
return `(() => {
|
||||
const items = [];
|
||||
const seen = new Set();
|
||||
const push = (value) => {
|
||||
const text = String(value || "").replace(/\\s+/g, " ").trim();
|
||||
const selectors = [
|
||||
"a",
|
||||
"button",
|
||||
"[role='menuitem']",
|
||||
"[role='menuitemcheckbox']",
|
||||
"[role='menuitemradio']",
|
||||
"[role='option']",
|
||||
"[role='tab']",
|
||||
"[role='link']",
|
||||
"[role='button']"
|
||||
].join(",");
|
||||
const push = (el) => {
|
||||
if (!(el instanceof HTMLElement)) return;
|
||||
const style = getComputedStyle(el);
|
||||
const rect = el.getBoundingClientRect();
|
||||
if (rect.width <= 0 || rect.height <= 0) return;
|
||||
if (style.display === "none" || style.visibility === "hidden" || style.opacity === "0") return;
|
||||
const text = String(
|
||||
el.getAttribute("aria-label") ||
|
||||
el.getAttribute("title") ||
|
||||
el.textContent ||
|
||||
""
|
||||
).replace(/\\s+/g, " ").trim();
|
||||
if (!text || seen.has(text)) return;
|
||||
seen.add(text);
|
||||
items.push(text);
|
||||
items.push({
|
||||
text,
|
||||
role: (el.getAttribute("role") || "").toLowerCase(),
|
||||
tag: el.tagName.toLowerCase(),
|
||||
x: rect.left,
|
||||
y: rect.top,
|
||||
});
|
||||
};
|
||||
for (const target of targets) {
|
||||
if (!(target instanceof Element)) continue;
|
||||
const style = getComputedStyle(target);
|
||||
if (style.display === "none" || style.visibility === "hidden" || style.opacity === "0") continue;
|
||||
const clickable = target.querySelectorAll('a, button, [role="menuitem"], [role="option"]');
|
||||
if (clickable.length > 0) {
|
||||
for (const child of Array.from(clickable)) push(child.textContent);
|
||||
} else {
|
||||
push(target.textContent);
|
||||
}
|
||||
}
|
||||
return items.slice(0, 12);
|
||||
for (const el of Array.from(document.querySelectorAll(selectors))) push(el);
|
||||
return items.slice(0, 400);
|
||||
})()`;
|
||||
}
|
||||
|
||||
@@ -453,45 +443,217 @@ async function clearHover(cdp: CdpRunner, tabId: number): Promise<void> {
|
||||
);
|
||||
}
|
||||
|
||||
async function probeHoverSurfaces(cdp: CdpRunner, tabId: number): Promise<CapturedSurfaceProbe[]> {
|
||||
function capturedText(node: CapturedNode): string | undefined {
|
||||
const value =
|
||||
node.attrs["aria-label"] ?? node.attrs.title ?? node.attrs.alt ?? node.textContent ?? "";
|
||||
const clean = value.replace(/\s+/g, " ").trim();
|
||||
return clean || undefined;
|
||||
}
|
||||
|
||||
function hasGraphicDescendant(
|
||||
node: CapturedNode,
|
||||
childrenByParentId: Map<number, CapturedNode[]>,
|
||||
depth = 0,
|
||||
): boolean {
|
||||
if (depth > 3) return false;
|
||||
for (const child of childrenByParentId.get(node.backendNodeId) ?? []) {
|
||||
const tag = child.tag.toLowerCase();
|
||||
if (["img", "svg", "use", "path", "i"].includes(tag)) return true;
|
||||
if (hasGraphicDescendant(child, childrenByParentId, depth + 1)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function roleOf(node: CapturedNode): string {
|
||||
return (node.attrs.role ?? "").toLowerCase();
|
||||
}
|
||||
|
||||
function scoreHoverCandidate(
|
||||
node: CapturedNode,
|
||||
childrenByParentId: Map<number, CapturedNode[]>,
|
||||
cssHoverPoints: Array<{ x: number; y: number }>,
|
||||
): HoverCandidate | null {
|
||||
const rect = node.rect;
|
||||
if (!rect) return null;
|
||||
const label = capturedText(node);
|
||||
const cssHoverMatch = cssHoverPoints.some(
|
||||
(point) =>
|
||||
point.x >= rect.x &&
|
||||
point.x <= rect.x + rect.w &&
|
||||
point.y >= rect.y &&
|
||||
point.y <= rect.y + rect.h,
|
||||
);
|
||||
const decision = evaluateHoverTrigger({
|
||||
tag: node.tag,
|
||||
role: roleOf(node),
|
||||
label,
|
||||
attrs: node.attrs,
|
||||
rect,
|
||||
cursor: node.cursor,
|
||||
pointerEvents: node.pointerEvents,
|
||||
hasGraphicDescendant: hasGraphicDescendant(node, childrenByParentId),
|
||||
cssHoverMatch,
|
||||
});
|
||||
|
||||
if (!decision.eligible) return null;
|
||||
return {
|
||||
backendNodeId: node.backendNodeId,
|
||||
label,
|
||||
x: rect.x + rect.w / 2,
|
||||
y: rect.y + rect.h / 2,
|
||||
score: decision.score,
|
||||
reasons: decision.reasons,
|
||||
};
|
||||
}
|
||||
|
||||
function buildHoverCandidates(
|
||||
nodes: CapturedNode[],
|
||||
cssHoverPoints: Array<{ x: number; y: number }>,
|
||||
): HoverCandidate[] {
|
||||
const childrenByParentId = new Map<number, CapturedNode[]>();
|
||||
const parentByBackendId = new Map<number, number | null>();
|
||||
for (const node of nodes) {
|
||||
parentByBackendId.set(node.backendNodeId, node.parentBackendNodeId);
|
||||
if (node.parentBackendNodeId === null) continue;
|
||||
const children = childrenByParentId.get(node.parentBackendNodeId) ?? [];
|
||||
children.push(node);
|
||||
childrenByParentId.set(node.parentBackendNodeId, children);
|
||||
}
|
||||
|
||||
const candidates = nodes
|
||||
.map((node) => scoreHoverCandidate(node, childrenByParentId, cssHoverPoints))
|
||||
.filter((candidate): candidate is HoverCandidate => candidate !== null)
|
||||
.sort((a, b) => b.score - a.score);
|
||||
|
||||
const deduped: HoverCandidate[] = [];
|
||||
const seen = new Set<number>();
|
||||
for (const candidate of candidates) {
|
||||
if (seen.has(candidate.backendNodeId)) continue;
|
||||
if (deduped.some((existing) => sameHoverCluster(existing, candidate, parentByBackendId))) {
|
||||
continue;
|
||||
}
|
||||
seen.add(candidate.backendNodeId);
|
||||
deduped.push(candidate);
|
||||
if (deduped.length >= MAX_HOVER_TRIGGERS) break;
|
||||
}
|
||||
return deduped;
|
||||
}
|
||||
|
||||
function sameHoverCluster(
|
||||
a: HoverCandidate,
|
||||
b: HoverCandidate,
|
||||
parentByBackendId: Map<number, number | null>,
|
||||
): boolean {
|
||||
if (Math.hypot(a.x - b.x, a.y - b.y) <= 8) return true;
|
||||
return (
|
||||
isBackendAncestor(a.backendNodeId, b.backendNodeId, parentByBackendId) ||
|
||||
isBackendAncestor(b.backendNodeId, a.backendNodeId, parentByBackendId)
|
||||
);
|
||||
}
|
||||
|
||||
function isBackendAncestor(
|
||||
ancestorId: number,
|
||||
nodeId: number,
|
||||
parentByBackendId: Map<number, number | null>,
|
||||
): boolean {
|
||||
let parentId = parentByBackendId.get(nodeId);
|
||||
let guard = 0;
|
||||
while (parentId !== null && parentId !== undefined && guard < parentByBackendId.size) {
|
||||
if (parentId === ancestorId) return true;
|
||||
parentId = parentByBackendId.get(parentId);
|
||||
guard += 1;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function diffHoverItems(before: HoverRuntimeItem[], after: HoverRuntimeItem[]): string[] {
|
||||
const beforeKeys = new Set(before.map((item) => item.text.toLowerCase()));
|
||||
const out: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const item of after) {
|
||||
const text = item.text.replace(/\s+/g, " ").trim();
|
||||
const key = text.toLowerCase();
|
||||
if (!text || beforeKeys.has(key) || seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
out.push(text);
|
||||
if (out.length >= MAX_HOVER_SUB_ITEMS) break;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function confidenceForHover(
|
||||
candidate: HoverCandidate,
|
||||
subItems: string[],
|
||||
): "high" | "medium" | "low" {
|
||||
if (subItems.length >= 2 && candidate.score >= 80) return "high";
|
||||
if (subItems.length >= 2 || candidate.score >= 80) return "medium";
|
||||
return "low";
|
||||
}
|
||||
|
||||
async function probeHoverSurfaces(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
nodes: CapturedNode[],
|
||||
options: CaptureViewModelOptions,
|
||||
): Promise<CapturedSurfaceProbe[]> {
|
||||
const started = Date.now();
|
||||
try {
|
||||
const scan = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: hoverCssScanExpression(),
|
||||
const cssScan = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: hoverCssTriggerScanExpression(),
|
||||
returnByValue: true,
|
||||
});
|
||||
const candidates = runtimeValue<HoverCandidate[]>(scan) ?? [];
|
||||
const cssHoverPoints = runtimeValue<Array<{ x: number; y: number }>>(cssScan) ?? [];
|
||||
const candidates = buildHoverCandidates(nodes, cssHoverPoints);
|
||||
if (candidates.length === 0) return [];
|
||||
|
||||
const results: CapturedSurfaceProbe[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const candidate of candidates.slice(0, MAX_HOVER_TRIGGERS)) {
|
||||
if (Date.now() - started > MAX_HOVER_PROBE_MS) break;
|
||||
if (!candidate.label || seen.has(candidate.label)) continue;
|
||||
try {
|
||||
await cdp.send(tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: candidate.x,
|
||||
y: candidate.y,
|
||||
});
|
||||
await wait(HOVER_SETTLE_MS);
|
||||
const collected = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: hoverCollectExpression(candidate),
|
||||
returnByValue: true,
|
||||
});
|
||||
const subItems = (runtimeValue<string[]>(collected) ?? [])
|
||||
.map((item) => item.replace(/\s+/g, " ").trim())
|
||||
.filter(Boolean);
|
||||
if (subItems.length === 0) continue;
|
||||
seen.add(candidate.label);
|
||||
results.push({
|
||||
triggerLabel: candidate.label,
|
||||
triggerAction: "hover",
|
||||
subItems,
|
||||
});
|
||||
} catch {
|
||||
continue;
|
||||
} finally {
|
||||
await clearHover(cdp, tabId);
|
||||
const seen = new Set<number>();
|
||||
await options.hoverProbeBypassOverlay?.(tabId, true).catch(() => undefined);
|
||||
try {
|
||||
for (const candidate of candidates.slice(0, MAX_HOVER_TRIGGERS)) {
|
||||
if (Date.now() - started > MAX_HOVER_PROBE_MS) break;
|
||||
if (results.length >= MAX_HOVER_SURFACES) break;
|
||||
if (seen.has(candidate.backendNodeId)) continue;
|
||||
try {
|
||||
await clearHover(cdp, tabId);
|
||||
await wait(HOVER_SETTLE_MS);
|
||||
const baselineReply = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: hoverStateExpression(),
|
||||
returnByValue: true,
|
||||
});
|
||||
const baselineItems = runtimeValue<HoverRuntimeItem[]>(baselineReply) ?? [];
|
||||
|
||||
await cdp.send(tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: candidate.x,
|
||||
y: candidate.y,
|
||||
});
|
||||
await wait(HOVER_SETTLE_MS);
|
||||
const collected = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: hoverStateExpression(),
|
||||
returnByValue: true,
|
||||
});
|
||||
const subItems = diffHoverItems(
|
||||
baselineItems,
|
||||
runtimeValue<HoverRuntimeItem[]>(collected) ?? [],
|
||||
);
|
||||
if (subItems.length === 0) continue;
|
||||
seen.add(candidate.backendNodeId);
|
||||
results.push({
|
||||
triggerBackendNodeId: candidate.backendNodeId,
|
||||
triggerPoint: { x: candidate.x, y: candidate.y },
|
||||
triggerAction: "hover",
|
||||
subItems,
|
||||
confidence: confidenceForHover(candidate, subItems),
|
||||
});
|
||||
} catch {
|
||||
continue;
|
||||
} finally {
|
||||
await clearHover(cdp, tabId);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await options.hoverProbeBypassOverlay?.(tabId, false).catch(() => undefined);
|
||||
}
|
||||
return results;
|
||||
} catch (err) {
|
||||
@@ -797,7 +959,9 @@ export async function captureViewModel(
|
||||
|
||||
await enrichFormControlStates(cdp, tabId, [nodes, ...iframeNodes.values()]);
|
||||
|
||||
const surfaceProbes = options.conditionalSurfaceProbe ? await probeHoverSurfaces(cdp, tabId) : [];
|
||||
const surfaceProbes = options.conditionalSurfaceProbe
|
||||
? await probeHoverSurfaces(cdp, tabId, nodes, options)
|
||||
: [];
|
||||
|
||||
return { nodes, viewport, iframeNodes, surfaceProbes, excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user