mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat(extension): support OOPIF refs with unified frame geometry
This commit is contained in:
Ljy-0827
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { type CdpDebuggerApi, ChromiumCdp } from "../chromium-cdp";
|
||||
import { type CdpDebuggee, type CdpDebuggerApi, ChromiumCdp } from "../chromium-cdp";
|
||||
|
||||
function fakeChromeEvent<TArgs extends unknown[]>() {
|
||||
const listeners = new Set<(...args: TArgs) => void>();
|
||||
@@ -14,7 +14,7 @@ function fakeChromeEvent<TArgs extends unknown[]>() {
|
||||
}
|
||||
|
||||
function fakeApi() {
|
||||
const onEvent = fakeChromeEvent<[chrome.debugger.Debuggee, string, unknown]>();
|
||||
const onEvent = fakeChromeEvent<[CdpDebuggee, string, unknown]>();
|
||||
const onDetach = fakeChromeEvent<[chrome.debugger.Debuggee, string]>();
|
||||
const api: CdpDebuggerApi = {
|
||||
attach: vi.fn(async () => {}),
|
||||
@@ -29,6 +29,135 @@ function fakeApi() {
|
||||
}
|
||||
|
||||
describe("ChromiumCdp", () => {
|
||||
it("discovers multiple iframe targets and recursively routes nested OOPIF commands", async () => {
|
||||
const { api, onEvent } = fakeApi();
|
||||
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
|
||||
async (
|
||||
target: chrome.debugger.Debuggee & { sessionId?: string },
|
||||
method: string,
|
||||
params?: { frameId?: string },
|
||||
) => {
|
||||
if (method === "DOM.getFrameOwner") {
|
||||
return { backendNodeId: params?.frameId === "nested" ? 300 : 200 };
|
||||
}
|
||||
if (method !== "Page.getFrameTree") return {};
|
||||
if (target.sessionId === "right-session") {
|
||||
return {
|
||||
frameTree: {
|
||||
frame: { id: "right", url: "https://right.test" },
|
||||
childFrames: [
|
||||
{ frame: { id: "nested", parentId: "right", url: "https://nested.test" } },
|
||||
],
|
||||
},
|
||||
};
|
||||
}
|
||||
if (target.sessionId === "nested-session") {
|
||||
return { frameTree: { frame: { id: "nested", url: "https://nested.test" } } };
|
||||
}
|
||||
return {
|
||||
frameTree: {
|
||||
frame: { id: "main", url: "https://app.test" },
|
||||
childFrames: [
|
||||
{ frame: { id: "left", parentId: "main", url: "https://left.test" } },
|
||||
{
|
||||
frame: { id: "right", parentId: "main", url: "https://right.test" },
|
||||
childFrames: [
|
||||
{ frame: { id: "nested", parentId: "right", url: "https://nested.test" } },
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
},
|
||||
);
|
||||
const cdp = new ChromiumCdp(api);
|
||||
await cdp.ensureAttached(4);
|
||||
onEvent.fire({ tabId: 4 }, "Target.attachedToTarget", { sessionId: "right-session" });
|
||||
onEvent.fire({ tabId: 4 } as chrome.debugger.Debuggee, "Target.attachedToTarget", {
|
||||
sessionId: "nested-session",
|
||||
});
|
||||
|
||||
const graph = await cdp.getFrameGraph(4);
|
||||
|
||||
expect(graph.frames).toHaveLength(4);
|
||||
expect(graph.frames.find((frame) => frame.frameId === "left")?.target).toEqual({ tabId: 4 });
|
||||
expect(graph.frames.find((frame) => frame.frameId === "right")?.target.sessionId).toBe(
|
||||
"right-session",
|
||||
);
|
||||
expect(graph.frames.find((frame) => frame.frameId === "nested")?.target.sessionId).toBe(
|
||||
"nested-session",
|
||||
);
|
||||
expect(graph.frames.find((frame) => frame.frameId === "right")?.ownerBackendNodeId).toBe(200);
|
||||
expect(graph.frames.find((frame) => frame.frameId === "nested")?.ownerBackendNodeId).toBe(300);
|
||||
expect(api.sendCommand).toHaveBeenCalledWith(
|
||||
{ tabId: 4, sessionId: "right-session" },
|
||||
"DOM.getFrameOwner",
|
||||
{ frameId: "nested" },
|
||||
);
|
||||
expect(api.sendCommand).toHaveBeenCalledWith(
|
||||
{ tabId: 4, sessionId: "nested-session" },
|
||||
"Target.setAutoAttach",
|
||||
expect.objectContaining({ flatten: true }),
|
||||
);
|
||||
});
|
||||
|
||||
it("waits for recursively attached iframe targets without a depth limit", async () => {
|
||||
const { api, onEvent } = fakeApi();
|
||||
const depth = 6;
|
||||
const scheduled = new Set<string>();
|
||||
const nestedTree = (index: number): Record<string, unknown> => ({
|
||||
frame: {
|
||||
id: index === 0 ? "main" : `frame-${index}`,
|
||||
...(index > 0 ? { parentId: index === 1 ? "main" : `frame-${index - 1}` } : {}),
|
||||
},
|
||||
...(index < depth ? { childFrames: [nestedTree(index + 1)] } : {}),
|
||||
});
|
||||
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
|
||||
async (target: CdpDebuggee, method: string, params?: { frameId?: string }) => {
|
||||
if (method === "Target.setAutoAttach") {
|
||||
const parentIndex = target.sessionId
|
||||
? Number(target.sessionId.replace("session-", ""))
|
||||
: 0;
|
||||
const nextIndex = parentIndex + 1;
|
||||
const key = `${target.sessionId ?? "root"}:${nextIndex}`;
|
||||
if (nextIndex <= depth && !scheduled.has(key)) {
|
||||
scheduled.add(key);
|
||||
setTimeout(() => {
|
||||
onEvent.fire(
|
||||
{ tabId: 4, ...(target.sessionId ? { sessionId: target.sessionId } : {}) },
|
||||
"Target.attachedToTarget",
|
||||
{
|
||||
sessionId: `session-${nextIndex}`,
|
||||
targetInfo: { type: "iframe" },
|
||||
},
|
||||
);
|
||||
}, 5);
|
||||
}
|
||||
return {};
|
||||
}
|
||||
if (method === "Page.getFrameTree") {
|
||||
if (!target.sessionId) return { frameTree: nestedTree(0) };
|
||||
const index = Number(target.sessionId.replace("session-", ""));
|
||||
return { frameTree: nestedTree(index) };
|
||||
}
|
||||
if (method === "DOM.getFrameOwner") {
|
||||
return { backendNodeId: Number(params?.frameId?.replace("frame-", "")) + 100 };
|
||||
}
|
||||
return {};
|
||||
},
|
||||
);
|
||||
const cdp = new ChromiumCdp(api);
|
||||
|
||||
const graph = await cdp.getFrameGraph(4);
|
||||
|
||||
expect(graph.frames).toHaveLength(depth + 1);
|
||||
for (let index = 1; index <= depth; index += 1) {
|
||||
expect(
|
||||
graph.frames.find((frame) => frame.frameId === `frame-${index}`)?.target.sessionId,
|
||||
).toBe(`session-${index}`);
|
||||
}
|
||||
});
|
||||
|
||||
it("coalesces concurrent attach calls for the same tab", async () => {
|
||||
const { api } = fakeApi();
|
||||
let releaseAttach!: () => void;
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { buildFrameGraph, type CdpFrameTreeNode, type CdpFrameTreeSource } from "../frame-graph";
|
||||
|
||||
describe("buildFrameGraph", () => {
|
||||
it("keeps sibling frames distinct and routes nested OOPIFs to their child sessions", () => {
|
||||
const graph = buildFrameGraph([
|
||||
{
|
||||
target: { tabId: 4 },
|
||||
tree: {
|
||||
frame: { id: "main", url: "https://app.test" },
|
||||
childFrames: [
|
||||
{ frame: { id: "left", parentId: "main", url: "https://left.test" } },
|
||||
{
|
||||
frame: { id: "right", parentId: "main", url: "https://right.test" },
|
||||
childFrames: [
|
||||
{ frame: { id: "nested", parentId: "right", url: "https://nested.test" } },
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
target: { tabId: 4, sessionId: "right-session" },
|
||||
tree: {
|
||||
frame: { id: "right", url: "https://right.test" },
|
||||
childFrames: [{ frame: { id: "nested", parentId: "right", url: "https://nested.test" } }],
|
||||
},
|
||||
},
|
||||
{
|
||||
target: { tabId: 4, sessionId: "nested-session" },
|
||||
tree: { frame: { id: "nested", url: "https://nested.test" } },
|
||||
},
|
||||
]);
|
||||
|
||||
expect(graph?.frames).toHaveLength(4);
|
||||
expect(graph?.frames.find((frame) => frame.frameId === "left")?.target).toEqual({ tabId: 4 });
|
||||
expect(graph?.frames.find((frame) => frame.frameId === "right")).toMatchObject({
|
||||
parentFrameId: "main",
|
||||
target: { tabId: 4, sessionId: "right-session" },
|
||||
});
|
||||
expect(graph?.frames.find((frame) => frame.frameId === "nested")).toMatchObject({
|
||||
parentFrameId: "right",
|
||||
target: { tabId: 4, sessionId: "nested-session" },
|
||||
});
|
||||
});
|
||||
|
||||
it("propagates each target boundary to its same-process descendants", () => {
|
||||
const sources: CdpFrameTreeSource[] = [
|
||||
{
|
||||
target: { tabId: 4 },
|
||||
tree: {
|
||||
frame: { id: "main" },
|
||||
childFrames: [
|
||||
{
|
||||
frame: { id: "oopif-a", parentId: "main" },
|
||||
childFrames: [
|
||||
{
|
||||
frame: { id: "same-process-b", parentId: "oopif-a" },
|
||||
childFrames: [
|
||||
{
|
||||
frame: { id: "oopif-c", parentId: "same-process-b" },
|
||||
childFrames: [{ frame: { id: "same-process-d", parentId: "oopif-c" } }],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
target: { tabId: 4, sessionId: "session-a" },
|
||||
tree: {
|
||||
frame: { id: "oopif-a" },
|
||||
childFrames: [
|
||||
{
|
||||
frame: { id: "same-process-b", parentId: "oopif-a" },
|
||||
childFrames: [{ frame: { id: "oopif-c", parentId: "same-process-b" } }],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
target: { tabId: 4, sessionId: "session-c" },
|
||||
tree: {
|
||||
frame: { id: "oopif-c" },
|
||||
childFrames: [{ frame: { id: "same-process-d", parentId: "oopif-c" } }],
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const graph = buildFrameGraph(sources);
|
||||
|
||||
expect(graph?.frames.map((frame) => [frame.frameId, frame.target.sessionId])).toEqual([
|
||||
["main", undefined],
|
||||
["oopif-a", "session-a"],
|
||||
["same-process-b", "session-a"],
|
||||
["oopif-c", "session-c"],
|
||||
["same-process-d", "session-c"],
|
||||
]);
|
||||
expect(graph?.frames.find((frame) => frame.frameId === "oopif-a")?.parentFrameId).toBe("main");
|
||||
expect(graph?.frames.find((frame) => frame.frameId === "oopif-c")?.parentFrameId).toBe(
|
||||
"same-process-b",
|
||||
);
|
||||
|
||||
const reordered = buildFrameGraph([sources[2], sources[1], sources[0]]);
|
||||
expect(reordered).toEqual(graph);
|
||||
});
|
||||
|
||||
it("walks deeply nested frame trees without consuming the JavaScript call stack", () => {
|
||||
const depth = 5_000;
|
||||
const root: CdpFrameTreeNode = { frame: { id: "frame-0" } };
|
||||
let parent = root;
|
||||
for (let index = 1; index <= depth; index += 1) {
|
||||
const child: CdpFrameTreeNode = {
|
||||
frame: { id: `frame-${index}`, parentId: `frame-${index - 1}` },
|
||||
};
|
||||
parent.childFrames = [child];
|
||||
parent = child;
|
||||
}
|
||||
|
||||
const graph = buildFrameGraph([{ target: { tabId: 4 }, tree: root }]);
|
||||
|
||||
expect(graph?.frames).toHaveLength(depth + 1);
|
||||
expect(graph?.frames.at(-1)?.frameId).toBe(`frame-${depth}`);
|
||||
});
|
||||
|
||||
it("does not loop when an in-memory frame tree contains an object cycle", () => {
|
||||
const root: CdpFrameTreeNode = { frame: { id: "main" } };
|
||||
root.childFrames = [root];
|
||||
|
||||
expect(buildFrameGraph([{ target: { tabId: 4 }, tree: root }])?.frames).toEqual([
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -31,6 +31,15 @@ import type {
|
||||
NetworkEntryKind,
|
||||
NetworkResult,
|
||||
} from "@/transport/types";
|
||||
import {
|
||||
buildFrameGraph,
|
||||
type CdpFrameGraph,
|
||||
type CdpFrameTreeNode,
|
||||
type CdpFrameTreeSource,
|
||||
type CdpTarget,
|
||||
} from "./frame-graph";
|
||||
|
||||
export type CdpDebuggee = chrome.debugger.Debuggee & { sessionId?: string };
|
||||
|
||||
/**
|
||||
* Minimal slice of `chrome.debugger` the rest of the extension
|
||||
@@ -38,21 +47,15 @@ import type {
|
||||
* fake without monkey-patching the real `chrome` global.
|
||||
*/
|
||||
export interface CdpDebuggerApi {
|
||||
attach(target: chrome.debugger.Debuggee, requiredVersion: string): Promise<void>;
|
||||
detach(target: chrome.debugger.Debuggee): Promise<void>;
|
||||
sendCommand(
|
||||
target: chrome.debugger.Debuggee,
|
||||
method: string,
|
||||
commandParams?: object,
|
||||
): Promise<unknown>;
|
||||
attach(target: CdpDebuggee, requiredVersion: string): Promise<void>;
|
||||
detach(target: CdpDebuggee): Promise<void>;
|
||||
sendCommand(target: CdpDebuggee, method: string, commandParams?: object): Promise<unknown>;
|
||||
/**
|
||||
* Fires for every CDP event (`Page.lifecycleEvent`, `DOM.documentUpdated`,
|
||||
* …). The first callback argument is the source debuggee; the second
|
||||
* is the CDP method name; the third is the payload.
|
||||
*/
|
||||
onEvent: chrome.events.Event<
|
||||
(source: chrome.debugger.Debuggee, method: string, params: unknown) => void
|
||||
>;
|
||||
onEvent: chrome.events.Event<(source: CdpDebuggee, method: string, params: unknown) => void>;
|
||||
/**
|
||||
* Fires when Chrome unilaterally detaches us — most commonly because
|
||||
* the tab navigated to a chrome:// URL or the user clicked
|
||||
@@ -110,6 +113,8 @@ const MAX_CONSOLE_STACK_FRAMES = 20;
|
||||
const MAX_NETWORK_BUFFER = 200;
|
||||
const MAX_NETWORK_FIELD_LENGTH = 4096;
|
||||
const MAX_NETWORK_REQUEST_META = 1024;
|
||||
const FRAME_DISCOVERY_TIMEOUT_MS = 1000;
|
||||
const FRAME_DISCOVERY_QUIET_MS = 20;
|
||||
|
||||
interface ParsedDialogOpening {
|
||||
type: JavaScriptDialogType;
|
||||
@@ -131,6 +136,27 @@ interface NetworkRequestMeta {
|
||||
truncated: boolean;
|
||||
}
|
||||
|
||||
interface FrameDiscoveryState {
|
||||
sessions: Set<string>;
|
||||
pending: Set<Promise<void>>;
|
||||
generation: number;
|
||||
}
|
||||
|
||||
async function settleBeforeDeadline(promises: Promise<void>[], deadline: number): Promise<boolean> {
|
||||
if (promises.length === 0) return true;
|
||||
const remaining = deadline - Date.now();
|
||||
if (remaining <= 0) return false;
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
const timedOut = await Promise.race([
|
||||
Promise.allSettled(promises).then(() => false),
|
||||
new Promise<boolean>((resolve) => {
|
||||
timer = setTimeout(() => resolve(true), remaining);
|
||||
}),
|
||||
]);
|
||||
if (timer) clearTimeout(timer);
|
||||
return !timedOut;
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrapper around `chrome.debugger` that owns the "attach once per
|
||||
* tabId" cache and exposes typed `send<T>()`.
|
||||
@@ -149,10 +175,12 @@ export class ChromiumCdp {
|
||||
private readonly networkSequences = new Map<number, number>();
|
||||
private readonly networkDomainsEnabledTabs = new Set<number>();
|
||||
private readonly networkRequestMeta = new Map<number, Map<string, NetworkRequestMeta>>();
|
||||
private readonly frameDiscovery = new Map<number, FrameDiscoveryState>();
|
||||
private detachSubscription: { dispose(): void } | null = null;
|
||||
private dialogSubscription: { dispose(): void } | null = null;
|
||||
private consoleSubscription: { dispose(): void } | null = null;
|
||||
private networkSubscription: { dispose(): void } | null = null;
|
||||
private frameTargetSubscription: { dispose(): void } | null = null;
|
||||
|
||||
constructor(api: CdpDebuggerApi = chromeDebuggerApi) {
|
||||
this.api = api;
|
||||
@@ -160,6 +188,7 @@ export class ChromiumCdp {
|
||||
this.bindDialogHandler();
|
||||
this.bindConsoleHandler();
|
||||
this.bindNetworkHandler();
|
||||
this.bindFrameTargetHandler();
|
||||
}
|
||||
|
||||
/** Attach to `tabId` if we haven't already in this driver. */
|
||||
@@ -177,6 +206,9 @@ export class ChromiumCdp {
|
||||
await this.enableConsoleDomains(tabId);
|
||||
await this.enableNetworkDomainBestEffort(tabId);
|
||||
this.attachedTabs.add(tabId);
|
||||
await this.enableFrameDiscovery({ tabId }).catch((err) => {
|
||||
console.debug("[bsk cdp] frame discovery unavailable", { tabId, err });
|
||||
});
|
||||
} catch (err) {
|
||||
// A CDP domain enable failed after the raw attach succeeded
|
||||
// (e.g. `Page.enable` rejects because the tab just navigated to
|
||||
@@ -222,6 +254,73 @@ export class ChromiumCdp {
|
||||
}
|
||||
}
|
||||
|
||||
async sendToTarget<T = unknown>(target: CdpTarget, method: string, params?: object): Promise<T> {
|
||||
if (!this.attachedTabs.has(target.tabId)) {
|
||||
await this.ensureAttached(target.tabId);
|
||||
}
|
||||
try {
|
||||
return (await this.api.sendCommand(target, method, params ?? {})) as T;
|
||||
} catch (err) {
|
||||
throw normalizeError(err);
|
||||
}
|
||||
}
|
||||
|
||||
async getFrameGraph(tabId: number): Promise<CdpFrameGraph> {
|
||||
await this.ensureAttached(tabId);
|
||||
await this.enableFrameDiscovery({ tabId }).catch(() => {});
|
||||
await this.drainFrameAttachTasks(tabId);
|
||||
|
||||
const sources: CdpFrameTreeSource[] = [];
|
||||
const root = await this.sendToTarget<{ frameTree?: CdpFrameTreeNode }>(
|
||||
{ tabId },
|
||||
"Page.getFrameTree",
|
||||
{},
|
||||
);
|
||||
if (root.frameTree) sources.push({ target: { tabId }, tree: root.frameTree });
|
||||
|
||||
const sessions = [...(this.frameDiscovery.get(tabId)?.sessions ?? [])];
|
||||
const childTrees = await Promise.all(
|
||||
sessions.map(async (sessionId): Promise<CdpFrameTreeSource | null> => {
|
||||
const target = { tabId, sessionId };
|
||||
try {
|
||||
const reply = await this.sendToTarget<{ frameTree?: CdpFrameTreeNode }>(
|
||||
target,
|
||||
"Page.getFrameTree",
|
||||
{},
|
||||
);
|
||||
return reply.frameTree ? { target, tree: reply.frameTree } : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}),
|
||||
);
|
||||
for (const source of childTrees) {
|
||||
if (source) sources.push(source);
|
||||
}
|
||||
|
||||
const graph = buildFrameGraph(sources);
|
||||
if (!graph) throw new Error("Page.getFrameTree returned no root frame");
|
||||
const frameById = new Map(graph.frames.map((frame) => [frame.frameId, frame]));
|
||||
await Promise.all(
|
||||
graph.frames.map(async (frame) => {
|
||||
if (!frame.parentFrameId) return;
|
||||
const parent = frameById.get(frame.parentFrameId);
|
||||
if (!parent) return;
|
||||
try {
|
||||
const owner = await this.sendToTarget<{ backendNodeId?: number }>(
|
||||
parent.target,
|
||||
"DOM.getFrameOwner",
|
||||
{ frameId: frame.frameId },
|
||||
);
|
||||
if (owner.backendNodeId !== undefined) frame.ownerBackendNodeId = owner.backendNodeId;
|
||||
} catch {
|
||||
// The frame may have navigated between tree capture and owner lookup.
|
||||
}
|
||||
}),
|
||||
);
|
||||
return graph;
|
||||
}
|
||||
|
||||
/** Return a cursor marking the current dialog sequence for `tabId`. */
|
||||
dialogCursor(tabId: number): DialogCursor {
|
||||
return this.dialogSequences.get(tabId) ?? 0;
|
||||
@@ -332,6 +431,7 @@ export class ChromiumCdp {
|
||||
this.clearDialogState(tabId);
|
||||
this.clearConsoleState(tabId);
|
||||
this.clearNetworkState(tabId);
|
||||
this.clearFrameState(tabId);
|
||||
try {
|
||||
await this.api.detach({ tabId });
|
||||
} catch (err) {
|
||||
@@ -354,7 +454,7 @@ export class ChromiumCdp {
|
||||
}
|
||||
|
||||
/** Subscribe to all CDP events. Returned disposable removes the listener. */
|
||||
onEvent(handler: (source: chrome.debugger.Debuggee, method: string, params: unknown) => void): {
|
||||
onEvent(handler: (source: CdpDebuggee, method: string, params: unknown) => void): {
|
||||
dispose(): void;
|
||||
} {
|
||||
this.api.onEvent.addListener(handler);
|
||||
@@ -378,6 +478,7 @@ export class ChromiumCdp {
|
||||
this.networkSequences.clear();
|
||||
this.networkDomainsEnabledTabs.clear();
|
||||
this.networkRequestMeta.clear();
|
||||
this.frameDiscovery.clear();
|
||||
await Promise.all(
|
||||
tabs.map(async (tabId) => {
|
||||
try {
|
||||
@@ -428,9 +529,104 @@ export class ChromiumCdp {
|
||||
}
|
||||
}
|
||||
|
||||
private async enableFrameDiscovery(target: CdpTarget): Promise<void> {
|
||||
await this.api.sendCommand(target, "Target.setAutoAttach", {
|
||||
autoAttach: true,
|
||||
waitForDebuggerOnStart: false,
|
||||
flatten: true,
|
||||
filter: [{ type: "iframe", exclude: false }],
|
||||
});
|
||||
}
|
||||
|
||||
private bindFrameTargetHandler(): void {
|
||||
if (this.frameTargetSubscription) return;
|
||||
const listener = (source: CdpDebuggee, method: string, params: unknown) => {
|
||||
const tabId = source.tabId;
|
||||
if (typeof tabId !== "number") return;
|
||||
const raw = (params ?? {}) as Record<string, unknown>;
|
||||
const sessionId = typeof raw.sessionId === "string" ? raw.sessionId : undefined;
|
||||
if (!sessionId) return;
|
||||
|
||||
if (method === "Target.detachedFromTarget") {
|
||||
const state = this.frameDiscovery.get(tabId);
|
||||
if (state?.sessions.delete(sessionId)) state.generation += 1;
|
||||
return;
|
||||
}
|
||||
if (method !== "Target.attachedToTarget") return;
|
||||
const targetInfo = raw.targetInfo as { type?: string } | undefined;
|
||||
if (targetInfo?.type && targetInfo.type !== "iframe") return;
|
||||
|
||||
const state = this.frameDiscoveryState(tabId);
|
||||
if (state.sessions.has(sessionId)) return;
|
||||
state.sessions.add(sessionId);
|
||||
state.generation += 1;
|
||||
|
||||
const task = this.initializeFrameTarget({ tabId, sessionId });
|
||||
state.pending.add(task);
|
||||
void task.finally(() => {
|
||||
state.pending.delete(task);
|
||||
});
|
||||
};
|
||||
this.api.onEvent.addListener(listener);
|
||||
this.frameTargetSubscription = {
|
||||
dispose: () => this.api.onEvent.removeListener(listener),
|
||||
};
|
||||
}
|
||||
|
||||
private async initializeFrameTarget(target: CdpTarget): Promise<void> {
|
||||
try {
|
||||
await this.enableFrameDiscovery(target);
|
||||
} catch (err) {
|
||||
const state = this.frameDiscovery.get(target.tabId);
|
||||
if (state?.sessions.delete(target.sessionId as string)) state.generation += 1;
|
||||
console.debug("[bsk cdp] child frame target initialization failed", { target, err });
|
||||
}
|
||||
}
|
||||
|
||||
private async drainFrameAttachTasks(tabId: number): Promise<void> {
|
||||
const deadline = Date.now() + FRAME_DISCOVERY_TIMEOUT_MS;
|
||||
while (Date.now() < deadline) {
|
||||
const state = this.frameDiscovery.get(tabId);
|
||||
const generation = state?.generation ?? 0;
|
||||
if (state?.pending.size && !(await settleBeforeDeadline([...state.pending], deadline))) break;
|
||||
|
||||
// Target.setAutoAttach may enqueue the next attachedToTarget event after
|
||||
// its command promise settles. Wait for a short quiet window, then finish
|
||||
// only if neither the state object nor its generation changed.
|
||||
const quietTime = Math.min(FRAME_DISCOVERY_QUIET_MS, deadline - Date.now());
|
||||
if (quietTime <= 0) break;
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, quietTime));
|
||||
const current = this.frameDiscovery.get(tabId);
|
||||
if (
|
||||
current === state &&
|
||||
(current?.generation ?? 0) === generation &&
|
||||
!current?.pending.size
|
||||
) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
console.debug("[bsk cdp] frame discovery did not reach quiescence before timeout", { tabId });
|
||||
}
|
||||
|
||||
private frameDiscoveryState(tabId: number): FrameDiscoveryState {
|
||||
const existing = this.frameDiscovery.get(tabId);
|
||||
if (existing) return existing;
|
||||
const created: FrameDiscoveryState = {
|
||||
sessions: new Set(),
|
||||
pending: new Set(),
|
||||
generation: 0,
|
||||
};
|
||||
this.frameDiscovery.set(tabId, created);
|
||||
return created;
|
||||
}
|
||||
|
||||
private clearFrameState(tabId: number): void {
|
||||
this.frameDiscovery.delete(tabId);
|
||||
}
|
||||
|
||||
private bindDialogHandler(): void {
|
||||
if (this.dialogSubscription) return;
|
||||
const listener = (source: chrome.debugger.Debuggee, method: string, params: unknown) => {
|
||||
const listener = (source: CdpDebuggee, method: string, params: unknown) => {
|
||||
if (method !== "Page.javascriptDialogOpening") return;
|
||||
const tabId = source.tabId;
|
||||
if (typeof tabId !== "number") return;
|
||||
@@ -614,6 +810,7 @@ export class ChromiumCdp {
|
||||
this.clearDialogState(source.tabId);
|
||||
this.clearConsoleState(source.tabId);
|
||||
this.clearNetworkState(source.tabId);
|
||||
this.clearFrameState(source.tabId);
|
||||
}
|
||||
};
|
||||
this.api.onDetach.addListener(listener);
|
||||
@@ -632,6 +829,8 @@ export class ChromiumCdp {
|
||||
this.consoleSubscription = null;
|
||||
this.networkSubscription?.dispose();
|
||||
this.networkSubscription = null;
|
||||
this.frameTargetSubscription?.dispose();
|
||||
this.frameTargetSubscription = null;
|
||||
}
|
||||
|
||||
/** Detach tabs only when no other live session has claimed them. */
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
export interface CdpTarget {
|
||||
tabId: number;
|
||||
sessionId?: string;
|
||||
}
|
||||
|
||||
export interface CdpFrame {
|
||||
frameId: string;
|
||||
parentFrameId?: string;
|
||||
ownerBackendNodeId?: number;
|
||||
url?: string;
|
||||
target: CdpTarget;
|
||||
}
|
||||
|
||||
export interface CdpFrameGraph {
|
||||
rootFrameId: string;
|
||||
frames: CdpFrame[];
|
||||
}
|
||||
|
||||
export interface CdpFrameTreeNode {
|
||||
frame: {
|
||||
id: string;
|
||||
parentId?: string;
|
||||
url?: string;
|
||||
};
|
||||
childFrames?: CdpFrameTreeNode[];
|
||||
}
|
||||
|
||||
export interface CdpFrameTreeSource {
|
||||
target: CdpTarget;
|
||||
tree: CdpFrameTreeNode;
|
||||
}
|
||||
|
||||
export function cdpTargetKey(target: CdpTarget): string {
|
||||
return `${target.tabId}:${target.sessionId ?? "root"}`;
|
||||
}
|
||||
|
||||
function mergeFrame(
|
||||
frames: Map<string, CdpFrame>,
|
||||
order: string[],
|
||||
node: CdpFrameTreeNode,
|
||||
target: CdpTarget,
|
||||
): void {
|
||||
const existing = frames.get(node.frame.id);
|
||||
if (!existing) order.push(node.frame.id);
|
||||
const parentFrameId = node.frame.parentId || existing?.parentFrameId;
|
||||
const url = node.frame.url || existing?.url;
|
||||
frames.set(node.frame.id, {
|
||||
frameId: node.frame.id,
|
||||
...(parentFrameId ? { parentFrameId } : {}),
|
||||
...(url ? { url } : {}),
|
||||
target,
|
||||
});
|
||||
}
|
||||
|
||||
function walkFrameTree(
|
||||
source: CdpFrameTreeSource,
|
||||
targetByRootFrameId: ReadonlyMap<string, CdpTarget>,
|
||||
frames: Map<string, CdpFrame>,
|
||||
order: string[],
|
||||
): void {
|
||||
const stack: Array<{ node: CdpFrameTreeNode; inheritedTarget: CdpTarget }> = [
|
||||
{ node: source.tree, inheritedTarget: source.target },
|
||||
];
|
||||
const expanded = new WeakSet<CdpFrameTreeNode>();
|
||||
|
||||
while (stack.length > 0) {
|
||||
const current = stack.pop();
|
||||
if (!current || expanded.has(current.node)) continue;
|
||||
expanded.add(current.node);
|
||||
|
||||
const target = targetByRootFrameId.get(current.node.frame.id) ?? current.inheritedTarget;
|
||||
mergeFrame(frames, order, current.node, target);
|
||||
|
||||
const children = current.node.childFrames ?? [];
|
||||
for (let index = children.length - 1; index >= 0; index -= 1) {
|
||||
stack.push({ node: children[index], inheritedTarget: target });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function buildFrameGraph(sources: CdpFrameTreeSource[]): CdpFrameGraph | null {
|
||||
const rootSource = sources.find((source) => source.target.sessionId === undefined) ?? sources[0];
|
||||
if (!rootSource) return null;
|
||||
|
||||
const frames = new Map<string, CdpFrame>();
|
||||
const order: string[] = [];
|
||||
const targetByRootFrameId = new Map(
|
||||
sources.map((source) => [source.tree.frame.id, source.target] as const),
|
||||
);
|
||||
// The top-level source is authoritative if malformed input reports the same
|
||||
// frame as both a root target and a child target.
|
||||
targetByRootFrameId.set(rootSource.tree.frame.id, rootSource.target);
|
||||
|
||||
walkFrameTree(rootSource, targetByRootFrameId, frames, order);
|
||||
for (const source of sources) {
|
||||
if (source === rootSource) continue;
|
||||
walkFrameTree(source, targetByRootFrameId, frames, order);
|
||||
}
|
||||
|
||||
return {
|
||||
rootFrameId: rootSource.tree.frame.id,
|
||||
frames: order.flatMap((frameId) => {
|
||||
const frame = frames.get(frameId);
|
||||
return frame ? [frame] : [];
|
||||
}),
|
||||
};
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
useRef,
|
||||
useState,
|
||||
} from "react";
|
||||
import type { HelpHighlightRect } from "@/lib/help-bridge";
|
||||
import logoUrl from "../../assets/logo.png";
|
||||
|
||||
export interface HelpRequestData {
|
||||
@@ -19,6 +20,10 @@ export interface HelpRequestData {
|
||||
displayMode?: "full" | "compact";
|
||||
/** CSS selectors to scroll to + flash-highlight. */
|
||||
selectors: string[];
|
||||
/** Pre-resolved top-viewport rectangles, used for cross-frame targets. */
|
||||
rects?: HelpHighlightRect[];
|
||||
/** Re-resolve cross-frame rectangles after scroll, resize, or layout changes. */
|
||||
refreshRects?: () => Promise<HelpHighlightRect[] | undefined>;
|
||||
onContinue: (note: string) => void;
|
||||
onCancel: () => void;
|
||||
}
|
||||
@@ -49,6 +54,7 @@ const PANEL_WIDTH = 420;
|
||||
const FALLBACK_PANEL_H = 180;
|
||||
const FALLBACK_PANEL_H_COLLAPSED = 44;
|
||||
const EMPTY_SELECTORS: string[] = [];
|
||||
const EMPTY_RECTS: HelpHighlightRect[] = [];
|
||||
|
||||
function clamp(value: number, min: number, max: number): number {
|
||||
return Math.max(min, Math.min(max, value));
|
||||
@@ -179,6 +185,8 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
const { t } = useTranslation("extension");
|
||||
const isCompact = request?.displayMode === "compact";
|
||||
const effectiveSelectors = isCompact ? EMPTY_SELECTORS : (request?.selectors ?? EMPTY_SELECTORS);
|
||||
const [liveRects, setLiveRects] = useState<HelpHighlightRect[] | null>(null);
|
||||
const effectiveRects = isCompact ? EMPTY_RECTS : (liveRects ?? request?.rects ?? EMPTY_RECTS);
|
||||
const [note, setNote] = useState("");
|
||||
const [collapsed, setCollapsed] = useState(false);
|
||||
const [boxes, setBoxes] = useState<Box[]>([]);
|
||||
@@ -201,6 +209,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
// Reset the note and collapse state whenever a new request appears.
|
||||
useEffect(() => {
|
||||
setNote("");
|
||||
setLiveRects(null);
|
||||
setCollapsed(false);
|
||||
placementRef.current = null;
|
||||
setDragPos(null);
|
||||
@@ -209,6 +218,32 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
dragStartRef.current = null;
|
||||
}, [request?.id]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!request || isCompact || !request.refreshRects) return;
|
||||
let disposed = false;
|
||||
let resolving = false;
|
||||
const refresh = async () => {
|
||||
if (resolving) return;
|
||||
resolving = true;
|
||||
try {
|
||||
const rects = await request.refreshRects?.();
|
||||
if (!disposed && rects) setLiveRects(rects);
|
||||
} finally {
|
||||
resolving = false;
|
||||
}
|
||||
};
|
||||
const onViewportChange = () => void refresh();
|
||||
window.addEventListener("scroll", onViewportChange, true);
|
||||
window.addEventListener("resize", onViewportChange);
|
||||
const interval = window.setInterval(onViewportChange, 500);
|
||||
return () => {
|
||||
disposed = true;
|
||||
window.removeEventListener("scroll", onViewportChange, true);
|
||||
window.removeEventListener("resize", onViewportChange);
|
||||
window.clearInterval(interval);
|
||||
};
|
||||
}, [request, isCompact]);
|
||||
|
||||
// Scroll the first matched target into view once per distinct request id.
|
||||
useLayoutEffect(() => {
|
||||
if (!request) return;
|
||||
@@ -231,7 +266,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
return;
|
||||
}
|
||||
const update = () => {
|
||||
const nextBoxes = measure(effectiveSelectors);
|
||||
const nextBoxes = [...measure(effectiveSelectors), ...effectiveRects];
|
||||
setBoxes((prev) => (boxesEqual(prev, nextBoxes) ? prev : nextBoxes));
|
||||
};
|
||||
update();
|
||||
@@ -243,7 +278,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
window.removeEventListener("resize", update);
|
||||
window.clearInterval(interval);
|
||||
};
|
||||
}, [request, effectiveSelectors]);
|
||||
}, [request, effectiveSelectors, effectiveRects]);
|
||||
|
||||
const onHeaderPointerDown = useCallback(
|
||||
(e: ReactPointerEvent<HTMLDivElement>) => {
|
||||
@@ -305,7 +340,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
return;
|
||||
}
|
||||
if (userMovedRef.current) return;
|
||||
const measured = boxes.length > 0 ? boxes : measure(effectiveSelectors);
|
||||
const measured = boxes.length > 0 ? boxes : [...measure(effectiveSelectors), ...effectiveRects];
|
||||
const anchor = unionBox(measured);
|
||||
const banner = bannerRef.current;
|
||||
if (!anchor || !banner) {
|
||||
@@ -329,7 +364,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
}
|
||||
const nextPos = placePanel(anchor, panelW, panelH, placementRef.current.placement);
|
||||
setPanelPos((prev) => (panelPosEqual(prev, nextPos) ? prev : nextPos));
|
||||
}, [request, boxes, collapsed, isCompact, effectiveSelectors]);
|
||||
}, [request, boxes, collapsed, isCompact, effectiveSelectors, effectiveRects]);
|
||||
|
||||
if (!request) return null;
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { i18n } from "@browser-skill/i18n";
|
||||
import { I18nextProvider } from "@browser-skill/i18n/react";
|
||||
import { cleanup, fireEvent, render, screen } from "@testing-library/react";
|
||||
import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react";
|
||||
import { createElement } from "react";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { type HelpRequestData, HelpRequestOverlay } from "../HelpRequestOverlay";
|
||||
@@ -38,6 +38,36 @@ describe("HelpRequestOverlay", () => {
|
||||
expect(screen.getByText("Please complete the captcha")).toBeTruthy();
|
||||
});
|
||||
|
||||
it("renders explicit viewport rectangles for cross-frame targets", () => {
|
||||
const { container } = renderOverlay(
|
||||
baseRequest({ rects: [{ top: 30, left: 40, width: 120, height: 50 }] }),
|
||||
);
|
||||
const highlight = container.querySelector<HTMLElement>("[data-slot='help-highlight']");
|
||||
expect(highlight?.style.top).toBe("30px");
|
||||
expect(highlight?.style.left).toBe("40px");
|
||||
expect(highlight?.style.width).toBe("120px");
|
||||
expect(highlight?.style.height).toBe("50px");
|
||||
});
|
||||
|
||||
it("re-resolves cross-frame rectangles after the viewport changes", async () => {
|
||||
const refreshRects = vi.fn(async () => [{ top: 80, left: 90, width: 140, height: 60 }]);
|
||||
const { container } = renderOverlay(
|
||||
baseRequest({
|
||||
rects: [{ top: 30, left: 40, width: 120, height: 50 }],
|
||||
refreshRects,
|
||||
}),
|
||||
);
|
||||
|
||||
window.dispatchEvent(new Event("scroll"));
|
||||
|
||||
await waitFor(() => {
|
||||
const highlight = container.querySelector<HTMLElement>("[data-slot='help-highlight']");
|
||||
expect(highlight?.style.top).toBe("80px");
|
||||
expect(highlight?.style.left).toBe("90px");
|
||||
});
|
||||
expect(refreshRects).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("keeps the inactive render stable", () => {
|
||||
const { container, rerender } = renderOverlay(null);
|
||||
|
||||
|
||||
@@ -350,6 +350,7 @@ export default defineContentScript({
|
||||
...(helpMsg.title ? { title: helpMsg.title } : {}),
|
||||
...(helpMsg.displayMode ? { displayMode: helpMsg.displayMode } : {}),
|
||||
selectors: helpMsg.selectors,
|
||||
rects: helpMsg.rects,
|
||||
onContinue: (note: string) =>
|
||||
void sendHelpFinish(helpMsg.requestId, "continued", note.trim() ? note : undefined),
|
||||
onCancel: () => void sendHelpFinish(helpMsg.requestId, "cancelled"),
|
||||
@@ -413,6 +414,15 @@ export default defineContentScript({
|
||||
...(helpMsg.title ? { title: helpMsg.title } : {}),
|
||||
...(helpMsg.displayMode ? { displayMode: helpMsg.displayMode } : {}),
|
||||
selectors: helpMsg.selectors,
|
||||
rects: helpMsg.rects,
|
||||
refreshRects: async () => {
|
||||
const response = (await chrome.runtime.sendMessage({
|
||||
type: HELP_QUERY,
|
||||
})) as HelpQueryResponse | undefined;
|
||||
return response?.active && response.request?.requestId === helpMsg.requestId
|
||||
? response.request.rects
|
||||
: undefined;
|
||||
},
|
||||
onContinue: (note: string) =>
|
||||
void sendHelpFinish(helpMsg.requestId, "continued", note.trim() ? note : undefined),
|
||||
onCancel: () => void sendHelpFinish(helpMsg.requestId, "cancelled"),
|
||||
|
||||
@@ -32,6 +32,7 @@ describe("help-bridge", () => {
|
||||
prompt: "log in",
|
||||
displayMode: "compact",
|
||||
selectors: ["#login"],
|
||||
rects: [{ top: 10, left: 20, width: 100, height: 40 }],
|
||||
timeoutMs: 1000,
|
||||
}),
|
||||
).toBe(true);
|
||||
|
||||
@@ -16,6 +16,13 @@ export const HELP_ACK = "bsk-help-ack";
|
||||
export const HELP_QUERY = "bsk-help-query";
|
||||
export const HELP_FINISH = "bsk-help-finish";
|
||||
|
||||
export interface HelpHighlightRect {
|
||||
top: number;
|
||||
left: number;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
export interface HelpRequestMessage {
|
||||
type: typeof HELP_REQUEST;
|
||||
requestId: string;
|
||||
@@ -26,6 +33,8 @@ export interface HelpRequestMessage {
|
||||
displayMode?: "full" | "compact";
|
||||
/** CSS selectors to scroll to + flash-highlight (may be empty). */
|
||||
selectors: string[];
|
||||
/** Top-viewport rectangles for targets that cannot be represented by a root selector. */
|
||||
rects?: HelpHighlightRect[];
|
||||
timeoutMs: number;
|
||||
}
|
||||
|
||||
@@ -74,6 +83,16 @@ export function isHelpRequestMessage(msg: unknown): msg is HelpRequestMessage {
|
||||
(m.displayMode === undefined || m.displayMode === "full" || m.displayMode === "compact") &&
|
||||
Array.isArray(m.selectors) &&
|
||||
m.selectors.every((selector) => typeof selector === "string") &&
|
||||
(m.rects === undefined ||
|
||||
(Array.isArray(m.rects) &&
|
||||
m.rects.every(
|
||||
(rect) =>
|
||||
typeof rect === "object" &&
|
||||
rect !== null &&
|
||||
["top", "left", "width", "height"].every((key) =>
|
||||
Number.isFinite((rect as Record<string, unknown>)[key]),
|
||||
),
|
||||
))) &&
|
||||
typeof m.timeoutMs === "number"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -33,4 +33,19 @@ describe("RefStore", () => {
|
||||
expect(s.resolve("@e11")).toBe(11);
|
||||
expect(s.size()).toBe(2);
|
||||
});
|
||||
|
||||
it("preserves the child CDP session as part of ref identity", () => {
|
||||
const s = new RefStore();
|
||||
s.set("e1", 42, {
|
||||
tabId: 7,
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
});
|
||||
expect(s.resolveEntry("e1")).toMatchObject({
|
||||
backendNodeId: 42,
|
||||
tabId: 7,
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
/**
|
||||
* Per-session map from `@e<N>` snapshot refs to CDP `backendNodeId`.
|
||||
* Per-session map from `@e<N>` snapshot refs to a CDP node address.
|
||||
*
|
||||
* Each fresh `tool.snapshot` resets the store: M6 will call
|
||||
* `replace(...)` with the new ref → backendNodeId pairs. Tools like
|
||||
* `tool.click` consume the store via `resolve("@e1")`.
|
||||
* `replace(...)` with the new ref → node address pairs. A node address
|
||||
* includes the owning flat CDP session/frame when the element lives in
|
||||
* an OOPIF; tools resolve live geometry from that identity at call time.
|
||||
*
|
||||
* Refs are session-scoped (§7): looking up a ref in the wrong session
|
||||
* returns `null`, never silently leaks. Storing values in different
|
||||
@@ -15,10 +16,19 @@ export type BackendNodeId = number;
|
||||
export interface RefEntry {
|
||||
backendNodeId: BackendNodeId;
|
||||
tabId: number | null;
|
||||
frameId?: string;
|
||||
cdpSessionId?: string;
|
||||
generation: number;
|
||||
}
|
||||
|
||||
type RefInput = BackendNodeId | { backendNodeId: BackendNodeId; tabId: number };
|
||||
export type RefInput =
|
||||
| BackendNodeId
|
||||
| {
|
||||
backendNodeId: BackendNodeId;
|
||||
tabId: number;
|
||||
frameId?: string;
|
||||
cdpSessionId?: string;
|
||||
};
|
||||
|
||||
export class RefStore {
|
||||
private readonly map = new Map<string, RefEntry>();
|
||||
@@ -44,7 +54,7 @@ export class RefStore {
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace the entire store with a new ref → backendNodeId mapping.
|
||||
* Replace the entire store with a new ref → CDP node identity mapping.
|
||||
* Used after every fresh `tool.snapshot`.
|
||||
*/
|
||||
replace(entries: Iterable<readonly [string, RefInput]>): void {
|
||||
@@ -53,10 +63,20 @@ export class RefStore {
|
||||
for (const [ref, input] of entries) this.map.set(normaliseRef(ref), this.entry(input));
|
||||
}
|
||||
|
||||
set(ref: string, id: BackendNodeId, opts: { tabId?: number } = {}): void {
|
||||
set(
|
||||
ref: string,
|
||||
id: BackendNodeId,
|
||||
opts: {
|
||||
tabId?: number;
|
||||
frameId?: string;
|
||||
cdpSessionId?: string;
|
||||
} = {},
|
||||
): void {
|
||||
this.map.set(normaliseRef(ref), {
|
||||
backendNodeId: id,
|
||||
tabId: opts.tabId ?? null,
|
||||
...(opts.frameId ? { frameId: opts.frameId } : {}),
|
||||
...(opts.cdpSessionId ? { cdpSessionId: opts.cdpSessionId } : {}),
|
||||
generation: this.generation,
|
||||
});
|
||||
}
|
||||
@@ -80,6 +100,8 @@ export class RefStore {
|
||||
return {
|
||||
backendNodeId: input.backendNodeId,
|
||||
tabId: input.tabId,
|
||||
...(input.frameId ? { frameId: input.frameId } : {}),
|
||||
...(input.cdpSessionId ? { cdpSessionId: input.cdpSessionId } : {}),
|
||||
generation: this.generation,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -19,6 +19,25 @@ describe("quadBoundingRect", () => {
|
||||
});
|
||||
|
||||
describe("nodeBoundingRect", () => {
|
||||
it("unions every visible content fragment", async () => {
|
||||
const send = async () => ({
|
||||
quads: [
|
||||
[10, 20, 60, 20, 60, 40, 10, 40],
|
||||
[15, 50, 90, 50, 90, 80, 15, 80],
|
||||
],
|
||||
});
|
||||
const cdp: CdpRunner = {
|
||||
send: send as CdpRunner["send"],
|
||||
};
|
||||
|
||||
await expect(nodeBoundingRect(cdp, 7, 555)).resolves.toEqual({
|
||||
x: 10,
|
||||
y: 20,
|
||||
width: 80,
|
||||
height: 60,
|
||||
});
|
||||
});
|
||||
|
||||
it("falls back to visible descendant bounds for zero-size containers", async () => {
|
||||
const calls: Array<{ method: string; params?: object }> = [];
|
||||
const send = async (_tabId: number, method: string, params?: object) => {
|
||||
|
||||
@@ -0,0 +1,222 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { type Quad, resolveFrameProjection, resolveNodeGeometry } from "../frame-geometry";
|
||||
import {
|
||||
clipPolygon,
|
||||
polygonBounds,
|
||||
projectAndClipRegion,
|
||||
projectUnitPoint,
|
||||
rectPolygon,
|
||||
} from "../geometry";
|
||||
import type { CdpRunner } from "../shared";
|
||||
|
||||
describe("frame geometry projection", () => {
|
||||
it("maps through the iframe content box instead of its border box", () => {
|
||||
const contentQuad: Quad = [
|
||||
{ x: 204, y: 306 },
|
||||
{ x: 604, y: 306 },
|
||||
{ x: 604, y: 506 },
|
||||
{ x: 204, y: 506 },
|
||||
];
|
||||
const projected = projectAndClipRegion(
|
||||
[rectPolygon({ x: 10, y: 20, w: 100, h: 40 })],
|
||||
[{ sourceViewport: { width: 200, height: 100 }, destinationQuad: contentQuad }],
|
||||
{ width: 1000, height: 800 },
|
||||
);
|
||||
expect(polygonBounds(projected.flat())).toEqual({ x: 224, y: 346, width: 200, height: 80 });
|
||||
});
|
||||
|
||||
it("uses a projective mapping for perspective-transformed iframe quads", () => {
|
||||
const quad: Quad = [
|
||||
{ x: 0, y: 0 },
|
||||
{ x: 200, y: 20 },
|
||||
{ x: 180, y: 140 },
|
||||
{ x: 20, y: 100 },
|
||||
];
|
||||
expect(projectUnitPoint({ x: 0, y: 0 }, quad)).toEqual(quad[0]);
|
||||
expect(projectUnitPoint({ x: 1, y: 0 }, quad)).toEqual(quad[1]);
|
||||
expect(projectUnitPoint({ x: 1, y: 1 }, quad)).toEqual(quad[2]);
|
||||
expect(projectUnitPoint({ x: 0, y: 1 }, quad)).toEqual(quad[3]);
|
||||
});
|
||||
|
||||
it("clips against the iframe polygon rather than its bounding box", () => {
|
||||
const frameQuad: Quad = [
|
||||
{ x: 50, y: 0 },
|
||||
{ x: 100, y: 50 },
|
||||
{ x: 50, y: 100 },
|
||||
{ x: 0, y: 50 },
|
||||
];
|
||||
expect(clipPolygon(rectPolygon({ x: 0, y: 0, w: 10, h: 10 }), frameQuad)).toEqual([]);
|
||||
});
|
||||
|
||||
it("does not reapply same-target frame transforms around an OOPIF", async () => {
|
||||
const send = vi.fn(async (_tabId, method, params) => {
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
|
||||
}
|
||||
if (method === "DOM.getBoxModel") {
|
||||
const backendNodeId = (params as { backendNodeId?: number }).backendNodeId;
|
||||
return backendNodeId === 20
|
||||
? { model: { content: [110, 70, 310, 70, 310, 170, 110, 170] } }
|
||||
: { model: { content: [100, 50, 400, 50, 400, 250, 100, 250] } };
|
||||
}
|
||||
throw new Error(`unexpected root command ${method}`);
|
||||
});
|
||||
const cdp: CdpRunner = {
|
||||
send: send as CdpRunner["send"],
|
||||
sendToTarget: vi.fn(async (_target, method) => {
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
||||
}
|
||||
throw new Error(`unexpected child command ${method}`);
|
||||
}) as CdpRunner["sendToTarget"],
|
||||
};
|
||||
const projection = await resolveFrameProjection(
|
||||
cdp,
|
||||
{
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "same-process-parent",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4 },
|
||||
},
|
||||
{
|
||||
frameId: "oopif",
|
||||
parentFrameId: "same-process-parent",
|
||||
ownerBackendNodeId: 20,
|
||||
target: { tabId: 4, sessionId: "oopif-session" },
|
||||
},
|
||||
],
|
||||
},
|
||||
"oopif",
|
||||
);
|
||||
|
||||
expect(projection?.edges).toEqual([
|
||||
{
|
||||
sourceViewport: { width: 200, height: 100 },
|
||||
destinationQuad: [
|
||||
{ x: 110, y: 70 },
|
||||
{ x: 310, y: 70 },
|
||||
{ x: 310, y: 170 },
|
||||
{ x: 110, y: 170 },
|
||||
],
|
||||
destinationClips: [
|
||||
[
|
||||
{ x: 100, y: 50 },
|
||||
{ x: 400, y: 50 },
|
||||
{ x: 400, y: 250 },
|
||||
{ x: 100, y: 250 },
|
||||
],
|
||||
],
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("clips live OOPIF geometry at every frame boundary", async () => {
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId, method) => {
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
|
||||
}
|
||||
if (method === "DOM.getBoxModel") {
|
||||
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } };
|
||||
}
|
||||
throw new Error(`unexpected root command ${method}`);
|
||||
}) as CdpRunner["send"],
|
||||
sendToTarget: vi.fn(async (_target, method) => {
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
||||
}
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[-50, -20, 250, -20, 250, 120, -50, 120]] };
|
||||
}
|
||||
throw new Error(`unexpected child command ${method}`);
|
||||
}) as CdpRunner["sendToTarget"],
|
||||
getFrameGraph: vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
})),
|
||||
};
|
||||
|
||||
const geometry = await resolveNodeGeometry(cdp, 4, {
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
frameId: "child",
|
||||
backendNodeId: 101,
|
||||
});
|
||||
|
||||
expect(geometry).toMatchObject({
|
||||
topBounds: { x: 100, y: 100, width: 200, height: 100 },
|
||||
actionPoint: { x: 200, y: 150 },
|
||||
});
|
||||
});
|
||||
|
||||
it("clips live geometry to same-target iframe ancestors", async () => {
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId, method) => {
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
|
||||
}
|
||||
if (method === "DOM.getBoxModel") {
|
||||
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } };
|
||||
}
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[50, 50, 350, 50, 350, 250, 50, 250]] };
|
||||
}
|
||||
throw new Error(`unexpected ${method}`);
|
||||
}) as CdpRunner["send"],
|
||||
getFrameGraph: vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4 },
|
||||
},
|
||||
],
|
||||
})),
|
||||
};
|
||||
|
||||
const geometry = await resolveNodeGeometry(cdp, 4, {
|
||||
target: { tabId: 4 },
|
||||
frameId: "child",
|
||||
backendNodeId: 101,
|
||||
});
|
||||
|
||||
expect(geometry).toMatchObject({
|
||||
topBounds: { x: 100, y: 100, width: 200, height: 100 },
|
||||
actionPoint: { x: 200, y: 150 },
|
||||
});
|
||||
});
|
||||
|
||||
it("fails closed when an OOPIF frame graph is unavailable", async () => {
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(),
|
||||
sendToTarget: vi.fn(async (_target, method) => {
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] };
|
||||
}
|
||||
throw new Error(`unexpected ${method}`);
|
||||
}) as CdpRunner["sendToTarget"],
|
||||
};
|
||||
|
||||
await expect(
|
||||
resolveNodeGeometry(cdp, 4, {
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
frameId: "child",
|
||||
backendNodeId: 101,
|
||||
}),
|
||||
).resolves.toMatchObject({ code: "cdp_failed" });
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,6 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import { RefStore } from "@/session-manager/ref-store";
|
||||
import type { RequestHelpParams } from "@/transport/types";
|
||||
import { handleRequestHelp, type RequestHelpDeps, resetHelpLifecycleForTests } from "../human-loop";
|
||||
|
||||
@@ -50,11 +51,10 @@ function installHelpLifecycleChrome() {
|
||||
}
|
||||
|
||||
function fakeManager(sessionId: string, agentWindowId: number, tabId: number) {
|
||||
const refStore = new RefStore();
|
||||
const mgr = {
|
||||
get: (id: string) =>
|
||||
id === sessionId
|
||||
? { sessionId, agentWindowId, refStore: { resolve: () => null }, borrowedTabs: new Map() }
|
||||
: null,
|
||||
id === sessionId ? { sessionId, agentWindowId, refStore, borrowedTabs: new Map() } : null,
|
||||
findByWindowId: (wid: number) => (wid === agentWindowId ? { sessionId } : null),
|
||||
} as unknown as SessionManager;
|
||||
return mgr;
|
||||
@@ -339,26 +339,23 @@ describe("handleRequestHelp", () => {
|
||||
|
||||
ac.abort();
|
||||
await expect(pending).resolves.toMatchObject({ code: "cancelled" });
|
||||
expect(cdpSend).toHaveBeenCalledWith(
|
||||
5,
|
||||
"DOM.querySelectorAll",
|
||||
expect.objectContaining({ selector: "[data-bsk-help]" }),
|
||||
);
|
||||
expect(cdpSend).toHaveBeenCalledWith(
|
||||
6,
|
||||
"DOM.querySelectorAll",
|
||||
expect.objectContaining({ selector: "[data-bsk-help]" }),
|
||||
expect(cdpSend).not.toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
"DOM.removeAttribute",
|
||||
expect.anything(),
|
||||
);
|
||||
});
|
||||
|
||||
it("tags ref targets via CDP and reports them matched", async () => {
|
||||
it("resolves ref targets to explicit viewport rectangles without mutating the page", async () => {
|
||||
const refStore = new RefStore();
|
||||
refStore.set("e1", 42, { tabId: 5 });
|
||||
const mgr = {
|
||||
get: (id: string) =>
|
||||
id === "abcd"
|
||||
? {
|
||||
sessionId: "abcd",
|
||||
agentWindowId: 99,
|
||||
refStore: { resolve: () => 42 },
|
||||
refStore,
|
||||
borrowedTabs: new Map(),
|
||||
}
|
||||
: null,
|
||||
@@ -366,7 +363,17 @@ describe("handleRequestHelp", () => {
|
||||
} as unknown as SessionManager;
|
||||
const deps = baseDeps({
|
||||
cdp: {
|
||||
send: vi.fn(async () => ({ object: { objectId: "obj-1" } })),
|
||||
send: vi.fn(async (_tabId, method) => {
|
||||
if (method === "DOM.getDocument") return { root: { nodeId: 1 } };
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||
}
|
||||
throw new Error(`unexpected ${method}`);
|
||||
}),
|
||||
} as unknown as RequestHelpDeps["cdp"],
|
||||
});
|
||||
const res = await handleRequestHelp(
|
||||
@@ -375,14 +382,97 @@ describe("handleRequestHelp", () => {
|
||||
deps,
|
||||
);
|
||||
const sentMsg = (deps.sendToTab as ReturnType<typeof vi.fn>).mock.calls[0][1];
|
||||
expect(sentMsg.selectors).toContain('[data-bsk-help="0"]');
|
||||
expect(sentMsg.selectors).toEqual([]);
|
||||
expect(sentMsg.rects).toEqual([{ top: 20, left: 10, width: 100, height: 40 }]);
|
||||
expect(res).toMatchObject({
|
||||
outcome: "continued",
|
||||
tab_id: 5,
|
||||
resolved_targets: [{ matched: true, ref: "@e1" }],
|
||||
resolved_targets: [
|
||||
{
|
||||
matched: true,
|
||||
ref: "@e1",
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it("projects an OOPIF ref through its live content quad for the help overlay", async () => {
|
||||
const mgr = {
|
||||
get: (id: string) =>
|
||||
id === "abcd"
|
||||
? {
|
||||
sessionId: "abcd",
|
||||
agentWindowId: 99,
|
||||
refStore: {
|
||||
resolveEntry: () => ({
|
||||
backendNodeId: 42,
|
||||
tabId: 5,
|
||||
frameId: "child",
|
||||
cdpSessionId: "child-session",
|
||||
generation: 1,
|
||||
}),
|
||||
},
|
||||
borrowedTabs: new Map(),
|
||||
}
|
||||
: null,
|
||||
findByWindowId: (wid: number) => (wid === 99 ? { sessionId: "abcd" } : null),
|
||||
} as unknown as SessionManager;
|
||||
const send = vi.fn(async (_tabId, method) => {
|
||||
if (method === "DOM.getDocument") return { root: { nodeId: 1 } };
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "DOM.getBoxModel") {
|
||||
return { model: { content: [204, 306, 604, 306, 604, 506, 204, 506] } };
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||
}
|
||||
throw new Error(`unexpected root ${method}`);
|
||||
});
|
||||
const sendToTarget = vi.fn(async (_target, method) => {
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
||||
}
|
||||
throw new Error(`unexpected child ${method}`);
|
||||
});
|
||||
const deps = baseDeps({
|
||||
cdp: {
|
||||
send,
|
||||
sendToTarget,
|
||||
getFrameGraph: vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 5 } },
|
||||
{
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 99,
|
||||
target: { tabId: 5, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
})),
|
||||
} as unknown as NonNullable<RequestHelpDeps["cdp"]>,
|
||||
});
|
||||
|
||||
const res = await handleRequestHelp(
|
||||
mgr,
|
||||
baseParams({ tab_id: 5, targets: [{ ref: "@e1" }] }),
|
||||
deps,
|
||||
);
|
||||
|
||||
const sentMsg = (deps.sendToTab as ReturnType<typeof vi.fn>).mock.calls[0][1];
|
||||
expect(sentMsg.rects).toEqual([{ top: 346, left: 224, width: 200, height: 80 }]);
|
||||
expect(res).toMatchObject({ resolved_targets: [{ matched: true, ref: "@e1" }] });
|
||||
expect(sendToTarget).toHaveBeenCalledWith(
|
||||
{ tabId: 5, sessionId: "child-session" },
|
||||
"DOM.getContentQuads",
|
||||
{ backendNodeId: 42 },
|
||||
);
|
||||
});
|
||||
|
||||
it("reports ref target unmatched when ref does not resolve", async () => {
|
||||
const deps = baseDeps();
|
||||
const res = await handleRequestHelp(
|
||||
@@ -398,16 +488,15 @@ describe("handleRequestHelp", () => {
|
||||
});
|
||||
|
||||
it("reports ref target unmatched when ref is for another tab", async () => {
|
||||
const refStore = new RefStore();
|
||||
refStore.set("e1", 42, { tabId: 4 });
|
||||
const mgr = {
|
||||
get: (id: string) =>
|
||||
id === "abcd"
|
||||
? {
|
||||
sessionId: "abcd",
|
||||
agentWindowId: 99,
|
||||
refStore: {
|
||||
resolve: (ref: string, opts: { tabId?: number }) =>
|
||||
ref === "e1" && opts.tabId === 4 ? 42 : null,
|
||||
},
|
||||
refStore,
|
||||
borrowedTabs: new Map(),
|
||||
}
|
||||
: null,
|
||||
|
||||
@@ -30,6 +30,9 @@ function makeFakeCdp(handlers: Record<string, (params: unknown) => unknown>) {
|
||||
const sendImpl = async (tabId: number, method: string, params?: object) => {
|
||||
sent.push({ tabId, method, params });
|
||||
const h = handlers[method];
|
||||
if (!h && method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||
}
|
||||
if (!h) throw new Error(`unexpected CDP call ${method}`);
|
||||
return h(params);
|
||||
};
|
||||
@@ -164,6 +167,63 @@ describe("handleClick", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("resolves frame refs in their CDP session and dispatches input in top coordinates", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e3", 1234, {
|
||||
tabId: 4,
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
});
|
||||
const fake = makeFakeCdp({
|
||||
"Input.dispatchMouseEvent": () => ({}),
|
||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||
"DOM.getBoxModel": () => ({
|
||||
model: { content: [204, 306, 604, 306, 604, 506, 204, 506] },
|
||||
}),
|
||||
});
|
||||
fake.cdp.getFrameGraph = vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child-frame",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 99,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
}));
|
||||
const targetCalls: Array<{ sessionId?: string; method: string }> = [];
|
||||
fake.cdp.sendToTarget = vi.fn(async (target, method) => {
|
||||
targetCalls.push({ sessionId: target.sessionId, method });
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
||||
}
|
||||
throw new Error(`unexpected child CDP call ${method}`);
|
||||
}) as CdpRunner["sendToTarget"];
|
||||
|
||||
const res = await handleClick(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e3" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
// The iframe content box starts at (204, 306) after its border and is scaled 2x.
|
||||
expect(res).toMatchObject({ x: 324, y: 386 });
|
||||
expect(targetCalls).toEqual([
|
||||
{ sessionId: "child-session", method: "DOM.scrollIntoViewIfNeeded" },
|
||||
{ sessionId: "child-session", method: "DOM.getContentQuads" },
|
||||
{ sessionId: "child-session", method: "Page.getLayoutMetrics" },
|
||||
]);
|
||||
expect(fake.sent.filter((call) => call.method === "Input.dispatchMouseEvent")).toHaveLength(3);
|
||||
});
|
||||
|
||||
it("enables overlay bypass before mouse events when overlay blocks the click point", async () => {
|
||||
const order: string[] = [];
|
||||
const bypassOverlay = vi.fn(async (_tabId: number, enabled: boolean) => {
|
||||
|
||||
@@ -63,6 +63,9 @@ function makeFakeCdp(handlers: Record<string, (params?: object) => unknown>) {
|
||||
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
sent.push({ method, params });
|
||||
const handler = handlers[method];
|
||||
if (!handler && method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||
}
|
||||
if (!handler) throw new Error(`unexpected CDP call ${method}`);
|
||||
return handler(params);
|
||||
});
|
||||
@@ -586,6 +589,7 @@ describe("buildVomScene", () => {
|
||||
{
|
||||
frameId: "main",
|
||||
contextScopeId: "main",
|
||||
target: { tabId: 7 },
|
||||
domNodes: mainNodes,
|
||||
axNodes: [
|
||||
axNode("root", 1, "RootWebArea"),
|
||||
@@ -598,6 +602,7 @@ describe("buildVomScene", () => {
|
||||
contextScopeId: "first",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 7, sessionId: "session-first" },
|
||||
domNodes: firstNodes,
|
||||
axNodes: [
|
||||
axNode("root", 100, "RootWebArea"),
|
||||
@@ -609,6 +614,7 @@ describe("buildVomScene", () => {
|
||||
contextScopeId: "second",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 20,
|
||||
target: { tabId: 7, sessionId: "session-second" },
|
||||
domNodes: secondNodes,
|
||||
axNodes: [
|
||||
axNode("root", 200, "RootWebArea"),
|
||||
@@ -621,6 +627,7 @@ describe("buildVomScene", () => {
|
||||
contextScopeId: "nested",
|
||||
parentFrameId: "second",
|
||||
ownerBackendNodeId: 210,
|
||||
target: { tabId: 7, sessionId: "session-nested" },
|
||||
domNodes: nestedNodes,
|
||||
axNodes: [
|
||||
axNode("root", 300, "RootWebArea"),
|
||||
@@ -673,6 +680,7 @@ describe("buildVomScene", () => {
|
||||
{
|
||||
frameId: "main",
|
||||
contextScopeId: "main",
|
||||
target: { tabId: 7 },
|
||||
domNodes: [],
|
||||
axNodes: [],
|
||||
},
|
||||
@@ -680,6 +688,7 @@ describe("buildVomScene", () => {
|
||||
frameId: "child",
|
||||
contextScopeId: "child",
|
||||
parentFrameId: "main",
|
||||
target: { tabId: 7 },
|
||||
domNodes: [childNode],
|
||||
axNodes: [
|
||||
{
|
||||
@@ -740,6 +749,7 @@ describe("buildVomScene", () => {
|
||||
{
|
||||
frameId: "main",
|
||||
contextScopeId: "main",
|
||||
target: { tabId: 7 },
|
||||
domNodes: mainNodes,
|
||||
axNodes: [
|
||||
{
|
||||
@@ -761,6 +771,7 @@ describe("buildVomScene", () => {
|
||||
contextScopeId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 7, sessionId: "child-session" },
|
||||
domNodes: [],
|
||||
axNodes: [
|
||||
{
|
||||
@@ -3407,6 +3418,32 @@ describe("handleGetHtml", () => {
|
||||
expect(deps.send).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("reads a frame ref through its child CDP session", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e7", 4242, {
|
||||
tabId: 4,
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
});
|
||||
const deps = makeDeps({});
|
||||
const sendToTarget = vi.fn(async (target, method, params) => {
|
||||
expect(target).toEqual({ tabId: 4, sessionId: "child-session" });
|
||||
expect(method).toBe("DOM.getOuterHTML");
|
||||
expect(params).toEqual({ backendNodeId: 4242 });
|
||||
return { outerHTML: "<button>frame</button>" };
|
||||
});
|
||||
(deps.cdp as CdpRunner).sendToTarget = sendToTarget as unknown as NonNullable<
|
||||
CdpRunner["sendToTarget"]
|
||||
>;
|
||||
|
||||
const res = await handleGetHtml(sm, { session_id: "aa11", ref: "@e7" }, deps);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(res.html).toBe("<button>frame</button>");
|
||||
expect(deps.send).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("returns not_found when a ref belongs to another tab", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
|
||||
@@ -46,6 +46,25 @@ describe("lookupSnapshotRef", () => {
|
||||
});
|
||||
|
||||
describe("resolveSnapshotRef", () => {
|
||||
it("preserves the frame and child session needed to route iframe refs", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e3", 1234, {
|
||||
tabId: 4,
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
});
|
||||
|
||||
const expected = {
|
||||
backendNodeId: 1234,
|
||||
refKey: "e3",
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
};
|
||||
expect(lookupSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
|
||||
expect(resolveSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
|
||||
});
|
||||
|
||||
it("returns not_found for unknown ref", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
|
||||
@@ -122,10 +122,14 @@ export async function nodeBoundingRect(
|
||||
tabId: number,
|
||||
backendNodeId: number,
|
||||
): Promise<ViewportRect | RpcError> {
|
||||
const polygon = await visibleContentPolygon(cdp, tabId, backendNodeId);
|
||||
if (isRpcError(polygon)) return polygon;
|
||||
// visibleContentPolygon only returns quads with positive area.
|
||||
return quadBoundingRect(polygon)!;
|
||||
const polygons = await visibleContentPolygons(cdp, tabId, backendNodeId);
|
||||
if (isRpcError(polygons)) return polygons;
|
||||
const bounds = polygons.flatMap((polygon) => quadBoundingRect(polygon) ?? []);
|
||||
const x = Math.min(...bounds.map((rect) => rect.x));
|
||||
const y = Math.min(...bounds.map((rect) => rect.y));
|
||||
const right = Math.max(...bounds.map((rect) => rect.x + rect.width));
|
||||
const bottom = Math.max(...bounds.map((rect) => rect.y + rect.height));
|
||||
return { x, y, width: right - x, height: bottom - y };
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -143,17 +147,17 @@ export function boxCentre(box: number[]): { x: number; y: number } {
|
||||
return quadCentre(box);
|
||||
}
|
||||
|
||||
async function visibleContentPolygon(
|
||||
async function visibleContentPolygons(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
backendNodeId: number,
|
||||
): Promise<number[] | RpcError> {
|
||||
): Promise<number[][] | RpcError> {
|
||||
try {
|
||||
const quads = await cdp.send<{ quads?: number[][] }>(tabId, "DOM.getContentQuads", {
|
||||
backendNodeId,
|
||||
});
|
||||
const quad = quads.quads?.find((q) => q.length === 8 && quadBoundingRect(q) !== null);
|
||||
if (quad) return quad;
|
||||
const visible = quads.quads?.filter((quad) => quad.length === 8 && quadBoundingRect(quad));
|
||||
if (visible && visible.length > 0) return visible;
|
||||
} catch (err) {
|
||||
console.debug("[bsk element-geometry] getContentQuads failed", err);
|
||||
}
|
||||
@@ -162,15 +166,27 @@ async function visibleContentPolygon(
|
||||
backendNodeId,
|
||||
});
|
||||
const content = box.model?.content;
|
||||
if (content && content.length === 8 && quadBoundingRect(content)) return content;
|
||||
if (content && content.length === 8 && quadBoundingRect(content)) return [content];
|
||||
} catch (err) {
|
||||
console.debug("[bsk element-geometry] getBoxModel failed", err);
|
||||
}
|
||||
const descendantRect = await descendantBoundingRect(cdp, tabId, backendNodeId);
|
||||
if (descendantRect) return rectToQuad(descendantRect);
|
||||
if (descendantRect) return [rectToQuad(descendantRect)];
|
||||
return elementNotVisibleError();
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a live content quad for a backend node in the target's own viewport.
|
||||
* Frame-aware callers can project this quad through the owning frame chain.
|
||||
*/
|
||||
export async function nodeContentQuads(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
backendNodeId: number,
|
||||
): Promise<number[][] | RpcError> {
|
||||
return visibleContentPolygons(cdp, tabId, backendNodeId);
|
||||
}
|
||||
|
||||
async function descendantBoundingRect(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
@@ -254,7 +270,7 @@ export async function nodeCentre(
|
||||
tabId: number,
|
||||
backendNodeId: number,
|
||||
): Promise<{ x: number; y: number } | RpcError> {
|
||||
const polygon = await visibleContentPolygon(cdp, tabId, backendNodeId);
|
||||
if (isRpcError(polygon)) return polygon;
|
||||
return quadCentre(polygon);
|
||||
const polygons = await visibleContentPolygons(cdp, tabId, backendNodeId);
|
||||
if (isRpcError(polygons)) return polygons;
|
||||
return quadCentre(polygons[0]);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,289 @@
|
||||
import { type CdpFrame, type CdpFrameGraph, type CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import type { RpcError } from "@/transport/types";
|
||||
import { nodeContentQuads, scrollNodeIntoView, type ViewportRect } from "./element-geometry";
|
||||
import {
|
||||
clipPolygon,
|
||||
type GeometryProjection,
|
||||
type Point,
|
||||
type Polygon,
|
||||
type ProjectiveEdge,
|
||||
parseCdpQuad,
|
||||
polygonBounds,
|
||||
polygonCentroid,
|
||||
projectRegionToViewport,
|
||||
type Quad,
|
||||
type Region,
|
||||
rectPolygon,
|
||||
type Size,
|
||||
} from "./geometry";
|
||||
import { type CdpRunner, cdpRunnerForTarget, isRpcError, sendToCdpTarget } from "./shared";
|
||||
|
||||
export type { Point, Quad, Region } from "./geometry";
|
||||
export { parseCdpQuad } from "./geometry";
|
||||
|
||||
export type FrameProjection = GeometryProjection;
|
||||
|
||||
export interface NodeAddress {
|
||||
target: CdpTarget;
|
||||
backendNodeId: number;
|
||||
frameId?: string;
|
||||
}
|
||||
|
||||
export interface ResolvedNodeGeometry {
|
||||
topVisibleRegions: Region;
|
||||
topBounds: ViewportRect;
|
||||
actionPoint: Point;
|
||||
}
|
||||
|
||||
function geometryError(message: string): RpcError {
|
||||
return { code: "cdp_failed", message };
|
||||
}
|
||||
|
||||
function frameMap(graph: CdpFrameGraph): Map<string, CdpFrame> {
|
||||
return new Map(graph.frames.map((frame) => [frame.frameId, frame]));
|
||||
}
|
||||
|
||||
function targetRootFrame(byId: Map<string, CdpFrame>, frame: CdpFrame): CdpFrame | null {
|
||||
const seen = new Set<string>();
|
||||
let current = frame;
|
||||
while (current.parentFrameId) {
|
||||
if (seen.has(current.frameId)) return null;
|
||||
seen.add(current.frameId);
|
||||
const parent = byId.get(current.parentFrameId);
|
||||
if (!parent || parent.target.sessionId !== current.target.sessionId) break;
|
||||
current = parent;
|
||||
}
|
||||
return current;
|
||||
}
|
||||
|
||||
function frameAncestry(graph: CdpFrameGraph, frameId: string): CdpFrame[] | null {
|
||||
const byId = frameMap(graph);
|
||||
const path: CdpFrame[] = [];
|
||||
const seen = new Set<string>();
|
||||
let current = byId.get(frameId);
|
||||
if (!current) return null;
|
||||
while (current.parentFrameId) {
|
||||
if (seen.has(current.frameId)) return null;
|
||||
seen.add(current.frameId);
|
||||
path.push(current);
|
||||
const parent = byId.get(current.parentFrameId);
|
||||
if (!parent) return null;
|
||||
current = parent;
|
||||
}
|
||||
return path;
|
||||
}
|
||||
|
||||
async function targetViewport(cdp: CdpRunner, target: CdpTarget): Promise<Size | null> {
|
||||
const metrics = await sendToCdpTarget<{
|
||||
cssLayoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||
layoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||
}>(cdp, target, "Page.getLayoutMetrics", {});
|
||||
const viewport = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||
const width = viewport.clientWidth ?? 0;
|
||||
const height = viewport.clientHeight ?? 0;
|
||||
return width > 0 && height > 0 ? { width, height } : null;
|
||||
}
|
||||
|
||||
async function ownerContentQuad(
|
||||
cdp: CdpRunner,
|
||||
parent: CdpFrame,
|
||||
ownerBackendNodeId: number,
|
||||
): Promise<Quad | null> {
|
||||
const result = await sendToCdpTarget<{ model?: { content?: number[] } }>(
|
||||
cdp,
|
||||
parent.target,
|
||||
"DOM.getBoxModel",
|
||||
{ backendNodeId: ownerBackendNodeId },
|
||||
);
|
||||
return parseCdpQuad(result.model?.content);
|
||||
}
|
||||
|
||||
async function sameTargetFrameClips(
|
||||
cdp: CdpRunner,
|
||||
byId: Map<string, CdpFrame>,
|
||||
frame: CdpFrame,
|
||||
targetRoot: CdpFrame,
|
||||
): Promise<Polygon[] | null> {
|
||||
const clips: Polygon[] = [];
|
||||
let current = frame;
|
||||
const seen = new Set<string>();
|
||||
while (current.frameId !== targetRoot.frameId) {
|
||||
if (seen.has(current.frameId) || !current.parentFrameId) return null;
|
||||
seen.add(current.frameId);
|
||||
const parent = byId.get(current.parentFrameId);
|
||||
if (!parent || current.ownerBackendNodeId === undefined) return null;
|
||||
const clip = await ownerContentQuad(cdp, parent, current.ownerBackendNodeId);
|
||||
if (!clip) return null;
|
||||
clips.push(clip);
|
||||
current = parent;
|
||||
}
|
||||
return clips;
|
||||
}
|
||||
|
||||
export async function resolveFrameProjection(
|
||||
cdp: CdpRunner,
|
||||
graph: CdpFrameGraph,
|
||||
frameId: string,
|
||||
): Promise<FrameProjection | null> {
|
||||
const byId = frameMap(graph);
|
||||
const frame = byId.get(frameId);
|
||||
if (!frame) return null;
|
||||
let targetRoot = targetRootFrame(byId, frame);
|
||||
if (!targetRoot) return null;
|
||||
const sourceViewport = await targetViewport(cdp, targetRoot.target);
|
||||
if (!sourceViewport) return null;
|
||||
const sourceClips = await sameTargetFrameClips(cdp, byId, frame, targetRoot);
|
||||
if (!sourceClips) return null;
|
||||
|
||||
const edges: ProjectiveEdge[] = [];
|
||||
while (targetRoot.parentFrameId) {
|
||||
const parent = byId.get(targetRoot.parentFrameId);
|
||||
if (!parent || targetRoot.ownerBackendNodeId === undefined) return null;
|
||||
const destinationQuad = await ownerContentQuad(cdp, parent, targetRoot.ownerBackendNodeId);
|
||||
if (!destinationQuad) return null;
|
||||
const source =
|
||||
edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
|
||||
if (!source) return null;
|
||||
const parentTargetRoot = targetRootFrame(byId, parent);
|
||||
if (!parentTargetRoot) return null;
|
||||
const destinationClips = await sameTargetFrameClips(cdp, byId, parent, parentTargetRoot);
|
||||
if (!destinationClips) return null;
|
||||
edges.push({ sourceViewport: source, destinationQuad, destinationClips });
|
||||
targetRoot = parentTargetRoot;
|
||||
}
|
||||
|
||||
const topViewport =
|
||||
edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
|
||||
return topViewport ? { sourceViewport, sourceClips, edges, topViewport } : null;
|
||||
}
|
||||
|
||||
async function loadFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFrameGraph | null> {
|
||||
if (!cdp.getFrameGraph) return null;
|
||||
try {
|
||||
return await cdp.getFrameGraph(tabId);
|
||||
} catch (error) {
|
||||
console.debug("[bsk frame-geometry] frame graph resolution failed", error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function scrollFrameOwners(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
graph: CdpFrameGraph,
|
||||
frameId: string,
|
||||
): Promise<RpcError | null> {
|
||||
const byId = frameMap(graph);
|
||||
const ancestry = frameAncestry(graph, frameId);
|
||||
if (!ancestry) return geometryError(`could not resolve frame ancestry for ${frameId}`);
|
||||
for (const child of [...ancestry].reverse()) {
|
||||
const parent = child.parentFrameId ? byId.get(child.parentFrameId) : undefined;
|
||||
if (!parent || child.ownerBackendNodeId === undefined) {
|
||||
return geometryError(`could not resolve frame owner for ${child.frameId}`);
|
||||
}
|
||||
const error = await scrollNodeIntoView(
|
||||
cdpRunnerForTarget(cdp, parent.target),
|
||||
tabId,
|
||||
child.ownerBackendNodeId,
|
||||
);
|
||||
if (error) return error;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export async function scrollElementAndFramesIntoView(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
target: CdpTarget,
|
||||
backendNodeId: number,
|
||||
frameId?: string,
|
||||
): Promise<RpcError | null> {
|
||||
if (frameId) {
|
||||
const graph = await loadFrameGraph(cdp, tabId);
|
||||
if (graph) {
|
||||
const error = await scrollFrameOwners(cdp, tabId, graph, frameId);
|
||||
if (error) return error;
|
||||
} else {
|
||||
return geometryError(`could not resolve frame graph for ${frameId}`);
|
||||
}
|
||||
} else if (target.sessionId) {
|
||||
return geometryError("an OOPIF node address requires frameId");
|
||||
}
|
||||
return scrollNodeIntoView(cdpRunnerForTarget(cdp, target), tabId, backendNodeId);
|
||||
}
|
||||
|
||||
function largestRegion(regions: Region): Polygon | null {
|
||||
let largest: { polygon: Polygon; area: number } | null = null;
|
||||
for (const polygon of regions) {
|
||||
const bounds = polygonBounds(polygon);
|
||||
if (!bounds) continue;
|
||||
const area = bounds.width * bounds.height;
|
||||
if (!largest || area > largest.area) largest = { polygon, area };
|
||||
}
|
||||
return largest?.polygon ?? null;
|
||||
}
|
||||
|
||||
export async function resolveNodeGeometry(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
address: NodeAddress,
|
||||
options: { scrollIntoView?: boolean } = {},
|
||||
): Promise<ResolvedNodeGeometry | RpcError> {
|
||||
try {
|
||||
if (options.scrollIntoView) {
|
||||
const scrollError = await scrollElementAndFramesIntoView(
|
||||
cdp,
|
||||
tabId,
|
||||
address.target,
|
||||
address.backendNodeId,
|
||||
address.frameId,
|
||||
);
|
||||
if (scrollError) return scrollError;
|
||||
}
|
||||
|
||||
const local = await nodeContentQuads(
|
||||
cdpRunnerForTarget(cdp, address.target),
|
||||
tabId,
|
||||
address.backendNodeId,
|
||||
);
|
||||
if (isRpcError(local)) return local;
|
||||
const localRegion = local
|
||||
.map((quad) => parseCdpQuad(quad))
|
||||
.filter((quad): quad is Quad => quad !== null);
|
||||
if (localRegion.length === 0) return geometryError("DOM returned no valid content regions");
|
||||
|
||||
const graph = address.frameId ? await loadFrameGraph(cdp, tabId) : null;
|
||||
if (address.target.sessionId && !address.frameId) {
|
||||
return geometryError("an OOPIF node address requires frameId");
|
||||
}
|
||||
if (address.frameId && !graph) {
|
||||
return geometryError(`could not resolve frame graph for ${address.frameId}`);
|
||||
}
|
||||
|
||||
let topVisibleRegions: Region;
|
||||
if (address.frameId && graph) {
|
||||
const projection = await resolveFrameProjection(cdp, graph, address.frameId);
|
||||
if (!projection)
|
||||
return geometryError(`could not resolve frame geometry for ${address.frameId}`);
|
||||
topVisibleRegions = projectRegionToViewport(localRegion, projection);
|
||||
} else {
|
||||
const viewport = await targetViewport(cdp, address.target);
|
||||
if (!viewport) return geometryError("could not resolve top viewport geometry");
|
||||
topVisibleRegions = localRegion
|
||||
.map((polygon) =>
|
||||
clipPolygon(polygon, rectPolygon({ x: 0, y: 0, w: viewport.width, h: viewport.height })),
|
||||
)
|
||||
.filter((polygon) => polygon.length >= 3);
|
||||
}
|
||||
|
||||
const topBounds = polygonBounds(topVisibleRegions.flat());
|
||||
const actionRegion = largestRegion(topVisibleRegions);
|
||||
const actionPoint = actionRegion ? polygonCentroid(actionRegion) : null;
|
||||
if (!topBounds || !actionPoint) {
|
||||
return { code: "permission_denied", message: "element not visible" };
|
||||
}
|
||||
return { topVisibleRegions, topBounds, actionPoint };
|
||||
} catch (error) {
|
||||
return geometryError(error instanceof Error ? error.message : String(error));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,255 @@
|
||||
import type { ViewportRect } from "./element-geometry";
|
||||
|
||||
export interface Point {
|
||||
x: number;
|
||||
y: number;
|
||||
}
|
||||
|
||||
export type Polygon = Point[];
|
||||
export type Region = Polygon[];
|
||||
export type Quad = [Point, Point, Point, Point];
|
||||
|
||||
export interface Size {
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
export interface ProjectiveEdge {
|
||||
sourceViewport: Size;
|
||||
destinationQuad: Quad;
|
||||
destinationClips?: Polygon[];
|
||||
}
|
||||
|
||||
export interface GeometryProjection {
|
||||
sourceViewport: Size;
|
||||
sourceClips: Polygon[];
|
||||
edges: ProjectiveEdge[];
|
||||
topViewport: Size;
|
||||
}
|
||||
|
||||
export function parseCdpQuad(raw: number[] | undefined): Quad | null {
|
||||
if (!raw || raw.length !== 8 || !raw.every(Number.isFinite)) return null;
|
||||
return [
|
||||
{ x: raw[0], y: raw[1] },
|
||||
{ x: raw[2], y: raw[3] },
|
||||
{ x: raw[4], y: raw[5] },
|
||||
{ x: raw[6], y: raw[7] },
|
||||
];
|
||||
}
|
||||
|
||||
export function rectPolygon(rect: { x: number; y: number; w: number; h: number }): Quad {
|
||||
return [
|
||||
{ x: rect.x, y: rect.y },
|
||||
{ x: rect.x + rect.w, y: rect.y },
|
||||
{ x: rect.x + rect.w, y: rect.y + rect.h },
|
||||
{ x: rect.x, y: rect.y + rect.h },
|
||||
];
|
||||
}
|
||||
|
||||
export function viewportPolygon(viewport: Size): Quad {
|
||||
return rectPolygon({ x: 0, y: 0, w: viewport.width, h: viewport.height });
|
||||
}
|
||||
|
||||
/** Project a normalized unit-square point into an arbitrary content quad. */
|
||||
export function projectUnitPoint(point: Point, quad: Quad): Point {
|
||||
const [p0, p1, p2, p3] = quad;
|
||||
const dx1 = p1.x - p2.x;
|
||||
const dx2 = p3.x - p2.x;
|
||||
const dx3 = p0.x - p1.x + p2.x - p3.x;
|
||||
const dy1 = p1.y - p2.y;
|
||||
const dy2 = p3.y - p2.y;
|
||||
const dy3 = p0.y - p1.y + p2.y - p3.y;
|
||||
|
||||
let a: number;
|
||||
let b: number;
|
||||
let c: number;
|
||||
let d: number;
|
||||
let e: number;
|
||||
let f: number;
|
||||
let g = 0;
|
||||
let h = 0;
|
||||
|
||||
if (Math.abs(dx3) < 1e-9 && Math.abs(dy3) < 1e-9) {
|
||||
a = p1.x - p0.x;
|
||||
b = p3.x - p0.x;
|
||||
c = p0.x;
|
||||
d = p1.y - p0.y;
|
||||
e = p3.y - p0.y;
|
||||
f = p0.y;
|
||||
} else {
|
||||
const denominator = dx1 * dy2 - dx2 * dy1;
|
||||
if (Math.abs(denominator) < 1e-9) {
|
||||
return {
|
||||
x: p0.x + (p1.x - p0.x) * point.x + (p3.x - p0.x) * point.y,
|
||||
y: p0.y + (p1.y - p0.y) * point.x + (p3.y - p0.y) * point.y,
|
||||
};
|
||||
}
|
||||
g = (dx3 * dy2 - dx2 * dy3) / denominator;
|
||||
h = (dx1 * dy3 - dx3 * dy1) / denominator;
|
||||
a = p1.x - p0.x + g * p1.x;
|
||||
b = p3.x - p0.x + h * p3.x;
|
||||
c = p0.x;
|
||||
d = p1.y - p0.y + g * p1.y;
|
||||
e = p3.y - p0.y + h * p3.y;
|
||||
f = p0.y;
|
||||
}
|
||||
|
||||
const scale = g * point.x + h * point.y + 1;
|
||||
return {
|
||||
x: (a * point.x + b * point.y + c) / scale,
|
||||
y: (d * point.x + e * point.y + f) / scale,
|
||||
};
|
||||
}
|
||||
|
||||
export function projectPolygon(polygon: Polygon, edge: ProjectiveEdge): Polygon {
|
||||
const { width, height } = edge.sourceViewport;
|
||||
if (width <= 0 || height <= 0) return [];
|
||||
return polygon.map((point) =>
|
||||
projectUnitPoint({ x: point.x / width, y: point.y / height }, edge.destinationQuad),
|
||||
);
|
||||
}
|
||||
|
||||
function polygonSignedArea(points: Polygon): number {
|
||||
let area = 0;
|
||||
for (let index = 0; index < points.length; index += 1) {
|
||||
const current = points[index];
|
||||
const next = points[(index + 1) % points.length];
|
||||
area += current.x * next.y - next.x * current.y;
|
||||
}
|
||||
return area / 2;
|
||||
}
|
||||
|
||||
function edgeIntersection(start: Point, end: Point, clipStart: Point, clipEnd: Point): Point {
|
||||
const subjectX = end.x - start.x;
|
||||
const subjectY = end.y - start.y;
|
||||
const clipX = clipEnd.x - clipStart.x;
|
||||
const clipY = clipEnd.y - clipStart.y;
|
||||
const denominator = subjectX * clipY - subjectY * clipX;
|
||||
if (Math.abs(denominator) < 1e-9) return end;
|
||||
const offsetX = clipStart.x - start.x;
|
||||
const offsetY = clipStart.y - start.y;
|
||||
const scale = (offsetX * clipY - offsetY * clipX) / denominator;
|
||||
return { x: start.x + scale * subjectX, y: start.y + scale * subjectY };
|
||||
}
|
||||
|
||||
/** Clip a polygon to a convex polygon, preserving every resulting vertex. */
|
||||
export function clipPolygon(subject: Polygon, clip: Polygon): Polygon {
|
||||
if (subject.length < 3 || clip.length < 3) return [];
|
||||
const orientation = polygonSignedArea(clip) >= 0 ? 1 : -1;
|
||||
let output = [...subject];
|
||||
for (let edge = 0; edge < clip.length && output.length > 0; edge += 1) {
|
||||
const clipStart = clip[edge];
|
||||
const clipEnd = clip[(edge + 1) % clip.length];
|
||||
const input = output;
|
||||
output = [];
|
||||
const inside = (point: Point) =>
|
||||
orientation *
|
||||
((clipEnd.x - clipStart.x) * (point.y - clipStart.y) -
|
||||
(clipEnd.y - clipStart.y) * (point.x - clipStart.x)) >=
|
||||
-1e-9;
|
||||
let start = input[input.length - 1];
|
||||
for (const end of input) {
|
||||
const startInside = inside(start);
|
||||
const endInside = inside(end);
|
||||
if (endInside) {
|
||||
if (!startInside) output.push(edgeIntersection(start, end, clipStart, clipEnd));
|
||||
output.push(end);
|
||||
} else if (startInside) {
|
||||
output.push(edgeIntersection(start, end, clipStart, clipEnd));
|
||||
}
|
||||
start = end;
|
||||
}
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
export function polygonBounds(points: Polygon): ViewportRect | null {
|
||||
if (points.length < 3) return null;
|
||||
const xs = points.map((point) => point.x);
|
||||
const ys = points.map((point) => point.y);
|
||||
const x = Math.min(...xs);
|
||||
const y = Math.min(...ys);
|
||||
const right = Math.max(...xs);
|
||||
const bottom = Math.max(...ys);
|
||||
if (right <= x || bottom <= y) return null;
|
||||
return { x, y, width: right - x, height: bottom - y };
|
||||
}
|
||||
|
||||
export function polygonCentroid(points: Polygon): Point | null {
|
||||
if (points.length < 3) return null;
|
||||
let twiceArea = 0;
|
||||
let x = 0;
|
||||
let y = 0;
|
||||
for (let index = 0; index < points.length; index += 1) {
|
||||
const current = points[index];
|
||||
const next = points[(index + 1) % points.length];
|
||||
const cross = current.x * next.y - next.x * current.y;
|
||||
twiceArea += cross;
|
||||
x += (current.x + next.x) * cross;
|
||||
y += (current.y + next.y) * cross;
|
||||
}
|
||||
if (Math.abs(twiceArea) < 1e-9) return null;
|
||||
return { x: x / (3 * twiceArea), y: y / (3 * twiceArea) };
|
||||
}
|
||||
|
||||
export function projectAndClipRegion(
|
||||
region: Region,
|
||||
edges: ProjectiveEdge[],
|
||||
topViewport: Size,
|
||||
): Region {
|
||||
let projected = region;
|
||||
for (const edge of edges) {
|
||||
const sourceClip = viewportPolygon(edge.sourceViewport);
|
||||
projected = projected
|
||||
.map((polygon) => clipPolygon(polygon, sourceClip))
|
||||
.filter((polygon) => polygon.length >= 3)
|
||||
.map((polygon) =>
|
||||
[edge.destinationQuad, ...(edge.destinationClips ?? [])].reduce(
|
||||
(current, clip) => clipPolygon(current, clip),
|
||||
projectPolygon(polygon, edge),
|
||||
),
|
||||
)
|
||||
.filter((polygon) => polygon.length >= 3);
|
||||
}
|
||||
const topClip = viewportPolygon(topViewport);
|
||||
return projected
|
||||
.map((polygon) => clipPolygon(polygon, topClip))
|
||||
.filter((polygon) => polygon.length >= 3);
|
||||
}
|
||||
|
||||
export function projectRegionToViewport(region: Region, projection: GeometryProjection): Region {
|
||||
const clipped = region
|
||||
.map((polygon) =>
|
||||
projection.sourceClips.reduce((current, clip) => clipPolygon(current, clip), polygon),
|
||||
)
|
||||
.filter((polygon) => polygon.length >= 3);
|
||||
return projectAndClipRegion(clipped, projection.edges, projection.topViewport);
|
||||
}
|
||||
|
||||
export function projectRectToViewport(
|
||||
rect: { x: number; y: number; w: number; h: number } | null,
|
||||
projection: GeometryProjection,
|
||||
): ViewportRect | null {
|
||||
if (!rect) return null;
|
||||
return polygonBounds(projectRegionToViewport([rectPolygon(rect)], projection).flat());
|
||||
}
|
||||
|
||||
export function childFrameProjection(
|
||||
parent: GeometryProjection,
|
||||
ownerRectInParent: { x: number; y: number; w: number; h: number },
|
||||
): GeometryProjection {
|
||||
const destinationQuad = rectPolygon(ownerRectInParent);
|
||||
return {
|
||||
sourceViewport: { width: ownerRectInParent.w, height: ownerRectInParent.h },
|
||||
sourceClips: [],
|
||||
edges: [
|
||||
{
|
||||
sourceViewport: { width: ownerRectInParent.w, height: ownerRectInParent.h },
|
||||
destinationQuad,
|
||||
destinationClips: parent.sourceClips,
|
||||
},
|
||||
...parent.edges,
|
||||
],
|
||||
topViewport: parent.topViewport,
|
||||
};
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
HELP_REQUEST,
|
||||
type HelpCancelMessage,
|
||||
type HelpFinishMessage,
|
||||
type HelpHighlightRect,
|
||||
type HelpQueryResponse,
|
||||
type HelpRequestMessage,
|
||||
isHelpAckMessage,
|
||||
@@ -25,6 +26,7 @@ import type {
|
||||
ResolvedTarget,
|
||||
RpcError,
|
||||
} from "@/transport/types";
|
||||
import { resolveNodeGeometry } from "./frame-geometry";
|
||||
import {
|
||||
type CdpRunner,
|
||||
type ChromeTabsApi,
|
||||
@@ -37,7 +39,6 @@ import {
|
||||
import { lookupSnapshotRef } from "./snapshot-ref";
|
||||
|
||||
const DEFAULT_HELP_TIMEOUT_MS = 300_000;
|
||||
const HELP_ATTR = "data-bsk-help";
|
||||
const HELP_SEND_RETRIES = 3;
|
||||
const HELP_SEND_RETRY_DELAY_MS = 350;
|
||||
const HELP_REARM_DEBOUNCE_MS = 150;
|
||||
@@ -74,6 +75,7 @@ interface ActiveHelpRequest {
|
||||
title?: string;
|
||||
targets: HelpTarget[];
|
||||
selectors: string[];
|
||||
rects: HelpHighlightRect[];
|
||||
timeoutMs: number;
|
||||
notificationId: string;
|
||||
resolvedTargets?: ResolvedTarget[];
|
||||
@@ -134,6 +136,7 @@ function helpRequestMessage(help: ActiveHelpRequest, tabId: number): HelpRequest
|
||||
...(help.title ? { title: help.title } : {}),
|
||||
displayMode: isPrimaryTab ? "full" : "compact",
|
||||
selectors: isPrimaryTab ? help.selectors : [],
|
||||
rects: isPrimaryTab ? help.rects : [],
|
||||
timeoutMs: help.timeoutMs,
|
||||
};
|
||||
}
|
||||
@@ -165,33 +168,6 @@ async function sendHelpRequestWithAck(
|
||||
throw lastError ?? new Error("failed to show help overlay");
|
||||
}
|
||||
|
||||
async function tagRefTarget(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
backendNodeId: number,
|
||||
index: number,
|
||||
): Promise<string | null> {
|
||||
let objectId: string | undefined;
|
||||
try {
|
||||
const resolved = await cdp.send<{ object?: { objectId?: string } }>(tabId, "DOM.resolveNode", {
|
||||
backendNodeId,
|
||||
});
|
||||
objectId = resolved.object?.objectId;
|
||||
if (!objectId) return null;
|
||||
await cdp.send(tabId, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
functionDeclaration: `function(){ this.setAttribute("${HELP_ATTR}", "${index}"); }`,
|
||||
});
|
||||
return `[${HELP_ATTR}="${index}"]`;
|
||||
} catch {
|
||||
return null;
|
||||
} finally {
|
||||
if (objectId) {
|
||||
await cdp.send(tabId, "Runtime.releaseObject", { objectId }).catch(() => {});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function selectorExists(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
@@ -209,36 +185,22 @@ async function selectorExists(
|
||||
}
|
||||
}
|
||||
|
||||
async function clearRefTags(cdp: CdpRunner | undefined, tabId: number): Promise<void> {
|
||||
if (!cdp) return;
|
||||
try {
|
||||
const doc = await cdp.send<{ root?: { nodeId?: number } }>(tabId, "DOM.getDocument", {
|
||||
depth: 0,
|
||||
});
|
||||
const rootId = doc.root?.nodeId;
|
||||
if (rootId === undefined) return;
|
||||
const { nodeIds } = await cdp.send<{ nodeIds: number[] }>(tabId, "DOM.querySelectorAll", {
|
||||
nodeId: rootId,
|
||||
selector: `[${HELP_ATTR}]`,
|
||||
});
|
||||
for (const nodeId of nodeIds ?? []) {
|
||||
await cdp.send(tabId, "DOM.removeAttribute", { nodeId, name: HELP_ATTR }).catch(() => {});
|
||||
}
|
||||
} catch {
|
||||
// Best-effort cleanup.
|
||||
}
|
||||
}
|
||||
|
||||
async function resolveHelpTargets(
|
||||
ctx: ReturnType<SessionManager["get"]>,
|
||||
tabId: number,
|
||||
targets: HelpTarget[],
|
||||
deps: RequestHelpDeps,
|
||||
): Promise<{ selectors: string[]; resolvedTargets?: ResolvedTarget[] }> {
|
||||
options: { scrollIntoView: boolean },
|
||||
): Promise<{
|
||||
selectors: string[];
|
||||
rects: HelpHighlightRect[];
|
||||
resolvedTargets?: ResolvedTarget[];
|
||||
}> {
|
||||
const selectors: string[] = [];
|
||||
const rects: HelpHighlightRect[] = [];
|
||||
const resolved: ResolvedTarget[] = [];
|
||||
let rootNodeId: number | undefined;
|
||||
if (deps.cdp && targets.length > 0) {
|
||||
if (deps.cdp && targets.some((target) => Boolean(target.selector))) {
|
||||
try {
|
||||
const doc = await deps.cdp.send<{ root?: { nodeId?: number } }>(tabId, "DOM.getDocument", {
|
||||
depth: 0,
|
||||
@@ -262,28 +224,55 @@ async function resolveHelpTargets(
|
||||
resolved.push({ matched, selector: tgt.selector });
|
||||
} else if (tgt.ref) {
|
||||
const looked = ctx ? lookupSnapshotRef(ctx, tgt.ref, tabId) : null;
|
||||
const backendNodeId = looked?.backendNodeId ?? null;
|
||||
let sel: string | null = null;
|
||||
if (backendNodeId !== null && deps.cdp) {
|
||||
sel = await tagRefTarget(deps.cdp, tabId, backendNodeId, i);
|
||||
let rect: HelpHighlightRect | null = null;
|
||||
if (looked && deps.cdp) {
|
||||
const target = {
|
||||
tabId,
|
||||
...(looked.cdpSessionId ? { sessionId: looked.cdpSessionId } : {}),
|
||||
};
|
||||
const geometry = await resolveNodeGeometry(
|
||||
deps.cdp,
|
||||
tabId,
|
||||
{
|
||||
target,
|
||||
backendNodeId: looked.backendNodeId,
|
||||
...(looked.frameId ? { frameId: looked.frameId } : {}),
|
||||
},
|
||||
{ scrollIntoView: options.scrollIntoView },
|
||||
);
|
||||
if (!isRpcError(geometry)) {
|
||||
rect = {
|
||||
top: geometry.topBounds.y,
|
||||
left: geometry.topBounds.x,
|
||||
width: geometry.topBounds.width,
|
||||
height: geometry.topBounds.height,
|
||||
};
|
||||
}
|
||||
}
|
||||
if (sel) selectors.push(sel);
|
||||
resolved.push({ matched: sel !== null, ref: tgt.ref });
|
||||
if (rect) rects.push(rect);
|
||||
resolved.push({
|
||||
matched: rect !== null,
|
||||
ref: tgt.ref,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return { selectors, resolvedTargets: resolved.length > 0 ? resolved : undefined };
|
||||
return { selectors, rects, resolvedTargets: resolved.length > 0 ? resolved : undefined };
|
||||
}
|
||||
|
||||
async function refreshHelpTargets(help: ActiveHelpRequest): Promise<void> {
|
||||
await clearRefTags(help.deps.cdp, help.primaryTabId);
|
||||
const { selectors, resolvedTargets } = await resolveHelpTargets(
|
||||
async function refreshHelpTargets(
|
||||
help: ActiveHelpRequest,
|
||||
options: { scrollIntoView: boolean } = { scrollIntoView: true },
|
||||
): Promise<void> {
|
||||
const { selectors, rects, resolvedTargets } = await resolveHelpTargets(
|
||||
help.ctx,
|
||||
help.primaryTabId,
|
||||
help.targets,
|
||||
help.deps,
|
||||
options,
|
||||
);
|
||||
help.selectors = selectors;
|
||||
help.rects = rects;
|
||||
help.resolvedTargets = resolvedTargets;
|
||||
}
|
||||
|
||||
@@ -294,12 +283,7 @@ async function cleanupHelp(help: ActiveHelpRequest): Promise<void> {
|
||||
const tabsToCancel = new Set([help.primaryTabId, ...help.overlayTabIds]);
|
||||
await Promise.all(
|
||||
[...tabsToCancel].map((tabId) =>
|
||||
Promise.all([
|
||||
clearRefTags(help.deps.cdp, tabId),
|
||||
help.deps
|
||||
.sendToTab(tabId, { type: HELP_CANCEL, requestId: help.requestId })
|
||||
.catch(() => {}),
|
||||
]),
|
||||
help.deps.sendToTab(tabId, { type: HELP_CANCEL, requestId: help.requestId }).catch(() => {}),
|
||||
),
|
||||
);
|
||||
}
|
||||
@@ -476,7 +460,9 @@ function attachHelpRuntimeListener(deps: RequestHelpDeps): () => void {
|
||||
sendResponse({ active: false });
|
||||
return;
|
||||
}
|
||||
if (tabId === help.primaryTabId) await refreshHelpTargets(help);
|
||||
if (tabId === help.primaryTabId) {
|
||||
await refreshHelpTargets(help, { scrollIntoView: false });
|
||||
}
|
||||
help.overlayTabIds.add(tabId);
|
||||
sendResponse({
|
||||
active: true,
|
||||
@@ -486,6 +472,7 @@ function attachHelpRuntimeListener(deps: RequestHelpDeps): () => void {
|
||||
...(help.title ? { title: help.title } : {}),
|
||||
displayMode: tabId === help.primaryTabId ? "full" : "compact",
|
||||
selectors: tabId === help.primaryTabId ? help.selectors : [],
|
||||
rects: tabId === help.primaryTabId ? help.rects : [],
|
||||
timeoutMs: help.timeoutMs,
|
||||
},
|
||||
});
|
||||
@@ -686,7 +673,13 @@ export async function handleRequestHelp(
|
||||
if (deps.autoAttachLifecycle !== false) ensureHelpLifecycleListeners(deps);
|
||||
const tabId = target.tabId;
|
||||
const initialTargets = params.targets ?? [];
|
||||
const { selectors, resolvedTargets } = await resolveHelpTargets(ctx, tabId, initialTargets, deps);
|
||||
const { selectors, rects, resolvedTargets } = await resolveHelpTargets(
|
||||
ctx,
|
||||
tabId,
|
||||
initialTargets,
|
||||
deps,
|
||||
{ scrollIntoView: true },
|
||||
);
|
||||
|
||||
await deps.windows.update(target.windowId, { focused: true }).catch(() => {});
|
||||
await deps.activateTab(tabId).catch(() => {});
|
||||
@@ -729,6 +722,7 @@ export async function handleRequestHelp(
|
||||
...(params.title ? { title: params.title } : {}),
|
||||
targets: initialTargets,
|
||||
selectors,
|
||||
rects,
|
||||
timeoutMs,
|
||||
notificationId,
|
||||
resolvedTargets,
|
||||
|
||||
@@ -3,14 +3,15 @@
|
||||
//
|
||||
// All interaction tools:
|
||||
// 1. Resolve target tab (sandbox: must be inside Agent Window).
|
||||
// 2. Resolve target element by `ref` (RefStore.resolve with tabId
|
||||
// binding) or `selector` (DOM.querySelector + describeNode).
|
||||
// 3. Scroll the node into view, then dispatch the appropriate
|
||||
// `Input.*` CDP events.
|
||||
// 2. Resolve target element by `ref` (compound frame/session identity)
|
||||
// or `selector` (DOM.querySelector + describeNode).
|
||||
// 3. Scroll the node and its frame owners into view, project its live
|
||||
// content quad into the top viewport, then dispatch `Input.*` events.
|
||||
// 4. Honour `AbortSignal` so canceled calls don't issue follow-up CDP
|
||||
// commands.
|
||||
|
||||
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
||||
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import type { SessionContext, SessionManager } from "@/session-manager/manager";
|
||||
import type {
|
||||
ClickParams,
|
||||
@@ -28,17 +29,13 @@ import type {
|
||||
SelectResult,
|
||||
} from "@/transport/types";
|
||||
import { attachDialogs, markDialogCursor } from "./dialogs";
|
||||
import {
|
||||
backendNodeToObject,
|
||||
boxCentre,
|
||||
nodeCentre,
|
||||
quadCentre,
|
||||
scrollNodeIntoView,
|
||||
} from "./element-geometry";
|
||||
import { backendNodeToObject, boxCentre, nodeCentre, quadCentre } from "./element-geometry";
|
||||
import { rpcError } from "./errors";
|
||||
import { resolveNodeGeometry, scrollElementAndFramesIntoView } from "./frame-geometry";
|
||||
import {
|
||||
type CdpRunner,
|
||||
type ChromeTabsApi,
|
||||
cdpRunnerForTarget,
|
||||
chromeTabsApi,
|
||||
enforceAgentWindow,
|
||||
isRpcError,
|
||||
@@ -137,7 +134,16 @@ async function resolveBackendNode(
|
||||
target: { tabId: number },
|
||||
params: { ref?: string; selector?: string },
|
||||
toolName: string,
|
||||
): Promise<{ backendNodeId: number; usedRef?: string; usedSelector?: string } | RpcError> {
|
||||
): Promise<
|
||||
| {
|
||||
backendNodeId: number;
|
||||
cdpTarget: CdpTarget;
|
||||
frameId?: string;
|
||||
usedRef?: string;
|
||||
usedSelector?: string;
|
||||
}
|
||||
| RpcError
|
||||
> {
|
||||
const hasRef = typeof params.ref === "string" && params.ref.length > 0;
|
||||
const hasSelector = typeof params.selector === "string" && params.selector.length > 0;
|
||||
if (hasRef && hasSelector) {
|
||||
@@ -155,7 +161,15 @@ async function resolveBackendNode(
|
||||
if (hasRef) {
|
||||
const resolved = resolveSnapshotRef(ctx, params.ref as string, target.tabId);
|
||||
if (isRpcError(resolved)) return resolved;
|
||||
return { backendNodeId: resolved.backendNodeId, usedRef: resolved.refKey };
|
||||
return {
|
||||
backendNodeId: resolved.backendNodeId,
|
||||
cdpTarget: {
|
||||
tabId: target.tabId,
|
||||
...(resolved.cdpSessionId ? { sessionId: resolved.cdpSessionId } : {}),
|
||||
},
|
||||
...(resolved.frameId ? { frameId: resolved.frameId } : {}),
|
||||
usedRef: resolved.refKey,
|
||||
};
|
||||
}
|
||||
// selector path
|
||||
try {
|
||||
@@ -192,7 +206,7 @@ async function resolveBackendNode(
|
||||
message: "DOM.describeNode returned no backendNodeId",
|
||||
};
|
||||
}
|
||||
return { backendNodeId, usedSelector: params.selector };
|
||||
return { backendNodeId, cdpTarget: { tabId: target.tabId }, usedSelector: params.selector };
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
@@ -228,19 +242,19 @@ export async function handleClick(
|
||||
return { code: "cancelled", message: "click aborted" };
|
||||
}
|
||||
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
|
||||
if (scrollErr) return scrollErr;
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
}
|
||||
|
||||
const centre = await nodeCentre(deps.cdp, target.tabId, node.backendNodeId);
|
||||
if (isRpcError(centre)) return centre;
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const geometry = await resolveNodeGeometry(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
{
|
||||
target: node.cdpTarget,
|
||||
backendNodeId: node.backendNodeId,
|
||||
...(node.frameId ? { frameId: node.frameId } : {}),
|
||||
},
|
||||
{ scrollIntoView: true },
|
||||
);
|
||||
if (isRpcError(geometry)) return geometry;
|
||||
const centre = geometry.actionPoint;
|
||||
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "click aborted" };
|
||||
@@ -353,19 +367,19 @@ export async function handleHover(
|
||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "hover");
|
||||
if (isRpcError(node)) return node;
|
||||
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
|
||||
if (scrollErr) return scrollErr;
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
}
|
||||
|
||||
const centre = await nodeCentre(deps.cdp, target.tabId, node.backendNodeId);
|
||||
if (isRpcError(centre)) return centre;
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const geometry = await resolveNodeGeometry(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
{
|
||||
target: node.cdpTarget,
|
||||
backendNodeId: node.backendNodeId,
|
||||
...(node.frameId ? { frameId: node.frameId } : {}),
|
||||
},
|
||||
{ scrollIntoView: true },
|
||||
);
|
||||
if (isRpcError(geometry)) return geometry;
|
||||
const centre = geometry.actionPoint;
|
||||
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "hover aborted" };
|
||||
@@ -585,10 +599,11 @@ export async function handleFill(
|
||||
|
||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "fill");
|
||||
if (isRpcError(node)) return node;
|
||||
const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
|
||||
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const described = await deps.cdp.send<{ node?: DescribedNode }>(
|
||||
const described = await nodeCdp.send<{ node?: DescribedNode }>(
|
||||
target.tabId,
|
||||
"DOM.describeNode",
|
||||
{
|
||||
@@ -602,12 +617,18 @@ export async function handleFill(
|
||||
`element ${described.node?.nodeName ?? "?"} not fillable (need input/textarea/contenteditable)`,
|
||||
);
|
||||
}
|
||||
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
|
||||
const scrollErr = await scrollElementAndFramesIntoView(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
node.cdpTarget,
|
||||
node.backendNodeId,
|
||||
node.frameId,
|
||||
);
|
||||
if (scrollErr) return scrollErr;
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
await deps.cdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
@@ -619,7 +640,7 @@ export async function handleFill(
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
|
||||
const objectIdOrErr = await backendNodeToObject(deps.cdp, target.tabId, node.backendNodeId);
|
||||
const objectIdOrErr = await backendNodeToObject(nodeCdp, target.tabId, node.backendNodeId);
|
||||
if (isRpcError(objectIdOrErr)) return objectIdOrErr;
|
||||
const objectId = objectIdOrErr;
|
||||
const clearBefore = params.clear_before ?? true;
|
||||
@@ -628,7 +649,7 @@ export async function handleFill(
|
||||
if (clearBefore) {
|
||||
// Clear input/textarea value or wipe contenteditable innerText,
|
||||
// then fire `input` so frameworks observe the empty state.
|
||||
await deps.cdp.send(target.tabId, "Runtime.callFunctionOn", {
|
||||
await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
functionDeclaration: `function() {
|
||||
if (this.isContentEditable) { this.textContent = ''; }
|
||||
@@ -655,7 +676,7 @@ export async function handleFill(
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
// Fire `input` + `change` so React / Vue controlled inputs commit.
|
||||
await deps.cdp.send(target.tabId, "Runtime.callFunctionOn", {
|
||||
await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
functionDeclaration: `function() {
|
||||
this.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
@@ -853,14 +874,21 @@ export async function handlePress(
|
||||
if (params.ref || params.selector) {
|
||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "press");
|
||||
if (isRpcError(node)) return node;
|
||||
const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
|
||||
const scrollErr = await scrollElementAndFramesIntoView(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
node.cdpTarget,
|
||||
node.backendNodeId,
|
||||
node.frameId,
|
||||
);
|
||||
if (scrollErr) return scrollErr;
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "press aborted" };
|
||||
}
|
||||
await deps.cdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
@@ -965,10 +993,11 @@ export async function handleSelect(
|
||||
|
||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "select");
|
||||
if (isRpcError(node)) return node;
|
||||
const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
|
||||
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const described = await deps.cdp.send<{ node?: DescribedNode }>(
|
||||
const described = await nodeCdp.send<{ node?: DescribedNode }>(
|
||||
target.tabId,
|
||||
"DOM.describeNode",
|
||||
{ backendNodeId: node.backendNodeId },
|
||||
@@ -991,12 +1020,18 @@ export async function handleSelect(
|
||||
"single-select <select> requires exactly one value",
|
||||
);
|
||||
}
|
||||
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
|
||||
const scrollErr = await scrollElementAndFramesIntoView(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
node.cdpTarget,
|
||||
node.backendNodeId,
|
||||
node.frameId,
|
||||
);
|
||||
if (scrollErr) return scrollErr;
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "select aborted" };
|
||||
}
|
||||
await deps.cdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
@@ -1008,12 +1043,12 @@ export async function handleSelect(
|
||||
return { code: "cancelled", message: "select aborted" };
|
||||
}
|
||||
|
||||
const objectIdOrErr = await backendNodeToObject(deps.cdp, target.tabId, node.backendNodeId);
|
||||
const objectIdOrErr = await backendNodeToObject(nodeCdp, target.tabId, node.backendNodeId);
|
||||
if (isRpcError(objectIdOrErr)) return objectIdOrErr;
|
||||
const objectId = objectIdOrErr;
|
||||
|
||||
try {
|
||||
const evaluated = await deps.cdp.send<{
|
||||
const evaluated = await nodeCdp.send<{
|
||||
result?: { value?: SelectMutationResult | null };
|
||||
}>(target.tabId, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
|
||||
@@ -13,6 +13,7 @@ import {
|
||||
type VomScene,
|
||||
} from "@browser-skill/vom";
|
||||
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
||||
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import {
|
||||
type CaptureSuppressSendToTab,
|
||||
withOverlaysHiddenForCapture,
|
||||
@@ -30,8 +31,8 @@ import type {
|
||||
SnapshotResult,
|
||||
} from "@/transport/types";
|
||||
import { attachDialogs, markDialogCursor } from "./dialogs";
|
||||
import { nodeBoundingRect, scrollNodeIntoView } from "./element-geometry";
|
||||
import { rpcError } from "./errors";
|
||||
import { resolveNodeGeometry } from "./frame-geometry";
|
||||
import {
|
||||
type ChromeTabsApi,
|
||||
enforceToolTargetScope,
|
||||
@@ -40,6 +41,7 @@ import {
|
||||
type ResolvedTargetTab,
|
||||
resolveCdpAccessibleTargetTab,
|
||||
type CdpRunner as SharedCdpRunner,
|
||||
sendToCdpTarget,
|
||||
normaliseRef as sharedNormaliseRef,
|
||||
type ToolEffect,
|
||||
} from "./shared";
|
||||
@@ -163,26 +165,30 @@ function throwIfAborted(signal: AbortSignal | undefined, tool: string): void {
|
||||
async function captureElementScreenshot(
|
||||
cdp: SharedCdpRunner,
|
||||
tabId: number,
|
||||
target: CdpTarget,
|
||||
backendNodeId: number,
|
||||
frameId?: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<{ image_base64: string; width: number; height: number } | RpcError> {
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
const scrollErr = await scrollNodeIntoView(cdp, tabId, backendNodeId);
|
||||
if (scrollErr) return scrollErr;
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
|
||||
const rectOrErr = await nodeBoundingRect(cdp, tabId, backendNodeId);
|
||||
if (isRpcError(rectOrErr)) return rectOrErr;
|
||||
const geometry = await resolveNodeGeometry(
|
||||
cdp,
|
||||
tabId,
|
||||
{ target, backendNodeId, ...(frameId ? { frameId } : {}) },
|
||||
{ scrollIntoView: true },
|
||||
);
|
||||
if (isRpcError(geometry)) return geometry;
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
const rect = geometry.topBounds;
|
||||
|
||||
try {
|
||||
const shot = await cdp.send<{ data?: string }>(tabId, "Page.captureScreenshot", {
|
||||
format: "png",
|
||||
clip: {
|
||||
x: rectOrErr.x,
|
||||
y: rectOrErr.y,
|
||||
width: rectOrErr.width,
|
||||
height: rectOrErr.height,
|
||||
x: rect.x,
|
||||
y: rect.y,
|
||||
width: rect.width,
|
||||
height: rect.height,
|
||||
scale: 1,
|
||||
},
|
||||
});
|
||||
@@ -192,8 +198,8 @@ async function captureElementScreenshot(
|
||||
return { code: "cdp_failed", message: "Page.captureScreenshot returned no data" };
|
||||
}
|
||||
const dims = parsePngDimensions(image_base64) ?? {
|
||||
width: Math.round(rectOrErr.width),
|
||||
height: Math.round(rectOrErr.height),
|
||||
width: Math.round(rect.width),
|
||||
height: Math.round(rect.height),
|
||||
};
|
||||
return { image_base64, width: dims.width, height: dims.height };
|
||||
} catch (err) {
|
||||
@@ -297,9 +303,21 @@ export async function handleScreenshot(
|
||||
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
const cdp = deps.cdp;
|
||||
const nodeTarget = {
|
||||
tabId: target.tabId,
|
||||
...(node.cdpSessionId ? { sessionId: node.cdpSessionId } : {}),
|
||||
};
|
||||
const captured = await withOverlaysHiddenForCapture(
|
||||
target.tabId,
|
||||
() => captureElementScreenshot(cdp, target.tabId, node.backendNodeId, signal),
|
||||
() =>
|
||||
captureElementScreenshot(
|
||||
cdp,
|
||||
target.tabId,
|
||||
nodeTarget,
|
||||
node.backendNodeId,
|
||||
node.frameId,
|
||||
signal,
|
||||
),
|
||||
deps.sendToTab,
|
||||
);
|
||||
if (isRpcError(captured)) return captured;
|
||||
@@ -1280,7 +1298,7 @@ export function buildFrameVomScene(
|
||||
excludedBackendNodeIds:
|
||||
frameId === rootFrameId ? captured.excludedBackendNodeIds : new Set(),
|
||||
},
|
||||
{ pageUrl: options.pageUrl },
|
||||
{ pageUrl: document.url ?? options.pageUrl },
|
||||
),
|
||||
);
|
||||
}
|
||||
@@ -1452,9 +1470,15 @@ export async function handleGetHtml(
|
||||
if (params.ref) {
|
||||
const resolved = resolveSnapshotRef(ctx, params.ref, target.tabId);
|
||||
if (isRpcError(resolved)) return resolved;
|
||||
const resp = await deps.cdp.send<{ outerHTML?: string }>(target.tabId, "DOM.getOuterHTML", {
|
||||
backendNodeId: resolved.backendNodeId,
|
||||
});
|
||||
const resp = await sendToCdpTarget<{ outerHTML?: string }>(
|
||||
deps.cdp,
|
||||
{
|
||||
tabId: target.tabId,
|
||||
...(resolved.cdpSessionId ? { sessionId: resolved.cdpSessionId } : {}),
|
||||
},
|
||||
"DOM.getOuterHTML",
|
||||
{ backendNodeId: resolved.backendNodeId },
|
||||
);
|
||||
throwIfAborted(signal, "get_html");
|
||||
html = resp.outerHTML ?? "";
|
||||
} else {
|
||||
@@ -1604,10 +1628,22 @@ async function handleVomObservation(
|
||||
activeRegionPolicy: true,
|
||||
});
|
||||
throwIfAborted(signal, toolName);
|
||||
const targetByFrameId = new Map(
|
||||
frameDocuments.map((document) => [document.frameId, document.target]),
|
||||
);
|
||||
ctx.refStore.replace(
|
||||
rendered.refs.map(
|
||||
(ref) => [ref.ref, { backendNodeId: ref.backendNodeId, tabId: target.tabId }] as const,
|
||||
),
|
||||
rendered.refs.map((ref) => {
|
||||
const refTarget = ref.frameId ? targetByFrameId.get(ref.frameId) : undefined;
|
||||
return [
|
||||
ref.ref,
|
||||
{
|
||||
backendNodeId: ref.backendNodeId,
|
||||
tabId: target.tabId,
|
||||
...(ref.frameId ? { frameId: ref.frameId } : {}),
|
||||
...(refTarget?.sessionId ? { cdpSessionId: refTarget.sessionId } : {}),
|
||||
},
|
||||
] as const;
|
||||
}),
|
||||
);
|
||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||
text: rendered.text,
|
||||
|
||||
@@ -6,7 +6,8 @@
|
||||
// exactly the same sandbox + visibility rules as the M6 observation
|
||||
// handlers (review parity).
|
||||
|
||||
import type { DialogCursor } from "@/browser-driver/chromium-cdp";
|
||||
import type { CdpDebuggee, DialogCursor } from "@/browser-driver/chromium-cdp";
|
||||
import type { CdpFrameGraph, CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import type { SessionContext, SessionManager } from "@/session-manager/manager";
|
||||
import { normaliseRef } from "@/session-manager/ref-store";
|
||||
import type { ConsoleResult, JavaScriptDialogInfo, RpcError } from "@/transport/types";
|
||||
@@ -51,9 +52,11 @@ export type { DialogCursor };
|
||||
*/
|
||||
export interface CdpRunner {
|
||||
send<T = unknown>(tabId: number, method: string, params?: object): Promise<T>;
|
||||
sendToTarget?<T = unknown>(target: CdpTarget, method: string, params?: object): Promise<T>;
|
||||
getFrameGraph?(tabId: number): Promise<CdpFrameGraph>;
|
||||
ensureAttachedToUrl?(tabId: number, expectedUrl: string | undefined): Promise<void>;
|
||||
trackSessionTab?(sessionId: string, tabId: number): void;
|
||||
onEvent?(handler: (source: chrome.debugger.Debuggee, method: string, params: unknown) => void): {
|
||||
onEvent?(handler: (source: CdpDebuggee, method: string, params: unknown) => void): {
|
||||
dispose(): void;
|
||||
};
|
||||
dialogCursor?(tabId: number): DialogCursor;
|
||||
@@ -68,6 +71,25 @@ export interface CdpRunner {
|
||||
): ConsoleResult;
|
||||
}
|
||||
|
||||
export function sendToCdpTarget<T = unknown>(
|
||||
cdp: CdpRunner,
|
||||
target: CdpTarget,
|
||||
method: string,
|
||||
params?: object,
|
||||
): Promise<T> {
|
||||
if (target.sessionId && cdp.sendToTarget) {
|
||||
return cdp.sendToTarget<T>(target, method, params);
|
||||
}
|
||||
return cdp.send<T>(target.tabId, method, params);
|
||||
}
|
||||
|
||||
export function cdpRunnerForTarget(cdp: CdpRunner, target: CdpTarget): CdpRunner {
|
||||
return {
|
||||
send: <T = unknown>(_tabId: number, method: string, params?: object) =>
|
||||
sendToCdpTarget<T>(cdp, target, method, params),
|
||||
};
|
||||
}
|
||||
|
||||
export interface BufferedReadBounds {
|
||||
since: number | undefined;
|
||||
limit: number;
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// Snapshot ref resolution — normalise `@eN` / `eN`, look up
|
||||
// `backendNodeId` via the session RefStore, and build stable
|
||||
// compound CDP node identity via the session RefStore, and build stable
|
||||
// `ref_not_found` errors for hard-failure tool paths.
|
||||
|
||||
import type { SessionContext } from "@/session-manager/manager";
|
||||
@@ -10,6 +10,13 @@ import { rpcError } from "./errors";
|
||||
export interface SnapshotRefLookup {
|
||||
backendNodeId: number;
|
||||
refKey: string;
|
||||
frameId?: string;
|
||||
cdpSessionId?: string;
|
||||
}
|
||||
|
||||
function refEntryForTab(ctx: SessionContext, refKey: string, tabId: number) {
|
||||
const entry = ctx.refStore.resolveEntry(refKey);
|
||||
return entry?.tabId === tabId ? entry : null;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -23,9 +30,14 @@ export function lookupSnapshotRef(
|
||||
tabId: number,
|
||||
): SnapshotRefLookup | null {
|
||||
const refKey = normaliseRef(ref);
|
||||
const backendNodeId = ctx.refStore.resolve(refKey, { tabId });
|
||||
if (backendNodeId === null) return null;
|
||||
return { backendNodeId, refKey };
|
||||
const entry = refEntryForTab(ctx, refKey, tabId);
|
||||
if (!entry) return null;
|
||||
return {
|
||||
backendNodeId: entry.backendNodeId,
|
||||
refKey,
|
||||
...(entry.frameId ? { frameId: entry.frameId } : {}),
|
||||
...(entry.cdpSessionId ? { cdpSessionId: entry.cdpSessionId } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -37,7 +49,7 @@ export function resolveSnapshotRef(
|
||||
ctx: SessionContext,
|
||||
ref: string,
|
||||
tabId: number,
|
||||
): { backendNodeId: number; refKey: string } | RpcError {
|
||||
): SnapshotRefLookup | RpcError {
|
||||
const looked = lookupSnapshotRef(ctx, ref, tabId);
|
||||
if (looked === null) {
|
||||
return rpcError(
|
||||
|
||||
@@ -698,7 +698,7 @@ describe("captureViewModel", () => {
|
||||
const { nodes } = await captureViewModel(cdp, 4);
|
||||
const div = nodes.find((n) => n.backendNodeId === 12);
|
||||
expect(div?.localRect?.y).toBe(-200);
|
||||
expect(div?.rect).toMatchObject({ y: -200, h: 800 });
|
||||
expect(div?.rect).toMatchObject({ y: 0, h: 600 });
|
||||
});
|
||||
|
||||
it("normalizes device-pixel bounds by devicePixelRatio", async () => {
|
||||
@@ -919,7 +919,7 @@ describe("captureViewModel", () => {
|
||||
expect(input?.attrs.type).toBe("text");
|
||||
expect(input?.ownerFrameBackendNodeId).toBe(13);
|
||||
expect(input?.localRect).toEqual({ x: 0, y: 0, w: 200, h: 40 });
|
||||
expect(input?.rect).toBeNull();
|
||||
expect(input?.rect).toEqual({ x: 100, y: 300, w: 200, h: 40 });
|
||||
expect(rootFrameId).toBe("main-frame");
|
||||
expect(frameNodes?.get("child-frame")).toBe(subNodes);
|
||||
expect(frameOwnerBackendNodeIds?.get("child-frame")).toBe(13);
|
||||
@@ -1091,10 +1091,10 @@ describe("captureViewModel", () => {
|
||||
const input = iframeNodes.get(23)?.find((n) => n.backendNodeId === 31);
|
||||
|
||||
expect(nestedIframe?.localRect).toEqual({ x: 5, y: 6, w: 100, h: 80 });
|
||||
expect(nestedIframe?.rect).toBeNull();
|
||||
expect(nestedIframe?.rect).toEqual({ x: 15, y: 26, w: 100, h: 80 });
|
||||
expect(input?.ownerFrameBackendNodeId).toBe(23);
|
||||
expect(input?.localRect).toEqual({ x: 1, y: 2, w: 40, h: 20 });
|
||||
expect(input?.rect).toBeNull();
|
||||
expect(input?.rect).toEqual({ x: 16, y: 28, w: 40, h: 20 });
|
||||
});
|
||||
|
||||
it("normalizes bounds then subtracts CSS scroll at dpr>1", async () => {
|
||||
|
||||
@@ -3,61 +3,133 @@ import type { CdpRunner } from "../../shared";
|
||||
import type { CapturedNode, CapturedViewModel } from "../capture";
|
||||
import { captureFrameData } from "../frame-capture";
|
||||
|
||||
function node(frameId: string, backendNodeId: number): CapturedNode {
|
||||
function ownerNode(backendNodeId: number, x: number): CapturedNode {
|
||||
return {
|
||||
backendNodeId,
|
||||
parentBackendNodeId: null,
|
||||
frameId,
|
||||
tag: "div",
|
||||
parentBackendNodeId: 1,
|
||||
frameId: "main",
|
||||
ownerFrameBackendNodeId: null,
|
||||
tag: "iframe",
|
||||
attrs: {},
|
||||
rect: null,
|
||||
localRect: { x: 0, y: 0, w: 20, h: 20 },
|
||||
paintOrder: 0,
|
||||
rect: { x, y: 100, w: 300, h: 200 },
|
||||
localRect: { x, y: 100, w: 300, h: 200 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
};
|
||||
}
|
||||
|
||||
function childSnapshot(frameId: string, backendNodeId: number) {
|
||||
const strings = [frameId, "body", "button", "static", "auto", "pointer"];
|
||||
return {
|
||||
strings,
|
||||
documents: [
|
||||
{
|
||||
frameId,
|
||||
nodes: {
|
||||
parentIndex: [-1, 0],
|
||||
nodeName: [1, 2],
|
||||
backendNodeId: [backendNodeId - 1, backendNodeId],
|
||||
attributes: [[], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1],
|
||||
styles: [
|
||||
[3, 4, 4],
|
||||
[3, 4, 5],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 300, 200],
|
||||
[10, 20, 100, 40],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
describe("captureFrameData", () => {
|
||||
it("captures AX trees for every same-target frame found in the DOM snapshot", async () => {
|
||||
it("captures and positions multiple OOPIF documents missing from the root snapshot", async () => {
|
||||
const leftOwner = ownerNode(10, 50);
|
||||
const rightOwner = ownerNode(20, 500);
|
||||
const captured: CapturedViewModel = {
|
||||
nodes: [node("main", 1)],
|
||||
nodes: [leftOwner, rightOwner],
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map([[10, [node("child", 101)]]]),
|
||||
frameNodes: new Map([
|
||||
["main", [node("main", 1)]],
|
||||
["child", [node("child", 101)]],
|
||||
]),
|
||||
frameOwnerBackendNodeIds: new Map([["child", 10]]),
|
||||
frameParentIds: new Map([["child", "main"]]),
|
||||
iframeNodes: new Map(),
|
||||
frameNodes: new Map([["main", [leftOwner, rightOwner]]]),
|
||||
frameOwnerBackendNodeIds: new Map(),
|
||||
rootFrameId: "main",
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
if (method === "Accessibility.enable") return {};
|
||||
if (method === "Accessibility.getFullAXTree") {
|
||||
const frameId = (params as { frameId?: string })?.frameId ?? "main";
|
||||
return {
|
||||
nodes: [
|
||||
{
|
||||
nodeId: `${frameId}-root`,
|
||||
frameId,
|
||||
backendDOMNodeId: frameId === "main" ? 1 : 101,
|
||||
},
|
||||
],
|
||||
};
|
||||
const sendToTarget = vi.fn(async (target, method) => {
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 300, clientHeight: 200, pageX: 0, pageY: 0 } };
|
||||
}
|
||||
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") {
|
||||
return target.sessionId === "left-session"
|
||||
? childSnapshot("left", 101)
|
||||
: childSnapshot("right", 201);
|
||||
}
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
|
||||
throw new Error(`unexpected ${method}`);
|
||||
});
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId, method, params) => {
|
||||
if (method === "Accessibility.enable") return {};
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
|
||||
if (method === "DOM.getBoxModel") {
|
||||
const backendNodeId = (params as { backendNodeId?: number })?.backendNodeId;
|
||||
const x = backendNodeId === 10 ? 50 : 500;
|
||||
return { model: { content: [x, 100, x + 300, 100, x + 300, 300, x, 300] } };
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
|
||||
}
|
||||
throw new Error(`unexpected root ${method}`);
|
||||
}) as CdpRunner["send"],
|
||||
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
|
||||
getFrameGraph: vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "left",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4, sessionId: "left-session" },
|
||||
},
|
||||
{
|
||||
frameId: "right",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 20,
|
||||
target: { tabId: 4, sessionId: "right-session" },
|
||||
},
|
||||
],
|
||||
})),
|
||||
};
|
||||
|
||||
const documents = await captureFrameData({ send } as CdpRunner, 4, captured);
|
||||
const trees = await captureFrameData(cdp, 4, captured);
|
||||
|
||||
expect(send).toHaveBeenCalledWith(4, "Accessibility.enable", {});
|
||||
expect(documents.map((document) => document.frameId)).toEqual(["main", "child"]);
|
||||
expect(documents.find((document) => document.frameId === "child")).toMatchObject({
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
axNodes: [{ nodeId: "child-root", frameId: "child", backendDOMNodeId: 101 }],
|
||||
});
|
||||
expect(trees.map((tree) => tree.frameId)).toEqual(["main", "left", "right"]);
|
||||
expect(
|
||||
captured.frameNodes?.get("left")?.find((node) => node.backendNodeId === 101)?.rect,
|
||||
).toEqual({ x: 60, y: 120, w: 100, h: 40 });
|
||||
expect(
|
||||
captured.frameNodes?.get("right")?.find((node) => node.backendNodeId === 201)?.rect,
|
||||
).toEqual({ x: 510, y: 120, w: 100, h: 40 });
|
||||
expect(captured.frameOwnerBackendNodeIds).toEqual(
|
||||
new Map([
|
||||
["left", 10],
|
||||
["right", 20],
|
||||
]),
|
||||
);
|
||||
expect(captured.frameParentIds).toEqual(
|
||||
new Map([
|
||||
["left", "main"],
|
||||
["right", "main"],
|
||||
]),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { CdpFrameGraph } from "@/browser-driver/frame-graph";
|
||||
import type { CapturedNode, CapturedViewModel } from "../capture";
|
||||
import { buildFrameDocuments, type FrameAxBatch, type FrameOwnedAxNode } from "../frame-document";
|
||||
|
||||
@@ -9,8 +10,7 @@ function domNode(frameId: string, backendNodeId: number): CapturedNode {
|
||||
frameId,
|
||||
tag: "div",
|
||||
attrs: {},
|
||||
rect: null,
|
||||
localRect: { x: 0, y: 0, w: 20, h: 20 },
|
||||
rect: { x: 0, y: 0, w: 20, h: 20 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
@@ -30,9 +30,16 @@ function captured(frameNodes: Map<string, CapturedNode[]>): CapturedViewModel {
|
||||
|
||||
describe("buildFrameDocuments", () => {
|
||||
it("partitions mixed AX results by node ownership and cuts cross-frame parent edges", () => {
|
||||
const graph: CdpFrameGraph = {
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{ frameId: "child", parentFrameId: "main", target: { tabId: 4 } },
|
||||
],
|
||||
};
|
||||
const batches: FrameAxBatch<FrameOwnedAxNode>[] = [
|
||||
{
|
||||
frameId: "main",
|
||||
frame: graph.frames[0],
|
||||
nodes: [
|
||||
{ nodeId: "root", frameId: "main", backendDOMNodeId: 1, childIds: ["h"] },
|
||||
{
|
||||
@@ -48,6 +55,7 @@ describe("buildFrameDocuments", () => {
|
||||
];
|
||||
|
||||
const documents = buildFrameDocuments(
|
||||
graph,
|
||||
batches,
|
||||
captured(
|
||||
new Map([
|
||||
@@ -64,14 +72,28 @@ describe("buildFrameDocuments", () => {
|
||||
expect(childNodes.find((node) => node.nodeId === "virtual")?.frameId).toBe("child");
|
||||
});
|
||||
|
||||
it("deduplicates overlapping AX results using the strongest ownership", () => {
|
||||
it("deduplicates overlapping same-target AX results using the strongest ownership", () => {
|
||||
const graph: CdpFrameGraph = {
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{ frameId: "child", parentFrameId: "main", target: { tabId: 4 } },
|
||||
],
|
||||
};
|
||||
const childNode = { nodeId: "shared", backendDOMNodeId: 100 };
|
||||
const batches: FrameAxBatch<FrameOwnedAxNode>[] = [
|
||||
{ frameId: "main", nodes: [{ ...childNode, frameId: "child" }] },
|
||||
{ frameId: "child", nodes: [childNode] },
|
||||
{
|
||||
frame: graph.frames[0],
|
||||
nodes: [{ ...childNode, frameId: "child" }],
|
||||
},
|
||||
{
|
||||
frame: graph.frames[1],
|
||||
nodes: [childNode],
|
||||
},
|
||||
];
|
||||
|
||||
const documents = buildFrameDocuments(
|
||||
graph,
|
||||
batches,
|
||||
captured(
|
||||
new Map([
|
||||
@@ -85,22 +107,64 @@ describe("buildFrameDocuments", () => {
|
||||
expect(documents.find((document) => document.frameId === "child")?.axNodes).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("uses target-scoped backend ownership when OOPIF backend ids collide", () => {
|
||||
const graph: CdpFrameGraph = {
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "left",
|
||||
parentFrameId: "main",
|
||||
target: { tabId: 4, sessionId: "left-session" },
|
||||
},
|
||||
{
|
||||
frameId: "right",
|
||||
parentFrameId: "main",
|
||||
target: { tabId: 4, sessionId: "right-session" },
|
||||
},
|
||||
],
|
||||
};
|
||||
const batches: FrameAxBatch<FrameOwnedAxNode>[] = graph.frames.slice(1).map((frame) => ({
|
||||
frame,
|
||||
nodes: [{ nodeId: `${frame.frameId}-node`, backendDOMNodeId: 7 }],
|
||||
}));
|
||||
|
||||
const documents = buildFrameDocuments(
|
||||
graph,
|
||||
batches,
|
||||
captured(
|
||||
new Map([
|
||||
["main", []],
|
||||
["left", [domNode("left", 7)]],
|
||||
["right", [domNode("right", 7)]],
|
||||
]),
|
||||
),
|
||||
);
|
||||
|
||||
expect(documents.find((document) => document.frameId === "left")?.axNodes[0]?.nodeId).toBe(
|
||||
"left-node",
|
||||
);
|
||||
expect(documents.find((document) => document.frameId === "right")?.axNodes[0]?.nodeId).toBe(
|
||||
"right-node",
|
||||
);
|
||||
});
|
||||
|
||||
it("uses captured frame parent identity instead of guessing from backend ids", () => {
|
||||
const frameNodes = new Map([
|
||||
["main", [domNode("main", 1)]],
|
||||
["left", [domNode("left", 7)]],
|
||||
["right", [domNode("right", 8)]],
|
||||
["right", [domNode("right", 7)]],
|
||||
["child", [domNode("child", 9)]],
|
||||
]);
|
||||
const capture = captured(frameNodes);
|
||||
capture.frameOwnerBackendNodeIds = new Map([["child", 7]]);
|
||||
capture.frameParentIds = new Map([["child", "left"]]);
|
||||
const batches = ["main", "left", "right", "child"].map((frameId) => ({
|
||||
frameId,
|
||||
const frames = ["main", "left", "right", "child"].map((frameId) => ({
|
||||
frame: { frameId, target: { tabId: 4 } },
|
||||
nodes: [],
|
||||
}));
|
||||
|
||||
const documents = buildFrameDocuments(batches, capture);
|
||||
const documents = buildFrameDocuments(null, frames, capture);
|
||||
|
||||
expect(documents.find((document) => document.frameId === "child")).toMatchObject({
|
||||
parentFrameId: "left",
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
import type { Rect, Viewport } from "@browser-skill/vom";
|
||||
import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
|
||||
import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
|
||||
import { childFrameProjection, type GeometryProjection, projectRectToViewport } from "../geometry";
|
||||
import type { CdpRunner } from "../shared";
|
||||
|
||||
const REQUESTED_STYLES = ["position", "pointer-events", "cursor"] as const;
|
||||
@@ -22,9 +23,9 @@ export interface CapturedNode {
|
||||
ownerFrameBackendNodeId?: number | null;
|
||||
tag: string;
|
||||
attrs: Record<string, string>;
|
||||
/** Top-level viewport geometry when this node belongs to the root document. */
|
||||
/** Top-level viewport-relative CSS px, clipped to the owning frame viewport. */
|
||||
rect: Rect | null;
|
||||
/** Owning document viewport-relative CSS px. */
|
||||
/** Frame-local viewport-relative CSS px before top-level projection. */
|
||||
localRect?: Rect | null;
|
||||
paintOrder: number;
|
||||
position: string;
|
||||
@@ -317,6 +318,7 @@ interface ParseDocumentResult {
|
||||
interface FrameContext {
|
||||
frameId?: string;
|
||||
ownerFrameBackendNodeId: number | null;
|
||||
projection: GeometryProjection | null;
|
||||
scrollX: number;
|
||||
scrollY: number;
|
||||
}
|
||||
@@ -869,7 +871,10 @@ function parseDocumentNodes(
|
||||
w: b[2] / dpr,
|
||||
h: b[3] / dpr,
|
||||
};
|
||||
rect = context.ownerFrameBackendNodeId === null ? localRect : null;
|
||||
const bounds = context.projection
|
||||
? projectRectToViewport(localRect, context.projection)
|
||||
: null;
|
||||
rect = bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
|
||||
}
|
||||
paintOrder = dl?.paintOrders?.[li] ?? 0;
|
||||
const styleRow = dl?.styles?.[li] ?? [];
|
||||
@@ -939,6 +944,7 @@ function parseChildFrameDocuments(
|
||||
strings: string[],
|
||||
dpr: number,
|
||||
parentDocIndex: number,
|
||||
parentNodes: CapturedNode[],
|
||||
parentContext: FrameContext,
|
||||
visited = new Set<number>(),
|
||||
): ParseFrameDocumentsResult {
|
||||
@@ -953,6 +959,7 @@ function parseChildFrameDocuments(
|
||||
}
|
||||
|
||||
const parentBackendIds = parentDoc?.nodes?.backendNodeId ?? [];
|
||||
const parentNodeByBackendId = new Map(parentNodes.map((node) => [node.backendNodeId, node]));
|
||||
|
||||
for (const [nodeArrayIdx, childDocIndex] of cdi) {
|
||||
if (visited.has(childDocIndex)) continue;
|
||||
@@ -960,10 +967,16 @@ function parseChildFrameDocuments(
|
||||
if (!childDoc) continue;
|
||||
const iframeBackendId = parentBackendIds[nodeArrayIdx];
|
||||
if (iframeBackendId === undefined) continue;
|
||||
const iframeNode = parentNodeByBackendId.get(iframeBackendId);
|
||||
const projection =
|
||||
parentContext.projection && iframeNode?.localRect
|
||||
? childFrameProjection(parentContext.projection, iframeNode.localRect)
|
||||
: null;
|
||||
|
||||
const childContext: FrameContext = {
|
||||
frameId: snapshotFrameId(childDoc, strings),
|
||||
ownerFrameBackendNodeId: iframeBackendId,
|
||||
projection,
|
||||
scrollX: childDoc.scrollOffsetX ?? 0,
|
||||
scrollY: childDoc.scrollOffsetY ?? 0,
|
||||
};
|
||||
@@ -982,6 +995,7 @@ function parseChildFrameDocuments(
|
||||
strings,
|
||||
dpr,
|
||||
childDocIndex,
|
||||
parsed.nodes,
|
||||
childContext,
|
||||
nextVisited,
|
||||
);
|
||||
@@ -1042,6 +1056,12 @@ export async function captureViewModel(
|
||||
const topContext: FrameContext = {
|
||||
frameId: snapshotFrameId(doc0, strings),
|
||||
ownerFrameBackendNodeId: null,
|
||||
projection: {
|
||||
sourceViewport: viewport,
|
||||
sourceClips: [],
|
||||
edges: [],
|
||||
topViewport: viewport,
|
||||
},
|
||||
scrollX,
|
||||
scrollY,
|
||||
};
|
||||
@@ -1049,7 +1069,7 @@ export async function captureViewModel(
|
||||
const nodes = mainParsed.nodes;
|
||||
const excludedBackendNodeIds = new Set(mainParsed.excludedBackendNodeIds);
|
||||
|
||||
const frameParsed = parseChildFrameDocuments(documents, strings, dpr, 0, topContext);
|
||||
const frameParsed = parseChildFrameDocuments(documents, strings, dpr, 0, nodes, topContext);
|
||||
const iframeNodes = frameParsed.iframeNodes;
|
||||
for (const id of frameParsed.excludedBackendNodeIds) {
|
||||
excludedBackendNodeIds.add(id);
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
import type { CdpRunner } from "../shared";
|
||||
import type { CapturedViewModel } from "./capture";
|
||||
import type { CdpFrame, CdpFrameGraph } from "@/browser-driver/frame-graph";
|
||||
import { type FrameProjection, resolveFrameProjection } from "../frame-geometry";
|
||||
import { projectRectToViewport } from "../geometry";
|
||||
import { type CdpRunner, cdpRunnerForTarget, sendToCdpTarget } from "../shared";
|
||||
import { type CapturedNode, type CapturedViewModel, captureViewModel } from "./capture";
|
||||
import {
|
||||
buildFrameDocuments,
|
||||
type FrameAxBatch,
|
||||
@@ -18,39 +21,133 @@ function isAbortError(err: unknown): boolean {
|
||||
return err instanceof Error && err.name === "AbortError";
|
||||
}
|
||||
|
||||
function capturedFrameIds(captured: CapturedViewModel): string[] {
|
||||
const frameIds = new Set<string>();
|
||||
if (captured.rootFrameId) frameIds.add(captured.rootFrameId);
|
||||
for (const frameId of captured.frameNodes?.keys() ?? []) frameIds.add(frameId);
|
||||
if (frameIds.size === 0) frameIds.add("root");
|
||||
return [...frameIds];
|
||||
async function discoverFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFrameGraph | null> {
|
||||
if (!cdp.getFrameGraph) return null;
|
||||
try {
|
||||
return await cdp.getFrameGraph(tabId);
|
||||
} catch (err) {
|
||||
console.debug("[bsk observation] frame graph capture failed", err);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function transformFrameNodes(nodes: CapturedNode[], projection: FrameProjection): CapturedNode[] {
|
||||
return nodes.map((node) => ({
|
||||
...node,
|
||||
rect: (() => {
|
||||
const bounds = projectRectToViewport(node.rect, projection);
|
||||
return bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
|
||||
})(),
|
||||
}));
|
||||
}
|
||||
|
||||
function frameOwnerNode(captured: CapturedViewModel, frame: CdpFrame): CapturedNode | undefined {
|
||||
if (frame.ownerBackendNodeId === undefined) return undefined;
|
||||
const parentNodes = frame.parentFrameId
|
||||
? captured.frameNodes?.get(frame.parentFrameId)
|
||||
: captured.nodes;
|
||||
return parentNodes?.find((node) => node.backendNodeId === frame.ownerBackendNodeId);
|
||||
}
|
||||
|
||||
async function captureMissingFrameDocuments(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
graph: CdpFrameGraph,
|
||||
captured: CapturedViewModel,
|
||||
signal?: AbortSignal,
|
||||
): Promise<void> {
|
||||
if (!captured.frameNodes) captured.frameNodes = new Map();
|
||||
if (!captured.frameOwnerBackendNodeIds) captured.frameOwnerBackendNodeIds = new Map();
|
||||
if (!captured.frameParentIds) captured.frameParentIds = new Map();
|
||||
|
||||
for (const frame of graph.frames) {
|
||||
if (!frame.target.sessionId || captured.frameNodes.has(frame.frameId)) continue;
|
||||
const owner = frameOwnerNode(captured, frame);
|
||||
if (!owner?.rect || frame.ownerBackendNodeId === undefined) continue;
|
||||
try {
|
||||
const child = await captureViewModel(cdpRunnerForTarget(cdp, frame.target), tabId, {
|
||||
signal,
|
||||
});
|
||||
const projection = await resolveFrameProjection(cdp, graph, frame.frameId);
|
||||
if (!projection) continue;
|
||||
const rootFrameId = child.rootFrameId ?? frame.frameId;
|
||||
const childFrames = child.frameNodes ?? new Map<string, CapturedNode[]>();
|
||||
if (!childFrames.has(rootFrameId)) childFrames.set(rootFrameId, child.nodes);
|
||||
if (rootFrameId !== frame.frameId && !childFrames.has(frame.frameId)) {
|
||||
childFrames.set(
|
||||
frame.frameId,
|
||||
child.nodes.map((node) => ({ ...node, frameId: frame.frameId })),
|
||||
);
|
||||
}
|
||||
for (const [childFrameId, nodes] of childFrames) {
|
||||
captured.frameNodes.set(childFrameId, transformFrameNodes(nodes, projection));
|
||||
}
|
||||
captured.iframeNodes.set(
|
||||
frame.ownerBackendNodeId,
|
||||
captured.frameNodes.get(frame.frameId) ?? [],
|
||||
);
|
||||
captured.frameOwnerBackendNodeIds.set(frame.frameId, frame.ownerBackendNodeId);
|
||||
if (frame.parentFrameId) captured.frameParentIds.set(frame.frameId, frame.parentFrameId);
|
||||
for (const [childFrameId, ownerBackendNodeId] of child.frameOwnerBackendNodeIds ?? []) {
|
||||
captured.frameOwnerBackendNodeIds.set(childFrameId, ownerBackendNodeId);
|
||||
}
|
||||
for (const [childFrameId, parentFrameId] of child.frameParentIds ?? []) {
|
||||
captured.frameParentIds.set(childFrameId, parentFrameId);
|
||||
}
|
||||
} catch (err) {
|
||||
if (isAbortError(err)) throw err;
|
||||
console.debug("[bsk observation] child frame DOM capture failed", {
|
||||
frameId: frame.frameId,
|
||||
err,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function captureAxTrees<T extends FrameAxNode>(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
discoveredFrames: CdpFrame[],
|
||||
captured: CapturedViewModel,
|
||||
signal?: AbortSignal,
|
||||
): Promise<FrameAxBatch<T>[]> {
|
||||
const frameIds = capturedFrameIds(captured);
|
||||
let frames = [...discoveredFrames];
|
||||
const capturedFrameIds = [...(captured.frameNodes?.keys() ?? [])];
|
||||
if (frames.length === 0) {
|
||||
const rootFrameId = captured.rootFrameId ?? capturedFrameIds[0] ?? "root";
|
||||
frames = [{ frameId: rootFrameId, target: { tabId } }];
|
||||
}
|
||||
for (const frameId of capturedFrameIds) {
|
||||
if (!frames.some((frame) => frame.frameId === frameId)) {
|
||||
frames.push({ frameId, target: { tabId } });
|
||||
}
|
||||
}
|
||||
|
||||
let firstFailure: unknown;
|
||||
await cdp.send(tabId, "Accessibility.enable", {});
|
||||
const trees = await Promise.all(
|
||||
frameIds.map(async (frameId): Promise<FrameAxBatch<T>> => {
|
||||
frames.map(async (frame): Promise<FrameAxBatch<T>> => {
|
||||
try {
|
||||
throwIfAborted(signal);
|
||||
const result = await cdp.send<{ nodes?: T[] }>(
|
||||
tabId,
|
||||
await sendToCdpTarget(cdp, frame.target, "Accessibility.enable", {});
|
||||
const result = await sendToCdpTarget<{ nodes?: T[] }>(
|
||||
cdp,
|
||||
frame.target,
|
||||
"Accessibility.getFullAXTree",
|
||||
frameId === "root" ? {} : { frameId },
|
||||
frame.frameId === "root" ? {} : { frameId: frame.frameId },
|
||||
);
|
||||
throwIfAborted(signal);
|
||||
return { frameId, nodes: result.nodes ?? [] };
|
||||
return {
|
||||
frame,
|
||||
nodes: result.nodes ?? [],
|
||||
};
|
||||
} catch (err) {
|
||||
if (isAbortError(err)) throw err;
|
||||
firstFailure ??= err;
|
||||
console.debug("[bsk observation] frame AX capture failed", { frameId, err });
|
||||
return { frameId, nodes: [] };
|
||||
console.debug("[bsk observation] frame AX capture failed", {
|
||||
frameId: frame.frameId,
|
||||
err,
|
||||
});
|
||||
return { frame, nodes: [] };
|
||||
}
|
||||
}),
|
||||
);
|
||||
@@ -70,6 +167,10 @@ export async function captureFrameData<T extends FrameAxNode>(
|
||||
captured: CapturedViewModel,
|
||||
signal?: AbortSignal,
|
||||
): Promise<CapturedFrameDocument<T>[]> {
|
||||
const batches = await captureAxTrees<T>(cdp, tabId, captured, signal);
|
||||
return buildFrameDocuments(batches, captured);
|
||||
const graph = await discoverFrameGraph(cdp, tabId);
|
||||
const frames = graph?.frames ?? [];
|
||||
if (graph) await captureMissingFrameDocuments(cdp, tabId, graph, captured, signal);
|
||||
throwIfAborted(signal);
|
||||
const batches = await captureAxTrees<T>(cdp, tabId, frames, captured, signal);
|
||||
return buildFrameDocuments(graph, batches, captured);
|
||||
}
|
||||
|
||||
@@ -1,3 +1,9 @@
|
||||
import {
|
||||
type CdpFrame,
|
||||
type CdpFrameGraph,
|
||||
type CdpTarget,
|
||||
cdpTargetKey,
|
||||
} from "@/browser-driver/frame-graph";
|
||||
import type { CapturedNode, CapturedViewModel } from "./capture";
|
||||
|
||||
export interface FrameOwnedAxNode {
|
||||
@@ -9,14 +15,11 @@ export interface FrameOwnedAxNode {
|
||||
}
|
||||
|
||||
export interface FrameAxBatch<T extends FrameOwnedAxNode> {
|
||||
frameId: string;
|
||||
frame: CdpFrame;
|
||||
nodes: T[];
|
||||
}
|
||||
|
||||
export interface FrameDocument<T extends FrameOwnedAxNode> {
|
||||
frameId: string;
|
||||
parentFrameId?: string;
|
||||
ownerBackendNodeId?: number;
|
||||
export interface FrameDocument<T extends FrameOwnedAxNode> extends CdpFrame {
|
||||
contextScopeId: string;
|
||||
axNodes: T[];
|
||||
domNodes: CapturedNode[];
|
||||
@@ -29,30 +32,55 @@ interface Ownership {
|
||||
|
||||
interface OwnedCandidate<T> {
|
||||
node: T;
|
||||
target: CdpTarget;
|
||||
ownership: Ownership;
|
||||
}
|
||||
|
||||
function frameIds<T extends FrameOwnedAxNode>(
|
||||
function targetNodeKey(target: CdpTarget, nodeId: string): string {
|
||||
return `${cdpTargetKey(target)}:${nodeId}`;
|
||||
}
|
||||
|
||||
function targetBackendKey(target: CdpTarget, backendNodeId: number): string {
|
||||
return `${cdpTargetKey(target)}:${backendNodeId}`;
|
||||
}
|
||||
|
||||
function frameList<T extends FrameOwnedAxNode>(
|
||||
graph: CdpFrameGraph | null,
|
||||
batches: FrameAxBatch<T>[],
|
||||
captured: CapturedViewModel,
|
||||
): string[] {
|
||||
const ids = new Set<string>();
|
||||
if (captured.rootFrameId) ids.add(captured.rootFrameId);
|
||||
for (const frameId of captured.frameNodes?.keys() ?? []) ids.add(frameId);
|
||||
for (const batch of batches) ids.add(batch.frameId);
|
||||
return [...ids];
|
||||
): CdpFrame[] {
|
||||
const frames = new Map<string, CdpFrame>();
|
||||
for (const frame of graph?.frames ?? []) frames.set(frame.frameId, frame);
|
||||
for (const batch of batches) {
|
||||
if (!frames.has(batch.frame.frameId)) frames.set(batch.frame.frameId, batch.frame);
|
||||
}
|
||||
for (const frameId of captured.frameNodes?.keys() ?? []) {
|
||||
if (!frames.has(frameId))
|
||||
frames.set(frameId, { frameId, target: { tabId: batches[0]?.frame.target.tabId ?? 0 } });
|
||||
}
|
||||
return [...frames.values()].map((frame) => {
|
||||
const ownerBackendNodeId =
|
||||
frame.ownerBackendNodeId ?? captured.frameOwnerBackendNodeIds?.get(frame.frameId);
|
||||
const parentFrameId = frame.parentFrameId ?? captured.frameParentIds?.get(frame.frameId);
|
||||
return {
|
||||
...frame,
|
||||
...(ownerBackendNodeId !== undefined ? { ownerBackendNodeId } : {}),
|
||||
...(parentFrameId ? { parentFrameId } : {}),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export function buildFrameDocuments<T extends FrameOwnedAxNode>(
|
||||
graph: CdpFrameGraph | null,
|
||||
batches: FrameAxBatch<T>[],
|
||||
captured: CapturedViewModel,
|
||||
): FrameDocument<T>[] {
|
||||
const frames = frameIds(batches, captured);
|
||||
const knownFrames = new Set(frames);
|
||||
const backendOwner = new Map<number, string>();
|
||||
for (const frameId of frames) {
|
||||
for (const node of captured.frameNodes?.get(frameId) ?? []) {
|
||||
backendOwner.set(node.backendNodeId, frameId);
|
||||
const frames = frameList(graph, batches, captured);
|
||||
const frameById = new Map(frames.map((frame) => [frame.frameId, frame]));
|
||||
const backendOwner = new Map<string, string>();
|
||||
for (const frame of frames) {
|
||||
for (const node of captured.frameNodes?.get(frame.frameId) ?? []) {
|
||||
backendOwner.set(targetBackendKey(frame.target, node.backendNodeId), frame.frameId);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -64,15 +92,17 @@ export function buildFrameDocuments<T extends FrameOwnedAxNode>(
|
||||
const resolveOwnership = (node: T): Ownership => {
|
||||
const cached = ownershipByNodeId.get(node.nodeId);
|
||||
if (cached) return cached;
|
||||
if (node.frameId && knownFrames.has(node.frameId)) {
|
||||
if (node.frameId && frameById.has(node.frameId)) {
|
||||
const ownership = { frameId: node.frameId, strength: 4 };
|
||||
ownershipByNodeId.set(node.nodeId, ownership);
|
||||
return ownership;
|
||||
}
|
||||
if (typeof node.backendDOMNodeId === "number") {
|
||||
const owner = backendOwner.get(node.backendDOMNodeId);
|
||||
if (owner) {
|
||||
const ownership = { frameId: owner, strength: 3 };
|
||||
const frameId = backendOwner.get(
|
||||
targetBackendKey(batch.frame.target, node.backendDOMNodeId),
|
||||
);
|
||||
if (frameId) {
|
||||
const ownership = { frameId, strength: 3 };
|
||||
ownershipByNodeId.set(node.nodeId, ownership);
|
||||
return ownership;
|
||||
}
|
||||
@@ -91,28 +121,32 @@ export function buildFrameDocuments<T extends FrameOwnedAxNode>(
|
||||
return ownership;
|
||||
}
|
||||
}
|
||||
const ownership = { frameId: batch.frameId, strength: 1 };
|
||||
const ownership = { frameId: batch.frame.frameId, strength: 1 };
|
||||
ownershipByNodeId.set(node.nodeId, ownership);
|
||||
return ownership;
|
||||
};
|
||||
|
||||
for (const node of batch.nodes) {
|
||||
const ownership = resolveOwnership(node);
|
||||
const existing = candidates.get(node.nodeId);
|
||||
const key = targetNodeKey(batch.frame.target, node.nodeId);
|
||||
const existing = candidates.get(key);
|
||||
if (!existing || ownership.strength > existing.ownership.strength) {
|
||||
candidates.set(node.nodeId, { node, ownership });
|
||||
candidates.set(key, { node, target: batch.frame.target, ownership });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const ownershipByNodeId = new Map(
|
||||
[...candidates].map(([nodeId, candidate]) => [nodeId, candidate.ownership.frameId]),
|
||||
const ownershipByTargetNode = new Map(
|
||||
[...candidates].map(([key, candidate]) => [key, candidate.ownership.frameId]),
|
||||
);
|
||||
const axNodesByFrame = new Map<string, T[]>();
|
||||
for (const { node, ownership } of candidates.values()) {
|
||||
const parentFrameId = node.parentId ? ownershipByNodeId.get(node.parentId) : undefined;
|
||||
for (const candidate of candidates.values()) {
|
||||
const { node, target, ownership } = candidate;
|
||||
const parentFrameId = node.parentId
|
||||
? ownershipByTargetNode.get(targetNodeKey(target, node.parentId))
|
||||
: undefined;
|
||||
const childIds = node.childIds?.filter(
|
||||
(childId) => ownershipByNodeId.get(childId) === ownership.frameId,
|
||||
(childId) => ownershipByTargetNode.get(targetNodeKey(target, childId)) === ownership.frameId,
|
||||
);
|
||||
const ownedNode = {
|
||||
...node,
|
||||
@@ -127,16 +161,10 @@ export function buildFrameDocuments<T extends FrameOwnedAxNode>(
|
||||
axNodesByFrame.set(ownership.frameId, nodes);
|
||||
}
|
||||
|
||||
return frames.map((frameId) => ({
|
||||
frameId,
|
||||
...(captured.frameParentIds?.get(frameId)
|
||||
? { parentFrameId: captured.frameParentIds.get(frameId) }
|
||||
: {}),
|
||||
...(captured.frameOwnerBackendNodeIds?.get(frameId) !== undefined
|
||||
? { ownerBackendNodeId: captured.frameOwnerBackendNodeIds.get(frameId) }
|
||||
: {}),
|
||||
contextScopeId: frameId,
|
||||
axNodes: axNodesByFrame.get(frameId) ?? [],
|
||||
domNodes: captured.frameNodes?.get(frameId) ?? [],
|
||||
return frames.map((frame) => ({
|
||||
...frame,
|
||||
contextScopeId: frame.frameId,
|
||||
axNodes: axNodesByFrame.get(frame.frameId) ?? [],
|
||||
domNodes: captured.frameNodes?.get(frame.frameId) ?? [],
|
||||
}));
|
||||
}
|
||||
|
||||
@@ -25,10 +25,8 @@ export default defineConfig({
|
||||
name: "BrowserSkill",
|
||||
description:
|
||||
"Let AI agents use your logged-in browser in a separate Agent Window—without interrupting your work. Powered by the bsk CLI.",
|
||||
// Chrome 116 is the first version where WebSocket activity resets the
|
||||
// service-worker idle timer, which the 20s heartbeat relies on to
|
||||
// keep the daemon link alive.
|
||||
minimum_chrome_version: "116",
|
||||
// Flat debugger sessions are required to address out-of-process iframes.
|
||||
minimum_chrome_version: "125",
|
||||
permissions: [
|
||||
"alarms",
|
||||
"debugger",
|
||||
|
||||
Reference in New Issue
Block a user