Compare commits

...
Author SHA1 Message Date
Zhang GH cff1ea62da Merge pull request #126 from NianJiuZst/codex/focus-blur-primitive
feat(interaction): add explicit focus and blur primitives
2026-09-09 23:23:50 +08:00
drakezhang 20e8770006 feat(dsh-plugin): expose focus and blur actions 2026-09-09 22:47:47 +08:00
drakezhang ac9e7948bf fix(interaction): verify focus changes and honor cancellation 2026-09-09 22:47:42 +08:00
drakezhang aadc1376ff merge: integrate main into focus and blur PR 2026-09-09 22:33:48 +08:00
Zhang GH 0fe36bc0b1 Merge pull request #206 from Tencent/fix/oopif-scrollbar-geometry
fix(geometry): preserve OOPIF scale with occupied scrollbars
2026-09-09 17:45:21 +08:00
drakezhang a62a28d8cd fix(geometry): preserve OOPIF scale with occupied scrollbars 2026-09-09 16:58:25 +08:00
Hannah Liu 896c7944d8 Merge pull request #203 from Tencent/fix/capture-document-consistency
fix(observation): validate DOM identity across capture phases | 校验跨采集阶段的 DOM 身份一致性
2026-09-09 16:26:24 +08:00
Hannah Liu 7e5f3bae26 Merge pull request #200 from Tencent/fix/observation-document-consistency
refactor(observation):  unify cross-frame capture coordination and data ownership | 统一跨 frame 采集协调与数据归属
2026-09-09 16:25:52 +08:00
Hannah Liu af4c2fa229 Merge pull request #198 from Tencent/refactor/document-capture-facts
refactor(observation): decouple DOM processing from capture orchestration | 解耦 DOM 处理与采集流程
2026-09-09 16:24:42 +08:00
Hannah Liu ee10da6fe9 Merge pull request #195 from Tencent/fix/frame-geometry-contracts
fix(geometry): unify cross-frame coordinates and screenshot conversion | 统一跨 frame 坐标与截图转换
2026-09-09 16:24:06 +08:00
drakezhang 4834d8b5fa test(geometry): identify the regression browser attachment 2026-09-09 16:11:39 +08:00
Ljy-0827 b4006d5b85 perf(observation): validate document identity against snapshot roots 2026-09-09 16:09:09 +08:00
Ljy-0827 2d1e5c6d9d fix(observation): validate document consistency with bounded identity checks 2026-09-09 16:09:09 +08:00
drakezhang 282e6f5623 fix(observation): carry snapshot units through capture coordination 2026-09-09 16:05:42 +08:00
Ljy-0827 bd68c293b6 perf(observation): collect frame AX trees with bounded concurrency 2026-09-09 15:58:56 +08:00
Ljy-0827 35a20582ad refactor(observation): separate capture coordination from document consistency 2026-09-09 15:58:56 +08:00
Ljy-0827 4e3365450a refactor(observation): integrate capture orchestration and document consistency 2026-09-09 15:58:56 +08:00
Ljy-0827 df5ed1a495 fix(observation): validate document consistency during capture 2026-09-09 15:58:40 +08:00
NianJiuZst 0055e90f91 feat(interaction): add explicit focus and blur primitives 2026-08-21 19:14:40 +08:00
56 changed files with 5062 additions and 1677 deletions

No files matched your search

@@ -29,6 +29,24 @@ function fakeApi() {
} }
describe("ChromiumCdp", () => { describe("ChromiumCdp", () => {
it("changes attachment identity only across real debugger attachments", async () => {
const { api, onEvent, onDetach } = fakeApi();
const cdp = new ChromiumCdp(api);
expect(cdp.getAttachmentId(4)).toBeUndefined();
await cdp.ensureAttached(4);
const first = cdp.getAttachmentId(4);
expect(first).toBeTruthy();
await cdp.ensureAttached(4);
onEvent.fire({ tabId: 4 }, "Target.attachedToTarget", { sessionId: "child" });
expect(cdp.getAttachmentId(4)).toBe(first);
onDetach.fire({ tabId: 4 }, "target_closed");
expect(cdp.getAttachmentId(4)).toBeUndefined();
await cdp.ensureAttached(4);
expect(cdp.getAttachmentId(4)).not.toBe(first);
await cdp.detach(4);
expect(cdp.getAttachmentId(4)).toBeUndefined();
});
it("discovers multiple iframe targets and recursively routes nested OOPIF commands", async () => { it("discovers multiple iframe targets and recursively routes nested OOPIF commands", async () => {
const { api, onEvent } = fakeApi(); const { api, onEvent } = fakeApi();
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation( (api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
@@ -164,6 +164,7 @@ async function settleBeforeDeadline(promises: Promise<void>[], deadline: number)
export class ChromiumCdp { export class ChromiumCdp {
private readonly api: CdpDebuggerApi; private readonly api: CdpDebuggerApi;
private readonly attachedTabs = new Set<number>(); private readonly attachedTabs = new Set<number>();
private readonly attachmentIds = new Map<number, string>();
private readonly attachInFlight = new Map<number, Promise<void>>(); private readonly attachInFlight = new Map<number, Promise<void>>();
private readonly detachInFlight = new Map<number, Promise<void>>(); private readonly detachInFlight = new Map<number, Promise<void>>();
private readonly tabOwners = new Map<number, Set<string>>(); private readonly tabOwners = new Map<number, Set<string>>();
@@ -198,6 +199,11 @@ export class ChromiumCdp {
this.bindFrameTargetHandler(); this.bindFrameTargetHandler();
} }
/** Identity of this actual debugger attachment, independent of frame topology. */
getAttachmentId(tabId: number): string | undefined {
return this.attachmentIds.get(tabId);
}
/** Attach to `tabId` if we haven't already in this driver. */ /** Attach to `tabId` if we haven't already in this driver. */
async ensureAttached(tabId: number): Promise<void> { async ensureAttached(tabId: number): Promise<void> {
// Returning a tab clears the cache before Chrome finishes detaching. // Returning a tab clears the cache before Chrome finishes detaching.
@@ -217,6 +223,7 @@ export class ChromiumCdp {
await this.enableConsoleDomains(tabId); await this.enableConsoleDomains(tabId);
await this.enableNetworkDomainBestEffort(tabId); await this.enableNetworkDomainBestEffort(tabId);
this.attachedTabs.add(tabId); this.attachedTabs.add(tabId);
this.attachmentIds.set(tabId, crypto.randomUUID());
await this.enableFrameDiscovery({ tabId }).catch((err) => { await this.enableFrameDiscovery({ tabId }).catch((err) => {
console.debug("[bsk cdp] frame discovery unavailable", { tabId, err }); console.debug("[bsk cdp] frame discovery unavailable", { tabId, err });
}); });
@@ -444,6 +451,7 @@ export class ChromiumCdp {
this.attachInFlight.delete(tabId); this.attachInFlight.delete(tabId);
if (!this.attachedTabs.has(tabId)) return; if (!this.attachedTabs.has(tabId)) return;
this.attachedTabs.delete(tabId); this.attachedTabs.delete(tabId);
this.attachmentIds.delete(tabId);
this.clearDialogState(tabId); this.clearDialogState(tabId);
this.clearConsoleState(tabId); this.clearConsoleState(tabId);
this.clearNetworkState(tabId); this.clearNetworkState(tabId);
@@ -502,6 +510,7 @@ export class ChromiumCdp {
this.attachInFlight.clear(); this.attachInFlight.clear();
this.tabOwners.clear(); this.tabOwners.clear();
this.attachedTabs.clear(); this.attachedTabs.clear();
this.attachmentIds.clear();
this.dialogBuffers.clear(); this.dialogBuffers.clear();
this.dialogSequences.clear(); this.dialogSequences.clear();
this.consoleBuffers.clear(); this.consoleBuffers.clear();
@@ -848,6 +857,7 @@ export class ChromiumCdp {
const listener = (source: chrome.debugger.Debuggee, _reason: string) => { const listener = (source: chrome.debugger.Debuggee, _reason: string) => {
if (typeof source.tabId === "number") { if (typeof source.tabId === "number") {
this.attachedTabs.delete(source.tabId); this.attachedTabs.delete(source.tabId);
this.attachmentIds.delete(source.tabId);
this.attachInFlight.delete(source.tabId); this.attachInFlight.delete(source.tabId);
this.tabOwners.delete(source.tabId); this.tabOwners.delete(source.tabId);
this.clearDialogState(source.tabId); this.clearDialogState(source.tabId);
@@ -599,6 +599,70 @@ describe("ToolDispatcher", () => {
expect(onSessionsChanged).toHaveBeenCalledTimes(2); expect(onSessionsChanged).toHaveBeenCalledTimes(2);
}); });
it.each([
"focus",
"blur",
] as const)("routes %s with hover cleanup and cooperative cancellation", async (action) => {
const tab = { id: 7, windowId: 4242, active: true };
vi.stubGlobal("chrome", {
tabs: {
get: vi.fn(async () => tab),
query: vi.fn(async () => [tab]),
sendMessage: vi.fn(async () => undefined),
},
});
const sessions = new SessionManager({
agentWindow: {
create: async () => 4242,
remove: async () => {},
ensureActiveTab: async () => 7,
},
});
const ctx = await sessions.start("aa11");
ctx.refStore.set("e1", 12, { tabId: 7 });
const { transport, sent, deliver } = fakeTransport();
const onBrowserControlResumed = vi.fn();
let resolveNode: ((value: object) => void) | undefined;
const cdp = {
send: vi.fn(async (_tabId: number, method: string) => {
if (method === "DOM.resolveNode")
return new Promise((resolve) => {
resolveNode = resolve;
});
if (method === "Runtime.callFunctionOn") return { result: { value: true } };
return {};
}),
} as unknown as TestDispatcherCdp;
const dispatcher = new ToolDispatcher({ transport, sessions, cdp, onBrowserControlResumed });
const hover = dispatcher as unknown as {
rememberHover: (sessionId: string, result: object) => void;
setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
};
hover.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 });
await hover.setHoverBypass("aa11", 7, true);
dispatcher.start();
deliver(makeRequest(`tool.${action}`, { session_id: "aa11", ref: "e1" }));
await vi.waitFor(() => expect(resolveNode).toBeDefined());
expect(onBrowserControlResumed).toHaveBeenCalledWith("aa11");
deliver({ id: "cancel-focus", method: "cancel", params: { rpc_id: "r-1" } });
resolveNode!({ object: { objectId: "focus-target" } });
await vi.waitFor(() => expect(sent).toHaveLength(2));
expect(sent).toContainEqual({ id: "cancel-focus", result: { cancelled: true } });
expect(sent).toContainEqual({
id: "r-1",
error: expect.objectContaining({ code: "cancelled" }),
});
expect(cdp.send).not.toHaveBeenCalledWith(7, "DOM.focus", expect.anything());
expect(cdp.send).not.toHaveBeenCalledWith(7, "Runtime.callFunctionOn", expect.anything());
expect(cdp.send).toHaveBeenCalledWith(7, "Runtime.releaseObject", { objectId: "focus-target" });
expect(chrome.tabs.sendMessage).toHaveBeenCalledWith(
7,
expect.objectContaining({ enabled: false }),
);
expect(dispatcher.inflightAbortControllers.size).toBe(0);
dispatcher.stop();
});
it("invokes onBrowserControlResumed for browser-control tools but not passive reads", async () => { it("invokes onBrowserControlResumed for browser-control tools but not passive reads", async () => {
vi.stubGlobal("chrome", { vi.stubGlobal("chrome", {
tabs: { tabs: {
@@ -436,6 +436,8 @@ describe("file transfer tools", () => {
if (method === "DOM.getContentQuads") { if (method === "DOM.getContentQuads") {
return { quads: [[10, 20, 210, 20, 210, 120, 10, 120]] }; return { quads: [[10, 20, 210, 20, 210, 120, 10, 120]] };
} }
if (method === "Runtime.evaluate")
return { result: { value: { width: 400, height: 300 } } };
if (method === "Page.getLayoutMetrics") { if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 400, clientHeight: 300 } }; return { cssLayoutViewport: { clientWidth: 400, clientHeight: 300 } };
} }
@@ -0,0 +1,166 @@
// @vitest-environment node
// Opt in with BSK_FOCUS_CHROME=/path/to/chrome; each test owns its browser/profile.
import { describe, expect, it } from "vitest";
import { SessionManager } from "@/session-manager/manager";
import { handleBlur, handleFocus } from "../interaction";
import type { CdpRunner } from "../shared";
type Send = <T = Record<string, unknown>>(
method: string,
params?: object,
sessionId?: string,
) => Promise<T>;
async function withFocusBrowser(
run: (h: {
evaluate: (expression: string) => Promise<unknown>;
ref: () => Promise<void>;
focus: (signal?: AbortSignal) => ReturnType<typeof handleFocus>;
blur: (signal?: AbortSignal) => ReturnType<typeof handleBlur>;
calls: string[];
afterCommand: { current: (method: string) => void };
}) => Promise<void>,
) {
// Reuse the existing isolated Chrome launcher; no new browser dependency.
const { withChrome } = await import(
new URL(
"../../../../../evals/browser/cases/regression/snapshot-coordinates/chrome.mjs",
import.meta.url,
).href
);
await withChrome(
{ executable: process.env.BSK_FOCUS_CHROME, deviceScale: 1, zoom: 1 },
async (send: Send) => {
const { targetId } = await send<{ targetId: string }>("Target.createTarget", {
url: "about:blank",
});
const { sessionId } = await send<{ sessionId: string }>("Target.attachToTarget", {
targetId,
flatten: true,
});
await send("Page.bringToFront", {}, sessionId);
const evaluate = async (expression: string) => {
const reply = await send<{ result: { value: unknown }; exceptionDetails?: unknown }>(
"Runtime.evaluate",
{ expression, returnByValue: true },
sessionId,
);
expect(reply.exceptionDetails).toBeUndefined();
return reply.result.value;
};
await evaluate(
`document.body.innerHTML = '<input id="target"><input id="other">'; window.probe = document.querySelector('#target');`,
);
const manager = new SessionManager({
agentWindow: {
create: async () => 100,
remove: async () => {},
ensureActiveTab: async () => 4,
},
});
const ctx = await manager.start("aa11");
const calls: string[] = [];
const afterCommand = { current: (_method: string) => {} };
const cdp: CdpRunner = {
send: async (_tabId, method, params) => {
calls.push(method);
const result = await send(method, params, sessionId);
afterCommand.current(method);
return result as never;
},
};
const deps = {
cdp,
tabsApi: {
get: async (id: number) => ({ id, windowId: 100, active: true }) as chrome.tabs.Tab,
query: async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab],
},
};
await run({
evaluate,
calls,
afterCommand,
ref: async () => {
const { result } = await send<{ result: { objectId: string } }>(
"Runtime.evaluate",
{ expression: "window.probe" },
sessionId,
);
const { node } = await send<{ node: { backendNodeId: number } }>(
"DOM.describeNode",
{ objectId: result.objectId },
sessionId,
);
await send("Runtime.releaseObject", { objectId: result.objectId }, sessionId);
ctx.refStore.set("e1", node.backendNodeId, { tabId: 4 });
},
focus: (signal) =>
handleFocus(manager, { session_id: "aa11", ref: "e1" }, { ...deps, signal }),
blur: (signal) =>
handleBlur(manager, { session_id: "aa11", ref: "e1" }, { ...deps, signal }),
});
},
);
}
describe.skipIf(!process.env.BSK_FOCUS_CHROME)("real browser focus and blur", () => {
it.each(["light", "open", "closed"])("verifies focus and blur in %s DOM", async (mode) => {
await withFocusBrowser(async (h) => {
if (mode !== "light")
await h.evaluate(`(() => {
const host = document.createElement('div'); document.body.append(host);
const nested = document.createElement('div'); host.attachShadow({mode:'${mode}'}).append(nested);
const input = document.createElement('input'); nested.attachShadow({mode:'${mode}'}).append(input);
window.probe = input;
})()`);
await h.ref();
expect(await h.focus()).toMatchObject({ focused: true });
expect(await h.evaluate("window.probe.matches(':focus')")).toBe(true);
expect(await h.blur()).toMatchObject({ was_focused: true, focused: false });
expect(await h.evaluate("window.probe.matches(':focus')")).toBe(false);
expect(h.calls.filter((method) => method === "Runtime.releaseObject")).toHaveLength(2);
});
});
it("detects focus changes from event-handler microtasks", async () => {
await withFocusBrowser(async (h) => {
await h.ref();
await h.evaluate(
"window.probe.addEventListener('focus', () => queueMicrotask(() => document.querySelector('#other').focus()), {once:true})",
);
expect(await h.focus()).toMatchObject({ code: "cdp_failed" });
expect(await h.focus()).toMatchObject({ focused: true });
await h.evaluate(
"window.probe.addEventListener('blur', () => queueMicrotask(() => window.probe.focus()), {once:true})",
);
expect(await h.blur()).toMatchObject({ code: "cdp_failed" });
expect(await h.evaluate("window.probe.matches(':focus')")).toBe(true);
});
});
it("cancels before blur while still releasing the resolved object", async () => {
await withFocusBrowser(async (h) => {
await h.ref();
await h.focus();
const abort = new AbortController();
h.afterCommand.current = (method) => {
if (method === "DOM.resolveNode") abort.abort();
};
expect(await h.blur(abort.signal)).toMatchObject({ code: "cancelled" });
expect(await h.evaluate("window.probe.matches(':focus')")).toBe(true);
expect(h.calls.at(-1)).toBe("Runtime.releaseObject");
});
});
it("reports a page exception with its cause", async () => {
await withFocusBrowser(async (h) => {
await h.ref();
await h.evaluate("window.probe.blur = () => { throw new Error('page blur failed'); }");
expect(await h.blur()).toMatchObject({
code: "cdp_failed",
message: expect.stringContaining("page blur failed"),
});
expect(h.calls.at(-1)).toBe("Runtime.releaseObject");
});
});
});
@@ -0,0 +1,293 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { CdpTarget } from "@/browser-driver/frame-graph";
import { SessionManager } from "@/session-manager/manager";
import { handleBlur, handleFocus } from "../interaction";
import type { CdpRunner } from "../shared";
interface ScriptParams {
functionDeclaration: string;
}
interface Call {
target: CdpTarget;
method: string;
}
async function setup(element?: HTMLElement, childSession = false) {
if (!element) {
document.body.innerHTML = '<input id="target"><input id="other">';
element = document.querySelector<HTMLInputElement>("#target")!;
}
const targetElement = element;
const manager = new SessionManager({
agentWindow: {
create: async () => 100,
remove: async () => {},
ensureActiveTab: async () => 4,
},
});
const ctx = await manager.start("aa11");
ctx.refStore.set("e1", 12, {
tabId: 4,
...(childSession ? { frameId: "child", cdpSessionId: "child-session" } : {}),
});
const calls: Call[] = [];
const afterCommand = vi.fn<(call: Call) => void>();
const script = vi.fn(async (params: ScriptParams) => {
try {
const fn = new Function(`return (${params.functionDeclaration})`)();
return { result: { value: await fn.call(targetElement) } };
} catch (error) {
return { exceptionDetails: { text: "Uncaught", exception: { description: String(error) } } };
}
});
const focus = vi.fn(() => targetElement.focus());
const release = vi.fn(async () => ({}));
const send = async (target: CdpTarget, method: string, params?: object) => {
const call = { target, method };
calls.push(call);
try {
switch (method) {
case "DOM.getDocument":
return { root: { nodeId: 1 } };
case "DOM.querySelector":
return { nodeId: 2 };
case "DOM.describeNode":
return { node: { backendNodeId: 12 } };
case "DOM.scrollIntoViewIfNeeded":
return {};
case "DOM.resolveNode":
return { object: { objectId: "focus-target" } };
case "DOM.focus":
focus();
return {};
case "Runtime.callFunctionOn":
return await script(params as ScriptParams);
case "Runtime.releaseObject":
return await release();
default:
throw new Error(`unexpected ${method}`);
}
} finally {
afterCommand(call);
}
};
const cdp: CdpRunner = {
send: (tabId, method, params) => send({ tabId }, method, params) as never,
sendToTarget: send as CdpRunner["sendToTarget"],
getFrameGraph: vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
...(childSession
? [
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 99,
target: { tabId: 4, sessionId: "child-session" },
},
]
: []),
],
})),
};
const tabsApi = {
get: vi.fn(async (id: number) => ({ id, windowId: 100, active: true }) as chrome.tabs.Tab),
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
};
return {
element: targetElement,
calls,
afterCommand,
script,
focus,
release,
cdp,
tabsApi,
run: (
action: "focus" | "blur",
signal?: AbortSignal,
params: { ref?: string; selector?: string; tab_id?: number } = { ref: "e1" },
) =>
(action === "focus" ? handleFocus : handleBlur)(
manager,
{ session_id: "aa11", ...params },
{ cdp, tabsApi, signal },
),
};
}
afterEach(() => {
vi.restoreAllMocks();
document.body.innerHTML = "";
});
describe("focus and blur state verification", () => {
it.each(["open", "closed"] as const)("handles nested %s shadow roots", async (mode) => {
const host = document.createElement("div");
document.body.append(host);
const outer = host.attachShadow({ mode });
const innerHost = document.createElement("div");
outer.append(innerHost);
const inner = innerHost.attachShadow({ mode });
const element = document.createElement("input");
inner.append(element);
const h = await setup(element);
expect(await h.run("focus")).toMatchObject({ focused: true });
expect(inner.activeElement).toBe(element);
expect(await h.run("blur")).toMatchObject({ was_focused: true, focused: false });
expect(inner.activeElement).toBeNull();
expect(h.release).toHaveBeenCalledTimes(2);
});
it("keeps DOM focus usable when the document is in the background", async () => {
const h = await setup();
vi.spyOn(document, "hasFocus").mockReturnValue(false);
expect(await h.run("focus")).toMatchObject({ focused: true });
expect(await h.run("blur")).toMatchObject({ was_focused: true, focused: false });
});
it("blurring an unfocused target preserves the other element's focus", async () => {
const h = await setup();
const other = document.querySelector<HTMLInputElement>("#other")!;
other.focus();
expect(await h.run("blur")).toMatchObject({ was_focused: false, focused: false });
expect(document.activeElement).toBe(other);
});
it("detects focus redirected by a focus handler's microtask", async () => {
const h = await setup();
h.element.addEventListener("focus", () =>
queueMicrotask(() => document.querySelector<HTMLInputElement>("#other")!.focus()),
);
expect(await h.run("focus")).toMatchObject({
code: "cdp_failed",
message: "target element did not become focused",
});
expect(h.release).toHaveBeenCalledOnce();
});
it("does not report blur success when a microtask restores focus", async () => {
const h = await setup();
h.element.focus();
h.element.addEventListener("blur", () => queueMicrotask(() => h.element.focus()));
expect(await h.run("blur")).toMatchObject({
code: "cdp_failed",
message: "target element remained focused after blur()",
});
expect(document.activeElement).toBe(h.element);
expect(h.release).toHaveBeenCalledOnce();
});
it.each(["focus", "blur"] as const)("rejects a detached %s target", async (action) => {
const h = await setup();
h.element.remove();
expect(await h.run(action)).toMatchObject({ code: "cdp_failed" });
expect(h.release).toHaveBeenCalledOnce();
});
it("preserves a script exception's cause", async () => {
const h = await setup();
vi.spyOn(h.element, "blur").mockImplementation(() => {
throw new Error("page blur failed");
});
expect(await h.run("blur")).toMatchObject({
code: "cdp_failed",
message: expect.stringContaining("page blur failed"),
});
expect(h.release).toHaveBeenCalledOnce();
});
it.each(["focus", "blur"] as const)("rejects malformed %s script results", async (action) => {
const h = await setup();
h.script.mockResolvedValueOnce({ result: { value: null } });
expect(await h.run(action)).toMatchObject({
code: "cdp_failed",
message: expect.stringContaining("unexpected result"),
});
expect(h.release).toHaveBeenCalledOnce();
});
it.each([
"focus",
"blur",
] as const)("keeps %s, verification and cleanup in the OOPIF session", async (action) => {
const h = await setup(undefined, true);
if (action === "blur") h.element.focus();
expect(await h.run(action)).toMatchObject({ focused: action === "focus" });
const targetCalls = h.calls.filter((call) => call.method !== "DOM.scrollIntoViewIfNeeded");
expect(targetCalls.every((call) => call.target.sessionId === "child-session")).toBe(true);
expect(targetCalls.at(-1)?.method).toBe("Runtime.releaseObject");
});
it.each(["focus", "blur"] as const)("rejects %s on a user tab before CDP", async (action) => {
const h = await setup();
h.tabsApi.get.mockResolvedValue({ id: 9, windowId: 200, active: true } as chrome.tabs.Tab);
expect(await h.run(action, undefined, { ref: "e1", tab_id: 9 })).toMatchObject({
code: "permission_denied",
});
expect(h.calls).toEqual([]);
});
it("does not replace a successful result when navigation disposed the remote object", async () => {
const h = await setup();
h.release.mockRejectedValue(new Error("Cannot find context"));
expect(await h.run("focus")).toMatchObject({ focused: true });
});
});
describe("focus and blur cancellation", () => {
const boundaries = [
"DOM.getDocument",
"DOM.querySelector",
"DOM.describeNode",
"DOM.resolveNode",
"Runtime.callFunctionOn",
];
for (const action of ["focus", "blur"] as const) {
it.each(
action === "focus" ? [...boundaries, "DOM.scrollIntoViewIfNeeded", "DOM.focus"] : boundaries,
)(`${action} stops after cancellation during %s`, async (method) => {
const h = await setup();
const abort = new AbortController();
let cancelledAfter = 0;
h.afterCommand.mockImplementation((call) => {
if (call.method === method && !abort.signal.aborted) {
cancelledAfter = h.calls.length;
abort.abort();
}
});
expect(await h.run(action, abort.signal, { selector: "#target" })).toMatchObject({
code: "cancelled",
});
expect(abort.signal.aborted).toBe(true);
expect(
h.calls.slice(cancelledAfter).every((call) => call.method === "Runtime.releaseObject"),
).toBe(true);
if (h.calls.some((call) => call.method === "DOM.resolveNode"))
expect(h.release).toHaveBeenCalledOnce();
});
it(`${action} stops if cancellation arrives while resolving the tab`, async () => {
const h = await setup();
const abort = new AbortController();
h.tabsApi.query.mockImplementation(async () => {
abort.abort();
return [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab];
});
expect(await h.run(action, abort.signal)).toMatchObject({ code: "cancelled" });
expect(h.calls).toEqual([]);
});
}
it("does not scroll frame owners after cancellation in an OOPIF", async () => {
const h = await setup(undefined, true);
const abort = new AbortController();
h.afterCommand.mockImplementation((call) => {
if (call.method === "DOM.scrollIntoViewIfNeeded") abort.abort();
});
expect(await h.run("focus", abort.signal)).toMatchObject({ code: "cancelled" });
expect(h.calls.filter((call) => call.method === "DOM.scrollIntoViewIfNeeded")).toHaveLength(1);
expect(h.focus).not.toHaveBeenCalled();
});
});
@@ -13,7 +13,138 @@ import {
import { GeometryContext } from "../geometry/frame-context"; import { GeometryContext } from "../geometry/frame-context";
import type { CdpRunner } from "../shared"; import type { CdpRunner } from "../shared";
function scrollbarDriver(
visible = { width: 185, height: 89 },
frameSize: unknown = { width: 200, height: 100 },
) {
const send = vi.fn(async (target: { sessionId?: string }, method: string) => {
if (method === "Runtime.evaluate") return { result: { value: frameSize } };
if (method === "Page.getLayoutMetrics")
return {
cssLayoutViewport: {
clientWidth: target.sessionId ? visible.width : 800,
clientHeight: target.sessionId ? visible.height : 600,
},
};
if (method === "DOM.getBoxModel")
return {
model: {
content: target.sessionId
? [0, 0, 200, 0, 200, 100, 0, 100]
: [100, 100, 500, 100, 500, 300, 100, 300],
},
};
if (method === "DOM.getContentQuads")
return { quads: [[180, 80, 230, 80, 230, 120, 180, 120]] };
throw new Error(`unexpected ${method}`);
});
const cdp: CdpRunner = {
send: (tabId, method) => send({ tabId } as never, method) as never,
sendToTarget: send as CdpRunner["sendToTarget"],
getFrameGraph: async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4, sessionId: "child" },
},
{
frameId: "same-child",
parentFrameId: "child",
ownerBackendNodeId: 20,
target: { tabId: 4, sessionId: "child" },
},
],
}),
};
return { cdp, send };
}
describe("frame geometry projection", () => { describe("frame geometry projection", () => {
it.each([
{ width: 200, height: 100 },
{ width: 185, height: 100 },
{ width: 200, height: 89 },
{ width: 185, height: 89 },
])("preserves scale and clips scrollbar strips for visible viewport $width × $height", async (visible) => {
const { cdp, send } = scrollbarDriver(visible);
const geometry = await resolveNodeGeometry(cdp, 4, {
target: { tabId: 4, sessionId: "child" },
frameId: "child",
backendNodeId: 101,
});
expect(geometry).toMatchObject({
topBounds: {
x: 460,
y: 260,
width: (visible.width - 180) * 2,
height: (visible.height - 80) * 2,
},
actionPoint: { x: 460 + visible.width - 180, y: 260 + visible.height - 80 },
targetActionPoint: { x: (180 + visible.width) / 2, y: (80 + visible.height) / 2 },
});
expect(send.mock.calls.filter(([, method]) => method === "Runtime.evaluate")).toHaveLength(1);
expect(send.mock.calls.filter(([, method]) => method === "Page.getLayoutMetrics")).toHaveLength(
2,
);
});
it("shares the full viewport read across same-target frames, and refreshes it next operation", async () => {
const { cdp, send } = scrollbarDriver();
const context = new GeometryContext(cdp, 4);
await Promise.all([context.targetProjection("child"), context.targetProjection("same-child")]);
expect(send.mock.calls.filter(([, method]) => method === "Runtime.evaluate")).toHaveLength(1);
await new GeometryContext(cdp, 4).targetProjection("child");
expect(send.mock.calls.filter(([, method]) => method === "Runtime.evaluate")).toHaveLength(2);
});
it.each([
null,
{},
{ width: 0, height: 100 },
{ width: 200, height: NaN },
])("rejects unavailable full viewport dimensions: %j", async (size) => {
const { cdp } = scrollbarDriver(undefined, size);
expect(await new GeometryContext(cdp, 4).targetProjection("child")).toBeNull();
});
it("clips an inner OOPIF against an intermediate OOPIF's occupied scrollbar strips", async () => {
const { cdp } = scrollbarDriver();
const outer = cdp.sendToTarget!;
cdp.sendToTarget = async (target, method, params) => {
if (method === "DOM.getBoxModel")
return { model: { content: [150, 60, 250, 60, 250, 120, 150, 120] } } as never;
if (target.sessionId !== "inner") return outer(target, method, params);
if (method === "Runtime.evaluate")
return { result: { value: { width: 100, height: 60 } } } as never;
if (method === "Page.getLayoutMetrics")
return { cssLayoutViewport: { clientWidth: 90, clientHeight: 50 } } as never;
if (method === "DOM.getContentQuads")
return { quads: [[0, 0, 100, 0, 100, 60, 0, 60]] } as never;
throw new Error(`unexpected ${method}`);
};
const graph = await cdp.getFrameGraph!(4);
graph.frames[2] = {
frameId: "inner",
parentFrameId: "child",
ownerBackendNodeId: 20,
target: { tabId: 4, sessionId: "inner" },
};
cdp.getFrameGraph = async () => graph;
expect(
await resolveNodeGeometry(cdp, 4, {
target: { tabId: 4, sessionId: "inner" },
frameId: "inner",
backendNodeId: 101,
}),
).toMatchObject({
topBounds: { x: 400, y: 220, width: 70, height: 58 },
});
});
it("keeps region bounds separate from polygon area", () => { it("keeps region bounds separate from polygon area", () => {
const region = [ const region = [
rectPolygon({ x: 0, y: 0, w: 10, h: 10 }), rectPolygon({ x: 0, y: 0, w: 10, h: 10 }),
@@ -77,6 +208,8 @@ describe("frame geometry projection", () => {
const cdp: CdpRunner = { const cdp: CdpRunner = {
send: send as CdpRunner["send"], send: send as CdpRunner["send"],
sendToTarget: vi.fn(async (_target, method) => { sendToTarget: vi.fn(async (_target, method) => {
if (method === "Runtime.evaluate")
return { result: { value: { width: 200, height: 100 } } };
if (method === "Page.getLayoutMetrics") { if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } }; return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
} }
@@ -135,6 +268,8 @@ describe("frame geometry projection", () => {
throw new Error(`unexpected root command ${method}`); throw new Error(`unexpected root command ${method}`);
}) as CdpRunner["send"], }) as CdpRunner["send"],
sendToTarget: vi.fn(async (_target, method) => { sendToTarget: vi.fn(async (_target, method) => {
if (method === "Runtime.evaluate")
return { result: { value: { width: 200, height: 100 } } };
if (method === "Page.getLayoutMetrics") { if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } }; return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
} }
@@ -196,6 +331,8 @@ describe("frame geometry projection", () => {
}) as CdpRunner["send"], }) as CdpRunner["send"],
sendToTarget: vi.fn(async (_target, method) => { sendToTarget: vi.fn(async (_target, method) => {
if (method === "DOM.scrollIntoViewIfNeeded") return {}; if (method === "DOM.scrollIntoViewIfNeeded") return {};
if (method === "Runtime.evaluate")
return { result: { value: { width: 200, height: 100 } } };
if (method === "Page.getLayoutMetrics") { if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } }; return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
} }
@@ -433,6 +433,7 @@ describe("handleRequestHelp", () => {
if (method === "DOM.getContentQuads") { if (method === "DOM.getContentQuads") {
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }; return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
} }
if (method === "Runtime.evaluate") return { result: { value: { width: 200, height: 100 } } };
if (method === "Page.getLayoutMetrics") { if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } }; return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
} }
@@ -2,8 +2,10 @@ import { describe, expect, it, vi } from "vitest";
import { SessionManager } from "@/session-manager/manager"; import { SessionManager } from "@/session-manager/manager";
import type { CdpRunner } from "@/tools/shared"; import type { CdpRunner } from "@/tools/shared";
import { import {
handleBlur,
handleClick, handleClick,
handleFill, handleFill,
handleFocus,
handleHover, handleHover,
handlePress, handlePress,
handleSelect, handleSelect,
@@ -201,6 +203,7 @@ describe("handleClick", () => {
if (method === "DOM.getContentQuads") { if (method === "DOM.getContentQuads") {
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }; return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
} }
if (method === "Runtime.evaluate") return { result: { value: { width: 200, height: 100 } } };
if (method === "Page.getLayoutMetrics") { if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } }; return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
} }
@@ -220,6 +223,7 @@ describe("handleClick", () => {
{ sessionId: "child-session", method: "DOM.scrollIntoViewIfNeeded" }, { sessionId: "child-session", method: "DOM.scrollIntoViewIfNeeded" },
{ sessionId: "child-session", method: "DOM.getContentQuads" }, { sessionId: "child-session", method: "DOM.getContentQuads" },
{ sessionId: "child-session", method: "Page.getLayoutMetrics" }, { sessionId: "child-session", method: "Page.getLayoutMetrics" },
{ sessionId: "child-session", method: "Runtime.evaluate" },
]); ]);
expect(fake.sent.filter((call) => call.method === "Input.dispatchMouseEvent")).toHaveLength(3); expect(fake.sent.filter((call) => call.method === "Input.dispatchMouseEvent")).toHaveLength(3);
}); });
@@ -540,6 +544,161 @@ describe("handleHover", () => {
}); });
}); });
describe("handleFocus and handleBlur", () => {
it("focuses a ref and verifies the deep active element", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e3", 1234, { tabId: 4 });
const fake = makeFakeCdp({
"DOM.scrollIntoViewIfNeeded": () => ({}),
"DOM.focus": () => ({}),
"DOM.resolveNode": () => ({ object: { objectId: "focus-target" } }),
"Runtime.callFunctionOn": () => ({ result: { value: true } }),
});
const res = await handleFocus(
sm,
{ session_id: "aa11", ref: "@e3" },
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(res).toMatchObject({ tab_id: 4, used_ref: "e3", focused: true });
expect(fake.sent.map((call) => call.method)).toEqual([
"DOM.scrollIntoViewIfNeeded",
"DOM.resolveNode",
"DOM.focus",
"Runtime.callFunctionOn",
"Runtime.releaseObject",
]);
});
it("focuses an OOPIF ref in its CDP session", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e3", 1234, {
tabId: 4,
frameId: "child-frame",
cdpSessionId: "child-session",
});
const fake = makeFakeCdp({
"DOM.scrollIntoViewIfNeeded": () => ({}),
});
fake.cdp.getFrameGraph = vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child-frame",
parentFrameId: "main",
ownerBackendNodeId: 99,
target: { tabId: 4, sessionId: "child-session" },
},
],
}));
const targetCalls: Array<{ sessionId?: string; method: string }> = [];
fake.cdp.sendToTarget = vi.fn(async (target, method) => {
targetCalls.push({ sessionId: target.sessionId, method });
if (method === "DOM.scrollIntoViewIfNeeded") return {};
if (method === "DOM.focus" || method === "Runtime.releaseObject") return {};
if (method === "DOM.resolveNode") return { object: { objectId: "focus-target" } };
if (method === "Runtime.callFunctionOn") return { result: { value: true } };
throw new Error(`unexpected child CDP call ${method}`);
}) as CdpRunner["sendToTarget"];
const res = await handleFocus(
sm,
{ session_id: "aa11", ref: "@e3" },
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(res.focused).toBe(true);
expect(targetCalls).toEqual([
{ sessionId: "child-session", method: "DOM.scrollIntoViewIfNeeded" },
{ sessionId: "child-session", method: "DOM.resolveNode" },
{ sessionId: "child-session", method: "DOM.focus" },
{ sessionId: "child-session", method: "Runtime.callFunctionOn" },
{ sessionId: "child-session", method: "Runtime.releaseObject" },
]);
});
it("does not focus after cancellation during scrolling", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e3", 1234, { tabId: 4 });
const abort = new AbortController();
const fake = makeFakeCdp({
"DOM.scrollIntoViewIfNeeded": () => {
abort.abort();
return {};
},
});
const res = await handleFocus(
sm,
{ session_id: "aa11", ref: "@e3" },
{ cdp: fake.cdp, tabsApi: fake.tabsApi, signal: abort.signal },
);
expect(res).toMatchObject({ code: "cancelled" });
expect(fake.sent.some((call) => call.method === "DOM.focus")).toBe(false);
});
it("blurs a ref and returns its previous focus state", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e3", 1234, { tabId: 4 });
const fake = makeFakeCdp({
"DOM.resolveNode": () => ({ object: { objectId: "focus-target" } }),
"Runtime.callFunctionOn": vi
.fn()
.mockResolvedValueOnce({ result: { value: { ok: true, was_focused: true } } })
.mockResolvedValueOnce({ result: { value: false } }),
});
const res = await handleBlur(
sm,
{ session_id: "aa11", ref: "@e3" },
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(res).toMatchObject({
tab_id: 4,
used_ref: "e3",
was_focused: true,
focused: false,
});
expect(fake.sent.map((call) => call.method)).toEqual([
"DOM.resolveNode",
"Runtime.callFunctionOn",
"Runtime.callFunctionOn",
"Runtime.releaseObject",
]);
});
it("rejects a target that does not implement blur", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e3", 1234, { tabId: 4 });
const fake = makeFakeCdp({
"DOM.resolveNode": () => ({ object: { objectId: "focus-target" } }),
"Runtime.callFunctionOn": () => ({
result: { value: { ok: false, was_focused: false, focused: false } },
}),
});
const res = await handleBlur(
sm,
{ session_id: "aa11", ref: "@e3" },
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
);
expect(res).toMatchObject({ code: "invalid_params", message: /does not support blur/ });
});
});
function successfulFillScript(params: unknown) { function successfulFillScript(params: unknown) {
const script = params as { arguments?: Array<{ value: unknown }>; functionDeclaration: string }; const script = params as { arguments?: Array<{ value: unknown }>; functionDeclaration: string };
const args = script.arguments ?? []; const args = script.arguments ?? [];
File diff suppressed because it is too large. Load diff
@@ -1,10 +1,9 @@
// @vitest-environment node // @vitest-environment node
import { describe, expect, it } from "vitest"; import { describe, expect, it } from "vitest";
import type { CdpFrame, CdpFrameGraph, CdpTarget } from "@/browser-driver/frame-graph"; import type { CdpFrame, CdpFrameGraph, CdpTarget } from "@/browser-driver/frame-graph";
import { GeometryContext } from "../geometry/frame-context"; import { resolveNodeGeometry } from "../frame-geometry";
import type { CdpRunner } from "../shared"; import type { CdpRunner } from "../shared";
import { captureViewModel } from "../vom/capture"; import { captureObservationFacts } from "../vom/capture-coordinator";
import { captureFrameData } from "../vom/frame-capture";
type Send = <T = Record<string, unknown>>( type Send = <T = Record<string, unknown>>(
method: string, method: string,
@@ -13,12 +12,28 @@ type Send = <T = Record<string, unknown>>(
) => Promise<T>; ) => Promise<T>;
type Tree = { frame: { id: string; parentId?: string; name?: string }; childFrames?: Tree[] }; type Tree = { frame: { id: string; parentId?: string; name?: string }; childFrames?: Tree[] };
type Rect = { x: number; y: number; w: number; h: number }; type Rect = { x: number; y: number; w: number; h: number };
type Oracle = { probe: Rect; owners: Record<string, { x: number; y: number; scale: number }> }; type Oracle = {
probes: Record<string, Rect>;
viewport: { width: number; height: number };
owners: Record<string, { x: number; y: number; scale: number }>;
};
function clip(rect: Rect, viewport: Oracle["viewport"]): Rect | null {
const x = Math.max(0, rect.x),
y = Math.max(0, rect.y);
const right = Math.min(viewport.width, rect.x + rect.w);
const bottom = Math.min(viewport.height, rect.y + rect.h);
return right > x && bottom > y ? { x, y, w: right - x, h: bottom - y } : null;
}
// The independent oracle uses DOM border boxes and this fixture's axis-aligned // The independent oracle uses DOM border boxes and this fixture's axis-aligned
// iframe transforms. No production snapshot conversion/projection builds expectations. // iframe transforms. No production snapshot conversion/projection builds expectations.
const oracleExpression = `(() => { const oracleExpression = `(() => {
const box = document.querySelector('#probe').getBoundingClientRect(); const probes = {};
for (const node of document.querySelectorAll('[data-geometry-probe]')) {
const box = node.getBoundingClientRect();
probes[node.id] = { x: box.x, y: box.y, w: box.width, h: box.height };
}
const owners = {}; const owners = {};
for (const frame of document.querySelectorAll('iframe')) { for (const frame of document.querySelectorAll('iframe')) {
const rect = frame.getBoundingClientRect(); const rect = frame.getBoundingClientRect();
@@ -30,17 +45,31 @@ const oracleExpression = `(() => {
scale, scale,
}; };
} }
return { probe: { x: box.x, y: box.y, w: box.width, h: box.height }, owners }; return { probes, viewport: { width: document.documentElement.clientWidth, height: document.documentElement.clientHeight }, owners };
})()`; })()`;
describe.skipIf(!process.env.BSK_GEOMETRY_CHROME)("real DOMSnapshot coordinate contract", () => { describe.skipIf(!process.env.BSK_GEOMETRY_CHROME)("real DOMSnapshot coordinate contract", () => {
it.each([ it.each(
{ deviceScale: 1, zoom: 1 }, [
{ deviceScale: 0.8, zoom: 1 }, { deviceScale: 1, zoom: 1 },
{ deviceScale: 1, zoom: 1.25 }, { deviceScale: 0.8, zoom: 1 },
{ deviceScale: 2, zoom: 1 }, { deviceScale: 1, zoom: 1.25 },
{ deviceScale: 2, zoom: 0.8 }, { deviceScale: 2, zoom: 1 },
])("matches DOM geometry at device scale $deviceScale and zoom $zoom", async (configuration) => { { deviceScale: 2, zoom: 0.8 },
]
.flatMap((configuration) => [
{ ...configuration, fixture: "snapshot-coordinates", scrollbars: "none" },
{ ...configuration, fixture: "oopif-scrollbars", scrollbars: "both" },
])
.concat(
["vertical", "horizontal", "none"].map((scrollbars) => ({
deviceScale: 1,
zoom: 1,
fixture: "oopif-scrollbars",
scrollbars,
})),
),
)("$fixture: device scale $deviceScale, zoom $zoom, scrollbars $scrollbars", async (configuration) => {
const evalRoot = new URL("../../../../../evals/browser/", import.meta.url); const evalRoot = new URL("../../../../../evals/browser/", import.meta.url);
const { createEvalServer } = await import(new URL("lib/server.mjs", evalRoot).href); const { createEvalServer } = await import(new URL("lib/server.mjs", evalRoot).href);
const { withChrome } = await import( const { withChrome } = await import(
@@ -61,7 +90,9 @@ describe.skipIf(!process.env.BSK_GEOMETRY_CHROME)("real DOMSnapshot coordinate c
); );
await send( await send(
"Page.navigate", "Page.navigate",
{ url: `${baseUrl}/snapshot-coordinates?run=coordinates` }, {
url: `${baseUrl}/${configuration.fixture}?run=coordinates&scrollbars=${configuration.scrollbars}`,
},
rootSession, rootSession,
); );
await expect await expect
@@ -88,7 +119,7 @@ describe.skipIf(!process.env.BSK_GEOMETRY_CHROME)("real DOMSnapshot coordinate c
}); });
sessions.push(sessionId); sessions.push(sessionId);
} }
expect(sessions.length).toBe(2); // The cross-site fixture must actually be an OOPIF. expect(sessions.length).toBe(configuration.fixture === "oopif-scrollbars" ? 3 : 2);
const frames: CdpFrame[] = []; const frames: CdpFrame[] = [];
const names = new Map<string, string>(); const names = new Map<string, string>();
const sessionFor = (target: CdpTarget) => target.sessionId ?? rootSession; const sessionFor = (target: CdpTarget) => target.sessionId ?? rootSession;
@@ -109,7 +140,7 @@ describe.skipIf(!process.env.BSK_GEOMETRY_CHROME)("real DOMSnapshot coordinate c
}; };
visit(frameTree); visit(frameTree);
} }
expect(frames).toHaveLength(5); expect(frames).toHaveLength(configuration.fixture === "oopif-scrollbars" ? 3 : 5);
for (const frame of frames) { for (const frame of frames) {
if (!frame.parentFrameId) continue; if (!frame.parentFrameId) continue;
const parent = frames.find((item) => item.frameId === frame.parentFrameId)!; const parent = frames.find((item) => item.frameId === frame.parentFrameId)!;
@@ -129,9 +160,15 @@ describe.skipIf(!process.env.BSK_GEOMETRY_CHROME)("real DOMSnapshot coordinate c
}, },
sendToTarget: (target, method, params) => { sendToTarget: (target, method, params) => {
calls.push(`${target.sessionId ?? target.tabId}:${method}`); calls.push(`${target.sessionId ?? target.tabId}:${method}`);
if (
method === "Runtime.evaluate" &&
(params as { expression?: string })?.expression?.includes("window.innerWidth")
)
calls.push(`${target.sessionId ?? target.tabId}:viewport-size`);
return send(method, params, sessionFor(target)); return send(method, params, sessionFor(target));
}, },
getFrameGraph: async () => graph, getFrameGraph: async () => graph,
getAttachmentId: () => rootSession,
}; };
const oracles = new Map<string, Oracle>(); const oracles = new Map<string, Oracle>();
for (const frame of frames) { for (const frame of frames) {
@@ -180,49 +217,130 @@ describe.skipIf(!process.env.BSK_GEOMETRY_CHROME)("real DOMSnapshot coordinate c
); );
oracles.set(frame.frameId, reply.result.value); oracles.set(frame.frameId, reply.result.value);
} }
const geometry = new GeometryContext(cdp, 1, graph); const childViewport = await send<{
const captured = await captureViewModel(cdp, 1, { geometry }); result: {
await captureFrameData(cdp, 1, captured, undefined, geometry); value: { width: number; height: number; clientWidth: number; clientHeight: number };
expect(captured.frameGeometryIssues).toEqual([]); };
for (const frame of frames) { }>(
const node = captured.frameNodes "Runtime.evaluate",
?.get(frame.frameId) {
?.find((node) => node.attrs.id === "probe"); expression:
expect(node, `missing probe in ${names.get(frame.frameId) || "root"}`).toBeDefined(); "({ width: innerWidth, height: innerHeight, clientWidth: document.documentElement.clientWidth, clientHeight: document.documentElement.clientHeight })",
const local = oracles.get(frame.frameId)!.probe; returnByValue: true,
const top = { ...local }; },
let current = frame; sessions[1],
while (current.parentFrameId) { );
const owner = oracles.get(current.parentFrameId)!.owners[names.get(current.frameId)!]; for (const dimension of ["Width", "Height"] as const) {
top.x = owner.x + top.x * owner.scale; const occupied =
top.y = owner.y + top.y * owner.scale; childViewport.result.value[dimension === "Width" ? "width" : "height"] -
top.w *= owner.scale; childViewport.result.value[`client${dimension}`];
top.h *= owner.scale; if (
current = frames.find((item) => item.frameId === current.parentFrameId)!; configuration.scrollbars === "both" ||
} configuration.scrollbars === (dimension === "Width" ? "vertical" : "horizontal")
expect( )
node!.rect, expect(occupied).toBeGreaterThan(0);
JSON.stringify({ else expect(occupied).toBe(0);
frame: names.get(frame.frameId),
local,
top,
viewport: captured.viewport,
}),
).not.toBeNull();
for (const key of ["x", "y", "w", "h"] as const) {
expect(
Math.abs(node!.localRect![key] - local[key]),
`${names.get(frame.frameId)} local ${key}`,
).toBeLessThan(2);
expect(
Math.abs(node!.rect![key] - top[key]),
`${names.get(frame.frameId)} top ${key}`,
).toBeLessThan(2);
}
} }
const facts = await captureObservationFacts(cdp, 1);
expect(facts.issues).toEqual([]);
expect(calls.filter((call) => call.endsWith(":Page.getLayoutMetrics"))).toHaveLength( expect(calls.filter((call) => call.endsWith(":Page.getLayoutMetrics"))).toHaveLength(
sessions.length, sessions.length,
); );
// Identity verification also uses Runtime.evaluate; count the full viewport reads separately.
const viewportReadsBeforeLive = calls.filter((call) => call.endsWith(":viewport-size"));
expect(new Set(viewportReadsBeforeLive).size).toBe(viewportReadsBeforeLive.length);
expect(viewportReadsBeforeLive.length).toBeLessThanOrEqual(sessions.length - 1);
for (const frame of frames) {
for (const [id, local] of Object.entries(oracles.get(frame.frameId)!.probes)) {
const node = facts.documents
.find((doc) => doc.frame.frameId === frame.frameId)
?.domNodes.find((node) => node.attrs.id === id);
expect(node, `missing ${id} in ${names.get(frame.frameId) || "root"}`).toBeDefined();
let top = clip(local, oracles.get(frame.frameId)!.viewport);
let current = frame;
while (top && current.parentFrameId) {
const parentOracle = oracles.get(current.parentFrameId)!;
const owner = parentOracle.owners[names.get(current.frameId)!];
top = clip(
{
x: owner.x + top.x * owner.scale,
y: owner.y + top.y * owner.scale,
w: top.w * owner.scale,
h: top.h * owner.scale,
},
parentOracle.viewport,
);
current = frames.find((item) => item.frameId === current.parentFrameId)!;
}
for (const key of ["x", "y", "w", "h"] as const)
expect(
Math.abs(node!.localRect![key] - local[key]),
`${names.get(frame.frameId)} ${id} local ${key}`,
).toBeLessThan(2);
if (!top) {
expect(node!.rect, `${id} must be clipped out`).toBeNull();
if (configuration.fixture === "oopif-scrollbars")
expect(
await resolveNodeGeometry(cdp, 1, {
target: frame.target,
frameId: frame.frameId,
backendNodeId: node!.backendNodeId,
}),
).toMatchObject({ code: "permission_denied" });
continue;
}
expect(node!.rect).not.toBeNull();
for (const key of ["x", "y", "w", "h"] as const)
expect(
Math.abs(node!.rect![key] - top[key]),
`${names.get(frame.frameId)} ${id} top ${key}`,
).toBeLessThan(2);
if (configuration.fixture !== "oopif-scrollbars") continue;
const live = await resolveNodeGeometry(cdp, 1, {
target: frame.target,
frameId: frame.frameId,
backendNodeId: node!.backendNodeId,
});
if ("code" in live) throw new Error(live.message);
for (const [key, liveKey] of [
["x", "x"],
["y", "y"],
["w", "width"],
["h", "height"],
] as const)
expect(
Math.abs(live.topBounds[liveKey] - top[key]),
`${id} live ${key}`,
).toBeLessThan(2);
const before = server.snapshot("coordinates").events.length;
for (const type of ["mousePressed", "mouseReleased"])
await send(
"Input.dispatchMouseEvent",
{ type, ...live.actionPoint, button: "left", clickCount: 1 },
rootSession,
);
await expect
.poll(
() =>
server
.snapshot("coordinates")
.events.slice(before)
.some(
(event: { type: string; path: string; data: { probe?: string } }) =>
event.type === "geometry.clicked" &&
event.data.probe === id &&
event.path ===
(frame.parentFrameId
? names.get(frame.frameId) === "nested"
? "/oopif-scrollbars/nested"
: "/oopif-scrollbars/frame"
: "/oopif-scrollbars"),
),
{ timeout: 3000 },
)
.toBe(true);
}
}
const metrics = await send<{ const metrics = await send<{
cssVisualViewport: { zoom: number }; cssVisualViewport: { zoom: number };
visualViewport: { clientWidth: number }; visualViewport: { clientWidth: number };
+35 -1
View File
@@ -2,12 +2,14 @@ import { OVERLAY_AUTOMATION_BYPASS } from "@/lib/overlay-bridge";
import type { SessionManager } from "@/session-manager/manager"; import type { SessionManager } from "@/session-manager/manager";
import type { Transport } from "@/transport/transport"; import type { Transport } from "@/transport/transport";
import type { import type {
BlurParams,
ClickParams, ClickParams,
ConsoleParams, ConsoleParams,
DownloadParams, DownloadParams,
EmulateParams, EmulateParams,
EvaluateParams, EvaluateParams,
FillParams, FillParams,
FocusParams,
GetHtmlParams, GetHtmlParams,
HoverParams, HoverParams,
HoverResult, HoverResult,
@@ -39,7 +41,15 @@ import { type EmulateCdpRunner, handleEmulate } from "./emulate";
import { classifyCdpError } from "./errors"; import { classifyCdpError } from "./errors";
import { handleEvaluate } from "./evaluate"; import { handleEvaluate } from "./evaluate";
import { handleRequestHelp } from "./human-loop"; import { handleRequestHelp } from "./human-loop";
import { handleClick, handleFill, handleHover, handlePress, handleSelect } from "./interaction"; import {
handleBlur,
handleClick,
handleFill,
handleFocus,
handleHover,
handlePress,
handleSelect,
} from "./interaction";
import { import {
handleNavigate, handleNavigate,
handleNavigateBack, handleNavigateBack,
@@ -515,6 +525,28 @@ export class ToolDispatcher {
); );
return this.rememberHover((req.params as HoverParams).session_id, result); return this.rememberHover((req.params as HoverParams).session_id, result);
} }
case "tool.focus":
return this.withHoverReleaseForRequest(
req.params as FocusParams,
() =>
handleFocus(
this.sessions,
req.params as FocusParams,
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
),
signal,
);
case "tool.blur":
return this.withHoverReleaseForRequest(
req.params as BlurParams,
() =>
handleBlur(
this.sessions,
req.params as BlurParams,
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
),
signal,
);
case "tool.fill": case "tool.fill":
return this.withHoverReleaseForRequest( return this.withHoverReleaseForRequest(
req.params as FillParams, req.params as FillParams,
@@ -785,6 +817,8 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
case "tool.reload": case "tool.reload":
case "tool.click": case "tool.click":
case "tool.hover": case "tool.hover":
case "tool.focus":
case "tool.blur":
case "tool.fill": case "tool.fill":
case "tool.press": case "tool.press":
case "tool.select": case "tool.select":
+2 -1
View File
@@ -161,7 +161,8 @@ export async function resolveNodeGeometry(
return geometryError(`could not resolve frame geometry for ${address.frameId}`); return geometryError(`could not resolve frame geometry for ${address.frameId}`);
topVisibleRegions = projectRegionToViewport(localRegion, projection); topVisibleRegions = projectRegionToViewport(localRegion, projection);
if (address.target.sessionId) { if (address.target.sessionId) {
const localViewport = projection.edges[0]?.sourceViewport ?? projection.topViewport; const localViewport = await context.viewport(address.target);
if (!localViewport) return geometryError("could not resolve target viewport geometry");
const localVisibleRegions = localRegion const localVisibleRegions = localRegion
.map((polygon) => .map((polygon) =>
clipPolygon( clipPolygon(
@@ -9,8 +9,10 @@ import {
type Polygon, type Polygon,
type ProjectiveEdge, type ProjectiveEdge,
parseCdpQuad, parseCdpQuad,
projectPolygon,
type Quad, type Quad,
type Size, type Size,
viewportPolygon,
} from "../geometry"; } from "../geometry";
import { type CdpRunner, sendToCdpTarget } from "../shared"; import { type CdpRunner, sendToCdpTarget } from "../shared";
import type { CoordinateOwner, CssViewport, SnapshotProjectionResult } from "./coordinate-types"; import type { CoordinateOwner, CssViewport, SnapshotProjectionResult } from "./coordinate-types";
@@ -67,6 +69,7 @@ export function cssViewport(metrics: LayoutMetrics): CssViewport {
/** One read-only measurement phase. Discard after scrolling or any later operation. */ /** One read-only measurement phase. Discard after scrolling or any later operation. */
export class GeometryContext { export class GeometryContext {
private readonly metrics = new Map<string, Promise<LayoutMetrics>>(); private readonly metrics = new Map<string, Promise<LayoutMetrics>>();
private readonly frameViewports = new Map<string, Promise<Size | null>>();
private readonly owners = new Map<string, Promise<Quad | null>>(); private readonly owners = new Map<string, Promise<Quad | null>>();
private readonly projections = new Map<string, Promise<GeometryProjection | null>>(); private readonly projections = new Map<string, Promise<GeometryProjection | null>>();
private readonly snapshotOwners = new Map<string, Promise<{ quad: Quad; size: Size } | null>>(); private readonly snapshotOwners = new Map<string, Promise<{ quad: Quad; size: Size } | null>>();
@@ -157,6 +160,35 @@ export class GeometryContext {
: null; : null;
} }
/** The full OOPIF viewport maps to its owner's content quad. Layout metrics
* exclude occupied scrollbars and remain the clipping boundary, not the scale. */
private frameViewport(target: CdpTarget): Promise<Size | null> {
const key = cdpTargetKey(target);
let promise = this.frameViewports.get(key);
if (!promise) {
promise = this.request<{ result?: { value?: Size }; exceptionDetails?: unknown }>(
target,
"Runtime.evaluate",
{
expression: "({ width: window.innerWidth, height: window.innerHeight })",
returnByValue: true,
},
).then(({ result, exceptionDetails }) => {
const size = result?.value;
return !exceptionDetails &&
size &&
Number.isFinite(size.width) &&
Number.isFinite(size.height) &&
size.width > 0 &&
size.height > 0
? size
: null;
});
this.frameViewports.set(key, promise);
}
return promise;
}
ownerContent(target: CdpTarget, backendNodeId: number): Promise<Quad | null> { ownerContent(target: CdpTarget, backendNodeId: number): Promise<Quad | null> {
const key = `${cdpTargetKey(target)}:${backendNodeId}`; const key = `${cdpTargetKey(target)}:${backendNodeId}`;
let promise = this.owners.get(key); let promise = this.owners.get(key);
@@ -342,12 +374,19 @@ export class GeometryContext {
const parent = frames.get(root.parentFrameId); const parent = frames.get(root.parentFrameId);
if (!parent || root.ownerBackendNodeId === undefined) return null; if (!parent || root.ownerBackendNodeId === undefined) return null;
const destinationQuad = await this.ownerContent(parent.target, root.ownerBackendNodeId); const destinationQuad = await this.ownerContent(parent.target, root.ownerBackendNodeId);
const source = await this.viewport(root.target); if (!destinationQuad) return null;
const source = await this.frameViewport(root.target);
const visible = await this.viewport(root.target);
const parentRoot = this.targetRoot(frames, parent); const parentRoot = this.targetRoot(frames, parent);
if (!destinationQuad || !source || !parentRoot) return null; if (!source || !visible || !parentRoot) return null;
const destinationClips = await this.clips(frames, parent, parentRoot); const destinationClips = await this.clips(frames, parent, parentRoot);
if (!destinationClips) return null; if (!destinationClips) return null;
edges.push({ sourceViewport: source, destinationQuad, destinationClips }); const edge = { sourceViewport: source, destinationQuad, destinationClips };
// Keep scrollbar strips clipped at every target boundary, including
// intermediate OOPIFs. Project the clip with the same full-viewport scale.
if (source.width !== visible.width || source.height !== visible.height)
destinationClips.push(projectPolygon(viewportPolygon(visible), edge));
edges.push(edge);
root = parentRoot; root = parentRoot;
} }
const topViewport = await this.viewport(root.target); const topViewport = await this.viewport(root.target);
+206 -2
View File
@@ -1,5 +1,4 @@
// DOM interaction tools — `tool.click`, `tool.fill`, `tool.press`, and // DOM interaction tools — click, hover, focus/blur, fill, press, and select.
// `tool.select`.
// //
// All interaction tools: // All interaction tools:
// 1. Resolve target tab (sandbox: must be inside Agent Window). // 1. Resolve target tab (sandbox: must be inside Agent Window).
@@ -14,10 +13,14 @@ import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
import type { CdpTarget } from "@/browser-driver/frame-graph"; import type { CdpTarget } from "@/browser-driver/frame-graph";
import type { SessionContext, SessionManager } from "@/session-manager/manager"; import type { SessionContext, SessionManager } from "@/session-manager/manager";
import type { import type {
BlurParams,
BlurResult,
ClickParams, ClickParams,
ClickResult, ClickResult,
FillParams, FillParams,
FillResult, FillResult,
FocusParams,
FocusResult,
HoverParams, HoverParams,
HoverResult, HoverResult,
KeyModifier, KeyModifier,
@@ -227,6 +230,207 @@ export async function resolveBackendNode(
} }
} }
// Check the target's own root, then its hosts: closed shadow roots are not
// reachable via host.shadowRoot. DOM focus remains meaningful in background tabs.
const FOCUS_CHECK = `function() {
if (!this.isConnected) return false;
let element = this;
while (element) {
const root = element.getRootNode();
if (root.activeElement !== element) return false;
element = root.host;
}
return true;
}`;
const BLUR_TARGET = `function() {
if (!this.isConnected) throw new Error('blur target is detached');
const wasFocused = (${FOCUS_CHECK}).call(this);
if (typeof this.blur !== 'function') return { ok: false, was_focused: wasFocused };
this.blur();
return { ok: true, was_focused: wasFocused };
}`;
interface FocusScriptReply<T> {
result?: { value?: T };
exceptionDetails?: { text?: string; exception?: { description?: string } };
}
function checkFocusAbort(signal: AbortSignal | undefined): void {
if (signal?.aborted) throw new DOMException("interaction aborted", "AbortError");
}
// Only focus/blur use this view. Guard every CDP boundary, including commands
// inside selector resolution and frame scrolling, without changing other tools.
function focusCdp(cdp: CdpRunner, signal: AbortSignal | undefined): CdpRunner {
const send = <T = unknown>(target: CdpTarget, method: string, params?: object): Promise<T> => {
checkFocusAbort(signal);
return cdpRunnerForTarget(cdp, target).send<T>(target.tabId, method, params);
};
return {
send: (tabId, method, params) => send({ tabId }, method, params),
sendToTarget: send,
trackSessionTab: cdp.trackSessionTab?.bind(cdp),
getFrameGraph: cdp.getFrameGraph
? (tabId) => {
checkFocusAbort(signal);
return cdp.getFrameGraph!(tabId);
}
: undefined,
};
}
// ---------------------------------------------------------------------------
// tool.focus / tool.blur
// ---------------------------------------------------------------------------
export function handleFocus(
manager: SessionManager,
params: FocusParams,
deps: InteractionDeps = getDefaultDeps(),
): Promise<FocusResult | RpcError> {
return changeFocus(manager, params, deps, "focus");
}
export function handleBlur(
manager: SessionManager,
params: BlurParams,
deps: InteractionDeps = getDefaultDeps(),
): Promise<BlurResult | RpcError> {
return changeFocus(manager, params, deps, "blur");
}
function changeFocus(
manager: SessionManager,
params: FocusParams,
deps: InteractionDeps,
action: "focus",
): Promise<FocusResult | RpcError>;
function changeFocus(
manager: SessionManager,
params: BlurParams,
deps: InteractionDeps,
action: "blur",
): Promise<BlurResult | RpcError>;
async function changeFocus(
manager: SessionManager,
params: FocusParams | BlurParams,
deps: InteractionDeps,
action: "focus" | "blur",
): Promise<FocusResult | BlurResult | RpcError> {
const ctx = lookupSession(manager, params, action);
if (isRpcError(ctx)) return ctx;
const cdp = focusCdp(deps.cdp, deps.signal);
let objectId: string | undefined;
let cleanupCdp: CdpRunner | undefined;
let tabId: number | undefined;
try {
checkFocusAbort(deps.signal);
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
checkFocusAbort(deps.signal);
if (isRpcError(target)) return target;
const denied = enforceAgentWindow(ctx, target, action);
if (denied) return denied;
tabId = target.tabId;
const dialogCursor = markDialogCursor(deps.cdp, tabId);
const node = await resolveBackendNode(cdp, ctx, target, params, action);
checkFocusAbort(deps.signal);
if (isRpcError(node)) return node;
const nodeCdp = cdpRunnerForTarget(cdp, node.cdpTarget);
cleanupCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
deps.cdp.trackSessionTab?.(ctx.sessionId, tabId);
if (action === "focus") {
const scrollErr = await scrollElementAndFramesIntoView(
cdp,
tabId,
node.cdpTarget,
node.backendNodeId,
node.frameId,
);
checkFocusAbort(deps.signal);
if (scrollErr) return scrollErr;
}
const resolved = await backendNodeToObject(nodeCdp, tabId, node.backendNodeId);
// Keep the handle before checking cancellation so finally can release it.
if (!isRpcError(resolved)) objectId = resolved;
checkFocusAbort(deps.signal);
if (isRpcError(resolved)) return resolved;
const runScript = async <T>(functionDeclaration: string): Promise<T | undefined> => {
const reply = await nodeCdp.send<FocusScriptReply<T>>(
target.tabId,
"Runtime.callFunctionOn",
{
objectId,
functionDeclaration,
returnByValue: true,
},
);
checkFocusAbort(deps.signal);
if (reply.exceptionDetails) {
const details = reply.exceptionDetails;
throw new Error(
`${action} script failed: ${details.exception?.description ?? details.text ?? "unknown exception"}`,
);
}
return reply.result?.value;
};
let wasFocused: boolean | undefined;
if (action === "focus") {
await nodeCdp.send(tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
checkFocusAbort(deps.signal);
} else {
const mutation = await runScript<{ ok: boolean; was_focused: boolean }>(BLUR_TARGET);
if (typeof mutation?.ok !== "boolean" || typeof mutation.was_focused !== "boolean") {
throw new Error("blur script returned an unexpected result");
}
if (!mutation.ok) {
return { code: "invalid_params", message: "target element does not support blur()" };
}
wasFocused = mutation.was_focused;
}
// Use a separate call so microtasks from focus/blur handlers finish before
// we report success. In particular, a blur handler can restore focus.
const focused = await runScript<boolean>(FOCUS_CHECK);
if (typeof focused !== "boolean") {
throw new Error(`${action} verification returned an unexpected result`);
}
if (focused !== (action === "focus")) {
throw new Error(
action === "focus"
? "target element did not become focused"
: "target element remained focused after blur()",
);
}
return attachDialogs(deps.cdp, tabId, dialogCursor, {
tab_id: tabId,
used_ref: node.usedRef,
used_selector: node.usedSelector,
focused,
...(action === "blur" ? { was_focused: wasFocused! } : {}),
});
} catch (err) {
return (
throwIfAborted(deps.signal) ?? {
code: "cdp_failed",
message: err instanceof Error ? err.message : String(err),
}
);
} finally {
if (objectId !== undefined && cleanupCdp && tabId !== undefined) {
// Cleanup must also run after cancellation. Navigation may have already
// disposed the object, which must not replace the operation's result.
try {
await cleanupCdp.send(tabId, "Runtime.releaseObject", { objectId });
} catch {
// The target or execution context may no longer exist.
}
}
}
}
export async function resolveActionTarget( export async function resolveActionTarget(
cdp: CdpRunner, cdp: CdpRunner,
ctx: SessionContext, ctx: SessionContext,
+38 -227
View File
@@ -1,3 +1,4 @@
import type { CapturedSceneInput } from "./vom/facts";
// Observation handlers — `tool.snapshot`, `tool.get_html`, `tool.screenshot`, // Observation handlers — `tool.snapshot`, `tool.get_html`, `tool.screenshot`,
// and semantic `tool.observe` (design §7). Each handler resolves the target // and semantic `tool.observe` (design §7). Each handler resolves the target
// tab (defaulting to the Agent Window's active tab when omitted) and // tab (defaulting to the Agent Window's active tab when omitted) and
@@ -35,7 +36,6 @@ import { attachDialogs, markDialogCursor } from "./dialogs";
import { rpcError } from "./errors"; import { rpcError } from "./errors";
import { resolveNodeGeometry } from "./frame-geometry"; import { resolveNodeGeometry } from "./frame-geometry";
import { screenshotPageRect } from "./geometry/coordinate-types"; import { screenshotPageRect } from "./geometry/coordinate-types";
import { cssViewport, GeometryContext } from "./geometry/frame-context";
import { import {
type ChromeTabsApi, type ChromeTabsApi,
enforceToolTargetScope, enforceToolTargetScope,
@@ -49,15 +49,9 @@ import {
type ToolEffect, type ToolEffect,
} from "./shared"; } from "./shared";
import { resolveSnapshotRef } from "./snapshot-ref"; import { resolveSnapshotRef } from "./snapshot-ref";
import { import { type CapturedNode, type CapturedSurfaceProbe, probeHoverSurfaces } from "./vom/capture";
type CapturedNode, import { captureObservationFacts, semanticCapture } from "./vom/capture-coordinator";
type CapturedSurfaceProbe, import type { FrameDocument as CapturedFrameDocument } from "./vom/frame-document";
type CapturedViewModel,
captureViewModel,
collectOverlayExcludedBackendIds,
probeHoverSurfaces,
} from "./vom/capture";
import { type CapturedFrameDocument, captureFrameData } from "./vom/frame-capture";
import { withOverlayBypass } from "./vom/hover-perception"; import { withOverlayBypass } from "./vom/hover-perception";
import { probeTooltipNames } from "./vom/name-enrichment"; import { probeTooltipNames } from "./vom/name-enrichment";
import { import {
@@ -531,7 +525,7 @@ function buildActiveScopeBlocks(nodes: VomNode[], signals: VomNodeDomSignals): A
function buildConditionalSurfaces( function buildConditionalSurfaces(
nodes: VomNode[], nodes: VomNode[],
captured: CapturedViewModel, captured: CapturedSceneInput,
probes: CapturedSurfaceProbe[], probes: CapturedSurfaceProbe[],
): CondSurface[] { ): CondSurface[] {
if (probes.length === 0) return []; if (probes.length === 0) return [];
@@ -661,163 +655,9 @@ export interface BuildVomSceneOptions {
export type VomFrameDocument = CapturedFrameDocument<CdpAxNode>; export type VomFrameDocument = CapturedFrameDocument<CdpAxNode>;
function legacyFrameDocuments(
axNodes: CdpAxNode[],
captured: CapturedViewModel,
pageUrl?: string,
rootTarget: VomFrameDocument["target"] = { tabId: 0 },
): VomFrameDocument[] {
const rootFrameId = captured.rootFrameId ?? "root";
const documents: VomFrameDocument[] = [
{
frameId: rootFrameId,
contextScopeId: rootFrameId,
target: rootTarget,
...(pageUrl ? { url: pageUrl } : {}),
axNodes: [],
domNodes: captured.nodes.map((node) => ({ ...node, frameId: node.frameId ?? rootFrameId })),
excludedBackendNodeIds: captured.frameExcludedBackendNodeIds?.get(rootFrameId),
},
];
const pending = [...captured.iframeNodes.entries()];
let progress = true;
let nextSyntheticFrame = 1;
while (pending.length > 0 && progress) {
progress = false;
for (let index = pending.length - 1; index >= 0; index -= 1) {
const [ownerBackendNodeId, domNodes] = pending[index];
const parent = documents.find((document) =>
document.domNodes.some((node) => node.backendNodeId === ownerBackendNodeId),
);
if (!parent) continue;
const frameId =
domNodes.find((node) => node.frameId)?.frameId ?? `legacy-frame-${nextSyntheticFrame++}`;
documents.push({
frameId,
parentFrameId: parent.frameId,
ownerBackendNodeId,
contextScopeId: frameId,
target: parent.target,
axNodes: [],
domNodes: domNodes.map((node) => ({ ...node, frameId: node.frameId ?? frameId })),
excludedBackendNodeIds: captured.frameExcludedBackendNodeIds?.get(frameId),
});
pending.splice(index, 1);
progress = true;
}
}
const documentByFrameId = new Map(documents.map((document) => [document.frameId, document]));
const backendFrame = new Map<number, string>();
const childFrameByOwner = new Map<number, string>();
for (const document of documents) {
if (document.ownerBackendNodeId !== undefined) {
childFrameByOwner.set(document.ownerBackendNodeId, document.frameId);
}
for (const node of document.domNodes) backendFrame.set(node.backendNodeId, document.frameId);
}
const axById = new Map(axNodes.map((node) => [node.nodeId, node]));
const ownership = new Map<string, string>();
const resolving = new Set<string>();
const frameForAx = (node: CdpAxNode): string => {
const cached = ownership.get(node.nodeId);
if (cached) return cached;
let frameId: string | undefined;
if (node.frameId && documentByFrameId.has(node.frameId)) frameId = node.frameId;
if (!frameId && typeof node.backendDOMNodeId === "number") {
frameId = backendFrame.get(node.backendDOMNodeId);
}
if (!frameId && node.parentId && !resolving.has(node.nodeId)) {
const parent = axById.get(node.parentId);
if (parent) {
frameId =
typeof parent.backendDOMNodeId === "number"
? childFrameByOwner.get(parent.backendDOMNodeId)
: undefined;
if (!frameId) {
resolving.add(node.nodeId);
frameId = frameForAx(parent);
resolving.delete(node.nodeId);
}
}
}
frameId ??= rootFrameId;
ownership.set(node.nodeId, frameId);
return frameId;
};
for (const node of axNodes) frameForAx(node);
for (const node of axNodes) {
const frameId = ownership.get(node.nodeId) ?? rootFrameId;
const document = documentByFrameId.get(frameId) ?? documents[0];
document.axNodes.push({
...node,
frameId,
...(node.parentId && ownership.get(node.parentId) === frameId
? { parentId: node.parentId }
: { parentId: undefined }),
...(node.childIds
? { childIds: node.childIds.filter((childId) => ownership.get(childId) === frameId) }
: {}),
});
}
return documents;
}
function withLegacyBackendIds(scene: VomScene): VomScene {
const used = new Set<number>();
const idMap = new Map<number, number>();
let nextVirtualId = -1;
for (const node of scene.nodes) {
const preferred = node.backendNodeId;
const id = preferred !== undefined && !used.has(preferred) ? preferred : nextVirtualId--;
used.add(id);
idMap.set(node.id, id);
}
return {
...scene,
nodes: scene.nodes.map((node) => ({
...node,
id: idMap.get(node.id) as number,
parentId: node.parentId === null ? null : (idMap.get(node.parentId) ?? null),
...(node.domParentId !== undefined
? { domParentId: node.domParentId === null ? null : (idMap.get(node.domParentId) ?? null) }
: {}),
...(node.domAncestorIds
? { domAncestorIds: node.domAncestorIds.flatMap((id) => idMap.get(id) ?? []) }
: {}),
})),
...(scene.surfaces
? {
surfaces: scene.surfaces.map((surface) => ({
...surface,
triggerId: idMap.get(surface.triggerId) ?? surface.triggerId,
})),
}
: {}),
...(scene.activeScopeBlocks
? {
activeScopeBlocks: scene.activeScopeBlocks.map((block) => ({
...block,
triggerId: idMap.get(block.triggerId) ?? block.triggerId,
})),
}
: {}),
};
}
export function buildVomScene(
axNodes: CdpAxNode[],
captured: CapturedViewModel,
options: BuildVomSceneOptions = {},
): VomScene {
return withLegacyBackendIds(
buildFrameVomScene(legacyFrameDocuments(axNodes, captured, options.pageUrl), captured, options),
);
}
export function buildFrameVomScene( export function buildFrameVomScene(
documents: VomFrameDocument[], documents: VomFrameDocument[],
captured: CapturedViewModel, captured: CapturedSceneInput,
options: BuildVomSceneOptions = {}, options: BuildVomSceneOptions = {},
): VomScene { ): VomScene {
const scene = buildSemanticVomScene({ const scene = buildSemanticVomScene({
@@ -833,7 +673,7 @@ export function buildFrameVomScene(
function attachCapturedSceneAnnotations( function attachCapturedSceneAnnotations(
scene: VomScene, scene: VomScene,
documents: VomFrameDocument[], documents: VomFrameDocument[],
captured: CapturedViewModel, captured: CapturedSceneInput,
surfaceProbes: CapturedSurfaceProbe[], surfaceProbes: CapturedSurfaceProbe[],
): VomScene { ): VomScene {
const rootDocument = documents.find((document) => document.frameId === scene.rootFrameId); const rootDocument = documents.find((document) => document.frameId === scene.rootFrameId);
@@ -983,45 +823,6 @@ export async function handleGetHtml(
} }
} }
function emptyCapturedViewModel(viewport = { width: 0, height: 0 }): CapturedViewModel {
return { viewport, nodes: [], iframeNodes: new Map(), excludedBackendNodeIds: new Set() };
}
async function fallbackCapturedViewModel(
cdp: CdpRunner,
tabId: number,
geometry: GeometryContext,
signal?: AbortSignal,
): Promise<CapturedViewModel> {
throwIfAborted(signal, "observation");
let viewport = { width: 0, height: 0 };
try {
const metrics = await geometry.layoutMetrics({ tabId });
throwIfAborted(signal, "observation");
const source = cssViewport(metrics);
viewport = { width: source.width, height: source.height };
} catch (error) {
if (isAbortError(error)) throw error;
}
const excludedBackendNodeIds = await collectOverlayExcludedBackendIds(cdp, tabId, signal);
throwIfAborted(signal, "observation");
return { ...emptyCapturedViewModel(viewport), excludedBackendNodeIds };
}
async function captureForVom(
cdp: CdpRunner,
tabId: number,
options: CaptureVomObservationOptions,
geometry: GeometryContext,
): Promise<CapturedViewModel> {
try {
return await captureViewModel(cdp, tabId, { signal: options.signal, geometry });
} catch (error) {
if (isAbortError(error)) throw error;
return fallbackCapturedViewModel(cdp, tabId, geometry, options.signal);
}
}
export interface HoverProbeOutcome { export interface HoverProbeOutcome {
/** Whether any active hover was dispatched during this observation. */ /** Whether any active hover was dispatched during this observation. */
performed: boolean; performed: boolean;
@@ -1054,7 +855,7 @@ const NO_HOVER_PROBES: HoverProbeOutcome = {
async function runHoverProbes( async function runHoverProbes(
cdp: CdpRunner, cdp: CdpRunner,
tabId: number, tabId: number,
captured: CapturedViewModel, captured: CapturedSceneInput,
documents: VomFrameDocument[], documents: VomFrameDocument[],
staticSemantics: ReturnType<typeof resolveSemanticGraph>, staticSemantics: ReturnType<typeof resolveSemanticGraph>,
options: CaptureVomObservationOptions, options: CaptureVomObservationOptions,
@@ -1093,21 +894,9 @@ export async function captureVomObservation(
throwIfAborted(options.signal, "observation"); throwIfAborted(options.signal, "observation");
await cdp.ensureAttachedToUrl?.(tabId, url); await cdp.ensureAttachedToUrl?.(tabId, url);
throwIfAborted(options.signal, "observation"); throwIfAborted(options.signal, "observation");
const geometry = new GeometryContext(cdp, tabId, undefined, options.signal); const facts = await captureObservationFacts<CdpAxNode>(cdp, tabId, options.signal, url);
const captured = await captureForVom(cdp, tabId, options, geometry); const { captured, documents: normalizedDocuments } = semanticCapture(facts);
throwIfAborted(options.signal, "observation"); throwIfAborted(options.signal, "observation");
const documents = await captureFrameData<CdpAxNode>(
cdp,
tabId,
captured,
options.signal,
geometry,
);
throwIfAborted(options.signal, "observation");
const normalizedDocuments =
documents.length === 1 && captured.iframeNodes.size > 0
? legacyFrameDocuments(documents[0].axNodes, captured, url, documents[0].target)
: documents;
const semanticGraph = buildSemanticGraph({ const semanticGraph = buildSemanticGraph({
documents: normalizedDocuments, documents: normalizedDocuments,
viewport: captured.viewport, viewport: captured.viewport,
@@ -1137,19 +926,41 @@ export async function captureVomObservation(
); );
// Reserve space using the renderer's character-based token estimate. Like VOM // Reserve space using the renderer's character-based token estimate. Like VOM
// headers, this integrity notice remains visible even under a tiny token budget. // headers, this integrity notice remains visible even under a tiny token budget.
const geometryNotice = captured.frameGeometryIssues?.length const notices: string[] = [];
? "@warning geometry incomplete: some page or frame content has no top-level coordinates.\n" if (facts.issues.some((issue) => issue.stage === "geometry"))
: ""; notices.push(
"@warning geometry incomplete: some page or frame content has no top-level coordinates.",
);
const incompleteStages = ["dom", "ax", "forms", "ownership"].filter((stage) =>
facts.issues.some((issue) => issue.stage === stage),
);
if (incompleteStages.length)
notices.push(
`@warning observation incomplete: some ${incompleteStages.join(", ")} data is unavailable or omitted.`,
);
if (
facts.issues.some(
(issue) => issue.stage === "identity" && issue.reason !== "identity-unverified",
)
)
notices.push(
"@warning observation incomplete: some documents were omitted because their identity changed or could not be revalidated.",
);
if (facts.issues.some((issue) => issue.reason === "identity-unverified"))
notices.push(
"@warning document identity unverified: some retained documents could not be checked for changes during capture.",
);
const captureNotice = notices.join("\n");
const rendered = renderVom(decoratedScene, { const rendered = renderVom(decoratedScene, {
maxDepth: options.maxDepth, maxDepth: options.maxDepth,
maxTokens: maxTokens:
!geometryNotice || options.maxTokens === undefined !captureNotice || options.maxTokens === undefined
? options.maxTokens ? options.maxTokens
: Math.max(0, options.maxTokens - Math.ceil(geometryNotice.length / 4)), : Math.max(0, options.maxTokens - Math.ceil((captureNotice.length + 1) / 4)),
redactValues: options.redactValues, redactValues: options.redactValues,
activeRegionPolicy: options.activeRegionPolicy, activeRegionPolicy: options.activeRegionPolicy,
}); });
if (geometryNotice) rendered.text += `\n${geometryNotice.trimEnd()}`; if (captureNotice) rendered.text += `\n${captureNotice}`;
throwIfAborted(options.signal, "observation"); throwIfAborted(options.signal, "observation");
return projectRecordSafeObservation({ return projectRecordSafeObservation({
rootFrameId: captured.rootFrameId ?? normalizedDocuments[0]?.frameId ?? "root", rootFrameId: captured.rootFrameId ?? normalizedDocuments[0]?.frameId ?? "root",
+1
View File
@@ -55,6 +55,7 @@ export interface CdpRunner {
sendToTarget?<T = unknown>(target: CdpTarget, method: string, params?: object): Promise<T>; sendToTarget?<T = unknown>(target: CdpTarget, method: string, params?: object): Promise<T>;
detach?(tabId: number): Promise<void>; detach?(tabId: number): Promise<void>;
getFrameGraph?(tabId: number): Promise<CdpFrameGraph>; getFrameGraph?(tabId: number): Promise<CdpFrameGraph>;
getAttachmentId?(tabId: number): string | undefined;
ensureAttachedToUrl?(tabId: number, expectedUrl: string | undefined): Promise<void>; ensureAttachedToUrl?(tabId: number, expectedUrl: string | undefined): Promise<void>;
trackSessionTab?(sessionId: string, tabId: number): void; trackSessionTab?(sessionId: string, tabId: number): void;
releaseSessionTab?(sessionId: string, tabId: number): Promise<void>; releaseSessionTab?(sessionId: string, tabId: number): Promise<void>;
File diff suppressed because it is too large. Load diff
@@ -1,7 +1,7 @@
import { describe, expect, it, vi } from "vitest"; import { describe, expect, it, vi } from "vitest";
import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "../../../lib/overlay-bridge"; import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "../../../lib/overlay-bridge";
import type { CdpRunner } from "../../shared"; import { collectOverlayExcludedBackendIds, probeHoverSurfaces } from "../capture";
import { captureViewModel, collectOverlayExcludedBackendIds, probeHoverSurfaces } from "../capture"; import { captureObservationFacts, semanticCapture } from "../capture-coordinator";
// Minimal but format-accurate captureSnapshot reply: a body with one // Minimal but format-accurate captureSnapshot reply: a body with one
// fixed full-screen overlay div carrying a password input. // fixed full-screen overlay div carrying a password input.
@@ -27,6 +27,7 @@ function fakeSnapshotReply() {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
nodes: { nodes: {
parentIndex: [-1, 0, 1, 2], parentIndex: [-1, 0, 1, 2],
nodeType: [1, 1, 1, 1], nodeType: [1, 1, 1, 1],
@@ -74,6 +75,7 @@ function hoverTriggerSnapshotReply() {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
nodes: { nodes: {
parentIndex: [-1, 0, 1], parentIndex: [-1, 0, 1],
nodeType: [1, 1, 1], nodeType: [1, 1, 1],
@@ -118,6 +120,7 @@ function twoHoverTriggerSnapshotReply() {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
nodes: { nodes: {
parentIndex: [-1, 0, 1, 1], parentIndex: [-1, 0, 1, 1],
nodeType: [1, 1, 1, 1], nodeType: [1, 1, 1, 1],
@@ -171,6 +174,7 @@ function nestedHoverTriggerSnapshotReply() {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
nodes: { nodes: {
parentIndex: [-1, 0, 1, 2, 3], parentIndex: [-1, 0, 1, 2, 3],
nodeType: [1, 1, 1, 1, 1], nodeType: [1, 1, 1, 1, 1],
@@ -248,13 +252,11 @@ function makeCdp(snapshot: unknown) {
}; };
} }
describe("captureViewModel", () => { describe("snapshot capture and hover", () => {
it("parses nodes, attrs, rects, paint order and styles", async () => { it("parses nodes, attrs, rects, paint order and styles", async () => {
const { nodes, viewport, iframeNodes } = await captureViewModel( const { nodes, viewport } = semanticCapture(
makeCdp(fakeSnapshotReply()), await captureObservationFacts(makeCdp(fakeSnapshotReply()), 4),
4, ).captured;
);
expect(iframeNodes.size).toBe(0);
expect(viewport).toEqual({ width: 1000, height: 800 }); expect(viewport).toEqual({ width: 1000, height: 800 });
const div = nodes.find((n) => n.backendNodeId === 12); const div = nodes.find((n) => n.backendNodeId === 12);
expect(div).toMatchObject({ tag: "div", position: "fixed", parentBackendNodeId: 11 }); expect(div).toMatchObject({ tag: "div", position: "fixed", parentBackendNodeId: 11 });
@@ -266,35 +268,11 @@ describe("captureViewModel", () => {
it("never hovers the page while capturing", async () => { it("never hovers the page while capturing", async () => {
const cdp = makeCdp(fakeSnapshotReply()); const cdp = makeCdp(fakeSnapshotReply());
await captureViewModel(cdp, 4); semanticCapture(await captureObservationFacts(cdp, 4)).captured;
expect(cdp.send).not.toHaveBeenCalledWith(4, "Input.dispatchMouseEvent", expect.anything()); expect(cdp.send).not.toHaveBeenCalledWith(4, "Input.dispatchMouseEvent", expect.anything());
}); });
it("batch-enriches form controls without per-node object resolution", async () => {
const cdp = makeCdp(fakeSnapshotReply());
const result = await captureViewModel(cdp, 4);
const input = result.nodes.find((node) => node.backendNodeId === 13);
expect(input).toMatchObject({
formState: "filled",
formPlaceholder: "secret",
attrs: { type: "password" },
});
expect(input?.formValue).toBeUndefined();
expect(input?.formDefaultValue).toBeUndefined();
expect(cdp.send).toHaveBeenCalledWith(
4,
"Runtime.evaluate",
expect.objectContaining({
serializationOptions: expect.objectContaining({ serialization: "deep" }),
}),
);
expect(cdp.send).toHaveBeenCalledWith(4, "Runtime.releaseObjectGroup", expect.anything());
expect(cdp.send).not.toHaveBeenCalledWith(4, "DOM.resolveNode", expect.anything());
expect(cdp.send).not.toHaveBeenCalledWith(4, "Runtime.callFunctionOn", expect.anything());
});
it("runs hover surface probes when explicitly enabled", async () => { it("runs hover surface probes when explicitly enabled", async () => {
let hoverStateCalls = 0; let hoverStateCalls = 0;
const cdp = { const cdp = {
@@ -336,7 +314,7 @@ describe("captureViewModel", () => {
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>, }) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
}; };
const captured = await captureViewModel(cdp, 4); const captured = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes); const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes);
expect(surfaceProbes).toEqual([ expect(surfaceProbes).toEqual([
@@ -398,7 +376,7 @@ describe("captureViewModel", () => {
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>, }) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
}; };
const captured = await captureViewModel(cdp, 4); const captured = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes); const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes);
expect(surfaceProbes).toEqual([ expect(surfaceProbes).toEqual([
@@ -445,7 +423,7 @@ describe("captureViewModel", () => {
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>, }) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
}; };
const captured = await captureViewModel(cdp, 4); const captured = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
await probeHoverSurfaces(cdp, 4, captured.nodes); await probeHoverSurfaces(cdp, 4, captured.nodes);
expect(hoverMoves).toBe(1); expect(hoverMoves).toBe(1);
@@ -477,6 +455,7 @@ describe("captureViewModel", () => {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
nodes: { nodes: {
parentIndex: [-1, 0, 1, 0, 3, 4], parentIndex: [-1, 0, 1, 0, 3, 4],
nodeName: [i("html"), i("body"), i("div"), i(OVERLAY_HOST_NAME), i("div"), i("button")], nodeName: [i("html"), i("body"), i("div"), i(OVERLAY_HOST_NAME), i("div"), i("button")],
@@ -507,7 +486,7 @@ describe("captureViewModel", () => {
], ],
}; };
const { nodes } = await captureViewModel(makeCdp(snapshot), 4); const { nodes } = semanticCapture(await captureObservationFacts(makeCdp(snapshot), 4)).captured;
const ids = nodes.map((n) => n.backendNodeId); const ids = nodes.map((n) => n.backendNodeId);
// Real page nodes survive. // Real page nodes survive.
expect(ids).toContain(12); expect(ids).toContain(12);
@@ -538,9 +517,7 @@ describe("captureViewModel", () => {
strings: S, strings: S,
documents: [ documents: [
{ {
scrollOffsetX: 0, frameId: "root",
scrollOffsetY: 0,
frameId: "main",
nodes: { nodes: {
parentIndex: [-1, 0, 1, 0, 3, 4], parentIndex: [-1, 0, 1, 0, 3, 4],
nodeName: [i("html"), i("body"), i("div"), i(OVERLAY_HOST_NAME), i("div"), i("button")], nodeName: [i("html"), i("body"), i("div"), i(OVERLAY_HOST_NAME), i("div"), i("button")],
@@ -571,12 +548,10 @@ describe("captureViewModel", () => {
], ],
}; };
const { excludedBackendNodeIds, frameExcludedBackendNodeIds } = await captureViewModel( const { excludedBackendNodeIds } = semanticCapture(
makeCdp(snapshot), await captureObservationFacts(makeCdp(snapshot), 4),
4, ).captured;
);
expect(excludedBackendNodeIds).toEqual(new Set([13, 14, 15])); expect(excludedBackendNodeIds).toEqual(new Set([13, 14, 15]));
expect(frameExcludedBackendNodeIds?.get("main")).toEqual(excludedBackendNodeIds);
}); });
it("excludes overlay host matched by tag name only", async () => { it("excludes overlay host matched by tag name only", async () => {
@@ -596,6 +571,7 @@ describe("captureViewModel", () => {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
nodes: { nodes: {
parentIndex: [-1, 0, 1], parentIndex: [-1, 0, 1],
nodeName: [i("html"), i("body"), i(OVERLAY_HOST_NAME)], nodeName: [i("html"), i("body"), i(OVERLAY_HOST_NAME)],
@@ -618,7 +594,9 @@ describe("captureViewModel", () => {
], ],
}; };
const { nodes, excludedBackendNodeIds } = await captureViewModel(makeCdp(snapshot), 4); const { nodes, excludedBackendNodeIds } = semanticCapture(
await captureObservationFacts(makeCdp(snapshot), 4),
).captured;
expect(nodes.map((n) => n.backendNodeId)).not.toContain(13); expect(nodes.map((n) => n.backendNodeId)).not.toContain(13);
expect(excludedBackendNodeIds).toEqual(new Set([13])); expect(excludedBackendNodeIds).toEqual(new Set([13]));
}); });
@@ -641,6 +619,7 @@ describe("captureViewModel", () => {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
nodes: { nodes: {
parentIndex: [-1, 0, 1], parentIndex: [-1, 0, 1],
nodeName: [i("html"), i("body"), i("div")], nodeName: [i("html"), i("body"), i("div")],
@@ -663,7 +642,9 @@ describe("captureViewModel", () => {
], ],
}; };
const { nodes, excludedBackendNodeIds } = await captureViewModel(makeCdp(snapshot), 4); const { nodes, excludedBackendNodeIds } = semanticCapture(
await captureObservationFacts(makeCdp(snapshot), 4),
).captured;
expect(nodes.map((n) => n.backendNodeId)).not.toContain(13); expect(nodes.map((n) => n.backendNodeId)).not.toContain(13);
expect(excludedBackendNodeIds).toEqual(new Set([13])); expect(excludedBackendNodeIds).toEqual(new Set([13]));
}); });
@@ -686,6 +667,7 @@ describe("captureViewModel", () => {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
nodes: { nodes: {
parentIndex: [-1, 0, 1], parentIndex: [-1, 0, 1],
nodeName: [i("html"), i("body"), i("div")], nodeName: [i("html"), i("body"), i("div")],
@@ -708,7 +690,7 @@ describe("captureViewModel", () => {
}, },
], ],
}; };
const { nodes } = await captureViewModel(makeCdp(snapshot), 4); const { nodes } = semanticCapture(await captureObservationFacts(makeCdp(snapshot), 4)).captured;
expect(nodes.find((n) => n.backendNodeId === 12)?.cursor).toBe("pointer"); expect(nodes.find((n) => n.backendNodeId === 12)?.cursor).toBe("pointer");
expect(nodes.find((n) => n.backendNodeId === 11)?.cursor).toBe("auto"); expect(nodes.find((n) => n.backendNodeId === 11)?.cursor).toBe("auto");
}); });
@@ -720,6 +702,7 @@ describe("captureViewModel", () => {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
nodes: { nodes: {
parentIndex: [-1, 0, 1, 1, 1], parentIndex: [-1, 0, 1, 1, 1],
nodeName: [i("html"), i("body"), i("button"), i("button"), i("button")], nodeName: [i("html"), i("body"), i("button"), i("button"), i("button")],
@@ -746,7 +729,7 @@ describe("captureViewModel", () => {
], ],
}; };
const { nodes } = await captureViewModel(makeCdp(snapshot), 4); const { nodes } = semanticCapture(await captureObservationFacts(makeCdp(snapshot), 4)).captured;
expect(nodes.find((node) => node.backendNodeId === 12)?.rendered).toBe(true); expect(nodes.find((node) => node.backendNodeId === 12)?.rendered).toBe(true);
expect(nodes.find((node) => node.backendNodeId === 13)?.rendered).toBe(false); expect(nodes.find((node) => node.backendNodeId === 13)?.rendered).toBe(false);
@@ -768,51 +751,12 @@ describe("captureViewModel", () => {
throw new Error(method); throw new Error(method);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>, }) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
}; };
const { nodes } = await captureViewModel(cdp, 4); const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
const div = nodes.find((n) => n.backendNodeId === 12); const div = nodes.find((n) => n.backendNodeId === 12);
expect(div?.localRect?.y).toBe(-200); expect(div?.localRect?.y).toBe(-200);
expect(div?.rect).toMatchObject({ y: 0, h: 600 }); expect(div?.rect).toMatchObject({ y: 0, h: 600 });
}); });
it("normalizes snapshot-owned scroll before using a stale CSS metrics fallback", async () => {
const snapshot = {
strings: ["html", "button"],
documents: [
{
frameId: "root",
scrollOffsetX: 40,
scrollOffsetY: 200,
nodes: {
parentIndex: [-1, 0],
nodeName: [0, 1],
backendNodeId: [10, 11],
attributes: [[], []],
},
layout: { nodeIndex: [1], bounds: [[200, 800, 240, 80]] },
},
],
};
const cdp: CdpRunner = {
send: vi.fn(async (_tab: number, method: string) => {
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics")
return {
visualViewport: { clientWidth: 2000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 999, pageY: 999 },
};
return {};
}) as CdpRunner["send"],
};
const captured = await captureViewModel(cdp, 4);
expect(captured.nodes.find((node) => node.backendNodeId === 11)).toMatchObject({
localRect: { x: 80, y: 300, w: 120, h: 40 },
rect: { x: 80, y: 300, w: 120, h: 40 },
rendered: true,
});
expect(captured.frameGeometryIssues).toEqual([]);
});
it("normalizes raw snapshot bounds before viewport clipping", async () => { it("normalizes raw snapshot bounds before viewport clipping", async () => {
const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"]; const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"];
const i = (s: string) => S.indexOf(s); const i = (s: string) => S.indexOf(s);
@@ -820,6 +764,7 @@ describe("captureViewModel", () => {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
nodes: { nodes: {
parentIndex: [-1, 0, 1], parentIndex: [-1, 0, 1],
nodeName: [i("html"), i("body"), i("div")], nodeName: [i("html"), i("body"), i("div")],
@@ -856,7 +801,7 @@ describe("captureViewModel", () => {
throw new Error(method); throw new Error(method);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>, }) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
}; };
const { nodes } = await captureViewModel(cdp, 4); const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
expect(nodes.find((n) => n.backendNodeId === 12)?.rect).toEqual({ expect(nodes.find((n) => n.backendNodeId === 12)?.rect).toEqual({
x: 0, x: 0,
y: 0, y: 0,
@@ -892,6 +837,7 @@ describe("captureViewModel", () => {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
scrollOffsetX: 0, scrollOffsetX: 0,
scrollOffsetY: 0, scrollOffsetY: 0,
nodes: { nodes: {
@@ -932,7 +878,7 @@ describe("captureViewModel", () => {
throw new Error(method); throw new Error(method);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>, }) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
}; };
const { nodes } = await captureViewModel(cdp, 4); const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
const btn = nodes.find((n) => n.backendNodeId === 12); const btn = nodes.find((n) => n.backendNodeId === 12);
expect(btn?.textContent).toBe("登录"); expect(btn?.textContent).toBe("登录");
const link = nodes.find((n) => n.backendNodeId === 14); const link = nodes.find((n) => n.backendNodeId === 14);
@@ -942,7 +888,7 @@ describe("captureViewModel", () => {
expect(body?.textContent).toBeUndefined(); expect(body?.textContent).toBeUndefined();
}); });
it("parses iframe sub-documents and returns iframeNodes keyed by iframe backendNodeId", async () => { it("retains explicit iframe document ownership and projected nodes", async () => {
const S = [ const S = [
"html", "html",
"body", "body",
@@ -1030,30 +976,21 @@ describe("captureViewModel", () => {
throw new Error(method); throw new Error(method);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>, }) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
}; };
const { const facts = await captureObservationFacts(cdp, 4);
nodes, const root = facts.documents.find((doc) => doc.frame.frameId === "main-frame")!;
iframeNodes, const child = facts.documents.find((doc) => doc.frame.frameId === "child-frame")!;
frameNodes, expect(facts.documents).toHaveLength(2);
frameOwnerBackendNodeIds, expect(root.domNodes.find((n) => n.backendNodeId === 13)?.tag).toBe("iframe");
frameParentIds, const subNodes = child.domNodes;
rootFrameId,
} = await captureViewModel(cdp, 4);
// Main frame has 4 nodes; iframe is included
expect(nodes.find((n) => n.backendNodeId === 13)?.tag).toBe("iframe");
// iframeNodes keyed by the <iframe> element's backendNodeId=13
expect(iframeNodes.size).toBe(1);
expect(iframeNodes.has(13)).toBe(true);
const subNodes = iframeNodes.get(13)!;
const input = subNodes.find((n) => n.backendNodeId === 101); const input = subNodes.find((n) => n.backendNodeId === 101);
expect(input?.tag).toBe("input"); expect(input?.tag).toBe("input");
expect(input?.attrs.type).toBe("text"); expect(input?.attrs.type).toBe("text");
expect(input?.ownerFrameBackendNodeId).toBe(13); expect(input?.ownerFrameBackendNodeId).toBe(13);
expect(input?.localRect).toEqual({ x: 0, y: 0, w: 200, h: 40 }); expect(input?.localRect).toEqual({ x: 0, y: 0, w: 200, h: 40 });
expect(input?.rect).toEqual({ x: 100, y: 300, w: 200, h: 40 }); expect(input?.rect).toEqual({ x: 100, y: 300, w: 200, h: 40 });
expect(rootFrameId).toBe("main-frame"); expect(facts.rootFrameId).toBe("main-frame");
expect(frameNodes?.get("child-frame")).toBe(subNodes); expect(child.frame.ownerBackendNodeId).toBe(13);
expect(frameOwnerBackendNodeIds?.get("child-frame")).toBe(13); expect(child.frame.parentFrameId).toBe("main-frame");
expect(frameParentIds?.get("child-frame")).toBe("main-frame");
}); });
it("projects iframe documents independently of owner snapshot geometry", async () => { it("projects iframe documents independently of owner snapshot geometry", async () => {
@@ -1074,6 +1011,7 @@ describe("captureViewModel", () => {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
scrollOffsetX: 0, scrollOffsetX: 0,
scrollOffsetY: 0, scrollOffsetY: 0,
nodes: { nodes: {
@@ -1091,6 +1029,7 @@ describe("captureViewModel", () => {
}, },
}, },
{ {
frameId: "child",
scrollOffsetX: 0, scrollOffsetX: 0,
scrollOffsetY: 0, scrollOffsetY: 0,
nodes: { nodes: {
@@ -1129,8 +1068,10 @@ describe("captureViewModel", () => {
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>, }) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
}; };
const { iframeNodes } = await captureViewModel(cdp, 4); const { documents } = await captureObservationFacts(cdp, 4);
const input = iframeNodes.get(13)?.find((n) => n.backendNodeId === 101); const input = documents
.find((doc) => doc.frame.frameId === "child")
?.domNodes?.find((n) => n.backendNodeId === 101);
expect(input?.localRect).toEqual({ x: 20, y: 20, w: 120, h: 60 }); expect(input?.localRect).toEqual({ x: 20, y: 20, w: 120, h: 60 });
expect(input?.rect).toEqual({ x: 120, y: 320, w: 120, h: 60 }); expect(input?.rect).toEqual({ x: 120, y: 320, w: 120, h: 60 });
@@ -1154,6 +1095,7 @@ describe("captureViewModel", () => {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
nodes: { nodes: {
parentIndex: [-1, 0, 1], parentIndex: [-1, 0, 1],
nodeName: [i("html"), i("body"), i("iframe")], nodeName: [i("html"), i("body"), i("iframe")],
@@ -1177,6 +1119,7 @@ describe("captureViewModel", () => {
{ {
scrollOffsetX: 0, scrollOffsetX: 0,
scrollOffsetY: 0, scrollOffsetY: 0,
frameId: "child",
nodes: { nodes: {
parentIndex: [-1, 0, 1], parentIndex: [-1, 0, 1],
nodeName: [i("html"), i("body"), i("iframe")], nodeName: [i("html"), i("body"), i("iframe")],
@@ -1200,6 +1143,7 @@ describe("captureViewModel", () => {
{ {
scrollOffsetX: 0, scrollOffsetX: 0,
scrollOffsetY: 0, scrollOffsetY: 0,
frameId: "nested",
nodes: { nodes: {
parentIndex: [-1, 0], parentIndex: [-1, 0],
nodeName: [i("body"), i("input")], nodeName: [i("body"), i("input")],
@@ -1257,9 +1201,13 @@ describe("captureViewModel", () => {
) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>, ) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
}; };
const { iframeNodes } = await captureViewModel(cdp, 4); const { documents } = await captureObservationFacts(cdp, 4);
const nestedIframe = iframeNodes.get(13)?.find((n) => n.backendNodeId === 23); const nestedIframe = documents
const input = iframeNodes.get(23)?.find((n) => n.backendNodeId === 31); .find((doc) => doc.frame.frameId === "child")
?.domNodes?.find((n) => n.backendNodeId === 23);
const input = documents
.find((doc) => doc.frame.frameId === "nested")
?.domNodes?.find((n) => n.backendNodeId === 31);
expect(nestedIframe?.localRect).toEqual({ x: 5, y: 6, w: 100, h: 80 }); expect(nestedIframe?.localRect).toEqual({ x: 5, y: 6, w: 100, h: 80 });
expect(nestedIframe?.rect).toEqual({ x: 15, y: 26, w: 100, h: 80 }); expect(nestedIframe?.rect).toEqual({ x: 15, y: 26, w: 100, h: 80 });
@@ -1275,6 +1223,7 @@ describe("captureViewModel", () => {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
nodes: { nodes: {
parentIndex: [-1, 0, 1], parentIndex: [-1, 0, 1],
nodeName: [i("html"), i("body"), i("div")], nodeName: [i("html"), i("body"), i("div")],
@@ -1311,7 +1260,7 @@ describe("captureViewModel", () => {
throw new Error(method); throw new Error(method);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>, }) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
}; };
const { nodes } = await captureViewModel(cdp, 4); const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
expect(nodes.find((n) => n.backendNodeId === 12)?.rect?.y).toBe(400 / 2 - 100); expect(nodes.find((n) => n.backendNodeId === 12)?.rect?.y).toBe(400 / 2 - 100);
}); });
@@ -1337,200 +1286,3 @@ describe("captureViewModel", () => {
expect(excluded).toEqual(new Set([200, 201, 202])); expect(excluded).toEqual(new Set([200, 201, 202]));
}); });
}); });
// Six sibling frames and one nested frame exercise scheduling through the real capture entry.
function siblingCaptureFixture(
beforeReply: (method: string, params: Record<string, unknown>) => Promise<void> = async () => {},
) {
const document = (id: number, owners: number[], childIndexes: number[]) => ({
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: `frame-${id}`,
nodes: {
parentIndex: [-1, ...owners.map(() => 0)],
nodeName: [0, ...owners.map(() => 1)],
backendNodeId: [1000 + id, ...owners],
attributes: [[], ...owners.map(() => [])],
contentDocumentIndex: { index: owners.map((_, i) => i + 1), value: childIndexes },
},
layout: {
nodeIndex: [0, ...owners.map((_, i) => i + 1)],
bounds: [[0, 0, 200, 100], ...owners.map(() => [0, 0, 200, 100])],
},
});
const snapshot = {
strings: ["body", "iframe"],
documents: [
document(0, [100, 101, 102, 103, 104, 105], [1, 2, 3, 4, 5, 6]),
document(1, [200], [7]),
...Array.from({ length: 6 }, (_, i) => document(i + 2, [], [])),
],
};
let active = 0;
let peak = 0;
const send = vi.fn(async (_tabId: number, method: string, params: object = {}) => {
const args = params as Record<string, unknown>;
active++;
peak = Math.max(peak, active);
try {
await beforeReply(method, args);
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics")
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 },
};
if (method === "DOM.getBoxModel")
return { model: { content: [0, 0, 200, 0, 200, 100, 0, 100] } };
if (method === "DOM.resolveNode") return { object: { objectId: String(args.backendNodeId) } };
if (method === "Runtime.callFunctionOn")
return { result: { value: { width: 200, height: 100 } } };
return {};
} finally {
active--;
}
});
return {
cdp: { send: send as CdpRunner["send"] },
send,
peak: () => peak,
active: () => active,
measured: () =>
send.mock.calls
.filter(([, method]) => method === "DOM.getBoxModel")
.map(([, , params]) => (params as { backendNodeId: number }).backendNodeId),
};
}
describe("sibling frame measurement scheduling", () => {
it("batches same-target owner sizes without changing projected results", async () => {
const fixture = siblingCaptureFixture();
const original = fixture.cdp.send;
const methods: string[] = [];
fixture.cdp.send = async (tabId, method, params) => {
methods.push(method);
if (method === "Runtime.evaluate")
return {
result: {
deepSerializedValue: {
type: "array",
value: [100, 101, 102, 103, 104, 105, 200].map((backendNodeId) => ({
type: "array",
value: [
{ type: "node", value: { backendNodeId } },
{ type: "string", value: JSON.stringify({ width: 200, height: 100 }) },
],
})),
},
},
} as never;
return original(tabId, method, params);
};
const captured = await captureViewModel(fixture.cdp, 4);
expect(captured.frameGeometryIssues).toEqual([]);
expect([...captured.iframeNodes.keys()]).toEqual([100, 200, 101, 102, 103, 104, 105]);
expect(captured.iframeNodes.get(200)?.[0].rect).toEqual({ x: 0, y: 0, w: 200, h: 100 });
expect(methods.filter((method) => method === "DOM.getBoxModel")).toHaveLength(7);
expect(methods.filter((method) => method === "Runtime.evaluate")).toHaveLength(1);
expect(methods.filter((method) => method === "Runtime.releaseObjectGroup")).toHaveLength(1);
expect(methods).not.toContain("DOM.resolveNode");
expect(methods).not.toContain("Runtime.callFunctionOn");
});
it("bounds concurrent reads, fills free slots and preserves depth-first output despite reordered replies", async () => {
const pending = new Map<number, () => void>();
const fixture = siblingCaptureFixture(async (method, params) => {
if (method === "DOM.getBoxModel" && Number(params.backendNodeId) < 200)
await new Promise<void>((resolve) => pending.set(Number(params.backendNodeId), resolve));
});
const capture = captureViewModel(fixture.cdp, 4);
await vi.waitFor(() => expect(pending.size).toBe(4));
expect(fixture.measured()).toEqual([100, 101, 102, 103]);
pending.get(103)!();
await vi.waitFor(() => expect(pending.has(104)).toBe(true));
pending.get(104)!();
await vi.waitFor(() => expect(pending.has(105)).toBe(true));
expect(fixture.measured()).not.toContain(200);
for (const id of [105, 102, 101, 100]) pending.get(id)!();
const captured = await capture;
expect(captured.frameGeometryIssues).toEqual([]);
expect(fixture.peak()).toBe(4);
expect(fixture.active()).toBe(0);
expect(fixture.measured()).toEqual([100, 101, 102, 103, 104, 105, 200]);
expect([...captured.iframeNodes.keys()]).toEqual([100, 200, 101, 102, 103, 104, 105]);
expect(captured.iframeNodes.get(200)?.[0].rect).toEqual({ x: 0, y: 0, w: 200, h: 100 });
for (const method of [
"DOM.getBoxModel",
"DOM.resolveNode",
"Runtime.callFunctionOn",
"Runtime.releaseObject",
])
expect(fixture.send.mock.calls.filter(([, name]) => name === method)).toHaveLength(7);
});
it("does not measure descendants of a failed owner and retains sibling geometry and local nodes", async () => {
const fixture = siblingCaptureFixture(async (method, params) => {
if (method === "DOM.resolveNode" && params.backendNodeId === 100)
throw new Error("owner replaced");
});
const captured = await captureViewModel(fixture.cdp, 4);
expect(captured.frameGeometryIssues).toEqual([
{
status: "unavailable",
source: { target: { tabId: 4 }, frameId: "frame-1" },
ownerBackendNodeId: 100,
},
{
status: "blocked",
source: { target: { tabId: 4 }, frameId: "frame-7" },
cause: captured.frameGeometryIssues?.[0],
},
]);
expect(fixture.measured()).not.toContain(200);
expect(captured.iframeNodes.get(200)?.[0]).toMatchObject({
tag: "body",
rect: null,
localRect: { x: 0, y: 0, w: 200, h: 100 },
});
expect(captured.iframeNodes.get(101)?.[0].rect).toEqual({ x: 0, y: 0, w: 200, h: 100 });
expect([...captured.iframeNodes.keys()]).toEqual([100, 200, 101, 102, 103, 104, 105]);
});
it("stops scheduling on cancellation and waits for resolved objects to be released", async () => {
const controller = new AbortController();
const resolutions = new Map<string, () => void>();
const releases = new Map<string, () => void>();
const fixture = siblingCaptureFixture(async (method, params) => {
if (method === "DOM.resolveNode")
await new Promise<void>((resolve) =>
resolutions.set(String(params.backendNodeId), resolve),
);
if (method === "Runtime.releaseObject")
await new Promise<void>((resolve) => releases.set(String(params.objectId), resolve));
});
let settled = false;
const capture = captureViewModel(fixture.cdp, 4, { signal: controller.signal });
const rejected = expect(capture).rejects.toMatchObject({ name: "AbortError" });
void capture.then(
() => {
settled = true;
},
() => {
settled = true;
},
);
await vi.waitFor(() => expect(resolutions.size).toBe(4));
controller.abort();
for (const resolve of resolutions.values()) resolve();
await vi.waitFor(() => expect(releases.size).toBe(4));
expect(settled).toBe(false);
expect(fixture.measured()).toEqual([100, 101, 102, 103]);
for (const release of releases.values()) release();
await rejected;
expect(fixture.active()).toBe(0);
expect(fixture.send.mock.calls.some(([, method]) => method === "Runtime.callFunctionOn")).toBe(
false,
);
});
});
@@ -1,289 +0,0 @@
import { describe, expect, it, vi } from "vitest";
import { OVERLAY_HOST_MARKER_ATTR } from "@/lib/overlay-bridge";
import type { CdpRunner } from "../../shared";
import type { CapturedNode, CapturedViewModel } from "../capture";
import { captureFrameData } from "../frame-capture";
function ownerNode(backendNodeId: number, x: number): CapturedNode {
return {
backendNodeId,
parentBackendNodeId: 1,
frameId: "main",
ownerFrameBackendNodeId: null,
tag: "iframe",
attrs: {},
rect: { x, y: 100, w: 300, h: 200 },
localRect: { x, y: 100, w: 300, h: 200 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
};
}
function childSnapshot(frameId: string, backendNodeId: number) {
const strings = [frameId, "body", "button", "static", "auto", "pointer"];
return {
strings,
documents: [
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId,
nodes: {
parentIndex: [-1, 0],
nodeName: [1, 2],
backendNodeId: [backendNodeId - 1, backendNodeId],
attributes: [[], []] as number[][],
},
layout: {
nodeIndex: [0, 1],
styles: [
[3, 4, 4],
[3, 4, 5],
],
bounds: [
[0, 0, 300, 200],
[10, 20, 100, 40],
],
paintOrders: [0, 1],
},
},
],
};
}
describe("captureFrameData", () => {
it("retains nested owner failures when merging a captured OOPIF", async () => {
const owner = ownerNode(10, 50);
const captured: CapturedViewModel = {
nodes: [owner],
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
frameNodes: new Map([["main", [owner]]]),
rootFrameId: "main",
excludedBackendNodeIds: new Set(),
};
const child = childSnapshot("child", 101);
const nested = childSnapshot("nested", 201);
const document = child.documents[0];
const snapshot = {
strings: child.strings,
documents: [
{
...document,
nodes: { ...document.nodes, contentDocumentIndex: { index: [1], value: [1] } },
},
nested.documents[0],
],
};
const reply = async (_target: unknown, method: string) => {
if (method === "Page.getLayoutMetrics")
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 300, clientHeight: 200 },
};
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "DOM.getBoxModel")
return { model: { content: [50, 100, 350, 100, 350, 300, 50, 300] } };
if (method === "DOM.resolveNode") throw new Error("nested owner replaced");
return {};
};
const cdp: CdpRunner = {
send: vi.fn(reply) as CdpRunner["send"],
sendToTarget: vi.fn(reply) as NonNullable<CdpRunner["sendToTarget"]>,
getFrameGraph: vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4, sessionId: "child-session" },
},
],
})),
};
await captureFrameData(cdp, 4, captured);
expect(captured.frameGeometryIssues).toEqual([
{
status: "unavailable",
source: { target: { tabId: 4, sessionId: "child-session" }, frameId: "nested" },
ownerBackendNodeId: 101,
},
]);
expect(captured.frameNodes?.get("nested")?.[1]).toMatchObject({
rect: null,
localRect: { x: 10, y: 20, w: 100, h: 40 },
});
});
it("captures and positions multiple OOPIF documents missing from the root snapshot", async () => {
const leftOwner = ownerNode(10, 50);
const rightOwner = ownerNode(20, 500);
const captured: CapturedViewModel = {
nodes: [leftOwner, rightOwner],
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
frameNodes: new Map([["main", [leftOwner, rightOwner]]]),
frameOwnerBackendNodeIds: new Map(),
rootFrameId: "main",
excludedBackendNodeIds: new Set(),
};
const sendToTarget = vi.fn(async (target, method) => {
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 300, clientHeight: 200, pageX: 0, pageY: 0 },
};
}
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") {
if (target.sessionId !== "left-session") return childSnapshot("right", 201);
const snapshot = childSnapshot("left", 101);
const marker = snapshot.strings.push(OVERLAY_HOST_MARKER_ATTR) - 1;
const nodes = snapshot.documents[0].nodes;
nodes.backendNodeId.push(201); // Same id as a real button in the other target.
nodes.parentIndex.push(0);
nodes.nodeName.push(2);
nodes.attributes.push([marker, -1]);
return snapshot;
}
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
throw new Error(`unexpected ${method}`);
});
const cdp: CdpRunner = {
send: vi.fn(async (_tabId, method, params) => {
if (method === "Accessibility.enable") return {};
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
if (method === "DOM.getBoxModel") {
const backendNodeId = (params as { backendNodeId?: number })?.backendNodeId;
const x = backendNodeId === 10 ? 50 : 500;
return { model: { content: [x, 100, x + 300, 100, x + 300, 300, x, 300] } };
}
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 },
};
}
throw new Error(`unexpected root ${method}`);
}) as CdpRunner["send"],
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
getFrameGraph: vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "left",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4, sessionId: "left-session" },
},
{
frameId: "right",
parentFrameId: "main",
ownerBackendNodeId: 20,
target: { tabId: 4, sessionId: "right-session" },
},
],
})),
};
const trees = await captureFrameData(cdp, 4, captured);
expect(trees.find((tree) => tree.frameId === "left")?.excludedBackendNodeIds).toEqual(
new Set([201]),
);
expect(trees.find((tree) => tree.frameId === "right")?.excludedBackendNodeIds?.size).toBe(0);
expect(trees.map((tree) => tree.frameId)).toEqual(["main", "left", "right"]);
expect(
captured.frameNodes?.get("left")?.find((node) => node.backendNodeId === 101)?.rect,
).toEqual({ x: 60, y: 120, w: 100, h: 40 });
expect(
captured.frameNodes?.get("right")?.find((node) => node.backendNodeId === 201)?.rect,
).toEqual({ x: 510, y: 120, w: 100, h: 40 });
expect(captured.frameOwnerBackendNodeIds).toEqual(
new Map([
["left", 10],
["right", 20],
]),
);
expect(captured.frameParentIds).toEqual(
new Map([
["left", "main"],
["right", "main"],
]),
);
});
it("keeps OOPIF semantics when viewport projection is unavailable", async () => {
const owner = { ...ownerNode(10, 50), rect: null, localRect: null };
const captured: CapturedViewModel = {
nodes: [owner],
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
frameNodes: new Map([["main", [owner]]]),
rootFrameId: "main",
excludedBackendNodeIds: new Set(),
};
const cdp: CdpRunner = {
send: vi.fn(async (_tabId, method) => {
if (method === "Accessibility.enable") return {};
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
if (method === "DOM.getBoxModel") throw new Error("owner geometry unavailable");
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 },
};
}
throw new Error(`unexpected root ${method}`);
}) as CdpRunner["send"],
sendToTarget: vi.fn(async (_target, method) => {
if (method === "Page.getLayoutMetrics") throw new Error("viewport unavailable");
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") return childSnapshot("child", 101);
if (method === "Accessibility.getFullAXTree") {
return {
nodes: [
{
nodeId: "button",
backendDOMNodeId: 101,
role: { type: "role", value: "button" },
name: { type: "computedString", value: "Continue" },
},
],
};
}
throw new Error(`unexpected ${method}`);
}) as unknown as NonNullable<CdpRunner["sendToTarget"]>,
getFrameGraph: vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4, sessionId: "child-session" },
},
],
})),
};
const documents = await captureFrameData(cdp, 4, captured);
const childDocument = documents.find((document) => document.frameId === "child");
expect(childDocument?.axNodes).toEqual([
expect.objectContaining({ backendDOMNodeId: 101, frameId: "child" }),
]);
expect(childDocument?.domNodes.find((node) => node.backendNodeId === 101)).toEqual(
expect.objectContaining({ rect: null, localRect: null, rendered: true }),
);
});
});
@@ -1,7 +1,12 @@
import { describe, expect, it } from "vitest"; import { describe, expect, it, vi } from "vitest";
import type { CdpFrameGraph } from "@/browser-driver/frame-graph"; import type { CdpFrameGraph } from "@/browser-driver/frame-graph";
import type { CapturedNode, CapturedViewModel } from "../capture"; import type { CapturedNode } from "../capture";
import { buildFrameDocuments, type FrameAxBatch, type FrameOwnedAxNode } from "../frame-document"; import {
buildFrameDocuments,
type FrameAxBatch,
type FrameDomInput,
type FrameOwnedAxNode,
} from "../frame-document";
function domNode(frameId: string, backendNodeId: number): CapturedNode { function domNode(frameId: string, backendNodeId: number): CapturedNode {
return { return {
@@ -17,14 +22,11 @@ function domNode(frameId: string, backendNodeId: number): CapturedNode {
}; };
} }
function captured(frameNodes: Map<string, CapturedNode[]>): CapturedViewModel { function captured(frameNodes: Map<string, CapturedNode[]>): FrameDomInput {
return { return {
nodes: frameNodes.get("main") ?? [], nodes: frameNodes.get("main") ?? [],
viewport: { width: 800, height: 600 },
iframeNodes: new Map(),
frameNodes, frameNodes,
rootFrameId: "main", rootFrameId: "main",
excludedBackendNodeIds: new Set(),
}; };
} }
@@ -157,10 +159,12 @@ describe("buildFrameDocuments", () => {
["child", [domNode("child", 9)]], ["child", [domNode("child", 9)]],
]); ]);
const capture = captured(frameNodes); const capture = captured(frameNodes);
capture.frameOwnerBackendNodeIds = new Map([["child", 7]]);
capture.frameParentIds = new Map([["child", "left"]]);
const frames = ["main", "left", "right", "child"].map((frameId) => ({ const frames = ["main", "left", "right", "child"].map((frameId) => ({
frame: { frameId, target: { tabId: 4 } }, frame: {
frameId,
target: { tabId: 4 },
...(frameId === "child" ? { parentFrameId: "left", ownerBackendNodeId: 7 } : {}),
},
nodes: [], nodes: [],
})); }));
@@ -171,6 +175,23 @@ describe("buildFrameDocuments", () => {
ownerBackendNodeId: 7, ownerBackendNodeId: 7,
}); });
}); });
it("rejects explicit AX frame ownership in a different target", async () => {
const frames = [
{ frameId: "main", target: { tabId: 4 } },
{ frameId: "remote", target: { tabId: 4, sessionId: "remote" } },
];
const unresolved = vi.fn();
const documents = await buildFrameDocuments(
{ rootFrameId: "main", frames },
[{ frame: frames[0], nodes: [{ nodeId: "wrong", frameId: "remote" }] }],
captured(new Map()),
undefined,
unresolved,
);
expect(documents.every((doc) => doc.axNodes.length === 0)).toBe(true);
expect(unresolved).toHaveBeenCalledOnce();
});
it("resolves a reverse-ordered deep AX parent chain without recursion", async () => { it("resolves a reverse-ordered deep AX parent chain without recursion", async () => {
const frame = { frameId: "main", target: { tabId: 4 } }; const frame = { frameId: "main", target: { tabId: 4 } };
const nodes: FrameOwnedAxNode[] = Array.from({ length: 10000 }, (_, i) => ({ const nodes: FrameOwnedAxNode[] = Array.from({ length: 10000 }, (_, i) => ({
@@ -0,0 +1,413 @@
import {
type CdpFrame,
type CdpFrameGraph,
type CdpTarget,
cdpTargetKey,
} from "@/browser-driver/frame-graph";
import { cssViewport, GeometryContext } from "../geometry/frame-context";
import { type CdpRunner, cdpRunnerForTarget } from "../shared";
import { collectOverlayExcludedBackendIds } from "./capture";
import {
isAbortError as isCaptureAbort,
throwIfAborted as throwCaptureAborted,
} from "./capture-abort";
import { verifyDocumentIdentity } from "./document-identity";
import type { CapturedSceneInput, DocumentIdentity } from "./facts";
import { buildDocumentIndex, type CaptureIssue, type ObservationFacts } from "./facts";
import { enrichFormControlStates } from "./form-capture";
import {
buildFrameDocuments,
type FrameAxBatch,
type FrameDocument,
type FrameOwnedAxNode,
} from "./frame-document";
import { type NormalizedFrameDocument, normalizeSnapshot } from "./normalize";
import { describeSnapshotFrames, REQUESTED_STYLES, type SnapshotReply } from "./snapshot";
interface TargetBatch<T extends FrameOwnedAxNode> {
target: CdpTarget;
frames: CdpFrame[];
snapshotAttachmentId?: string;
documents: NormalizedFrameDocument[];
ax: FrameAxBatch<T>[];
fallbackExcluded: Set<number>;
snapshotFrameIds?: Set<string>;
rootFrameId?: string;
}
/** A fixed worker pool scoped to one capture; each worker issues at most one
* collection request at a time. Geometry has its own measurement-phase bound. */
async function collectTasks<T>(
items: readonly T[],
collect: (item: T) => Promise<void>,
signal?: AbortSignal,
): Promise<void> {
let cursor = 0;
let failed = false;
let failure: unknown;
await Promise.all(
Array.from({ length: Math.min(4, items.length) }, async () => {
try {
while (cursor < items.length && !failed && !signal?.aborted) {
const item = items[cursor++];
await collect(item);
}
} catch (error) {
if (!failed) failure = error;
failed = true;
}
}),
);
// A worker's rejection must not let the capture outlive its own task scope.
// In-flight reads and their finally blocks settle before cancellation/failure.
throwCaptureAborted(signal);
if (failed) throw failure;
}
export async function captureObservationFacts<T extends FrameOwnedAxNode>(
cdp: CdpRunner,
tabId: number,
signal?: AbortSignal,
pageUrl?: string,
): Promise<ObservationFacts<T>> {
const startedAt = Date.now();
throwCaptureAborted(signal);
let graph: CdpFrameGraph | undefined;
try {
graph = await cdp.getFrameGraph?.(tabId);
} catch (error) {
if (isCaptureAbort(error)) throw error;
}
throwCaptureAborted(signal);
const geometry = new GeometryContext(cdp, tabId, graph, signal);
const issues: CaptureIssue[] = [];
const graphFrames = new Map(graph?.frames.map((frame) => [frame.frameId, frame]) ?? []);
const groups = new Map<string, TargetBatch<T>>();
for (const frame of graph?.frames ?? []) {
const key = cdpTargetKey(frame.target);
let group = groups.get(key);
if (!group) {
group = {
target: frame.target,
frames: [],
documents: [],
ax: [],
fallbackExcluded: new Set(),
};
groups.set(key, group);
}
group.frames.push(
!frame.target.sessionId && frame.frameId === graph?.rootFrameId && !frame.url && pageUrl
? { ...frame, url: pageUrl }
: frame,
);
}
if (!groups.has(cdpTargetKey({ tabId })))
groups.set(cdpTargetKey({ tabId }), {
target: { tabId },
frames: [],
documents: [],
ax: [],
fallbackExcluded: new Set(),
});
const batches = [...groups.values()];
const unavailable = (batch: TargetBatch<T>, stage: CaptureIssue["stage"], frameId?: string) =>
issues.push({ target: batch.target, frameId, stage, reason: "capture-unavailable" });
let firstFailure: unknown;
const axReady = new Set<TargetBatch<T>>();
await collectTasks(
batches,
async (batch) => {
const scoped = cdpRunnerForTarget(cdp, batch.target);
try {
throwCaptureAborted(signal);
await scoped.send(tabId, "DOMSnapshot.enable", {});
throwCaptureAborted(signal);
batch.snapshotAttachmentId = cdp.getAttachmentId?.(tabId);
const snapshot = await scoped.send<SnapshotReply>(tabId, "DOMSnapshot.captureSnapshot", {
computedStyles: REQUESTED_STYLES,
includePaintOrder: true,
includeDOMRects: true,
});
throwCaptureAborted(signal);
const observed = describeSnapshotFrames(snapshot, batch.target, graphFrames, pageUrl);
batch.snapshotFrameIds = observed.ids;
batch.rootFrameId = observed.rootFrameId;
// Keep graph-only frames for AX fallback, never in preference to observed membership.
batch.frames = [
...observed.frames,
...batch.frames.filter((frame) => !observed.ids.has(frame.frameId)),
];
batch.documents = await normalizeSnapshot(
snapshot,
batch.target,
batch.frames.filter((frame) => observed.ids.has(frame.frameId)),
geometry,
issues,
signal,
observed.rootFrameId,
);
const formsAvailable = await enrichFormControlStates(
scoped,
tabId,
batch.documents.map((doc) => doc.nodes),
signal,
);
throwCaptureAborted(signal);
if (!formsAvailable) unavailable(batch, "forms");
const present = new Set(batch.documents.map((doc) => doc.frame.frameId));
for (const frame of batch.frames)
if (!present.has(frame.frameId)) unavailable(batch, "dom", frame.frameId);
} catch (error) {
throwCaptureAborted(signal);
if (isCaptureAbort(error)) throw error;
firstFailure ??= error;
unavailable(batch, "dom");
batch.fallbackExcluded = await collectOverlayExcludedBackendIds(scoped, tabId, signal);
}
if (!batch.frames.length && !graph)
batch.frames = [
{
frameId: "root",
target: batch.target,
...(!batch.target.sessionId && pageUrl ? { url: pageUrl } : {}),
},
];
try {
throwCaptureAborted(signal);
await scoped.send(tabId, "Accessibility.enable", {});
} catch (error) {
throwCaptureAborted(signal);
if (isCaptureAbort(error)) throw error;
firstFailure ??= error;
unavailable(batch, "ax");
return;
}
axReady.add(batch);
},
signal,
);
// Schedule AX by frame across all ready targets, rather than nesting pools.
// Keep result slots in source order even when CDP replies arrive out of order.
const axTasks: { batch: TargetBatch<T>; frame: CdpFrame; result?: FrameAxBatch<T> }[] =
batches.flatMap((batch) =>
axReady.has(batch) ? batch.frames.map((frame) => ({ batch, frame })) : [],
);
await collectTasks(
axTasks,
async (task) => {
const { batch, frame } = task;
try {
throwCaptureAborted(signal);
const result = await cdpRunnerForTarget(cdp, batch.target).send<{ nodes?: T[] }>(
tabId,
"Accessibility.getFullAXTree",
frame.frameId === "root" ? {} : { frameId: frame.frameId },
);
throwCaptureAborted(signal);
task.result = { frame, nodes: result.nodes ?? [] };
} catch (error) {
throwCaptureAborted(signal);
if (isCaptureAbort(error)) throw error;
firstFailure ??= error;
unavailable(batch, "ax", frame.frameId);
}
},
signal,
);
for (const task of axTasks) if (task.result) task.batch.ax.push(task.result);
let viewport = { width: 0, height: 0 };
try {
const measured = cssViewport(await geometry.layoutMetrics({ tabId }));
viewport = { width: measured.width, height: measured.height };
} catch (error) {
if (isCaptureAbort(error)) throw error;
}
// A snapshot claim outranks an old graph hint. Conflicting actual claims
// cannot be resolved by completion order or by overwriting a frameId map.
const claims = new Map<string, string>();
const invalid = new Set<string>();
for (const batch of batches) {
const targetKey = cdpTargetKey(batch.target);
for (const id of batch.snapshotFrameIds ?? []) {
const previous = claims.get(id);
if (previous !== undefined && previous !== targetKey) invalid.add(id);
claims.set(id, targetKey);
}
const accepted = new Set(batch.frames.map((frame) => frame.frameId));
for (const id of batch.snapshotFrameIds ?? []) if (!accepted.has(id)) invalid.add(id);
}
const currentFrames = batches.flatMap((batch) =>
batch.frames.filter(
(frame) =>
!claims.has(frame.frameId) || claims.get(frame.frameId) === cdpTargetKey(batch.target),
),
);
const rootBatch = batches.find((batch) => !batch.target.sessionId);
const rootFrameId =
rootBatch?.rootFrameId ?? graph?.rootFrameId ?? rootBatch?.frames[0]?.frameId ?? "root";
const ownershipInvalid = new Set(invalid);
const identities = new Map<string, DocumentIdentity>();
const selected = new Map(currentFrames.map((frame) => [frame.frameId, frame]));
const checks = batches.flatMap((batch) => {
const docs = new Map(batch.documents.map((doc) => [doc.frame.frameId, doc]));
return batch.frames
.filter(
(frame) =>
!invalid.has(frame.frameId) &&
cdpTargetKey(selected.get(frame.frameId)!.target) === cdpTargetKey(batch.target),
)
.map((frame) => ({ batch, frame, doc: docs.get(frame.frameId) }));
});
await collectTasks(
checks,
async ({ batch, frame, doc }) => {
throwCaptureAborted(signal);
if (!batch.snapshotAttachmentId || doc?.documentElementBackendNodeId === undefined) {
issues.push({
target: frame.target,
frameId: frame.frameId,
stage: "identity",
reason: "identity-unverified",
});
return;
}
const identity: DocumentIdentity = {
attachmentId: batch.snapshotAttachmentId,
target: frame.target,
frameId: frame.frameId,
documentElementBackendNodeId: doc.documentElementBackendNodeId,
};
const verified = await verifyDocumentIdentity(cdp, identity, signal);
const status = cdp.getAttachmentId?.(tabId) === identity.attachmentId ? verified : "changed";
if (status === "current") identities.set(frame.frameId, identity);
else {
invalid.add(frame.frameId);
issues.push({
target: frame.target,
frameId: frame.frameId,
stage: "identity",
reason: status === "changed" ? "document-changed" : "identity-unavailable",
});
}
},
signal,
);
const children = new Map<string, string[]>();
for (const frame of currentFrames) {
if (!frame.parentFrameId) continue;
const siblings = children.get(frame.parentFrameId);
if (siblings) siblings.push(frame.frameId);
else children.set(frame.parentFrameId, [frame.frameId]);
}
const blocked = [...invalid];
for (let i = 0; i < blocked.length; i++) {
for (const child of children.get(blocked[i]) ?? []) {
if (!invalid.has(child)) {
invalid.add(child);
blocked.push(child);
}
}
}
for (const frame of currentFrames) {
if (ownershipInvalid.has(frame.frameId))
issues.push({
target: frame.target,
frameId: frame.frameId,
stage: "ownership",
reason: "frame-ownership-unresolved",
});
}
if (invalid.has(rootFrameId))
throw new Error(
ownershipInvalid.has(rootFrameId)
? "observation root document ownership is ambiguous"
: "observation document identity changed or could not be verified; observe again",
);
const frames = currentFrames.filter((frame) => !invalid.has(frame.frameId));
const frameById = new Map(frames.map((frame) => [frame.frameId, frame]));
const belongs = (frame: CdpFrame) => {
const current = frameById.get(frame.frameId);
return current && cdpTargetKey(current.target) === cdpTargetKey(frame.target);
};
const decoded = new Map(
batches
.flatMap((batch) => batch.documents)
.filter((doc) => belongs(doc.frame))
.map((doc) => [doc.frame.frameId, doc]),
);
const ax = batches.flatMap((batch) => batch.ax).filter((batch) => belongs(batch.frame));
const documents = await buildFrameDocuments(
{ rootFrameId, frames },
ax,
{
nodes: [],
rootFrameId,
frameNodes: new Map([...decoded].map(([id, doc]) => [id, doc.nodes])),
frameExcludedBackendNodeIds: new Map(
[...decoded].map(([id, doc]) => [id, doc.index.excludedBackendNodeIds]),
),
},
signal,
(frame) =>
issues.push({
target: frame.target,
frameId: frame.frameId,
stage: "ownership",
reason: "frame-ownership-unresolved",
}),
);
const facts: ObservationFacts<T>["documents"][number][] = [];
for (const document of documents) {
const doc = decoded.get(document.frameId);
const index = doc?.index ?? (await buildDocumentIndex([], signal));
const fallback = groups.get(cdpTargetKey(document.target))?.fallbackExcluded;
const {
domNodes,
axNodes,
contextScopeId: _scope,
excludedBackendNodeIds: _excluded,
...frame
} = document;
facts.push({
frame,
identity: identities.get(document.frameId),
index: fallback?.size ? { ...index, excludedBackendNodeIds: fallback } : index,
domNodes,
axNodes,
});
}
throwCaptureAborted(signal);
if (firstFailure && facts.every((doc) => !doc.domNodes.length && !doc.axNodes.length))
throw firstFailure;
return { rootFrameId, viewport, documents: facts, issues, startedAt, finishedAt: Date.now() };
}
/** Existing semantic consumers get a narrow view; no raw snapshot
* data can leak through recording's allowlisted output DTO. */
export function semanticCapture<T extends FrameOwnedAxNode>(
facts: ObservationFacts<T>,
): { captured: CapturedSceneInput; documents: FrameDocument<T>[] } {
const documents = facts.documents.map((doc) => ({
...doc.frame,
contextScopeId: doc.frame.frameId,
domNodes: doc.domNodes.filter(
(node) => !doc.index.excludedBackendNodeIds.has(node.backendNodeId),
),
axNodes: doc.axNodes,
excludedBackendNodeIds: doc.index.excludedBackendNodeIds,
}));
const root = documents.find((doc) => doc.frameId === facts.rootFrameId);
return {
documents,
captured: {
nodes: root?.domNodes ?? [],
viewport: facts.viewport,
rootFrameId: facts.rootFrameId,
excludedBackendNodeIds: root?.excludedBackendNodeIds ?? new Set(),
},
};
}
+10 -351
View File
@@ -1,67 +1,18 @@
// Coordinate DOMSnapshot acquisition, frame geometry and hover probes. // Browser-side hover probes and overlay exclusion fallback. Static collection
// Snapshot decoding and form enrichment live in their own modules. // lives in capture-coordinator; snapshot decoding has no browser dependency.
import type { Viewport } from "@browser-skill/vom";
import type { CdpTarget } from "@/browser-driver/frame-graph";
import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy"; import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
import { OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge"; import { OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
import {
type FrameProjectionIssue,
type FrameProjectionState,
snapshotCoordinates,
} from "../geometry/coordinate-types";
import {
GeometryContext,
type LayoutMetrics,
snapshotLayoutScale,
} from "../geometry/frame-context";
import type { CdpRunner } from "../shared"; import type { CdpRunner } from "../shared";
import { isAbortError, throwIfAborted } from "./capture-abort";
import type { CapturedNode } from "./capture-types"; import type { CapturedNode } from "./capture-types";
import { enrichFormControlStates } from "./form-capture"; import type { CapturedSurfaceProbe } from "./facts";
export type {
CapturedNode,
CapturedSurfaceProbe,
} from "./facts";
import { isAbortError, throwIfAborted } from "./capture-abort";
import { clearHover, ProbeBudget, waitForHover } from "./hover-perception"; import { clearHover, ProbeBudget, waitForHover } from "./hover-perception";
import { type FrameContext, normalizeDocument } from "./normalize";
import {
REQUESTED_STYLES,
type SnapshotDocument,
type SnapshotReply,
snapshotFrameId,
sparseIndexMap,
} from "./snapshot";
export type { CapturedNode } from "./capture-types";
export type CapturedIframeNodes = Map<number, CapturedNode[]>;
export interface CapturedSurfaceProbe {
triggerBackendNodeId: number;
triggerPoint?: { x: number; y: number };
triggerAction: "hover" | "focus" | string;
subItems: string[];
confidence?: "high" | "medium" | "low";
}
export interface CapturedViewModel {
nodes: CapturedNode[];
viewport: Viewport;
iframeNodes: CapturedIframeNodes;
frameNodes?: Map<string, CapturedNode[]>;
frameExcludedBackendNodeIds?: Map<string, ReadonlySet<number>>;
frameOwnerBackendNodeIds?: Map<string, number>;
/** Explicit DOMSnapshot frame ancestry; never inferred from backend node ids. */
frameParentIds?: Map<string, string>;
rootFrameId?: string;
/** Failed boundaries and descendants blocked by them; clipping is not a failure. */
frameGeometryIssues?: FrameProjectionIssue[];
/** Backend node ids belonging to the agent overlay host + its shadow subtree. */
excludedBackendNodeIds: Set<number>;
}
export interface CaptureViewModelOptions {
signal?: AbortSignal;
geometry?: GeometryContext;
target?: CdpTarget;
}
interface RuntimeEvaluateReply { interface RuntimeEvaluateReply {
result?: { result?: {
@@ -492,295 +443,3 @@ export async function collectOverlayExcludedBackendIds(
} }
return excluded; return excluded;
} }
interface ParseFrameDocumentsResult {
frameGeometryIssues: FrameProjectionIssue[];
iframeNodes: CapturedIframeNodes;
frameOwnerBackendNodeIds: Map<string, number>;
frameParentIds: Map<string, string>;
frameExcludedBackendNodeIds: Map<string, ReadonlySet<number>>;
excludedBackendNodeIds: Set<number>;
}
async function parseChildFrameDocuments(
documents: SnapshotDocument[],
strings: string[],
parentDocIndex: number,
parentContext: FrameContext,
geometry: GeometryContext,
signal?: AbortSignal,
visited = new Set<number>(),
): Promise<ParseFrameDocumentsResult> {
const frameGeometryIssues: FrameProjectionIssue[] = [];
const iframeNodes: CapturedIframeNodes = new Map();
const frameOwnerBackendNodeIds = new Map<string, number>();
const frameParentIds = new Map<string, string>();
const frameExcludedBackendNodeIds = new Map<string, ReadonlySet<number>>();
const excludedBackendNodeIds = new Set<number>();
const parentDoc = documents[parentDocIndex];
const cdi = sparseIndexMap(parentDoc?.nodes?.contentDocumentIndex);
if (cdi.size === 0) {
return {
iframeNodes,
frameOwnerBackendNodeIds,
frameParentIds,
frameExcludedBackendNodeIds,
excludedBackendNodeIds,
frameGeometryIssues,
};
}
const parentBackendIds = parentDoc?.nodes?.backendNodeId ?? [];
const children = [...cdi].flatMap(([nodeArrayIdx, childDocIndex]) => {
const childDoc = documents[childDocIndex];
const iframeBackendId = parentBackendIds[nodeArrayIdx];
if (visited.has(childDocIndex) || !childDoc || iframeBackendId === undefined) return [];
const source = { target: parentContext.target, frameId: snapshotFrameId(childDoc, strings) };
const coordinates = snapshotCoordinates(childDoc, parentContext.layoutUnitsPerCssPixel);
const parent = parentContext.projection;
const projection: FrameProjectionState =
parent.status === "available"
? coordinates
? { status: "unavailable", source, ownerBackendNodeId: iframeBackendId }
: { status: "unavailable", source, reason: "snapshot-coordinates-unavailable" }
: { status: "blocked", source, cause: parent.status === "blocked" ? parent.cause : parent };
return [
{
childDocIndex,
childDoc,
iframeBackendId,
source,
coordinates,
projection: projection as FrameProjectionState,
},
];
});
const parentProjection =
parentContext.projection.status === "available"
? parentContext.projection.projection.geometry
: null;
if (parentProjection) {
// Measure only direct siblings. Recursion starts after these workers finish,
// so descendants never hold or recursively acquire a measurement slot.
const edge = parentProjection.edges[0];
const clips = edge
? [edge.destinationQuad, ...(edge.destinationClips ?? [])]
: parentProjection.sourceClips;
let cursor = 0;
let failure: unknown;
await Promise.all(
Array.from({ length: Math.min(4, children.length) }, async () => {
while (cursor < children.length && !signal?.aborted && !failure) {
const child = children[cursor++];
if (!child.coordinates) continue;
try {
// Owner quads are target-relative; do not apply the parent transform twice.
child.projection = await geometry.snapshotProjection(
child.source,
child.iframeBackendId,
clips,
parentProjection.topViewport,
);
} catch (error) {
failure ??= error;
}
}
}),
);
// Join all active reads, including object cleanup, before propagating abort.
if (failure) throw failure;
}
throwIfAborted(signal);
// Completion order must not change document traversal or result insertion order.
for (const {
childDocIndex,
childDoc,
iframeBackendId,
source,
projection,
coordinates,
} of children) {
throwIfAborted(signal);
if (projection.status !== "available") frameGeometryIssues.push(projection);
const childContext: FrameContext = {
frameId: source.frameId,
target: source.target,
ownerFrameBackendNodeId: iframeBackendId,
projection,
coordinates,
layoutUnitsPerCssPixel: parentContext.layoutUnitsPerCssPixel,
};
const nextVisited = new Set(visited);
nextVisited.add(childDocIndex);
const parsed = await normalizeDocument(childDoc, strings, childContext, signal);
iframeNodes.set(iframeBackendId, parsed.nodes);
if (childContext.frameId) {
frameOwnerBackendNodeIds.set(childContext.frameId, iframeBackendId);
frameExcludedBackendNodeIds.set(childContext.frameId, parsed.index.excludedBackendNodeIds);
if (parentContext.frameId) frameParentIds.set(childContext.frameId, parentContext.frameId);
}
for (const id of parsed.index.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
const nested = await parseChildFrameDocuments(
documents,
strings,
childDocIndex,
childContext,
geometry,
signal,
nextVisited,
);
frameGeometryIssues.push(...nested.frameGeometryIssues);
for (const [nestedFrameId, nestedNodes] of nested.iframeNodes) {
iframeNodes.set(nestedFrameId, nestedNodes);
}
for (const [frameId, ownerBackendNodeId] of nested.frameOwnerBackendNodeIds) {
frameOwnerBackendNodeIds.set(frameId, ownerBackendNodeId);
}
for (const [frameId, parentFrameId] of nested.frameParentIds) {
frameParentIds.set(frameId, parentFrameId);
}
for (const [frameId, excluded] of nested.frameExcludedBackendNodeIds)
frameExcludedBackendNodeIds.set(frameId, excluded);
for (const id of nested.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
}
return {
iframeNodes,
frameOwnerBackendNodeIds,
frameParentIds,
frameExcludedBackendNodeIds,
excludedBackendNodeIds,
frameGeometryIssues,
};
}
export async function captureViewModel(
cdp: CdpRunner,
tabId: number,
options: CaptureViewModelOptions = {},
): Promise<CapturedViewModel> {
throwIfAborted(options.signal);
const target = options.target ?? { tabId };
const geometry = options.geometry ?? new GeometryContext(cdp, tabId, undefined, options.signal);
let metrics: LayoutMetrics = {};
try {
metrics = await geometry.layoutMetrics(target);
} catch (error) {
if (isAbortError(error)) throw error;
console.debug("[bsk capture] layout metrics unavailable", error);
}
throwIfAborted(options.signal);
const viewport: Viewport = {
width: metrics.cssLayoutViewport?.clientWidth ?? 0,
height: metrics.cssLayoutViewport?.clientHeight ?? 0,
};
const layoutUnitsPerCssPixel = snapshotLayoutScale(metrics);
await cdp.send(tabId, "DOMSnapshot.enable", {});
throwIfAborted(options.signal);
const snap = await cdp.send<SnapshotReply>(tabId, "DOMSnapshot.captureSnapshot", {
computedStyles: REQUESTED_STYLES,
includePaintOrder: true,
includeDOMRects: true,
});
throwIfAborted(options.signal);
const strings = snap.strings ?? [];
const documents = snap.documents ?? [];
const doc0 = documents[0];
if (!doc0?.nodes?.backendNodeId) {
return {
nodes: [],
viewport,
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
};
}
const source = { target, frameId: snapshotFrameId(doc0, strings) };
const coordinates = snapshotCoordinates(doc0, layoutUnitsPerCssPixel, {
x: metrics.cssLayoutViewport?.pageX,
y: metrics.cssLayoutViewport?.pageY,
});
const topContext: FrameContext = {
frameId: snapshotFrameId(doc0, strings),
ownerFrameBackendNodeId: null,
target,
projection:
coordinates &&
[viewport.width, viewport.height].every((size) => Number.isFinite(size) && size > 0)
? {
status: "available",
projection: {
source,
geometry: { sourceClips: [], edges: [], topViewport: viewport },
},
}
: { status: "unavailable", source, reason: "snapshot-coordinates-unavailable" },
coordinates,
layoutUnitsPerCssPixel,
};
const mainParsed = await normalizeDocument(doc0, strings, topContext, options.signal);
const nodes = mainParsed.nodes;
const excludedBackendNodeIds = new Set(mainParsed.index.excludedBackendNodeIds);
const ownerIds = new Set<number>();
for (const document of documents) {
for (const index of document.nodes?.contentDocumentIndex?.index ?? []) {
const id = document.nodes?.backendNodeId?.[index];
if (id !== undefined) ownerIds.add(id);
}
}
geometry.registerSnapshotOwners(target, ownerIds);
const frameParsed = await parseChildFrameDocuments(
documents,
strings,
0,
topContext,
geometry,
options.signal,
);
const iframeNodes = frameParsed.iframeNodes;
if (topContext.frameId)
frameParsed.frameExcludedBackendNodeIds.set(
topContext.frameId,
mainParsed.index.excludedBackendNodeIds,
);
for (const id of frameParsed.excludedBackendNodeIds) {
excludedBackendNodeIds.add(id);
}
await enrichFormControlStates(cdp, tabId, [nodes, ...iframeNodes.values()], options.signal);
throwIfAborted(options.signal);
const frameNodes = new Map<string, CapturedNode[]>();
if (topContext.frameId) frameNodes.set(topContext.frameId, nodes);
for (const iframeFrameNodes of iframeNodes.values()) {
const frameId = iframeFrameNodes.find((node) => node.frameId)?.frameId;
if (frameId) frameNodes.set(frameId, iframeFrameNodes);
}
for (const [frameId, ownerBackendNodeId] of frameParsed.frameOwnerBackendNodeIds) {
if (!frameNodes.has(frameId)) {
frameNodes.set(frameId, iframeNodes.get(ownerBackendNodeId) ?? []);
}
}
return {
nodes,
viewport,
iframeNodes,
frameNodes,
frameGeometryIssues: [
...(topContext.projection.status === "available" ? [] : [topContext.projection]),
...frameParsed.frameGeometryIssues,
],
frameOwnerBackendNodeIds: frameParsed.frameOwnerBackendNodeIds,
frameExcludedBackendNodeIds: frameParsed.frameExcludedBackendNodeIds,
frameParentIds: frameParsed.frameParentIds,
...(topContext.frameId ? { rootFrameId: topContext.frameId } : {}),
excludedBackendNodeIds,
};
}
@@ -0,0 +1,53 @@
import type { CdpRunner } from "../shared";
import { sendToCdpTarget } from "../shared";
import { isAbortError, throwIfAborted } from "./capture-abort";
import type { DocumentIdentity } from "./facts";
/** Compare the snapshot root with the current root in that exact frame.
* Deep serialization supplies the backend ID without a separate describeNode. */
export async function verifyDocumentIdentity(
cdp: CdpRunner,
identity: DocumentIdentity,
signal?: AbortSignal,
): Promise<"current" | "changed" | "unavailable"> {
const attached = () => cdp.getAttachmentId?.(identity.target.tabId) === identity.attachmentId;
throwIfAborted(signal);
if (!attached()) return "changed";
const objectGroup = `bsk-document-identity-${crypto.randomUUID()}`;
const send = <T>(method: string, params: object) => {
throwIfAborted(signal);
return sendToCdpTarget<T>(cdp, identity.target, method, params);
};
try {
const world = await send<{ executionContextId: number }>("Page.createIsolatedWorld", {
frameId: identity.frameId,
worldName: "bsk-document-identity",
});
const reply = await send<{
result?: { deepSerializedValue?: { type: string; value?: { backendNodeId?: number } } };
}>("Runtime.evaluate", {
expression: "document.documentElement",
contextId: world.executionContextId,
objectGroup,
serializationOptions: {
serialization: "deep",
additionalParameters: { maxNodeDepth: 0, includeShadowTree: "none" },
},
});
if (!attached()) return "changed";
const root = reply.result?.deepSerializedValue;
if (root?.type === "null") return "changed";
const id = root?.type === "node" ? root.value?.backendNodeId : undefined;
if (id === undefined) return "unavailable";
return id === identity.documentElementBackendNodeId ? "current" : "changed";
} catch (error) {
throwIfAborted(signal);
if (isAbortError(error)) throw error;
return attached() ? "unavailable" : "changed";
} finally {
await sendToCdpTarget(cdp, identity.target, "Runtime.releaseObjectGroup", {
objectGroup,
}).catch(() => {});
throwIfAborted(signal);
}
}
+61
View File
@@ -1,6 +1,10 @@
import type { Viewport } from "@browser-skill/vom";
import type { CdpFrame, CdpTarget } from "@/browser-driver/frame-graph";
import { isOverlayHostNode } from "@/lib/overlay-bridge"; import { isOverlayHostNode } from "@/lib/overlay-bridge";
import type { FrameProjectionIssue } from "../geometry/coordinate-types";
import { createCaptureCheckpoint } from "./capture-abort"; import { createCaptureCheckpoint } from "./capture-abort";
import type { CapturedNode } from "./capture-types"; import type { CapturedNode } from "./capture-types";
import type { FrameOwnedAxNode } from "./frame-document";
/** Bounds retain their raw snapshot document layout units until normalization. */ /** Bounds retain their raw snapshot document layout units until normalization. */
export interface SnapshotLayout { export interface SnapshotLayout {
@@ -92,3 +96,60 @@ export async function buildDocumentIndex<T extends DecodedNode>(
} }
return { nodes, excludedBackendNodeIds }; return { nodes, excludedBackendNodeIds };
} }
export interface DocumentIdentity {
attachmentId: string;
target: CdpTarget;
frameId: string;
documentElementBackendNodeId: number;
}
/** Narrow input shared by the existing semantic scene/hover consumers. */
export interface CapturedSceneInput {
nodes: CapturedNode[];
viewport: Viewport;
rootFrameId?: string;
excludedBackendNodeIds: ReadonlySet<number>;
}
export interface CaptureIssue {
/** Owner failure or inherited blockage; absent for other geometry failures. */
projectionIssue?: FrameProjectionIssue;
target: CdpTarget;
frameId?: string;
stage: "dom" | "ax" | "identity" | "ownership" | "geometry" | "forms";
reason:
| "document-changed"
| "identity-unavailable"
| "identity-unverified"
| "capture-unavailable"
| "frame-ownership-unresolved"
| "geometry-unavailable";
}
export interface DocumentFacts<T extends FrameOwnedAxNode> {
readonly frame: CdpFrame;
readonly identity?: DocumentIdentity;
readonly index: DocumentIndex;
readonly domNodes: CapturedNode[];
readonly axNodes: T[];
}
export interface ObservationFacts<T extends FrameOwnedAxNode> {
readonly rootFrameId: string;
readonly viewport: Viewport;
readonly documents: readonly DocumentFacts<T>[];
readonly issues: readonly CaptureIssue[];
readonly startedAt: number;
readonly finishedAt: number;
}
export interface CapturedSurfaceProbe {
triggerBackendNodeId: number;
triggerPoint?: { x: number; y: number };
triggerAction: "hover" | "focus" | string;
subItems: string[];
confidence?: "high" | "medium" | "low";
}
export type { CapturedNode } from "./capture-types";
@@ -1,183 +0,0 @@
import type { CdpFrame, CdpFrameGraph } from "@/browser-driver/frame-graph";
import { type GeometryProjection, projectRectToViewport } from "../geometry";
import { GeometryContext } from "../geometry/frame-context";
import { type CdpRunner, cdpRunnerForTarget, sendToCdpTarget } from "../shared";
import { type CapturedNode, type CapturedViewModel, captureViewModel } from "./capture";
import {
buildFrameDocuments,
type FrameAxBatch,
type FrameDocument,
type FrameOwnedAxNode,
} from "./frame-document";
export type FrameAxNode = FrameOwnedAxNode;
export type CapturedFrameDocument<T extends FrameAxNode> = FrameDocument<T>;
function throwIfAborted(signal?: AbortSignal): void {
if (signal?.aborted) throw new DOMException("observation aborted", "AbortError");
}
function isAbortError(err: unknown): boolean {
return err instanceof Error && err.name === "AbortError";
}
function transformFrameNodes(
nodes: CapturedNode[],
projection: GeometryProjection | null,
): CapturedNode[] {
return nodes.map((node) => ({
...node,
rect: projection
? (() => {
const bounds = projectRectToViewport(node.rect, projection);
return bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
})()
: null,
}));
}
async function captureMissingFrameDocuments(
cdp: CdpRunner,
tabId: number,
graph: CdpFrameGraph,
captured: CapturedViewModel,
geometry: GeometryContext,
signal?: AbortSignal,
): Promise<void> {
if (!captured.frameNodes) captured.frameNodes = new Map();
if (!captured.frameOwnerBackendNodeIds) captured.frameOwnerBackendNodeIds = new Map();
if (!captured.frameParentIds) captured.frameParentIds = new Map();
for (const frame of graph.frames) {
if (!frame.target.sessionId || captured.frameNodes.has(frame.frameId)) continue;
try {
const child = await captureViewModel(cdpRunnerForTarget(cdp, frame.target), tabId, {
signal,
geometry,
target: frame.target,
});
if (child.frameExcludedBackendNodeIds?.size) {
captured.frameExcludedBackendNodeIds ??= new Map();
for (const [id, excluded] of child.frameExcludedBackendNodeIds)
captured.frameExcludedBackendNodeIds.set(id, excluded);
}
if (child.frameGeometryIssues?.length) {
(captured.frameGeometryIssues ??= []).push(...child.frameGeometryIssues);
}
let projection: GeometryProjection | null = null;
try {
projection = await geometry.targetProjection(frame.frameId);
} catch (err) {
if (isAbortError(err)) throw err;
console.debug("[bsk observation] child frame geometry projection failed", {
frameId: frame.frameId,
err,
});
}
const rootFrameId = child.rootFrameId ?? frame.frameId;
const childFrames = child.frameNodes ?? new Map<string, CapturedNode[]>();
if (!childFrames.has(rootFrameId)) childFrames.set(rootFrameId, child.nodes);
if (rootFrameId !== frame.frameId && !childFrames.has(frame.frameId)) {
childFrames.set(
frame.frameId,
child.nodes.map((node) => ({ ...node, frameId: frame.frameId })),
);
}
for (const [childFrameId, nodes] of childFrames) {
captured.frameNodes.set(childFrameId, transformFrameNodes(nodes, projection));
}
if (frame.ownerBackendNodeId !== undefined) {
captured.iframeNodes.set(
frame.ownerBackendNodeId,
captured.frameNodes.get(frame.frameId) ?? [],
);
captured.frameOwnerBackendNodeIds.set(frame.frameId, frame.ownerBackendNodeId);
}
if (frame.parentFrameId) captured.frameParentIds.set(frame.frameId, frame.parentFrameId);
for (const [childFrameId, ownerBackendNodeId] of child.frameOwnerBackendNodeIds ?? []) {
captured.frameOwnerBackendNodeIds.set(childFrameId, ownerBackendNodeId);
}
for (const [childFrameId, parentFrameId] of child.frameParentIds ?? []) {
captured.frameParentIds.set(childFrameId, parentFrameId);
}
} catch (err) {
if (isAbortError(err)) throw err;
console.debug("[bsk observation] child frame DOM capture failed", {
frameId: frame.frameId,
err,
});
}
}
}
async function captureAxTrees<T extends FrameAxNode>(
cdp: CdpRunner,
tabId: number,
discoveredFrames: CdpFrame[],
captured: CapturedViewModel,
signal?: AbortSignal,
): Promise<FrameAxBatch<T>[]> {
let frames = [...discoveredFrames];
const capturedFrameIds = [...(captured.frameNodes?.keys() ?? [])];
if (frames.length === 0) {
const rootFrameId = captured.rootFrameId ?? capturedFrameIds[0] ?? "root";
frames = [{ frameId: rootFrameId, target: { tabId } }];
}
for (const frameId of capturedFrameIds) {
if (!frames.some((frame) => frame.frameId === frameId)) {
frames.push({ frameId, target: { tabId } });
}
}
let firstFailure: unknown;
const trees = await Promise.all(
frames.map(async (frame): Promise<FrameAxBatch<T>> => {
try {
throwIfAborted(signal);
await sendToCdpTarget(cdp, frame.target, "Accessibility.enable", {});
const result = await sendToCdpTarget<{ nodes?: T[] }>(
cdp,
frame.target,
"Accessibility.getFullAXTree",
frame.frameId === "root" ? {} : { frameId: frame.frameId },
);
throwIfAborted(signal);
return {
frame,
nodes: result.nodes ?? [],
};
} catch (err) {
if (isAbortError(err)) throw err;
firstFailure ??= err;
console.debug("[bsk observation] frame AX capture failed", {
frameId: frame.frameId,
err,
});
return { frame, nodes: [] };
}
}),
);
const capturedNodeCount = [...captured.iframeNodes.values()].reduce(
(count, nodes) => count + nodes.length,
captured.nodes.length,
);
if (firstFailure && capturedNodeCount === 0 && trees.every((tree) => tree.nodes.length === 0)) {
throw firstFailure;
}
return trees;
}
export async function captureFrameData<T extends FrameAxNode>(
cdp: CdpRunner,
tabId: number,
captured: CapturedViewModel,
signal?: AbortSignal,
geometry = new GeometryContext(cdp, tabId, undefined, signal),
): Promise<CapturedFrameDocument<T>[]> {
const graph = await geometry.graph();
const frames = graph?.frames ?? [];
if (graph) await captureMissingFrameDocuments(cdp, tabId, graph, captured, geometry, signal);
throwIfAborted(signal);
const batches = await captureAxTrees<T>(cdp, tabId, frames, captured, signal);
return buildFrameDocuments(graph, batches, captured, signal);
}
+17 -20
View File
@@ -32,8 +32,6 @@ export interface FrameDomInput {
nodes: CapturedNode[]; nodes: CapturedNode[];
rootFrameId?: string; rootFrameId?: string;
frameNodes?: ReadonlyMap<string, CapturedNode[]>; frameNodes?: ReadonlyMap<string, CapturedNode[]>;
frameOwnerBackendNodeIds?: ReadonlyMap<string, number>;
frameParentIds?: ReadonlyMap<string, string>;
frameExcludedBackendNodeIds?: ReadonlyMap<string, ReadonlySet<number>>; frameExcludedBackendNodeIds?: ReadonlyMap<string, ReadonlySet<number>>;
} }
@@ -59,27 +57,13 @@ function targetBackendKey(target: CdpTarget, backendNodeId: number): string {
function frameList<T extends FrameOwnedAxNode>( function frameList<T extends FrameOwnedAxNode>(
graph: CdpFrameGraph | null, graph: CdpFrameGraph | null,
batches: FrameAxBatch<T>[], batches: FrameAxBatch<T>[],
captured: FrameDomInput,
): CdpFrame[] { ): CdpFrame[] {
const frames = new Map<string, CdpFrame>(); const frames = new Map<string, CdpFrame>();
for (const frame of graph?.frames ?? []) frames.set(frame.frameId, frame); for (const frame of graph?.frames ?? []) frames.set(frame.frameId, frame);
for (const batch of batches) { for (const batch of batches) {
if (!frames.has(batch.frame.frameId)) frames.set(batch.frame.frameId, batch.frame); if (!frames.has(batch.frame.frameId)) frames.set(batch.frame.frameId, batch.frame);
} }
for (const frameId of captured.frameNodes?.keys() ?? []) { return [...frames.values()];
if (!frames.has(frameId))
frames.set(frameId, { frameId, target: { tabId: batches[0]?.frame.target.tabId ?? 0 } });
}
return [...frames.values()].map((frame) => {
const ownerBackendNodeId =
frame.ownerBackendNodeId ?? captured.frameOwnerBackendNodeIds?.get(frame.frameId);
const parentFrameId = frame.parentFrameId ?? captured.frameParentIds?.get(frame.frameId);
return {
...frame,
...(ownerBackendNodeId !== undefined ? { ownerBackendNodeId } : {}),
...(parentFrameId ? { parentFrameId } : {}),
};
});
} }
export async function buildFrameDocuments<T extends FrameOwnedAxNode>( export async function buildFrameDocuments<T extends FrameOwnedAxNode>(
@@ -87,10 +71,11 @@ export async function buildFrameDocuments<T extends FrameOwnedAxNode>(
batches: FrameAxBatch<T>[], batches: FrameAxBatch<T>[],
captured: FrameDomInput, captured: FrameDomInput,
signal?: AbortSignal, signal?: AbortSignal,
unresolved?: (frame: CdpFrame) => void,
): Promise<FrameDocument<T>[]> { ): Promise<FrameDocument<T>[]> {
const checkpoint = createCaptureCheckpoint(signal); const checkpoint = createCaptureCheckpoint(signal);
let work = 0; let work = 0;
const frames = frameList(graph, batches, captured); const frames = frameList(graph, batches);
const frameById = new Map(frames.map((frame) => [frame.frameId, frame])); const frameById = new Map(frames.map((frame) => [frame.frameId, frame]));
const rootFrameId = graph?.rootFrameId ?? captured.rootFrameId ?? frames[0]?.frameId; const rootFrameId = graph?.rootFrameId ?? captured.rootFrameId ?? frames[0]?.frameId;
const domNodesForFrame = (frameId: string): CapturedNode[] => const domNodesForFrame = (frameId: string): CapturedNode[] =>
@@ -116,6 +101,7 @@ export async function buildFrameDocuments<T extends FrameOwnedAxNode>(
} }
nodeById.set(node.nodeId, node); nodeById.set(node.nodeId, node);
} }
let reported = false;
const ownershipByNodeId = new Map<string, Ownership>(); const ownershipByNodeId = new Map<string, Ownership>();
for (const node of batch.nodes) { for (const node of batch.nodes) {
const path: T[] = []; const path: T[] = [];
@@ -134,8 +120,18 @@ export async function buildFrameDocuments<T extends FrameOwnedAxNode>(
} }
if (visiting.has(current.nodeId)) break; if (visiting.has(current.nodeId)) break;
visiting.add(current.nodeId); visiting.add(current.nodeId);
if (current.frameId && frameById.has(current.frameId)) { if (current.frameId) {
ownership = { frameId: current.frameId, strength: 4 }; const frame = frameById.get(current.frameId);
// An explicit contradiction must not fall through to a guessed owner.
if (!frame || cdpTargetKey(frame.target) !== cdpTargetKey(batch.frame.target)) {
if (!reported) {
unresolved?.(batch.frame);
reported = true;
}
ownership = { frameId: "", strength: 0 };
break;
}
ownership = { frameId: frame.frameId, strength: 4 };
ownershipByNodeId.set(current.nodeId, ownership); ownershipByNodeId.set(current.nodeId, ownership);
break; break;
} }
@@ -160,6 +156,7 @@ export async function buildFrameDocuments<T extends FrameOwnedAxNode>(
ownershipByNodeId.set(path[i].nodeId, ownership); ownershipByNodeId.set(path[i].nodeId, ownership);
} }
if (!ownershipByNodeId.has(node.nodeId)) ownershipByNodeId.set(node.nodeId, ownership); if (!ownershipByNodeId.has(node.nodeId)) ownershipByNodeId.set(node.nodeId, ownership);
if (!ownership.frameId) continue;
const key = targetNodeKey(batch.frame.target, node.nodeId); const key = targetNodeKey(batch.frame.target, node.nodeId);
const existing = candidates.get(key); const existing = candidates.get(key);
if (!existing || ownership.strength > existing.ownership.strength) { if (!existing || ownership.strength > existing.ownership.strength) {
+255 -8
View File
@@ -1,26 +1,44 @@
import type { CdpTarget } from "@/browser-driver/frame-graph"; import type { Viewport } from "@browser-skill/vom";
import { type CdpFrame, type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
import { type GeometryProjection, projectRectToViewport } from "../geometry";
import { import {
type FrameProjectionState, type FrameProjectionState,
projectSnapshotRect, projectSnapshotRect,
type SnapshotCoordinates, type SnapshotCoordinates,
snapshotCoordinates,
snapshotViewportRect, snapshotViewportRect,
} from "../geometry/coordinate-types"; } from "../geometry/coordinate-types";
import { createCaptureCheckpoint } from "./capture-abort"; import {
import { buildDocumentIndex, type DocumentIndex, type NodeFacts } from "./facts"; GeometryContext,
import { decodeDocument, type SnapshotDocument } from "./snapshot"; type LayoutMetrics,
snapshotLayoutScale,
} from "../geometry/frame-context";
import {
createCaptureCheckpoint,
isAbortError as isCaptureAbort,
throwIfAborted as throwCaptureAborted,
} from "./capture-abort";
import { buildDocumentIndex, type CaptureIssue, type DocumentIndex, type NodeFacts } from "./facts";
import {
decodeDocument,
type SnapshotDocument,
type SnapshotReply,
snapshotFrameId,
} from "./snapshot";
export interface FrameContext { export interface FrameContext {
frameId?: string; frameId?: string;
ownerFrameBackendNodeId: number | null; ownerFrameBackendNodeId: number | null;
projection: FrameProjectionState; projection: FrameProjectionState | null;
targetProjection?: GeometryProjection | null;
target: CdpTarget; target: CdpTarget;
coordinates: SnapshotCoordinates | null; coordinates: SnapshotCoordinates | null;
layoutUnitsPerCssPixel: number | null;
} }
export interface NormalizedDocument { export interface NormalizedDocument {
nodes: NodeFacts[]; nodes: NodeFacts[];
index: DocumentIndex; index: DocumentIndex;
documentElementBackendNodeId?: number;
} }
/** Interpret one document using a supplied projection. No live reads or frame scheduling. */ /** Interpret one document using a supplied projection. No live reads or frame scheduling. */
@@ -46,10 +64,17 @@ export async function normalizeDocument(
context.coordinates, context.coordinates,
); );
const local = input?.rect; const local = input?.rect;
const rect = let rect =
input && context.projection.status === "available" input && context.projection?.status === "available"
? projectSnapshotRect(input, context.projection.projection) ? projectSnapshotRect(input, context.projection.projection)
: null; : null;
if (rect && context.targetProjection !== undefined)
rect = context.targetProjection
? projectRectToViewport(
{ x: rect.x, y: rect.y, w: rect.width, h: rect.height },
context.targetProjection,
)
: null;
const visibility = node.layout?.styles.visibility || "visible"; const visibility = node.layout?.styles.visibility || "visible";
const opacity = node.layout?.styles.opacity || "1"; const opacity = node.layout?.styles.opacity || "1";
nodes.push({ nodes.push({
@@ -74,5 +99,227 @@ export async function normalizeDocument(
(node) => !node.tag.startsWith("#") && !index.excludedBackendNodeIds.has(node.backendNodeId), (node) => !node.tag.startsWith("#") && !index.excludedBackendNodeIds.has(node.backendNodeId),
), ),
index, index,
documentElementBackendNodeId: nodes.find(
(node) =>
node.nodeType === 1 &&
node.parentBackendNodeId !== null &&
index.nodes.get(node.parentBackendNodeId)?.nodeType === 9,
)?.backendNodeId,
}; };
} }
export interface NormalizedFrameDocument extends NormalizedDocument {
frame: CdpFrame;
}
/** One target's snapshot. Source ownership is checked before interpreting any
* coordinates. Snapshot and live quad coordinates are deliberately not conflated. */
export async function normalizeSnapshot(
snapshot: SnapshotReply,
target: CdpTarget,
frames: readonly CdpFrame[],
geometry: GeometryContext,
issues: CaptureIssue[],
signal?: AbortSignal,
rootFrameId?: string,
): Promise<NormalizedFrameDocument[]> {
const strings = snapshot.strings ?? [];
const raw = snapshot.documents ?? [];
const ownerIds = new Set<number>();
for (const document of raw) {
for (const index of document.nodes?.contentDocumentIndex?.index ?? []) {
const id = document.nodes?.backendNodeId?.[index];
if (id !== undefined) ownerIds.add(id);
}
}
geometry.registerSnapshotOwners(target, ownerIds);
const frameById = new Map(frames.map((frame) => [frame.frameId, frame]));
const sources = new Map<string, SnapshotDocument>();
for (const doc of raw) {
const id = snapshotFrameId(doc, strings);
if (!id || !frameById.has(id) || sources.has(id)) {
issues.push({
target,
frameId: id,
stage: "ownership",
reason: "frame-ownership-unresolved",
});
continue;
}
sources.set(id, doc);
}
let metrics: LayoutMetrics = {};
try {
metrics = await geometry.layoutMetrics(target);
} catch (error) {
if (isCaptureAbort(error)) throw error;
}
const viewport: Viewport = {
width: metrics.cssLayoutViewport?.clientWidth ?? 0,
height: metrics.cssLayoutViewport?.clientHeight ?? 0,
};
const layoutUnitsPerCssPixel = snapshotLayoutScale(metrics);
const viewportAvailable = [viewport.width, viewport.height].every(
(size) => Number.isFinite(size) && size > 0,
);
const children = new Map<string, CdpFrame[]>();
const pending: CdpFrame[] = [];
for (const frame of frames) {
if (!frame.parentFrameId || !frameById.has(frame.parentFrameId)) pending.push(frame);
else {
const siblings = children.get(frame.parentFrameId);
if (siblings) siblings.push(frame);
else children.set(frame.parentFrameId, [frame]);
}
}
const projections = new Map<string, FrameProjectionState | null>();
const result: NormalizedFrameDocument[] = [];
let targetProjection: GeometryProjection | null = null;
const targetRoot = rootFrameId ? frameById.get(rootFrameId) : undefined;
if (target.sessionId && targetRoot) {
try {
const known = await geometry.frame(targetRoot.frameId);
if (known && cdpTargetKey(known.target) === cdpTargetKey(target))
targetProjection = await geometry.targetProjection(targetRoot.frameId);
} catch (error) {
if (isCaptureAbort(error)) throw error;
}
}
const checkpoint = createCaptureCheckpoint(signal);
const visited = new Set<string>();
const remaining = frames[Symbol.iterator]();
for (let cursor = 0; visited.size < frames.length; cursor++) {
if (cursor >= pending.length) {
let next = remaining.next();
while (!next.done && visited.has(next.value.frameId)) next = remaining.next();
if (next.done) break;
pending.push(next.value);
}
if (cursor % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
const frame = pending[cursor];
if (visited.has(frame.frameId)) continue;
visited.add(frame.frameId);
for (const child of children.get(frame.frameId) ?? []) pending.push(child);
const doc = sources.get(frame.frameId);
if (!doc) continue;
const source = { target, frameId: frame.frameId };
const coordinates = snapshotCoordinates(
doc,
layoutUnitsPerCssPixel,
frame.frameId === rootFrameId
? { x: metrics.cssLayoutViewport?.pageX, y: metrics.cssLayoutViewport?.pageY }
: undefined,
);
let state: FrameProjectionState | null =
frame.frameId === rootFrameId
? viewportAvailable && coordinates
? {
status: "available",
projection: {
source,
geometry: { sourceClips: [], edges: [], topViewport: viewport },
},
}
: { status: "unavailable", source, reason: "snapshot-coordinates-unavailable" }
: (projections.get(frame.frameId) ?? null);
if (!state && frame.frameId !== rootFrameId)
issues.push({
target,
frameId: frame.frameId,
stage: "ownership",
reason: "frame-ownership-unresolved",
});
if (!coordinates && (!state || state.status === "available"))
state = { status: "unavailable", source, reason: "snapshot-coordinates-unavailable" };
projections.set(frame.frameId, state);
const projection = state?.status === "available" ? state.projection : null;
if (!projection || (target.sessionId && !targetProjection))
issues.push({
target,
frameId: frame.frameId,
stage: "geometry",
reason: "geometry-unavailable",
...(state && state.status !== "available" ? { projectionIssue: state } : {}),
});
// Only direct siblings share this pool. Keep normalization in its existing
// breadth-first order, independent of owner read completion order.
const siblings = (children.get(frame.frameId) ?? []).filter((child) =>
sources.has(child.frameId),
);
const edge = projection?.geometry.edges[0];
const clips = edge
? [edge.destinationQuad, ...(edge.destinationClips ?? [])]
: (projection?.geometry.sourceClips ?? []);
let next = 0;
let failure: unknown;
await Promise.all(
Array.from({ length: Math.min(4, siblings.length) }, async () => {
while (next < siblings.length && !signal?.aborted && !failure) {
const child = siblings[next++];
const childSource = { target, frameId: child.frameId };
if (!projection) {
projections.set(
child.frameId,
state && state.status !== "available"
? {
status: "blocked",
source: childSource,
cause: state.status === "blocked" ? state.cause : state,
}
: null,
);
continue;
}
if (child.ownerBackendNodeId === undefined) {
projections.set(child.frameId, null);
continue;
}
if (!snapshotCoordinates(sources.get(child.frameId)!, layoutUnitsPerCssPixel)) {
projections.set(child.frameId, {
status: "unavailable",
source: childSource,
reason: "snapshot-coordinates-unavailable",
});
continue;
}
try {
projections.set(
child.frameId,
await geometry.snapshotProjection(
childSource,
child.ownerBackendNodeId,
clips,
viewport,
),
);
} catch (error) {
failure ??= error;
}
}
}),
);
// Join active reads and their cleanup before returning cancellation.
if (failure) throw failure;
throwCaptureAborted(signal);
const normalized = await normalizeDocument(
doc,
strings,
{
frameId: frame.frameId,
ownerFrameBackendNodeId: frame.ownerBackendNodeId ?? null,
projection: state,
target,
...(target.sessionId ? { targetProjection } : {}),
coordinates,
},
signal,
);
result.push({ ...normalized, frame });
}
return result;
}
+67
View File
@@ -1,3 +1,4 @@
import { type CdpFrame, type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
import { createCaptureCheckpoint } from "./capture-abort"; import { createCaptureCheckpoint } from "./capture-abort";
import type { CapturedNode } from "./capture-types"; import type { CapturedNode } from "./capture-types";
import type { DecodedDocument, DecodedNode } from "./facts"; import type { DecodedDocument, DecodedNode } from "./facts";
@@ -23,6 +24,7 @@ interface RareBooleanData {
export interface SnapshotDocument { export interface SnapshotDocument {
frameId?: string | number; frameId?: string | number;
documentURL?: number;
scrollOffsetX?: number; scrollOffsetX?: number;
scrollOffsetY?: number; scrollOffsetY?: number;
nodes?: { nodes?: {
@@ -222,3 +224,68 @@ export async function decodeDocument(
} }
return { nodes }; return { nodes };
} }
/** Snapshot membership and same-target edges are current response evidence.
* The earlier graph only supplies compatible metadata and cross-target edges. */
export function describeSnapshotFrames(
snapshot: SnapshotReply,
target: CdpTarget,
hintById: ReadonlyMap<string, CdpFrame>,
pageUrl?: string,
): { frames: CdpFrame[]; ids: Set<string>; rootFrameId?: string } {
const raw = snapshot.documents ?? [];
const strings = snapshot.strings ?? [];
const ids = new Set<string>();
const invalid = new Set<string>();
const frames = raw.map((doc) => {
const frameId = snapshotFrameId(doc, strings);
if (!frameId) return undefined;
if (ids.has(frameId)) invalid.add(frameId);
ids.add(frameId);
const hint = hintById.get(frameId);
const compatible =
hint && cdpTargetKey(hint.target) === cdpTargetKey(target) ? hint : undefined;
const url =
(doc.documentURL === undefined ? undefined : strings[doc.documentURL]) || compatible?.url;
return { frameId, target, ...(url ? { url } : {}) } as CdpFrame;
});
const root = frames[0];
if (root) {
const hint = hintById.get(root.frameId);
const parent = hint?.parentFrameId ? hintById.get(hint.parentFrameId) : undefined;
if (
target.sessionId &&
hint &&
cdpTargetKey(hint.target) === cdpTargetKey(target) &&
parent &&
cdpTargetKey(parent.target) !== cdpTargetKey(target)
) {
root.parentFrameId = parent.frameId;
root.ownerBackendNodeId = hint.ownerBackendNodeId;
}
if (!target.sessionId && !root.url && pageUrl) root.url = pageUrl;
}
const owned = new Set<string>();
for (let i = 0; i < raw.length; i++) {
const edges = raw[i].nodes?.contentDocumentIndex;
const parent = frames[i];
if (!parent || !edges) continue;
for (let e = 0; e < edges.index.length; e++) {
const child = frames[edges.value[e]];
const owner = raw[i].nodes?.backendNodeId?.[edges.index[e]];
if (!child || owner === undefined) continue;
if (child === root || child === parent || owned.has(child.frameId)) {
invalid.add(child.frameId);
continue;
}
owned.add(child.frameId);
child.parentFrameId = parent.frameId;
child.ownerBackendNodeId = owner;
}
}
return {
frames: frames.filter((frame): frame is CdpFrame => !!frame && !invalid.has(frame.frameId)),
ids,
rootFrameId: root?.frameId,
};
}
+33
View File
@@ -470,6 +470,39 @@ export interface HoverResult {
dialogs?: JavaScriptDialogInfo[]; dialogs?: JavaScriptDialogInfo[];
} }
export interface FocusParams {
session_id: string;
ref?: string;
selector?: string;
tab_id?: number;
timeout_ms?: number;
}
export interface FocusResult {
tab_id: number;
used_ref?: string;
used_selector?: string;
focused: boolean;
dialogs?: JavaScriptDialogInfo[];
}
export interface BlurParams {
session_id: string;
ref?: string;
selector?: string;
tab_id?: number;
timeout_ms?: number;
}
export interface BlurResult {
tab_id: number;
used_ref?: string;
used_selector?: string;
was_focused: boolean;
focused: boolean;
dialogs?: JavaScriptDialogInfo[];
}
export interface FillParams { export interface FillParams {
session_id: string; session_id: string;
value: string; value: string;
+5 -2
View File
@@ -55,10 +55,13 @@ Use this default loop:
```text ```text
bsk navigate <url> --session <id> bsk navigate <url> --session <id>
bsk observe --session <id> bsk observe --session <id>
bsk click|hover|fill|select|press ... --session <id> bsk click|hover|focus|blur|fill|select|press ... --session <id>
bsk observe --session <id> # after navigation or a meaningful DOM change bsk observe --session <id> # after navigation or a meaningful DOM change
``` ```
`bsk focus <ref>` explicitly focuses a target; `bsk blur <ref>` removes focus and reports whether
it was focused. Use these for UI states triggered by focus changes.
Prefer fresh `@eN` refs over CSS selectors. Navigation invalidates refs; large DOM changes may also Prefer fresh `@eN` refs over CSS selectors. Navigation invalidates refs; large DOM changes may also
make them stale. Observe again before the next interaction. make them stale. Observe again before the next interaction.
@@ -114,7 +117,7 @@ This list of names is complete. Never invent a command outside it; read
session start|stop|list browsers status doctor update logs session start|stop|list browsers status doctor update logs
navigate navigate-back navigate-forward reload wait-for-navigation wait-ms navigate navigate-back navigate-forward reload wait-for-navigation wait-ms
observe snapshot get-html screenshot console network observe snapshot get-html screenshot console network
click hover fill select press evaluate click hover focus blur fill select press evaluate
tab list|create|close|select|borrow|return window resize emulate tab list|create|close|select|borrow|return window resize emulate
upload download request-help record start|stop upload download request-help record start|stop
``` ```
+123 -3
View File
@@ -1,4 +1,5 @@
//! `bsk click` / `bsk fill` / `bsk press` (M7 interaction tools). //! `bsk click` / `bsk focus` / `bsk blur` / `bsk fill` / `bsk press`
//! interaction tools.
//! //!
//! The `<target>` positional accepts either a snapshot ref (`@e3`, //! The `<target>` positional accepts either a snapshot ref (`@e3`,
//! `e3`) or a CSS selector. Use `--ref` / `--selector` explicitly when //! `e3`) or a CSS selector. Use `--ref` / `--selector` explicitly when
@@ -10,8 +11,9 @@ use std::time::Duration;
use anyhow::Context; use anyhow::Context;
use bsk_protocol::Method; use bsk_protocol::Method;
use bsk_protocol::tools::{ use bsk_protocol::tools::{
ClickParams, ClickResult, FillParams, FillResult, HoverParams, HoverResult, KeyModifier, BlurParams, BlurResult, ClickParams, ClickResult, FillParams, FillResult, FocusParams,
MouseButton, PressParams, PressResult, SelectParams, SelectResult, FocusResult, HoverParams, HoverResult, KeyModifier, MouseButton, PressParams, PressResult,
SelectParams, SelectResult,
}; };
use clap::{Args, ValueEnum}; use clap::{Args, ValueEnum};
@@ -274,6 +276,124 @@ pub fn dispatch_hover(args: HoverArgs, format: Format) -> Result<(), CliError> {
Ok(()) Ok(())
} }
// ---------------------------------------------------------------------------
// bsk focus / bsk blur
// ---------------------------------------------------------------------------
#[derive(Debug, Clone, Args)]
pub struct FocusArgs {
/// Snapshot ref (`@e3`, `e3`) or CSS selector. Optional when `--ref`/`--selector` is used.
pub target: Option<String>,
#[arg(long = "ref")]
pub ref_: Option<String>,
#[arg(long = "selector")]
pub selector: Option<String>,
#[arg(long)]
pub session: String,
#[arg(long = "tab-id")]
pub tab_id: Option<i64>,
#[arg(long, default_value = "30s", value_parser = parse_timeout_ms)]
pub timeout: u32,
}
pub fn dispatch_focus(args: FocusArgs, format: Format) -> Result<(), CliError> {
let info = ensure_daemon().context("ensure daemon is running")?;
let (ref_, selector) = split_target(args.target, args.ref_, args.selector)?;
let params = FocusParams {
session_id: args.session,
ref_,
selector,
tab_id: args.tab_id,
timeout_ms: Some(args.timeout),
};
let reply: FocusResult = call(
info.sock_path,
Method::ToolFocus,
params,
"focus-1",
args.timeout,
)?;
match format {
Format::Json => println!(
"{}",
serde_json::to_string_pretty(&reply)
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?
),
Format::Human => {
let target =
format_used_target(reply.used_ref.as_deref(), reply.used_selector.as_deref());
println!(
"focus ok tab={} target={target} focused={}",
reply.tab_id, reply.focused
);
print_dialog_summaries(&reply.dialogs);
}
}
Ok(())
}
#[derive(Debug, Clone, Args)]
pub struct BlurArgs {
/// Snapshot ref (`@e3`, `e3`) or CSS selector. Optional when `--ref`/`--selector` is used.
pub target: Option<String>,
#[arg(long = "ref")]
pub ref_: Option<String>,
#[arg(long = "selector")]
pub selector: Option<String>,
#[arg(long)]
pub session: String,
#[arg(long = "tab-id")]
pub tab_id: Option<i64>,
#[arg(long, default_value = "30s", value_parser = parse_timeout_ms)]
pub timeout: u32,
}
pub fn dispatch_blur(args: BlurArgs, format: Format) -> Result<(), CliError> {
let info = ensure_daemon().context("ensure daemon is running")?;
let (ref_, selector) = split_target(args.target, args.ref_, args.selector)?;
let params = BlurParams {
session_id: args.session,
ref_,
selector,
tab_id: args.tab_id,
timeout_ms: Some(args.timeout),
};
let reply: BlurResult = call(
info.sock_path,
Method::ToolBlur,
params,
"blur-1",
args.timeout,
)?;
match format {
Format::Json => println!(
"{}",
serde_json::to_string_pretty(&reply)
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?
),
Format::Human => {
let target =
format_used_target(reply.used_ref.as_deref(), reply.used_selector.as_deref());
println!(
"blur ok tab={} target={target} was_focused={} focused={}",
reply.tab_id, reply.was_focused, reply.focused
);
print_dialog_summaries(&reply.dialogs);
}
}
Ok(())
}
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// bsk fill // bsk fill
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
+9 -1
View File
@@ -47,7 +47,9 @@ use crate::cli::evaluate::EvaluateArgs;
use crate::cli::get_html::GetHtmlArgs; use crate::cli::get_html::GetHtmlArgs;
use crate::cli::human_loop::RequestHelpArgs; use crate::cli::human_loop::RequestHelpArgs;
use crate::cli::install_skill::InstallSkillArgs; use crate::cli::install_skill::InstallSkillArgs;
use crate::cli::interaction::{ClickArgs, FillArgs, HoverArgs, PressArgs, SelectArgs}; use crate::cli::interaction::{
BlurArgs, ClickArgs, FillArgs, FocusArgs, HoverArgs, PressArgs, SelectArgs,
};
use crate::cli::navigate::{NavigateCommand, NavigateHistoryArgs, ReloadArgs}; use crate::cli::navigate::{NavigateCommand, NavigateHistoryArgs, ReloadArgs};
use crate::cli::network::NetworkArgs; use crate::cli::network::NetworkArgs;
use crate::cli::observe::ObserveArgs; use crate::cli::observe::ObserveArgs;
@@ -173,6 +175,12 @@ pub enum Command {
/// Hover a snapshot ref or CSS selector. /// Hover a snapshot ref or CSS selector.
Hover(HoverArgs), Hover(HoverArgs),
/// Focus a snapshot ref or CSS selector.
Focus(FocusArgs),
/// Remove focus from a snapshot ref or CSS selector.
Blur(BlurArgs),
/// Fill an input / textarea / contenteditable. /// Fill an input / textarea / contenteditable.
Fill(FillArgs), Fill(FillArgs),
+2
View File
@@ -253,6 +253,8 @@ pub fn full_handler(status: DaemonStatus, state: Arc<DaemonState>) -> RpcHandler
| Method::ToolReload | Method::ToolReload
| Method::ToolClick | Method::ToolClick
| Method::ToolHover | Method::ToolHover
| Method::ToolFocus
| Method::ToolBlur
| Method::ToolFill | Method::ToolFill
| Method::ToolPress | Method::ToolPress
| Method::ToolSelect | Method::ToolSelect
+2
View File
@@ -94,6 +94,8 @@ fn dispatch(cli: Cli, format: Format) -> Result<(), CliError> {
Command::Reload(args) => cli::navigate::dispatch_reload(args, format), Command::Reload(args) => cli::navigate::dispatch_reload(args, format),
Command::Click(args) => cli::interaction::dispatch_click(args, format), Command::Click(args) => cli::interaction::dispatch_click(args, format),
Command::Hover(args) => cli::interaction::dispatch_hover(args, format), Command::Hover(args) => cli::interaction::dispatch_hover(args, format),
Command::Focus(args) => cli::interaction::dispatch_focus(args, format),
Command::Blur(args) => cli::interaction::dispatch_blur(args, format),
Command::Fill(args) => cli::interaction::dispatch_fill(args, format), Command::Fill(args) => cli::interaction::dispatch_fill(args, format),
Command::Press(args) => cli::interaction::dispatch_press(args, format), Command::Press(args) => cli::interaction::dispatch_press(args, format),
Command::Select(args) => cli::interaction::dispatch_select(args, format), Command::Select(args) => cli::interaction::dispatch_select(args, format),
+18
View File
@@ -255,6 +255,24 @@ fn parses_hover_with_settle() {
assert_eq!(args.settle, 300); assert_eq!(args.settle, 300);
} }
#[test]
fn parses_focus_target() {
let cli = parse(&["bsk", "focus", "@e2", "--session", "s1"]);
let Command::Focus(args) = cli.command else {
panic!("expected focus command");
};
assert_eq!(args.target.as_deref(), Some("@e2"));
}
#[test]
fn parses_blur_selector() {
let cli = parse(&["bsk", "blur", "--selector", "#search", "--session", "s1"]);
let Command::Blur(args) = cli.command else {
panic!("expected blur command");
};
assert_eq!(args.selector.as_deref(), Some("#search"));
}
#[test] #[test]
fn rejects_zero_click_count() { fn rejects_zero_click_count() {
assert!( assert!(
+92 -6
View File
@@ -1,5 +1,4 @@
//! M7 integration: drive the 7 new tool RPCs (navigate / //! M7 integration: drive interaction and navigation tool RPCs
//! navigate_back / navigate_forward / reload / click / fill / press)
//! through the IPC + per-session queue + fake extension and assert //! through the IPC + per-session queue + fake extension and assert
//! the wire shapes line up. The extension stub mirrors each method's //! the wire shapes line up. The extension stub mirrors each method's
//! `*Result` so we exercise the daemon's serialise → forward → //! `*Result` so we exercise the daemon's serialise → forward →
@@ -13,10 +12,11 @@ use bsk::daemon::{self, DaemonConfig};
use bsk::ipc_client::IpcClient; use bsk::ipc_client::IpcClient;
use bsk_protocol::system::{HandshakeParams, HandshakeResult}; use bsk_protocol::system::{HandshakeParams, HandshakeResult};
use bsk_protocol::tools::{ use bsk_protocol::tools::{
ClickParams, ClickResult, FillParams, FillResult, KeyModifier, MouseButton, NavigateBackParams, BlurParams, BlurResult, ClickParams, ClickResult, FillParams, FillResult, FocusParams,
NavigateBackResult, NavigateForwardParams, NavigateForwardResult, NavigateParams, FocusResult, KeyModifier, MouseButton, NavigateBackParams, NavigateBackResult,
NavigateResult, PressParams, PressResult, ReloadParams, ReloadResult, SelectParams, NavigateForwardParams, NavigateForwardResult, NavigateParams, NavigateResult, PressParams,
SelectResult, SessionStartParams, SessionStartResult, WaitUntil, PressResult, ReloadParams, ReloadResult, SelectParams, SelectResult, SessionStartParams,
SessionStartResult, WaitUntil,
}; };
use bsk_protocol::{ use bsk_protocol::{
BrowserPeerInfo, ErrorCode, Frame, Method, RequestFrame, ResponseBody, ResponseFrame, RpcError, BrowserPeerInfo, ErrorCode, Frame, Method, RequestFrame, ResponseBody, ResponseFrame, RpcError,
@@ -382,6 +382,92 @@ async fn click_round_trips_ref_and_modifiers() {
handle.shutdown().await; handle.shutdown().await;
} }
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn focus_round_trips_verified_state() {
let (handle, sock) = spawn_daemon().await;
let mut ws = connect_ext(handle.ws_addr()).await;
let _ = do_handshake(&mut ws).await;
run_extension(ws, |req| {
assert_eq!(req.method, Method::ToolFocus);
let params = req.params.clone().unwrap();
let p: FocusParams = serde_json::from_value(params).unwrap();
assert_eq!(p.ref_.as_deref(), Some("@e3"));
ResponseBody::Ok(
serde_json::to_value(FocusResult {
tab_id: 9,
used_ref: Some("e3".into()),
used_selector: None,
focused: true,
dialogs: vec![],
})
.unwrap(),
)
});
let session_id = ipc_session_start(&sock).await;
let result: FocusResult = ipc_tool_call(
&sock,
Method::ToolFocus,
FocusParams {
session_id,
ref_: Some("@e3".into()),
selector: None,
tab_id: None,
timeout_ms: Some(5_000),
},
)
.await
.expect("focus ok");
assert!(result.focused);
assert_eq!(result.used_ref.as_deref(), Some("e3"));
handle.shutdown().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn blur_round_trips_previous_focus_state() {
let (handle, sock) = spawn_daemon().await;
let mut ws = connect_ext(handle.ws_addr()).await;
let _ = do_handshake(&mut ws).await;
run_extension(ws, |req| {
assert_eq!(req.method, Method::ToolBlur);
let params = req.params.clone().unwrap();
let p: BlurParams = serde_json::from_value(params).unwrap();
assert_eq!(p.selector.as_deref(), Some("#search"));
ResponseBody::Ok(
serde_json::to_value(BlurResult {
tab_id: 9,
used_ref: None,
used_selector: Some("#search".into()),
was_focused: true,
focused: false,
dialogs: vec![],
})
.unwrap(),
)
});
let session_id = ipc_session_start(&sock).await;
let result: BlurResult = ipc_tool_call(
&sock,
Method::ToolBlur,
BlurParams {
session_id,
ref_: None,
selector: Some("#search".into()),
tab_id: None,
timeout_ms: Some(5_000),
},
)
.await
.expect("blur ok");
assert!(result.was_focused);
assert!(!result.focused);
assert_eq!(result.used_selector.as_deref(), Some("#search"));
handle.shutdown().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn fill_round_trips_clear_before_default() { async fn fill_round_trips_clear_before_default() {
let (handle, sock) = spawn_daemon().await; let (handle, sock) = spawn_daemon().await;
@@ -0,0 +1,42 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "BlurParams",
"type": "object",
"required": [
"session_id"
],
"properties": {
"ref": {
"description": "Optional `@e<N>` ref allocated by the last observation. Mutually exclusive with `selector`.",
"type": [
"string",
"null"
]
},
"selector": {
"type": [
"string",
"null"
]
},
"session_id": {
"type": "string"
},
"tab_id": {
"description": "Target tab. Defaults to the Agent Window's active tab.",
"type": [
"integer",
"null"
],
"format": "int64"
},
"timeout_ms": {
"type": [
"integer",
"null"
],
"format": "uint32",
"minimum": 1.0
}
}
}
@@ -0,0 +1,112 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "BlurResult",
"type": "object",
"required": [
"focused",
"tab_id",
"was_focused"
],
"properties": {
"dialogs": {
"type": "array",
"items": {
"$ref": "#/definitions/JavaScriptDialogInfo"
}
},
"focused": {
"description": "Always false for a successful blur; included for machine verification.",
"type": "boolean"
},
"tab_id": {
"type": "integer",
"format": "int64"
},
"used_ref": {
"type": [
"string",
"null"
]
},
"used_selector": {
"type": [
"string",
"null"
]
},
"was_focused": {
"description": "Whether the target held focus before the call.",
"type": "boolean"
}
},
"definitions": {
"JavaScriptDialogHandledAction": {
"description": "How the extension resolved the dialog so CDP could continue.",
"type": "string",
"enum": [
"accepted",
"dismissed"
]
},
"JavaScriptDialogInfo": {
"description": "One observed + handled JavaScript dialog during a tool call.",
"type": "object",
"required": [
"handled",
"message",
"sequence",
"tab_id",
"type"
],
"properties": {
"default_prompt": {
"type": [
"string",
"null"
]
},
"handled": {
"$ref": "#/definitions/JavaScriptDialogHandledAction"
},
"has_browser_handler": {
"type": [
"boolean",
"null"
]
},
"message": {
"type": "string"
},
"sequence": {
"description": "Monotonic per-tab sequence for ordering within a session.",
"type": "integer",
"format": "uint64",
"minimum": 0.0
},
"tab_id": {
"type": "integer",
"format": "int64"
},
"type": {
"$ref": "#/definitions/JavaScriptDialogType"
},
"url": {
"type": [
"string",
"null"
]
}
}
},
"JavaScriptDialogType": {
"description": "Native JS dialog kind reported by CDP.",
"type": "string",
"enum": [
"alert",
"confirm",
"prompt",
"beforeunload"
]
}
}
}
@@ -0,0 +1,42 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "FocusParams",
"type": "object",
"required": [
"session_id"
],
"properties": {
"ref": {
"description": "Optional `@e<N>` ref allocated by the last observation. Mutually exclusive with `selector`.",
"type": [
"string",
"null"
]
},
"selector": {
"type": [
"string",
"null"
]
},
"session_id": {
"type": "string"
},
"tab_id": {
"description": "Target tab. Defaults to the Agent Window's active tab.",
"type": [
"integer",
"null"
],
"format": "int64"
},
"timeout_ms": {
"type": [
"integer",
"null"
],
"format": "uint32",
"minimum": 1.0
}
}
}
@@ -0,0 +1,107 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "FocusResult",
"type": "object",
"required": [
"focused",
"tab_id"
],
"properties": {
"dialogs": {
"type": "array",
"items": {
"$ref": "#/definitions/JavaScriptDialogInfo"
}
},
"focused": {
"description": "Whether the target holds DOM focus in its document or shadow root after the call.",
"type": "boolean"
},
"tab_id": {
"type": "integer",
"format": "int64"
},
"used_ref": {
"type": [
"string",
"null"
]
},
"used_selector": {
"type": [
"string",
"null"
]
}
},
"definitions": {
"JavaScriptDialogHandledAction": {
"description": "How the extension resolved the dialog so CDP could continue.",
"type": "string",
"enum": [
"accepted",
"dismissed"
]
},
"JavaScriptDialogInfo": {
"description": "One observed + handled JavaScript dialog during a tool call.",
"type": "object",
"required": [
"handled",
"message",
"sequence",
"tab_id",
"type"
],
"properties": {
"default_prompt": {
"type": [
"string",
"null"
]
},
"handled": {
"$ref": "#/definitions/JavaScriptDialogHandledAction"
},
"has_browser_handler": {
"type": [
"boolean",
"null"
]
},
"message": {
"type": "string"
},
"sequence": {
"description": "Monotonic per-tab sequence for ordering within a session.",
"type": "integer",
"format": "uint64",
"minimum": 0.0
},
"tab_id": {
"type": "integer",
"format": "int64"
},
"type": {
"$ref": "#/definitions/JavaScriptDialogType"
},
"url": {
"type": [
"string",
"null"
]
}
}
},
"JavaScriptDialogType": {
"description": "Native JS dialog kind reported by CDP.",
"type": "string",
"enum": [
"alert",
"confirm",
"prompt",
"beforeunload"
]
}
}
}
@@ -79,6 +79,10 @@ fn main() {
dump!(ClickResult, "tool_click_result"); dump!(ClickResult, "tool_click_result");
dump!(HoverParams, "tool_hover_params"); dump!(HoverParams, "tool_hover_params");
dump!(HoverResult, "tool_hover_result"); dump!(HoverResult, "tool_hover_result");
dump!(FocusParams, "tool_focus_params");
dump!(FocusResult, "tool_focus_result");
dump!(BlurParams, "tool_blur_params");
dump!(BlurResult, "tool_blur_result");
dump!(FillParams, "tool_fill_params"); dump!(FillParams, "tool_fill_params");
dump!(FillResult, "tool_fill_result"); dump!(FillResult, "tool_fill_result");
dump!(PressParams, "tool_press_params"); dump!(PressParams, "tool_press_params");
+12
View File
@@ -72,6 +72,10 @@ pub enum Method {
ToolClick, ToolClick,
#[serde(rename = "tool.hover")] #[serde(rename = "tool.hover")]
ToolHover, ToolHover,
#[serde(rename = "tool.focus")]
ToolFocus,
#[serde(rename = "tool.blur")]
ToolBlur,
#[serde(rename = "tool.fill")] #[serde(rename = "tool.fill")]
ToolFill, ToolFill,
#[serde(rename = "tool.press")] #[serde(rename = "tool.press")]
@@ -168,6 +172,8 @@ impl Method {
| Method::ToolNavigateForward | Method::ToolNavigateForward
| Method::ToolReload | Method::ToolReload
| Method::ToolClick | Method::ToolClick
| Method::ToolFocus
| Method::ToolBlur
| Method::ToolFill | Method::ToolFill
| Method::ToolPress | Method::ToolPress
| Method::ToolSelect | Method::ToolSelect
@@ -305,6 +311,8 @@ mod tests {
assert!(Method::ToolNavigateForward.is_mutating()); assert!(Method::ToolNavigateForward.is_mutating());
assert!(Method::ToolReload.is_mutating()); assert!(Method::ToolReload.is_mutating());
assert!(Method::ToolClick.is_mutating()); assert!(Method::ToolClick.is_mutating());
assert!(Method::ToolFocus.is_mutating());
assert!(Method::ToolBlur.is_mutating());
assert!(Method::ToolFill.is_mutating()); assert!(Method::ToolFill.is_mutating());
assert!(Method::ToolPress.is_mutating()); assert!(Method::ToolPress.is_mutating());
assert!(Method::ToolSelect.is_mutating()); assert!(Method::ToolSelect.is_mutating());
@@ -349,6 +357,8 @@ mod tests {
assert_eq!(Method::ToolHover.effect(), MethodEffect::TransientInput); assert_eq!(Method::ToolHover.effect(), MethodEffect::TransientInput);
assert_eq!(Method::ToolObserve.effect(), MethodEffect::TransientInput); assert_eq!(Method::ToolObserve.effect(), MethodEffect::TransientInput);
assert_eq!(Method::ToolClick.effect(), MethodEffect::BrowserMutation); assert_eq!(Method::ToolClick.effect(), MethodEffect::BrowserMutation);
assert_eq!(Method::ToolFocus.effect(), MethodEffect::BrowserMutation);
assert_eq!(Method::ToolBlur.effect(), MethodEffect::BrowserMutation);
assert_eq!(Method::Cancel.effect(), MethodEffect::ControlPlane); assert_eq!(Method::Cancel.effect(), MethodEffect::ControlPlane);
} }
@@ -358,6 +368,8 @@ mod tests {
assert!(Method::ToolHover.requires_interrupt_gate()); assert!(Method::ToolHover.requires_interrupt_gate());
assert!(Method::ToolObserve.requires_interrupt_gate()); assert!(Method::ToolObserve.requires_interrupt_gate());
assert!(Method::ToolClick.requires_interrupt_gate()); assert!(Method::ToolClick.requires_interrupt_gate());
assert!(Method::ToolFocus.requires_interrupt_gate());
assert!(Method::ToolBlur.requires_interrupt_gate());
assert!(!Method::Cancel.requires_interrupt_gate()); assert!(!Method::Cancel.requires_interrupt_gate());
} }
} }
+109 -2
View File
@@ -1,5 +1,5 @@
//! DOM interaction tools (`tool.click`, `tool.fill`, `tool.press`, //! DOM interaction tools (`tool.click`, `tool.hover`, `tool.focus`,
//! `tool.select`). //! `tool.blur`, `tool.fill`, `tool.press`, `tool.select`).
//! //!
//! Element-targeted tools accept either a snapshot `ref` (`@e<N>` form, //! Element-targeted tools accept either a snapshot `ref` (`@e<N>` form,
//! normalised against the session's RefStore) **or** a CSS selector //! normalised against the session's RefStore) **or** a CSS selector
@@ -131,6 +131,82 @@ pub struct HoverResult {
pub dialogs: Vec<JavaScriptDialogInfo>, pub dialogs: Vec<JavaScriptDialogInfo>,
} }
// ---------------------------------------------------------------------------
// focus / blur
// ---------------------------------------------------------------------------
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct FocusParams {
pub session_id: String,
/// Optional `@e<N>` ref allocated by the last observation.
/// Mutually exclusive with `selector`.
#[serde(
rename = "ref",
alias = "ref_",
default,
skip_serializing_if = "Option::is_none"
)]
pub ref_: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub selector: Option<String>,
/// Target tab. Defaults to the Agent Window's active tab.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub tab_id: Option<i64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(range(min = 1))]
pub timeout_ms: Option<u32>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct FocusResult {
pub tab_id: i64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub used_ref: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub used_selector: Option<String>,
/// Whether the target holds DOM focus in its document or shadow root after the call.
pub focused: bool,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub dialogs: Vec<JavaScriptDialogInfo>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct BlurParams {
pub session_id: String,
/// Optional `@e<N>` ref allocated by the last observation.
/// Mutually exclusive with `selector`.
#[serde(
rename = "ref",
alias = "ref_",
default,
skip_serializing_if = "Option::is_none"
)]
pub ref_: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub selector: Option<String>,
/// Target tab. Defaults to the Agent Window's active tab.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub tab_id: Option<i64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(range(min = 1))]
pub timeout_ms: Option<u32>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct BlurResult {
pub tab_id: i64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub used_ref: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub used_selector: Option<String>,
/// Whether the target held focus before the call.
pub was_focused: bool,
/// Always false for a successful blur; included for machine verification.
pub focused: bool,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub dialogs: Vec<JavaScriptDialogInfo>,
}
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// fill // fill
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@@ -325,6 +401,37 @@ mod tests {
assert_eq!(round, p); assert_eq!(round, p);
} }
#[test]
fn focus_params_serialise_ref_field_name() {
let p = FocusParams {
session_id: "abcd".into(),
ref_: Some("@e4".into()),
selector: None,
tab_id: Some(42),
timeout_ms: Some(5_000),
};
let v = serde_json::to_value(&p).unwrap();
assert_eq!(v.get("ref").and_then(|v| v.as_str()), Some("@e4"));
assert!(v.get("ref_").is_none());
let round: FocusParams = serde_json::from_value(v).unwrap();
assert_eq!(round, p);
}
#[test]
fn blur_result_round_trips_focus_state() {
let result = BlurResult {
tab_id: 42,
used_ref: Some("e4".into()),
used_selector: None,
was_focused: true,
focused: false,
dialogs: vec![],
};
let value = serde_json::to_value(&result).unwrap();
let round: BlurResult = serde_json::from_value(value).unwrap();
assert_eq!(round, result);
}
#[test] #[test]
fn modifiers_render_as_lowercase_strings() { fn modifiers_render_as_lowercase_strings() {
let v = serde_json::to_value(KeyModifier::Ctrl).unwrap(); let v = serde_json::to_value(KeyModifier::Ctrl).unwrap();
@@ -0,0 +1,41 @@
# OOPIF occupied-scrollbar regression
An iframe's content quad includes its child viewport's scrollbar space, while
`Page.getLayoutMetrics().cssLayoutViewport` excludes that space. Mapping the latter
onto the whole quad stretches positions and sizes. The full target-local viewport
must determine scale; the visible viewport must still clip content at every OOPIF
boundary and constrain target-local action points.
This fixture alternates loopback hostnames across two nested OOPIFs. It includes
scrolling, borders, padding, scaled iframe owners, a partially clipped button and
a fully clipped button. Custom scrollbars occupy different widths and heights.
The `scrollbars` query parameter accepts `both` (default), `vertical`, `horizontal`
or `none`.
Run the numerical regression with Node 22+ and a local Chrome executable:
```sh
BSK_GEOMETRY_CHROME=/path/to/chrome pnpm --filter @browser-skill/extension exec vitest run \
src/tools/__tests__/snapshot-coordinates.browser.test.ts
```
The existing browser runner is shared with the snapshot-unit regression. The test:
- Verifies actual OOPIF targets and occupied scrollbar dimensions.
- Compares snapshot and live geometry with independent DOM rectangles and clipping.
- Dispatches real root-target clicks and verifies the intended frame/button received them.
- Rejects fully clipped controls and checks snapshot measurement reuse per target.
- Covers five device-scale/browser-zoom combinations plus single-axis and no-scrollbar cases.
The runner creates and removes isolated browser profiles. It is opt-in and skipped
in normal unit runs when `BSK_GEOMETRY_CHROME` is unset.
Run the CLI fixture smoke with a connected test extension:
```sh
BSK_AUTO_UPDATE=off pnpm eval:browser smoke --case oopif-scrollbars --bsk ./target/debug/bsk
```
The smoke assertions verify both nested frames have occupied scrollbars, that the
marker is observed and that the session is closed. Numeric geometry and actual
click assertions are exercised by the browser test above, not by smoke alone.
@@ -0,0 +1,39 @@
{
"$schema": "../../../schemas/case.schema.json",
"schemaVersion": 1,
"id": "oopif-scrollbars",
"title": "Nested OOPIFs with occupied scrollbars and clipped controls",
"suite": "regression",
"tags": ["geometry", "iframe", "scroll"],
"fixture": { "startPath": "/oopif-scrollbars" },
"prompts": {
"en": "Open {url}, observe the page and its frames, report the OOPIF-SCROLLBARS marker, then close the session.",
"zh-CN": "打开 {url},观察页面及其 iframe,报告 OOPIF-SCROLLBARS 标记,然后关闭会话。"
},
"coverage": ["session.start", "session.stop", "page.navigate", "inspect.observe"],
"assertions": {
"site": [
{
"label": "nested frame fixture settled",
"type": "geometry.ready",
"where": { "data.root": true },
"minCount": 1
},
{
"label": "both child frames have occupied scrollbars",
"type": "geometry.scrollbars",
"where": { "data.vertical": true, "data.horizontal": true },
"minCount": 2
}
],
"response": [{ "label": "observed the fixture marker", "includes": "OOPIF-SCROLLBARS" }],
"adapter": [{ "label": "browser session was closed", "key": "sessionStopped" }]
},
"smoke": {
"steps": [
{ "action": "navigate" },
{ "action": "wait-site-event", "type": "geometry.ready", "where": { "root": true } },
{ "action": "observe" }
]
}
}
@@ -0,0 +1,56 @@
import { page, withRun } from "../../../lib/fixtures.mjs";
export default {
id: "oopif-scrollbars",
routes: ["/oopif-scrollbars", "/oopif-scrollbars/frame", "/oopif-scrollbars/nested"],
render({ pathname, runId, query }) {
const root = pathname === "/oopif-scrollbars";
const nested = pathname.endsWith("/nested");
const mode = query.get("scrollbars") ?? "both";
return page({
title: "OOPIF scrollbar geometry regression",
body: `
<style>
main { margin: 0; width: auto; }
body { width: 2000px; height: 2400px; }
html { overflow-anchor: none; }
${!root ? `html { overflow-x: ${mode === "vertical" || mode === "none" ? "hidden" : "scroll"}; overflow-y: ${mode === "horizontal" || mode === "none" ? "hidden" : "scroll"}; } html::-webkit-scrollbar { width: 17px; height: 11px; }` : ""}
button { position: absolute; padding: 0; border: 0; border-radius: 0; }
#probe { left: ${root ? 180 : nested ? 30 : 100}px; top: ${root ? 360 : nested ? 40 : 160}px; width: 60px; height: 20px; }
#edge { position: fixed; right: -20px; bottom: -15px; width: 80px; height: 60px; }
#outside { position: fixed; left: calc(100% + 2px); top: calc(100% + 2px); width: 10px; height: 10px; }
iframe { position: absolute; box-sizing: content-box; width: 420px; height: 320px;
left: 340px; top: 400px; border: 6px solid black; padding: 8px; transform: scale(1.2); transform-origin: 0 0; }
#nested { left: 150px; top: 210px; width: 200px; height: 120px; border-width: 4px; padding: 6px; transform: scale(0.8); }
</style>
<button id="probe" data-geometry-probe>OOPIF-SCROLLBARS</button>
${!root ? '<button id="edge" data-geometry-probe>EDGE</button><button id="outside" data-geometry-probe>OUTSIDE</button>' : ""}
${!nested ? `<iframe id="${root ? "cross" : "nested"}" name="${root ? "cross" : "nested"}" title="${root ? "outer" : "nested"} cross-site frame"></iframe>` : ""}
`,
script: `
history.scrollRestoration = "manual";
const child = document.querySelector("iframe");
if (child) {
const url = new URL(${JSON.stringify(withRun(root ? "/oopif-scrollbars/frame" : "/oopif-scrollbars/nested", runId, { scrollbars: mode }))}, location.href);
url.hostname = location.hostname === "127.0.0.1" ? "localhost" : "127.0.0.1";
child.src = url.href;
}
document.addEventListener("click", event => {
const probe = event.target.closest("[data-geometry-probe]");
if (probe) browserEval.send("geometry.clicked", { probe: probe.id });
});
window.addEventListener("load", () => {
scrollTo(${root ? "80, 240" : nested ? "10, 20" : "40, 100"});
requestAnimationFrame(() => requestAnimationFrame(() => {
document.documentElement.dataset.geometryReady = "true";
if (!${root}) browserEval.send("geometry.scrollbars", {
vertical: innerWidth > document.documentElement.clientWidth,
horizontal: innerHeight > document.documentElement.clientHeight,
});
browserEval.send("geometry.ready", { root: ${root} });
}));
});
`,
});
},
};
@@ -33,12 +33,9 @@ BSK_AUTO_UPDATE=off pnpm eval:browser smoke --case snapshot-coordinates --bsk ./
Smoke alone does **not** certify coordinate accuracy; CLI observations do not expose raw boxes. Smoke alone does **not** certify coordinate accuracy; CLI observations do not expose raw boxes.
Use the geometric test above for the regression's numeric assertions. Use the geometric test above for the regression's numeric assertions.
## Separate existing boundary ## Related scrollbar regression
The OOPIF root uses non-occupying scrollbars in the unit-conversion regression. Root and The unit-conversion fixture keeps non-occupying OOPIF scrollbars by default. The same
same-process frames retain normal scrollbars. Append `&classic-scrollbars` to the fixture URL browser test also runs the dedicated [OOPIF scrollbar fixture](../oopif-scrollbars/README.md),
to reproduce the separate existing OOPIF projection issue: `targetProjection()` maps the CSS which covers occupied scrollbars, two OOPIF boundaries, clipping and real clicks.
layout viewport (which excludes occupying scrollbars) onto the entire owner content quad. Append `&classic-scrollbars` to this fixture's URL to enable its normal OOPIF scrollbars.
This inflates the projected coordinates when scrollbars occupy space. Main already has that
mapping; fixing it also changes shared live geometry and is outside this snapshot-unit fix.
The test does not relax its numeric tolerance to absorb that error.
+5 -4
View File
@@ -41,22 +41,23 @@ test("case manifests are discovered, ordered, and grouped into suites", () => {
"diagnostics", "diagnostics",
"mobile-emulation", "mobile-emulation",
"generated-form", "generated-form",
"oopif-scrollbars",
"snapshot-coordinates", "snapshot-coordinates",
], ],
); );
assert.deepEqual(repositorySummary(cases, fixtureRegistry).suites, { assert.deepEqual(repositorySummary(cases, fixtureRegistry).suites, {
core: 6, core: 6,
matrix: 1, matrix: 1,
regression: 1, regression: 2,
}); });
}); });
test("repository validation links every case to a fixture and valid workflow evidence", () => { test("repository validation links every case to a fixture and valid workflow evidence", () => {
assert.deepEqual(validateRepositoryCases(cases, fixtureRegistry), []); assert.deepEqual(validateRepositoryCases(cases, fixtureRegistry), []);
const summary = repositorySummary(cases, fixtureRegistry); const summary = repositorySummary(cases, fixtureRegistry);
assert.equal(summary.cases, 8); assert.equal(summary.cases, 9);
assert.equal(summary.fixtureModules, 9); assert.equal(summary.fixtureModules, 10);
assert.equal(summary.fixtureRoutes, 15); assert.equal(summary.fixtureRoutes, 18);
}); });
test("manifest validation rejects unknown operations and incomplete workflow steps", () => { test("manifest validation rejects unknown operations and incomplete workflow steps", () => {
+1 -1
View File
@@ -54,7 +54,7 @@ the `bsk` CLI and browser extension separately when a release requires it.
| `browser_session` | `start`, `stop`, `list` | Manage plugin-owned Agent Window sessions. | | `browser_session` | `start`, `stop`, `list` | Manage plugin-owned Agent Window sessions. |
| `browser_page` | `navigate`, `back`, `forward`, `reload`, `wait` | Navigate the active tab and wait for page lifecycle events. | | `browser_page` | `navigate`, `back`, `forward`, `reload`, `wait` | Navigate the active tab and wait for page lifecycle events. |
| `browser_inspect` | `observe`, `snapshot`, `html`, `screenshot`, `console`, `network` | Read semantic or diagnostic page state and capture screenshots. | | `browser_inspect` | `observe`, `snapshot`, `html`, `screenshot`, `console`, `network` | Read semantic or diagnostic page state and capture screenshots. |
| `browser_interact` | `click`, `hover`, `fill`, `select`, `press` | Interact with controls using fresh refs or selectors. | | `browser_interact` | `click`, `hover`, `focus`, `blur`, `fill`, `select`, `press` | Interact with controls using fresh refs or selectors. |
| `browser_tabs` | `list`, `create`, `select`, `close`, `borrow`, `return` | Manage Agent Window tabs and temporarily borrow user tabs. | | `browser_tabs` | `list`, `create`, `select`, `close`, `borrow`, `return` | Manage Agent Window tabs and temporarily borrow user tabs. |
| `browser_assist` | `resize`, `emulate`, `request-help` | Resize or emulate the browser and pause for human-only steps. | | `browser_assist` | `resize`, `emulate`, `request-help` | Resize or emulate the browser and pause for human-only steps. |
@@ -51,8 +51,8 @@ Use `browser_inspect` action `observe` as the primary semantic page view. It ret
text, and `@eN` refs. Prefer fresh refs over raw selectors. Refs invalidate after navigation and may text, and `@eN` refs. Prefer fresh refs over raw selectors. Refs invalidate after navigation and may
also become stale after large DOM changes, so observe again before the next interaction. also become stale after large DOM changes, so observe again before the next interaction.
Use `browser_interact` for click, hover, fill, select, and key actions. An observation marks a Use `browser_interact` for click, hover, focus, blur, fill, select, and key actions. Focus and blur
hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`. The listed items are explicitly enter or leave UI states triggered by focus changes. An observation marks a hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`. The listed items are
labels, not usable refs: hover the trigger, observe again, then act on the revealed item's own ref. labels, not usable refs: hover the trigger, observe again, then act on the revealed item's own ref.
Do not click the trigger itself unless the user wants the trigger's action. Do not click the trigger itself unless the user wants the trigger's action.
@@ -167,12 +167,14 @@ const BROWSER_TOOL_SPECS: BrowserToolSpec[] = [
{ {
name: "browser_interact", name: "browser_interact",
description: description:
"Interact with an element in the active Agent Window tab. Actions: click, hover, fill, select, " + "Interact with an element in the active Agent Window tab. Actions: click, hover, focus, blur, fill, select, " +
"press. click/hover/fill/select require target; fill also requires value; select requires " + "press. click/hover/focus/blur/fill/select require target; fill also requires value; select requires " +
"values; press requires key and may optionally focus target first.", "values; press requires key and may optionally focus target first.",
actions: { actions: {
click: "interact.click", click: "interact.click",
hover: "interact.hover", hover: "interact.hover",
focus: "interact.focus",
blur: "interact.blur",
fill: "interact.fill", fill: "interact.fill",
select: "interact.select", select: "interact.select",
press: "interact.press", press: "interact.press",
@@ -562,6 +562,10 @@ export function actionForLabel(label: string): string {
return "clicking"; return "clicking";
case "hover": case "hover":
return "hovering"; return "hovering";
case "focus":
return "focusing";
case "blur":
return "blurring";
case "fill": case "fill":
return "filling"; return "filling";
case "select": case "select":
@@ -13,7 +13,7 @@ import type { ToolDeps } from "./tools";
const MODIFIERS = ["alt", "ctrl", "meta", "shift"] as const; const MODIFIERS = ["alt", "ctrl", "meta", "shift"] as const;
/** Add hover and select without bypassing session ownership or observation. */ /** Add hover, focus, blur and select without bypassing session ownership or observation. */
export function registerPhaseOneInteractionTools( export function registerPhaseOneInteractionTools(
deps: ToolDeps, deps: ToolDeps,
register: ToolRegistrar, register: ToolRegistrar,
@@ -100,6 +100,88 @@ export function registerPhaseOneInteractionTools(
}), }),
); );
for (const action of ["focus", "blur"] as const) {
register(
defineTool({
name: `interact.${action}`,
description:
action === "focus"
? "Focus an element and verify its DOM focus state."
: "Remove focus from an element and report whether it was focused.",
parameters: {
target: {
type: "string",
required: true,
description: "Snapshot ref (@e3 / e3) or CSS selector of the element.",
},
session: SESSION_PARAM,
tabId: TAB_ID_PARAM,
timeoutMs: TIMEOUT_MS_PARAM,
},
output: {
schema: {
type: "object",
additionalProperties: false,
properties: {
session: { type: "string", required: true },
tabId: { type: "integer", required: true },
focused: { type: "boolean", required: true },
wasFocused: {
type: "boolean",
description: "Whether the target was focused before blur.",
},
},
},
render: (_args, value) => [
{
type: "text",
text:
`[session ${value.session}] ${action} on tab ${value.tabId}: focused=${value.focused}` +
(value.wasFocused === undefined ? "" : `, wasFocused=${value.wasFocused}`),
},
],
},
async execute(args, exec) {
requireNonEmpty(args.target, "target");
requirePositive(args.timeoutMs, "timeoutMs");
const sessionId = registry.resolve(args.session, `browser_interact(action=${action})`);
const cmdArgs = [action, "--session", sessionId];
appendTabId(cmdArgs, args.tabId);
if (args.timeoutMs !== undefined) cmdArgs.push("--timeout", `${args.timeoutMs}ms`);
appendTarget(cmdArgs, args.target);
const reply = (await runtime.run(
exec,
cmdArgs,
action,
sessionId,
runnerTimeout(deps, args.timeoutMs),
)) as {
tab_id: number;
focused: boolean;
was_focused?: boolean;
};
return {
session: sessionId,
tabId: reply.tab_id,
focused: reply.focused,
...(action === "blur" ? { wasFocused: reply.was_focused } : {}),
};
},
presentCall: (args) => ({
card: "terminal",
title: runtime.commandLine([
action,
args.target,
"--session",
args.session ?? "(current)",
]),
description: action === "focus" ? "Focus an element" : "Remove focus from an element",
}),
presentResult: runtime.presentTerminalResult,
}),
);
}
register( register(
defineTool({ defineTool({
name: "interact.select", name: "interact.select",
@@ -102,6 +102,8 @@ const ACTION_ROUTES: Record<string, readonly [string, string]> = {
"inspect.network": ["browser_inspect", "network"], "inspect.network": ["browser_inspect", "network"],
"interact.click": ["browser_interact", "click"], "interact.click": ["browser_interact", "click"],
"interact.hover": ["browser_interact", "hover"], "interact.hover": ["browser_interact", "hover"],
"interact.focus": ["browser_interact", "focus"],
"interact.blur": ["browser_interact", "blur"],
"interact.fill": ["browser_interact", "fill"], "interact.fill": ["browser_interact", "fill"],
"interact.select": ["browser_interact", "select"], "interact.select": ["browser_interact", "select"],
"interact.press": ["browser_interact", "press"], "interact.press": ["browser_interact", "press"],
@@ -200,7 +202,7 @@ const EXPECTED_ACTIONS = {
browser_session: ["start", "stop", "list"], browser_session: ["start", "stop", "list"],
browser_page: ["navigate", "back", "forward", "reload", "wait"], browser_page: ["navigate", "back", "forward", "reload", "wait"],
browser_inspect: ["observe", "snapshot", "html", "screenshot", "console", "network"], browser_inspect: ["observe", "snapshot", "html", "screenshot", "console", "network"],
browser_interact: ["click", "hover", "fill", "select", "press"], browser_interact: ["click", "hover", "focus", "blur", "fill", "select", "press"],
browser_tabs: ["list", "create", "select", "close", "borrow", "return"], browser_tabs: ["list", "create", "select", "close", "borrow", "return"],
browser_assist: ["resize", "emulate", "request-help"], browser_assist: ["resize", "emulate", "request-help"],
} as const; } as const;
@@ -499,6 +501,60 @@ describe("interaction tools", () => {
]); ]);
}); });
it.each([
"focus",
"blur",
] as const)("interact.%s maps focus state, target, tab and timeout", async (action) => {
const { tools, calls } = setup({
"session start": START_REPLY("s1"),
[action]: {
tab_id: 7,
focused: action === "focus",
...(action === "blur" ? { was_focused: true } : {}),
},
});
await startSession(tools);
const target = action === "focus" ? "@e3" : "#field";
const value = await tools
.get(`interact.${action}`)
?.execute({ target, tabId: 7, timeoutMs: 150_000 }, makeExec());
expect(value).toEqual({
session: "s1",
tabId: 7,
focused: action === "focus",
...(action === "blur" ? { wasFocused: true } : {}),
});
expect(calls[1].args).toEqual([
action,
"--session",
"s1",
"--tab-id",
"7",
"--timeout",
"150000ms",
target,
]);
expect(calls[1].options.timeoutMs).toBe(165_000);
});
it.each([
"focus",
"blur",
] as const)("interact.%s validates arguments and session ownership before running", async (action) => {
const { tools, calls } = setup({ "session start": START_REPLY("s1") });
await startSession(tools);
const tool = tools.get(`interact.${action}`)!;
await expect(tool.execute({}, makeExec())).rejects.toThrow(/invalid arguments/i);
await expect(tool.execute({ target: " " }, makeExec())).rejects.toThrow(/non-empty/i);
await expect(tool.execute({ target: "e1", timeoutMs: 0 }, makeExec())).rejects.toThrow(
/greater than zero/i,
);
await expect(tool.execute({ target: "e1", session: "foreign" }, makeExec())).rejects.toThrow(
/session/i,
);
expect(calls).toHaveLength(1);
});
it("interact.fill passes value and --no-clear", async () => { it("interact.fill passes value and --no-clear", async () => {
const { tools, calls } = setup(responses); const { tools, calls } = setup(responses);
await startSession(tools); await startSession(tools);
+5 -2
View File
@@ -55,10 +55,13 @@ Use this default loop:
```text ```text
bsk navigate <url> --session <id> bsk navigate <url> --session <id>
bsk observe --session <id> bsk observe --session <id>
bsk click|hover|fill|select|press ... --session <id> bsk click|hover|focus|blur|fill|select|press ... --session <id>
bsk observe --session <id> # after navigation or a meaningful DOM change bsk observe --session <id> # after navigation or a meaningful DOM change
``` ```
`bsk focus <ref>` explicitly focuses a target; `bsk blur <ref>` removes focus and reports whether
it was focused. Use these for UI states triggered by focus changes.
Prefer fresh `@eN` refs over CSS selectors. Navigation invalidates refs; large DOM changes may also Prefer fresh `@eN` refs over CSS selectors. Navigation invalidates refs; large DOM changes may also
make them stale. Observe again before the next interaction. make them stale. Observe again before the next interaction.
@@ -114,7 +117,7 @@ This list of names is complete. Never invent a command outside it; read
session start|stop|list browsers status doctor update logs session start|stop|list browsers status doctor update logs
navigate navigate-back navigate-forward reload wait-for-navigation wait-ms navigate navigate-back navigate-forward reload wait-for-navigation wait-ms
observe snapshot get-html screenshot console network observe snapshot get-html screenshot console network
click hover fill select press evaluate click hover focus blur fill select press evaluate
tab list|create|close|select|borrow|return window resize emulate tab list|create|close|select|borrow|return window resize emulate
upload download request-help record start|stop upload download request-help record start|stop
``` ```